diff --git a/.clang-tidy b/.clang-tidy index 2d495081c8d..4e1463affba 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,86 +1,58 @@ -Checks: '-*, - bugprone-* +Checks: ' + -*, + bugprone-*, -bugprone-easily-swappable-parameters, -bugprone-implicit-widening-of-multiplication-result, -bugprone-misplaced-widening-cast, -bugprone-unchecked-optional-access, - cert-* + cert-*, -cert-err58-cpp, - cppcoreguidelines-avoid-goto, - cppcoreguidelines-interfaces-global-init, + clang-diagnostic-*, + cppcoreguidelines-*, + -cppcoreguidelines-avoid-c-arrays, + -cppcoreguidelines-avoid-magic-numbers, -cppcoreguidelines-avoid-non-const-global-variables, -cppcoreguidelines-init-variables, -cppcoreguidelines-macro-usage, -cppcoreguidelines-narrowing-conversions, - -cppcoreguidelines-no-malloc, + -cppcoreguidelines-non-private-member-variables-in-classes, -cppcoreguidelines-owning-memory, + -cppcoreguidelines-pro-*, google-build-explicit-make-pair, google-build-namespaces, google-global-names-in-headers, - misc-const-correctness, - misc-misleading-bidirectional, - misc-misleading-identifier, - misc-misplaced-const, - misc-uniqueptr-reset-release, - misc-unused-alias-decls, - misc-unused-parameters, - misc-unused-using-decls, - -misc-definitions-in-headers, - modernize-avoid-bind, - modernize-concat-nested-namespaces, - modernize-deprecated-headers, - modernize-deprecated-ios-base-aliases, - modernize-loop-convert, - modernize-make-shared, - modernize-make-unique, - modernize-pass-by-value, - modernize-raw-string-literal, - modernize-redundant-void-arg, - modernize-replace-auto-ptr, - modernize-replace-disallow-copy-and-assign-macro, - modernize-replace-random-shuffle, - modernize-return-braced-init-list, - modernize-shrink-to-fit, - modernize-unary-static-assert, - modernize-use-nullptr, + misc-*, + -misc-no-recursion, + -misc-non-private-member-variables-in-classes, + modernize-*, + -modernize-avoid-c-arrays, + -modernize-return-braced-init-list, + -modernize-use-trailing-return-type, mpi-*, - performance-faster-string-find, - performance-for-range-copy, - performance-implicit-conversion-in-loop, - performance-inefficient-algorithm, - performance-inefficient-string-concatenation, - performance-inefficient-vector-operation, - performance-move-const-arg, - performance-move-constructor-init, - performance-no-automatic-move, - performance-no-int-to-ptr, - readability-avoid-const-params-in-decls, - readability-const-return-type, - readability-container-contains, - readability-container-data-pointer, - readability-container-size-empty, - readability-non-const-parameter, - readability-redundant-control-flow, - readability-redundant-declaration, - readability-redundant-function-ptr-dereference, - readability-redundant-member-init, - readability-redundant-preprocessor, - readability-redundant-smartptr-get, - readability-redundant-string-cstr, - readability-redundant-string-init, - readability-simplify-boolean-expr, - readability-simplify-subscript-expr, - readability-static-accessed-through-instance, - readability-static-definition-in-anonymous-namespace, - readability-string-compare, - readability-suspicious-call-argument, - readability-uniqueptr-delete-release, - readability-use-anyofallof, + performance-*, + -performance-unnecessary-copy-initialization, + -performance-unnecessary-value-param, + portability-*, + readability-*, + -readability-convert-member-functions-to-static, + -readability-else-after-return, + -readability-function-cognitive-complexity, + -readability-identifier-length, + -readability-implicit-bool-conversion, + -readability-isolate-declaration, + -readability-magic-numbers, + -readability-make-member-function-const, + -readability-named-parameter, -readability-uppercase-literal-suffix ' CheckOptions: +- key: bugprone-narrowing-conversions.WarnOnIntegerToFloatingPointNarrowingConversion + value: "false" +- key: misc-definitions-in-headers.HeaderFileExtensions + value: "H," - key: modernize-pass-by-value.ValuesOnly - value: 'true' + value: "true" + HeaderFilterRegex: 'Source[a-z_A-Z0-9\/]+\.H$' diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index adfb827e4d8..b8b0053adaa 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -16,28 +16,26 @@ jobs: - name: install dependencies run: | .github/workflows/dependencies/clang14.sh - - name: build WarpX using clang-tidy + - name: set up cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX & run clang-tidy run: | - + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=300M + export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy + export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt + ccache -z export CXX=$(which clang++) export CC=$(which clang) - # The following wrapper ensures that only source files - # in WarpX/Source/* are actually processed by clang-tidy - #_______________________________ - cat > clang_tidy_wrapper << EOF - #!/bin/bash - REGEX="[a-z_A-Z0-9\/]*WarpX\/Source[a-z_A-Z0-9\/]+.cpp" - if [[ \$4 =~ \$REGEX ]];then - clang-tidy \$@ - fi - EOF - chmod +x clang_tidy_wrapper - #_____________________________________ - cmake -S . -B build_clang_tidy \ - -DCMAKE_CXX_CLANG_TIDY="$PWD/clang_tidy_wrapper;--system-headers=0;--config-file=$PWD/.clang-tidy" \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="1;2;3;RZ" \ -DWarpX_MPI=ON \ @@ -46,9 +44,30 @@ jobs: -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ - -DWarpX_PRECISION=SINGLE + -DWarpX_PRECISION=SINGLE \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + + cmake --build build_clang_tidy -j 2 - cmake --build build_clang_tidy -j 2 2> build_clang_tidy/clang-tidy.log + ${{github.workspace}}/.github/workflows/source/makeMakefileForClangTidy.py --input ${{github.workspace}}/ccache.log.txt + make -j2 --keep-going -f clang-tidy-ccache-misses.mak \ + CLANG_TIDY=clang-tidy \ + CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*" - cat build_clang_tidy/clang-tidy.log - if [[ $(wc -m pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml new file mode 100644 index 00000000000..978e9c28f04 --- /dev/null +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -0,0 +1,40 @@ +name: CleanUpCachePostPR + +on: + workflow_run: + workflows: [PostPR] + types: + - completed + +jobs: + CleanUpCcacheCachePostPR: + name: Clean Up Ccache Cache Post PR + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Clean up ccache + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1) + # $keys might contain spaces. Thus we set IFS to \n. + IFS=$'\n' + for k in $keys + do + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm + done + unset IFS diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml new file mode 100644 index 00000000000..6421bbf4215 --- /dev/null +++ b/.github/workflows/cleanup-cache.yml @@ -0,0 +1,63 @@ +name: CleanUpCache + +on: + workflow_run: + workflows: [🧹 clang-tidy, 🔍 CodeQL, 🐧 CUDA, 🐧 HIP, 🐧 Intel, 🍏 macOS, 🐧 OpenMP] + types: + - completed + +jobs: + CleanUpCcacheCache: + name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }} + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Clean up ccache + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + # push or pull_request or schedule or ... + EVENT=${{ github.event.workflow_run.event }} + + # Triggering workflow run name (e.g., LinuxClang) + WORKFLOW_NAME="${{ github.event.workflow_run.name }}" + + if [[ $EVENT == "pull_request" ]]; then + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + else + BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} + fi + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + # In our cache keys, substring after `-git-` is git hash, substring + # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`). + # The goal is to keep the last used key of each job and delete all others. + + # something like ccache-LinuxClang- + keyprefix="ccache-${WORKFLOW_NAME}-" + + cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq) + + # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d". + # It might also contain spaces. Thus we set IFS to \n. + IFS=$'\n' + for j in $cached_jobs + do + old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) + for k in $old_keys + do + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm + done + done + unset IFS diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 55d0e473d27..436df798d3b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -35,13 +35,22 @@ jobs: if: ${{ matrix.language == 'cpp' }} run: | sudo apt-get update - sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libadios-openmpi-dev + sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libadios-openmpi-dev ccache python -m pip install --upgrade pip python -m pip install --upgrade wheel python -m pip install --upgrade cmake export CMAKE="$HOME/.local/bin/cmake" && echo "CMAKE=$CMAKE" >> $GITHUB_ENV + - name: Set Up Cache + if: ${{ matrix.language == 'cpp' }} + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: Configure (C++) if: ${{ matrix.language == 'cpp' }} run: | @@ -61,6 +70,19 @@ jobs: - name: Build (C++) if: ${{ matrix.language == 'cpp' }} run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + $CMAKE --build build -j 2 + + ccache -s + du -hs ~/.cache/ccache + + # Make sure CodeQL has something to do + touch Source/Utils/WarpXVersion.cpp + export CCACHE_DISABLE=1 $CMAKE --build build -j 2 - name: Perform CodeQL Analysis @@ -88,3 +110,18 @@ jobs: uses: github/codeql-action/upload-sarif@v2 with: sarif_file: sarif-results/${{ matrix.language }}.sarif + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 88037297d35..29a434dc579 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -18,10 +18,6 @@ jobs: env: CXXFLAGS: "-Werror" CMAKE_GENERATOR: Ninja - # setuptools/mp4py work-around, see - # https://github.com/mpi4py/mpi4py/pull/159 - # https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274 - SETUPTOOLS_USE_DISTUTILS: stdlib steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -33,18 +29,18 @@ jobs: .github/workflows/dependencies/nvcc11-3.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-cuda-nvcc-${{ hashFiles('.github/workflows/cuda.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-cuda-nvcc-${{ hashFiles('.github/workflows/cuda.yml') }}- - ccache-cuda-nvcc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: install openPMD-api run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" cmake-easyinstall --prefix=/usr/local \ @@ -57,6 +53,10 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} which nvcc || echo "nvcc not in PATH!" @@ -75,12 +75,16 @@ jobs: -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build_sp -j 2 - python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade build packaging setuptools wheel export WARPX_MPI=ON export PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel . python3 -m pip install *.whl + ccache -s + du -hs ~/.cache/ccache + # make sure legacy build system continues to build, i.e., that we don't forget # to add new .cpp files build_nvcc_gnumake: @@ -94,55 +98,54 @@ jobs: .github/workflows/dependencies/nvcc11-8.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-cuda-gnumake-${{ hashFiles('.github/workflows/cuda.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-cuda-gnumake-${{ hashFiles('.github/workflows/cuda.yml') }}- - ccache-cuda-gnumake- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach b98bdae9fb67e5d9aafc488de92c53001bd323ec && cd - + cd ../amrex && git checkout --detach 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 + ccache -s + du -hs ~/.cache/ccache + build_nvhpc21-11-nvcc: name: NVHPC@21.11 NVCC/NVC++ Release [tests] runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false - env: - # For NVHPC, Ninja is slower than the default: - #CMAKE_GENERATOR: Ninja - # setuptools/mp4py work-around, see - # https://github.com/mpi4py/mpi4py/pull/159 - # https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274 - SETUPTOOLS_USE_DISTUTILS: stdlib + #env: + # # For NVHPC, Ninja is slower than the default: + # CMAKE_GENERATOR: Ninja steps: - uses: actions/checkout@v3 - name: Dependencies run: .github/workflows/dependencies/nvhpc.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-cuda-nvhpc-${{ hashFiles('.github/workflows/cuda.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-cuda-nvhpc-${{ hashFiles('.github/workflows/cuda.yml') }}- - ccache-cuda-nvhpc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + source /etc/profile.d/modules.sh module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11 which nvcc || echo "nvcc not in PATH!" @@ -175,8 +178,27 @@ jobs: # https://github.com/mpi4py/mpi4py/issues/114 #export CFLAGS="-noswitcherror" - #python3 -m pip install --upgrade pip setuptools wheel + #python3 -m pip install --upgrade pip + #python3 -m pip install --upgrade build packaging setuptools wheel #export WARPX_MPI=ON #export PYWARPX_LIB_DIR=$PWD/build/lib/site-packages/pywarpx/ #python3 -m pip wheel . #python3 -m pip install *.whl + + ccache -s + du -hs ~/.cache/ccache + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/dependencies/ccache.sh b/.github/workflows/dependencies/ccache.sh new file mode 100755 index 00000000000..30a155ff20c --- /dev/null +++ b/.github/workflows/dependencies/ccache.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +if [[ $# -eq 2 ]]; then + CVER=$1 +else + CVER=4.8.3 +fi + +wget https://github.com/ccache/ccache/releases/download/v${CVER}/ccache-${CVER}-linux-x86_64.tar.xz +tar xvf ccache-${CVER}-linux-x86_64.tar.xz +sudo cp -f ccache-${CVER}-linux-x86_64/ccache /usr/local/bin/ diff --git a/.github/workflows/dependencies/clang14.sh b/.github/workflows/dependencies/clang14.sh index 18274d5e46c..27597e06c5d 100755 --- a/.github/workflows/dependencies/clang14.sh +++ b/.github/workflows/dependencies/clang14.sh @@ -15,7 +15,6 @@ echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries sudo apt-get -qqq update sudo apt-get install -y \ cmake \ - ccache \ clang-14 \ clang-tidy-14 \ libblas-dev \ @@ -29,6 +28,9 @@ sudo apt-get install -y \ libomp-dev \ ninja-build +# ccache +$(dirname "$0")/ccache.sh + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh index 9ecc5e4ca19..3b146405b4b 100755 --- a/.github/workflows/dependencies/dpcpp.sh +++ b/.github/workflows/dependencies/dpcpp.sh @@ -34,7 +34,6 @@ for itry in {1..5} do sudo apt-get install -y --no-install-recommends \ build-essential \ - ccache \ cmake \ intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \ g++ gfortran \ @@ -58,3 +57,6 @@ sudo rm -rf /opt/intel/oneapi/mkl/latest/lib/intel64/*.a \ du -sh /opt/intel/oneapi/ du -sh /opt/intel/oneapi/*/* df -h + +# ccache +$(dirname "$0")/ccache.sh diff --git a/.github/workflows/dependencies/gcc.sh b/.github/workflows/dependencies/gcc.sh index d845463a682..fb2351f8a92 100755 --- a/.github/workflows/dependencies/gcc.sh +++ b/.github/workflows/dependencies/gcc.sh @@ -16,9 +16,11 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ gnupg \ ninja-build \ pkg-config \ wget + +# ccache +$(dirname "$0")/ccache.sh diff --git a/.github/workflows/dependencies/gcc12.sh b/.github/workflows/dependencies/gcc12.sh index 01f30f07a31..fb46c6b811b 100755 --- a/.github/workflows/dependencies/gcc12.sh +++ b/.github/workflows/dependencies/gcc12.sh @@ -16,7 +16,6 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ g++-12 \ gnupg \ @@ -28,3 +27,6 @@ sudo apt-get install -y \ ninja-build \ pkg-config \ wget + +# ccache +$(dirname "$0")/ccache.sh diff --git a/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh b/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh index 65f029ea67b..9ff3d615a3e 100755 --- a/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh +++ b/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh @@ -16,7 +16,6 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ g++-12 \ gnupg \ @@ -30,6 +29,9 @@ sudo apt-get install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 2e7830368ca..2225e670bb0 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -41,7 +41,11 @@ sudo apt-get install -y --no-install-recommends \ rocm-dev \ rocfft-dev \ rocprim-dev \ - rocrand-dev + rocrand-dev \ + hiprand-dev + +# ccache +$(dirname "$0")/ccache.sh # activate # @@ -56,12 +60,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/icc.sh b/.github/workflows/dependencies/icc.sh index 5242a836a42..74f0db4b46a 100755 --- a/.github/workflows/dependencies/icc.sh +++ b/.github/workflows/dependencies/icc.sh @@ -17,12 +17,14 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ gnupg \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + # Ref.: https://github.com/rscohn2/oneapi-ci sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB diff --git a/.github/workflows/dependencies/nvcc11-3.sh b/.github/workflows/dependencies/nvcc11-3.sh index ed9fd211128..92e2717e425 100755 --- a/.github/workflows/dependencies/nvcc11-3.sh +++ b/.github/workflows/dependencies/nvcc11-3.sh @@ -26,6 +26,9 @@ sudo apt-get install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb sudo dpkg -i cuda-keyring_1.0-1_all.deb @@ -55,12 +58,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/nvcc11-8.sh b/.github/workflows/dependencies/nvcc11-8.sh index 33af2c1ade5..6089360392b 100755 --- a/.github/workflows/dependencies/nvcc11-8.sh +++ b/.github/workflows/dependencies/nvcc11-8.sh @@ -26,6 +26,9 @@ sudo apt-get install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb sudo dpkg -i cuda-keyring_1.0-1_all.deb @@ -55,12 +58,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/nvhpc.sh b/.github/workflows/dependencies/nvhpc.sh index 9c6166d386d..be2ff29ca1e 100755 --- a/.github/workflows/dependencies/nvhpc.sh +++ b/.github/workflows/dependencies/nvhpc.sh @@ -25,6 +25,9 @@ sudo apt install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | \ sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt update -y && \ @@ -46,12 +49,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/pyfull.sh b/.github/workflows/dependencies/pyfull.sh index b0a4f607a51..12678763da5 100755 --- a/.github/workflows/dependencies/pyfull.sh +++ b/.github/workflows/dependencies/pyfull.sh @@ -16,7 +16,6 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ clang \ cmake \ gnupg \ @@ -35,6 +34,9 @@ sudo apt-get install -y \ python3-setuptools \ wget +# ccache +$(dirname "$0")/ccache.sh + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 40e5055cd8b..f7378bfa775 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -21,19 +21,19 @@ jobs: run: .github/workflows/dependencies/hip.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-hip-3dsp-${{ hashFiles('.github/workflows/hip.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-hip-3dsp-${{ hashFiles('.github/workflows/hip.yml') }}- - ccache-hip-3dsp- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + source /etc/profile.d/rocm.sh hipcc --version which clang @@ -63,6 +63,9 @@ jobs: python3 -m pip wheel . python3 -m pip install *.whl + ccache -s + du -hs ~/.cache/ccache + build_hip_2d_dp: name: HIP 2D DP runs-on: ubuntu-20.04 @@ -77,19 +80,19 @@ jobs: run: .github/workflows/dependencies/hip.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-hip-2ddp-${{ hashFiles('.github/workflows/hip.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-hip-2ddp-${{ hashFiles('.github/workflows/hip.yml') }}- - ccache-hip-2ddp- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + source /etc/profile.d/rocm.sh hipcc --version which clang @@ -119,3 +122,21 @@ jobs: export PYWARPX_LIB_DIR=$PWD/build_2d/lib/site-packages/pywarpx/ python3 -m pip wheel . python3 -m pip install *.whl + + ccache -s + du -hs ~/.cache/ccache + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index aba8200f1b2..9124715fe18 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -23,25 +23,27 @@ jobs: .github/workflows/dependencies/icc.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-intel-icc-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-intel-icc-${{ hashFiles('.github/workflows/intel.yml') }}- - ccache-intel-icc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=200M + export CCACHE_DEPEND=1 + ccache -z + set +eu source /opt/intel/oneapi/setvars.sh set -eu export CXX=$(which icpc) export CC=$(which icc) - python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade build packaging setuptools wheel cmake -S . -B build_dp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -65,6 +67,9 @@ jobs: cmake --build build_sp -j 2 cmake --build build_sp --target pip_install + ccache -s + du -hs ~/.cache/ccache + build_icpx: name: oneAPI ICX SP runs-on: ubuntu-20.04 @@ -84,19 +89,20 @@ jobs: .github/workflows/dependencies/dpcpp.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-intel-icpx-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-intel-icpx-${{ hashFiles('.github/workflows/intel.yml') }}- - ccache-intel-icpx- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + export CCACHE_DEPEND=1 + ccache -z + set +e source /opt/intel/oneapi/setvars.sh set -e @@ -104,7 +110,7 @@ jobs: export CC=$(which icx) python3 -m pip install --upgrade pip - python3 -m pip install --upgrade setuptools wheel + python3 -m pip install --upgrade build packaging setuptools wheel cmake -S . -B build_sp \ -DCMAKE_CXX_FLAGS_RELEASE="-O1 -DNDEBUG" \ @@ -117,6 +123,9 @@ jobs: cmake --build build_sp -j 2 cmake --build build_sp --target pip_install + ccache -s + du -hs ~/.cache/ccache + - name: run pywarpx run: | set +e @@ -144,19 +153,20 @@ jobs: .github/workflows/dependencies/dpcpp.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-intel-dpcc-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-intel-dpcc-${{ hashFiles('.github/workflows/intel.yml') }}- - ccache-intel-dpcc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + export CCACHE_DEPEND=1 + ccache -z + set +e source /opt/intel/oneapi/setvars.sh set -e @@ -176,8 +186,26 @@ jobs: -DWarpX_PRECISION=SINGLE cmake --build build_sp -j 2 + ccache -s + du -hs ~/.cache/ccache + # Skip this as it will copy the binary artifacts and we are tight on disk space # python3 -m pip install --upgrade pip - # python3 -m pip install --upgrade setuptools wheel + # python3 -m pip install --upgrade build packaging setuptools wheel # PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel . # python3 -m pip install *.whl + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2fd979feff9..0e8819032e3 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -16,56 +16,50 @@ jobs: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: TRUE # For macOS, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja - # setuptools/mp4py work-around, see - # https://github.com/mpi4py/mpi4py/pull/159 - # https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274 - SETUPTOOLS_USE_DISTUTILS: stdlib steps: - uses: actions/checkout@v3 - - name: Brew Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key - with: - path: | - /usr/local/bin - /usr/local/lib - /usr/local/share - /Users/runner/Library/Caches/Homebrew - key: brew-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }} - restore-keys: | - brew-macos-appleclang- - name: install dependencies run: | - brew --cache set +e brew unlink gcc brew update + brew upgrade || true brew install --overwrite python brew install ccache brew install fftw brew install libomp - brew link --force libomp + brew link --overwrite --force libomp brew install ninja brew install open-mpi brew install pkg-config set -e brew tap openpmd/openpmd brew install openpmd-api + + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade virtualenv + + python3 -m venv py-venv + source py-venv/bin/activate + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade build packaging setuptools wheel + python3 -m pip install --upgrade mpi4py - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: path: /Users/runner/Library/Caches/ccache - key: ccache-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }}- - ccache-macos-appleclang- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | - python3 -m pip install --upgrade pip - python3 -m pip install --upgrade pip setuptools wheel + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + export CCACHE_DEPEND=1 + ccache -z + + source py-venv/bin/activate cmake -S . -B build_dp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -85,7 +79,26 @@ jobs: cmake --build build_sp -j 3 cmake --build build_sp --target pip_install + ccache -s + - name: run pywarpx run: | + source py-venv/bin/activate export OMP_NUM_THREADS=1 + mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml new file mode 100644 index 00000000000..f5b914033b7 --- /dev/null +++ b/.github/workflows/post-pr.yml @@ -0,0 +1,20 @@ +name: PostPR +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/source/makeMakefileForClangTidy.py b/.github/workflows/source/makeMakefileForClangTidy.py new file mode 100755 index 00000000000..07809187dbd --- /dev/null +++ b/.github/workflows/source/makeMakefileForClangTidy.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +""" +A script processing ccache log file to make Make file for Clang-Tidy + +This generates a makefile for clang-tidying ccache-cache-missing files. This +could be used to speed up clang-tidy in CIs. +""" + +import argparse +import re +import sys + + +def makeMakefileForClangTidy(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--input", + help="Ccache log file", + default="ccache.log.txt") + parser.add_argument("--identifier", + help="Unique identifier for finding compilation line in the log file", + default="WarpX/Source") + # We assume WarpX/Source can be used as an identifier to distinguish + # WarpX code from amrex, openMPD, and cmake's temporary files like + # build/CMakeFiles/CMakeScratch/TryCompile-hw3x4m/test_mpi.cpp + parser.add_argument("--output", + help="Make file for clang-tidy", + default="clang-tidy-ccache-misses.mak") + args = parser.parse_args() + + fin = open(args.input, "r") + fout = open(args.output, "w") + + fout.write("CLANG_TIDY ?= clang-tidy\n") + fout.write("override CLANG_TIDY_ARGS += --extra-arg=-Wno-unknown-warning-option --extra-arg-before=--driver-mode=g++\n") + fout.write("\n") + + fout.write(".SECONDEXPANSION:\n") + fout.write("clang-tidy: $$(all_targets)\n") + fout.write("\t@echo SUCCESS\n\n") + + exe_re = re.compile(r" Executing .*? (-.*{}.*) -c .* -o .* (\S*)".format(args.identifier)) + + count = 0 + for line in fin.readlines(): + ret_exe_re = exe_re.search(line) + if (ret_exe_re): + fout.write("target_{}: {}\n".format(count, ret_exe_re.group(2))) + fout.write("\t$(CLANG_TIDY) $(CLANG_TIDY_ARGS) $< -- {}\n".format + (ret_exe_re.group(1))) + fout.write("\ttouch target_{}\n\n".format(count)) + count = count + 1 + + fout.write("all_targets =") + for i in range(count): + fout.write(" target_{}".format(i)) + fout.write("\n\n") + + fout.write("clean:\n\t$(RM) $(all_targets)\n\n") + + fout.close() + fin.close() + +if __name__ == "__main__": + makeMakefileForClangTidy(sys.argv) diff --git a/.github/workflows/source/wrongFileNameInExamples b/.github/workflows/source/wrongFileNameInExamples index e116b9df084..0de69d69c9c 100755 --- a/.github/workflows/source/wrongFileNameInExamples +++ b/.github/workflows/source/wrongFileNameInExamples @@ -18,6 +18,7 @@ do [[ ${file:0:12} != PICMI_inputs ]] && [[ ${file:0:8 } != analysis ]] && [[ ${file: -4} != yaml ]] && + [[ ${file:0:4 } != plot ]] && [[ ${file:0:6 } != README ]] then files+=($file) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 0b43760b621..6b8e26111b8 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -20,18 +20,18 @@ jobs: .github/workflows/dependencies/gcc.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-cxxminimal-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-cxxminimal-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-cxxminimal- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;3" \ @@ -42,6 +42,9 @@ jobs: ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + ccache -s + du -hs ~/.cache/ccache + build_1D_2D: name: GCC 1D & 2D w/ MPI runs-on: ubuntu-22.04 @@ -57,18 +60,18 @@ jobs: .github/workflows/dependencies/gcc12.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-1D-2D-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-1D-2D-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-1D-2D- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + cmake -S . -B build \ -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -80,6 +83,9 @@ jobs: ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d + ccache -s + du -hs ~/.cache/ccache + build_3D_sp: name: GCC 3D & RZ w/ MPI, single precision runs-on: ubuntu-22.04 @@ -94,18 +100,17 @@ jobs: .github/workflows/dependencies/gcc12_blaspp_lapackpp.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-3D-RZ-SP-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-3D-RZ-SP-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-3D-RZ-SP- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=200M + ccache -z # we need to define this *after* having installed the dependencies, # because the compilation of blaspp raises warnings. @@ -125,6 +130,9 @@ jobs: ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + ccache -s + du -hs ~/.cache/ccache + build_gcc_ablastr: name: GCC ABLASTR w/o MPI runs-on: ubuntu-20.04 @@ -140,24 +148,27 @@ jobs: sudo apt-get install -y libopenmpi-dev openmpi-bin - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-gccablastr-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-gccablastr-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-gccablastr- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_APP=OFF \ -DWarpX_LIB=OFF cmake --build build -j 2 + ccache -s + du -hs ~/.cache/ccache + build_pyfull: name: Clang pywarpx runs-on: ubuntu-20.04 @@ -167,10 +178,6 @@ jobs: CXX: clang++ # On CI for this test, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja - # setuptools/mp4py work-around, see - # https://github.com/mpi4py/mpi4py/pull/159 - # https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274 - SETUPTOOLS_USE_DISTUTILS: stdlib steps: - uses: actions/checkout@v3 - name: install dependencies @@ -178,19 +185,20 @@ jobs: .github/workflows/dependencies/pyfull.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-pyfull-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-pyfull-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-pyfull- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | - python3 -m pip install --upgrade pip setuptools wheel + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade build packaging setuptools wheel export CXXFLAGS="-Werror -Wno-error=pass-failed" @@ -202,7 +210,25 @@ jobs: -DWarpX_QED_TABLE_GEN=ON cmake --build build -j 2 --target pip_install + ccache -s + du -hs ~/.cache/ccache + - name: run pywarpx run: | export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e0629e3abca..8e2bb00f1db 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -41,7 +41,9 @@ jobs: cmake --build build --config Debug --parallel 2 if(!$?) { Exit $LASTEXITCODE } - python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install --upgrade pip + if(!$?) { Exit $LASTEXITCODE } + python3 -m pip install --upgrade build packaging setuptools wheel if(!$?) { Exit $LASTEXITCODE } cmake --build build --config Debug --target install if(!$?) { Exit $LASTEXITCODE } @@ -64,7 +66,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: '3.8' - uses: seanmiddleditch/gha-setup-ninja@master - name: CCache Cache uses: actions/cache@v3 @@ -100,7 +102,9 @@ jobs: cmake --build build --config Release --target install if errorlevel 1 exit 1 - python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install --upgrade pip + if errorlevel 1 exit 1 + python3 -m pip install --upgrade build packaging setuptools wheel if errorlevel 1 exit 1 python3 -m pip install --upgrade -r requirements.txt if errorlevel 1 exit 1 diff --git a/.gitignore b/.gitignore index d614c1ae2d9..4fa65351331 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ Docs/warpx-doxygen-web.tag.xml Docs/openpmd-api-doxygen-web.tag.xml Docs/doxyhtml/ Docs/doxyxml/ -Docs/source/_static/ +Docs/source/_static/doxyhtml #################### # Package Managers # diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1c32c94c99c..35c38ac1ad6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty' # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] @@ -68,7 +68,7 @@ repos: # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v2.2.2 + rev: v2.4.0 hooks: - id: pycln name: pycln (python) @@ -76,7 +76,7 @@ repos: # Sorts Python imports according to PEP8 # https://www.python.org/dev/peps/pep-0008/#imports - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2501c9dcaff..e6aa3c8174f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 23.09) +project(WarpX VERSION 23.12) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) @@ -127,7 +127,7 @@ option(WarpX_PYTHON_IPO ) set(pyWarpX_VERSION_INFO "" CACHE STRING - "PEP-440 conformant version (set by distutils)") + "PEP-440 conformant version (set by setup.py)") # enforce consistency of dependent options if(WarpX_APP OR WarpX_PYTHON) @@ -141,10 +141,14 @@ option(ABLASTR_POSITION_INDEPENDENT_CODE "Build ABLASTR with position independent code" ON) mark_as_advanced(ABLASTR_POSITION_INDEPENDENT_CODE) +option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_PSATD}) +if(WarpX_PSATD) + set(ABLASTR_FFT ON CACHE STRING "compile AnyFFT wrappers" FORCE) +endif() + # this defined the variable BUILD_TESTING which is ON by default #include(CTest) - # Dependencies ################################################################ # @@ -372,6 +376,7 @@ if(WarpX_PYTHON) endif() add_subdirectory(Source/ablastr) + if(WarpX_LIB) add_subdirectory(Source/AcceleratorLattice) add_subdirectory(Source/BoundaryConditions) @@ -380,6 +385,7 @@ if(WarpX_LIB) add_subdirectory(Source/Evolve) add_subdirectory(Source/FieldSolver) add_subdirectory(Source/Filter) + add_subdirectory(Source/Fluids) add_subdirectory(Source/Initialization) add_subdirectory(Source/Laser) add_subdirectory(Source/Parallelization) @@ -402,7 +408,7 @@ foreach(D IN LISTS WarpX_DIMS) target_link_libraries(ablastr_${SD} PUBLIC WarpX::thirdparty::amrex_${D}d) endif() - if(WarpX_PSATD) + if(ABLASTR_FFT) target_link_libraries(ablastr_${SD} PUBLIC WarpX::thirdparty::FFT) if(D STREQUAL "RZ") target_link_libraries(ablastr_${SD} PUBLIC blaspp) @@ -499,6 +505,10 @@ foreach(D IN LISTS WarpX_DIMS) if(WarpX_PSATD) target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_PSATD) endif() + if(ABLASTR_FFT) + # We need to enable FFT support in ABLASTR for PSATD solver + target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT) + endif() if(WarpX_PYTHON AND pyWarpX_VERSION_INFO) # for module __version__ @@ -644,7 +654,7 @@ if(WarpX_PYTHON) ${CMAKE_COMMAND} -E rm -f -r warpx-whl COMMAND ${CMAKE_COMMAND} -E env PYWARPX_LIB_DIR=$ - python3 -m pip wheel -v --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR} + ${Python_EXECUTABLE} -m pip wheel -v --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR} WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS @@ -658,7 +668,7 @@ if(WarpX_PYTHON) set(pyWarpX_REQUIREMENT_FILE "requirements.txt") endif() add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements - python3 -m pip install ${PYINSTALLOPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}" + ${Python_EXECUTABLE} -m pip install ${PYINSTALLOPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}" WORKING_DIRECTORY ${WarpX_BINARY_DIR} ) @@ -675,7 +685,7 @@ if(WarpX_PYTHON) # because otherwise pip would also force reinstall all dependencies. add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI} - python3 -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx + ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx WORKING_DIRECTORY ${WarpX_BINARY_DIR} DEPENDS @@ -684,6 +694,17 @@ if(WarpX_PYTHON) ${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements ${_EXTRA_INSTALL_DEPENDS} ) + + # this is for package managers only + add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_nodeps + ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI} + ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx + WORKING_DIRECTORY + ${WarpX_BINARY_DIR} + DEPENDS + pyWarpX_${WarpX_DIMS_LAST} + ${WarpX_CUSTOM_TARGET_PREFIX}pip_wheel + ) endif() diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 1495a98a96f..a3a6a3046a9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -117,7 +117,7 @@ A typical format is: This is a short, 40-character title - After a newline, you can write arbitray paragraphs. You + After a newline, you can write arbitrary paragraphs. You usually limit the lines to 70 characters, but if you don't, then nothing bad will happen. @@ -212,12 +212,34 @@ Style and conventions - Space before and after assignment operator (``=``) -- To define a function , for e.g., ``myfunction()`` use a space between the name of the function and the paranthesis - ``myfunction ()``. - To call the function, the space is not required, i.e., just use ``myfunction()``. +- To define a function, use a space between the name of the function and the paranthesis, e.g., ``myfunction ()``. + When calling a function, no space should be used, i.e., just use ``myfunction()``. + The reason this is beneficial is that when we do a ``git grep`` to search for ``myfunction ()``, we can clearly see the locations where ``myfunction ()`` is defined and where ``myfunction()`` is called. + Also, using ``git grep "myfunction ()"`` searches for files only in the git repo, which is more efficient compared to the ``grep "myfunction ()"`` command that searches through all the files in a directory, including plotfiles for example. -- The reason this is beneficial is that when we do a ``git grep`` to search for ``myfunction ()``, we can clearly see the locations where ``myfunction ()`` is defined and where ``myfunction()`` is called. +- To define a class, use ``class`` on the same line as the name of the class, e.g., ``class MyClass``. + The reason this is beneficial is that when we do a ``git grep`` to search for ``class MyClass``, we can clearly see the locations where ``class MyClass`` is defined and where ``MyClass`` is called. -- Also, using ``git grep "myfunction ()"`` searches for files only in the git repo, which is more efficient compared to the ``grep "myfunction ()"`` command that searches through all the files in a directory, including plotfiles for example. +- When defining a function or class, make sure the starting ``{`` token appears on a new line. + +- Use curly braces for single statement blocks. For example: + + .. code-block:: cpp + + for (int n = 0; n < 10; ++n) { + amrex::Print() << "Like this!"; + } + + for (int n = 0; n < 10; ++n) { amrex::Print() << "Or like this!"; } + + but not + + .. code-block:: cpp + + for (int n = 0; n < 10; ++n) + amrex::Print() << "Not like this."; + + for (int n = 0; n < 10; ++n) amrex::Print() << "Nor like this."; - It is recommended that style changes are not included in the PR where new code is added. This is to avoid any errors that may be introduced in a PR just to do style change. diff --git a/Docs/requirements.txt b/Docs/requirements.txt index 1f21a414a19..b16bf161144 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -9,12 +9,15 @@ breathe docutils>=0.17.1 +openpmd-viewer # for checksumAPI + # PICMI API docs # note: keep in sync with version in ../requirements.txt -picmistandard==0.26.0 +picmistandard==0.28.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python +pybtex pygments recommonmark # Sphinx<7.2 because we are waiting for @@ -25,3 +28,4 @@ sphinx-design sphinx_rtd_theme>=1.1.1 sphinxcontrib-bibtex sphinxcontrib-napoleon +yt # for checksumAPI diff --git a/Docs/source/_static/custom.css b/Docs/source/_static/custom.css new file mode 100644 index 00000000000..b31c9188d76 --- /dev/null +++ b/Docs/source/_static/custom.css @@ -0,0 +1,6 @@ +.eqno { + /* As of 2023 Dec, sphinx_rtd_theme has a bug where equation numbers appear above the equations instead of on the right */ + /* The following is a make-shift fix, but comes at the cost of "making the header link invisible," though I'm not sure what that means */ + /* Copied from https://github.com/readthedocs/sphinx_rtd_theme/issues/301#issuecomment-1205755904 */ + float: right; +} diff --git a/Docs/source/acknowledge_us.rst b/Docs/source/acknowledge_us.rst index 2c0b1ce35f2..79bb9fa8884 100644 --- a/Docs/source/acknowledge_us.rst +++ b/Docs/source/acknowledge_us.rst @@ -55,9 +55,8 @@ If your project uses a specific algorithm or component, please consider citing t - Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. - 14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023. - `preprint `__, - `DOI:10.18429/JACoW-IPAC-23-WEPA101 `__ + 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. + `DOI:10.18429/JACoW-IPAC2023-WEPA101 `__ - Huebl A, Lehe R, Zoni E, Shapoval O, Sandberg R T, Garten M, Formenti A, Jambunathan R, Kumar P, Gott K, Myers A, Zhang W, Almgren A, Mitchell C E, Qiang J, Sinn A, Diederichs S, Thevenet M, Grote D, Fedeli L, Clark T, Zaim N, Vincenti H, Vay JL. **From Compact Plasma Particle Sources to Advanced Accelerators with Modeling at Exascale**. @@ -75,6 +74,11 @@ If your project uses a specific algorithm or component, please consider citing t *New Journal of Physics* **24** 025009, 2022. `DOI:10.1088/1367-2630/ac4ef1 `__ +- Lehe R, Blelly A, Giacomel L, Jambunathan R, Vay JL. + **Absorption of charged particles in perfectly matched layers by optimal damping of the deposited current**. + *Physical Review E* **106** 045306, 2022. + `DOI:10.1103/PhysRevE.106.045306 `__ + - Zoni E, Lehe R, Shapoval O, Belkin D, Zaim N, Fedeli L, Vincenti H, Vay JL. **A hybrid nodal-staggered pseudo-spectral electromagnetic particle-in-cell method with finite-order centering**. *Computer Physics Communications* **279**, 2022. `DOI:10.1016/j.cpc.2022.108457 `__ diff --git a/Docs/source/conf.py b/Docs/source/conf.py index e2cdfe15cda..f21dea15f18 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -30,9 +30,11 @@ import sys import urllib.request +import pybtex.plugin +from pybtex.style.formatting.unsrt import Style as UnsrtStyle import sphinx_rtd_theme -sys.path.insert(0, os.path.join( os.path.abspath(__file__), '../Python') ) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../Regression/Checksum')) # -- General configuration ------------------------------------------------ @@ -43,7 +45,8 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', +extensions = [ + 'sphinx.ext.autodoc', 'sphinx.ext.mathjax', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode', @@ -57,8 +60,29 @@ templates_path = ['_templates'] # Relative path to bibliography file, bibliography style -bibtex_bibfiles = ['./refs.bib'] -bibtex_default_style = 'unsrt' +bibtex_bibfiles = ['latex_theory/allbibs.bib', 'refs.bib'] + +# An brief introduction to custom BibTex formatting can be found in the Sphinx documentation: +# https://sphinxcontrib-bibtex.readthedocs.io/en/latest/usage.html#bibtex-custom-formatting +# +# More details can be gleaned from looking at the pybtex dist-package files. +# Some examples include the following: +# BaseStyle class in pybtex/style/formatting/__init__.py +# UnsrtStyle class in pybtex/style/formating/unsrt.py +class WarpXBibStyle(UnsrtStyle): + # This option makes the family name, i.e, "last" name, of an author to appear first. + # default_name_style = 'lastfirst' + + def __init__(self, *args, **kwargs): + # This option makes the given names of an author abbreviated to just initials. + # Example: "Jean-Luc" becomes "J.-L." + # Set 'abbreviate_names' to True before calling the superclass (BaseStyle class) initializer + kwargs['abbreviate_names'] = True + super().__init__(*args, **kwargs) + +pybtex.plugin.register_plugin('pybtex.style.formatting', 'warpxbibstyle', WarpXBibStyle) + +bibtex_default_style = 'warpxbibstyle' # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: @@ -79,9 +103,9 @@ # built documents. # # The short X.Y version. -version = u'23.09' +version = u'23.12' # The full version, including alpha/beta/rc tags. -release = u'23.09' +release = u'23.12' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -110,6 +134,11 @@ html_theme = 'sphinx_rtd_theme' numfig = True +math_eqref_format = "{number}" +numfig_format = {'figure': 'Fig. %s', + 'table': 'Table %s', + 'code-block': 'Listing %s', + 'section': 'Section %s'} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -122,6 +151,9 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +html_css_files = [ + 'custom.css', +] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/Docs/source/dataanalysis/ascent.rst b/Docs/source/dataanalysis/ascent.rst index 1fbeeb043e2..6748b111a2f 100644 --- a/Docs/source/dataanalysis/ascent.rst +++ b/Docs/source/dataanalysis/ascent.rst @@ -274,7 +274,7 @@ Example Actions A visualization of the electric field component :math:`E_x` (variable: ``Ex``) with a contour plot and with added particles can be obtained with the following Ascent Action. This action can be used both in replay as well as in situ runs. -.. literalinclude:: examples/Physics_applications/laser_acceleration/ascent_actions.yaml +.. literalinclude:: ../../../Examples/Physics_applications/laser_acceleration/ascent_actions.yaml :language: yaml There are more `Ascent Actions examples available `_ for you to play. diff --git a/Docs/source/dataanalysis/examples b/Docs/source/dataanalysis/examples deleted file mode 120000 index 76b45c18c59..00000000000 --- a/Docs/source/dataanalysis/examples +++ /dev/null @@ -1 +0,0 @@ -../../../Examples \ No newline at end of file diff --git a/Docs/source/dataanalysis/formats.rst b/Docs/source/dataanalysis/formats.rst index 85518abebb7..a7836d41fef 100644 --- a/Docs/source/dataanalysis/formats.rst +++ b/Docs/source/dataanalysis/formats.rst @@ -19,7 +19,7 @@ When using the AMReX `plotfile` format, users can set the ``amrex.async_out=1`` option to perform the IO in a non-blocking fashion, meaning that the simulation will continue to run while an IO thread controls writing the data to disk. This can significantly reduce the overall time spent in IO. This is primarily intended for -large runs on supercomputers such as Summit and Cori; depending on the MPI +large runs on supercomputers (e.g. at OLCF or NERSC); depending on the MPI implementation you are using, you may not see a benefit on your workstation. When writing plotfiles, each rank will write to a separate file, up to some maximum number diff --git a/Docs/source/dataanalysis/sensei.rst b/Docs/source/dataanalysis/sensei.rst index 16826c3082c..5e4bfff10fd 100644 --- a/Docs/source/dataanalysis/sensei.rst +++ b/Docs/source/dataanalysis/sensei.rst @@ -30,7 +30,7 @@ and bridge code making it easy to use in AMReX based simulation codes. SENSEI provides a *configurable analysis adaptor* which uses an XML file to select and configure one or more back ends at run time. Run time selection of the back end via XML means one user can access Catalyst, another Libsim, yet -another Python with no changes to the code. This is depicted in figure +another Python with no changes to the code. This is depicted in :numref:`sensei_arch`. On the left side of the figure AMReX produces data, the bridge code pushes the data through the configurable analysis adaptor to the back end that was selected at run time. @@ -99,7 +99,7 @@ The back end is selected and configured at run time using the SENSEI XML file. The XML sets parameters specific to SENSEI and to the chosen back end. Many of the back ends have sophisticated configuration mechanisms which SENSEI makes use of. For example the following XML configuration was used on NERSC's Cori -with WarpX to render 10 iso surfaces, shown in figure :numref:`lpa_visit`, using +with WarpX to render 10 iso surfaces, shown in :numref:`lpa_visit`, using VisIt Libsim. .. code-block:: xml @@ -123,7 +123,7 @@ run of the desired simulation. quadrant has been clipped away to reveal innner structure. The same run and visualization was repeated using ParaView Catalyst, shown in -figure :numref:`lpa_pv`, by providing the following XML configuration. +:numref:`lpa_pv`, by providing the following XML configuration. .. code-block:: xml diff --git a/Docs/source/developers/checksum.rst b/Docs/source/developers/checksum.rst index 62a367a4866..4412c3e3cc4 100644 --- a/Docs/source/developers/checksum.rst +++ b/Docs/source/developers/checksum.rst @@ -9,14 +9,14 @@ The checksum module is located in ``Regression/Checksum/``, and the benchmarks a For more details on the implementation, the Python files in ``Regression/Checksum/`` should be well documented. -From a user point of view, you should only need to use ``checksumAPI.py``. It contains Python functions that can be imported and used from an analysis Python script. It can also be executed directly as a Python script. Here are recipies for the main tasks related to checksum regression tests in WarpX CI. +From a user point of view, you should only need to use ``checksumAPI.py``. It contains Python functions that can be imported and used from an analysis Python script. It can also be executed directly as a Python script. Here are recipes for the main tasks related to checksum regression tests in WarpX CI. Include a checksum regression test in an analysis Python script --------------------------------------------------------------- -This relies on function ``evaluate_checksum``: +This relies on the function ``evaluate_checksum``: -.. doxygenfunction:: evaluate_checksum +.. autofunction:: checksumAPI.evaluate_checksum For an example, see @@ -32,7 +32,7 @@ You can execute ``checksumAPI.py`` as a Python script for that, and pass the plo .. code-block:: bash - ./checksumAPI.py --evaluate --plotfile --test-name + ./checksumAPI.py --evaluate --output-file --format <'openpmd' or 'plotfile'> --test-name See additional options @@ -41,17 +41,17 @@ See additional options * ``--rtol`` relative tolerance for the comparison * ``--atol`` absolute tolerance for the comparison (a sum of both is used by ``numpy.isclose()``) -Reset a benchmark with new values that you know are correct ------------------------------------------------------------ +Create/Reset a benchmark with new values that you know are correct +------------------------------------------------------------------ -Reset a benchmark from a plotfile generated locally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Create/Reset a benchmark from a plotfile generated locally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This is using ``checksumAPI.py`` as a Python script. .. code-block:: bash - ./checksumAPI.py --reset-benchmark --plotfile --test-name + ./checksumAPI.py --reset-benchmark --output-file --format <'openpmd' or 'plotfile'> --test-name See additional options diff --git a/Docs/source/developers/documentation.rst b/Docs/source/developers/documentation.rst index 0f3ca9dd9cd..a5013299336 100644 --- a/Docs/source/developers/documentation.rst +++ b/Docs/source/developers/documentation.rst @@ -11,21 +11,21 @@ Whenever you create a new class, please document it where it is declared (typica .. code-block:: cpp - /** A brief title + /** \brief A brief title * - * few-line description explaining the purpose of my_class. + * few-line description explaining the purpose of MyClass. * - * If you are kind enough, also quickly explain how things in my_class work. + * If you are kind enough, also quickly explain how things in MyClass work. * (typically a few more lines) */ - class my_class + class MyClass { ... } Doxygen reads this docstring, so please be accurate with the syntax! See `Doxygen manual `__ for more information. Similarly, please document functions when you declare them (typically in a header file) like: .. code-block:: cpp - /** A brief title + /** \brief A brief title * * few-line description explaining the purpose of my_function. * @@ -33,7 +33,7 @@ Doxygen reads this docstring, so please be accurate with the syntax! See `Doxyge * my_function will operate. * \return what is the meaning and value range of the returned value */ - int my_class::my_function(int* my_int); + int MyClass::my_function (int* my_int); An online version of this documentation is :ref:`linked here `. diff --git a/Docs/source/developers/fields.rst b/Docs/source/developers/fields.rst index af834354dcd..d0af160afef 100644 --- a/Docs/source/developers/fields.rst +++ b/Docs/source/developers/fields.rst @@ -40,7 +40,7 @@ By default, the ``MultiFab`` are set to ``0`` at initialization. They can be ass Field solver ------------ -The field solver is performed in ``WarpX::EvolveE`` for the electric field and ``WarpX::EvolveB`` for the magnetic field, called from ``WarpX::OneStep_nosub`` in ``WarpX::EvolveEM``. This section describes the FDTD field push. It is implemented in ``Source/FieldSolver/FiniteDifferenceSolver/``. +The field solver is performed in ``WarpX::EvolveE`` for the electric field and ``WarpX::EvolveB`` for the magnetic field, called from ``WarpX::OneStep_nosub`` in ``WarpX::Evolve``. This section describes the FDTD field push. It is implemented in ``Source/FieldSolver/FiniteDifferenceSolver/``. As all cell-wise operation, the field push is done as follows (this is split in multiple functions in the actual implementation to avoid code duplication) : @@ -112,9 +112,9 @@ Bilinear filter The multi-pass bilinear filter (applied on the current density) is implemented in ``Source/Filter/``, and class ``WarpX`` holds an instance of this class in member variable ``WarpX::bilinear_filter``. For performance reasons (to avoid creating too many guard cells), this filter is directly applied in communication routines, see ``WarpX::AddCurrentFromFineLevelandSumBoundary`` above and -.. doxygenfunction:: WarpX::ApplyFilterJ(const amrex::Vector, 3>> ¤t, const int lev, const int idim) +.. doxygenfunction:: WarpX::ApplyFilterJ(const amrex::Vector, 3>> ¤t, int lev, int idim) -.. doxygenfunction:: WarpX::SumBoundaryJ(const amrex::Vector, 3>> ¤t, const int lev, const int idim, const amrex::Periodicity &period) +.. doxygenfunction:: WarpX::SumBoundaryJ(const amrex::Vector, 3>> ¤t, int lev, int idim, const amrex::Periodicity &period) Godfrey's anti-NCI filter for FDTD simulations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Docs/source/developers/particles.rst b/Docs/source/developers/particles.rst index e9ad7754771..c1a711e0a01 100644 --- a/Docs/source/developers/particles.rst +++ b/Docs/source/developers/particles.rst @@ -87,7 +87,7 @@ Main functions .. doxygenfunction:: PhysicalParticleContainer::PushPX -.. doxygenfunction:: WarpXParticleContainer::DepositCurrent(amrex::Vector, 3>> &J, const amrex::Real dt, const amrex::Real relative_time) +.. doxygenfunction:: WarpXParticleContainer::DepositCurrent(amrex::Vector, 3>> &J, amrex::Real dt, amrex::Real relative_time) .. note:: The current deposition is used both by ``PhysicalParticleContainer`` and ``LaserParticleContainer``, so it is in the parent class ``WarpXParticleContainer``. diff --git a/Docs/source/developers/profiling.rst b/Docs/source/developers/profiling.rst index 8fe68fac817..5acea786920 100644 --- a/Docs/source/developers/profiling.rst +++ b/Docs/source/developers/profiling.rst @@ -121,7 +121,7 @@ Perlmutter Example """""""""""""""""" Example on how to create traces on a multi-GPU system that uses the Slurm scheduler (e.g., NERSC's Perlmutter system). -You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. +You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. .. code-block:: bash @@ -247,7 +247,7 @@ node, or without a workload management system. .. note:: To collect full statistics, Nsight-Compute reruns kernels, - temporarilly saving device memory in host memory. This makes it + temporarily saving device memory in host memory. This makes it slower than Nsight-Systems, so the provided script profiles only a single step of a single process. This is generally enough to extract relevant information. diff --git a/Docs/source/developers/python.rst b/Docs/source/developers/python.rst index aa58b0a398a..108f957c2f3 100644 --- a/Docs/source/developers/python.rst +++ b/Docs/source/developers/python.rst @@ -56,10 +56,10 @@ This is part of the standard - each of the PICMI classes call the method ``handl The main purpose is to process application specific keyword arguments (those that start with ``warpx_`` for example). These are then passed into the ``init`` methods. In the WarpX implementation, in the ``init``, each of the WarpX specific arguments are saved as attributes of the implementation -class instancles. +class instances. It is in the second method, ``initialize_inputs``, where the PICMI input parameters are translated into WarpX input parameters. -This method is called later during the intialization. +This method is called later during the initialization. The prefix instances described above are all accessible in the implementation classes (via the ``pywarpx`` module). For each PICMI input quantity, the appropriate WarpX input parameters are set in the prefix classes. As needed, for example in the ``Species`` class, the dynamic prefix instances are created and the attributes set. diff --git a/Docs/source/developers/repo_organization.rst b/Docs/source/developers/repo_organization.rst index b8a964f9839..cb450cc4c95 100644 --- a/Docs/source/developers/repo_organization.rst +++ b/Docs/source/developers/repo_organization.rst @@ -8,7 +8,7 @@ Repo Organization All the WarpX source code is located in ``Source/``. All sub-directories have a pretty straightforward name. -The PIC loop is part of the WarpX class, in function ``WarpX::EvolveEM`` implemented in ``Source/WarpXEvolveEM.cpp``. +The PIC loop is part of the WarpX class, in function ``WarpX::Evolve`` implemented in ``Source/WarpXEvolve.cpp``. The core of the PIC loop (i.e., without diagnostics etc.) is in ``WarpX::OneStep_nosub`` (when subcycling is OFF) or ``WarpX::OneStep_sub1`` (when subcycling is ON, with method 1). Here is a `visual representation `__ of the repository structure. @@ -41,7 +41,7 @@ All WarpX header files need to be specified relative to the ``Source/`` director By default, in a ``MyName.cpp`` source file we do not include headers already included in ``MyName.H``. Besides this exception, if a function or a class is used in a source file, the header file containing its declaration must be included, unless the inclusion of a facade header is more appropriate. This is sometimes the case for AMReX headers. For instance ``AMReX_GpuLaunch.H`` is a façade header for ``AMReX_GpuLaunchFunctsC.H`` and ``AMReX_GpuLaunchFunctsG.H``, which -contain respectively the CPU and the GPU implemetation of some methods, and which should not be included directly. +contain respectively the CPU and the GPU implementation of some methods, and which should not be included directly. Whenever possible, forward declarations headers are included instead of the actual headers, in order to save compilation time (see dedicated section below). In WarpX forward declaration headers have the suffix ``*_fwd.H``, while in AMReX they have the suffix ``*Fwd.H``. The include order (see `PR #874 `__ and `PR #1947 `__) and `proper quotation marks `__ are: diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index 227c2fd97b0..2b3fe989f2f 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -18,14 +18,14 @@ We slightly modify this file in ``Regression/prepare_file_ci.py``. For example, if you like to change the compiler to compilation to build on Nvidia GPUs, modify this block to add ``-DWarpX_COMPUTE=CUDA``: -.. code-block:: toml +.. code-block:: ini [source] dir = /home/regtester/AMReX_RegTesting/warpx branch = development cmakeSetupOpts = -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON -DWarpX_COMPUTE=CUDA -We also support changing compilation options via the usual :ref:`build enviroment variables `. +We also support changing compilation options via the usual :ref:`build environment variables `. For instance, compiling with ``clang++ -Werror`` would be: .. code-block:: sh diff --git a/Docs/source/developers/warning_logger.rst b/Docs/source/developers/warning_logger.rst index 057aa21bb8e..d878b042502 100644 --- a/Docs/source/developers/warning_logger.rst +++ b/Docs/source/developers/warning_logger.rst @@ -47,15 +47,15 @@ Each entry of warning list respects the following format: .. code-block:: sh * --> [PRIORITY] [TOPIC] [raised COUNTER] - * MULTILINE MESSAGGE - * MULTILINE MESSAGGE + * MULTILINE MESSAGE + * MULTILINE MESSAGE * @ Raised by: WHICH_RANKS where: * ``[PRIORITY]`` can be ``[! ]`` (low priority), ``[!! ]`` (medium priority) or ``[!!!]`` (high priority). It indicates the importance of the warning. * ``[TOPIC]`` indicates which part of the code is concerned by the warning (e.g., particles, laser, parallelization...) -* ``MULTILINE MESSAGGE`` is an arbitrary text message. It can span multiple-lines. Text is wrapped automatically. +* ``MULTILINE MESSAGE`` is an arbitrary text message. It can span multiple-lines. Text is wrapped automatically. * ``COUNTER`` indicates the number of times the warning was raised **across all the MPI ranks**. This means that if we run WarpX with 2048 MPI ranks and each rank raises the same warning once, the displayed message will be ``[raised 2048 times]``. Possible values are ``once``, ``twice``, ``XX times`` * ``WHICH_RANKS`` can be either ``ALL`` or a sequence of rank IDs. It is the list of the MPI ranks which have raised the warning message. diff --git a/Docs/source/glossary.rst b/Docs/source/glossary.rst index cb376af481f..000b8354e15 100644 --- a/Docs/source/glossary.rst +++ b/Docs/source/glossary.rst @@ -17,7 +17,8 @@ Abbreviations * **AMR:** adaptive mesh-refinement * **BC:** boundary condition (of a simulation) * **BCK:** `Benkler-Chavannes-Kuster `__ method, a stabilization technique for small cells in the electromagnetic solver -* **BTD:** backtransformed diagnosics, a method to collect data for analysis from a *boosted frame* simulation +* **BTD:** backtransformed diagnostics, a method to collect data for analysis from a *boosted frame* simulation +* **CEX:** charge-exchange collisions * **CFL:** the Courant-Friedrichs-Lewy condition, a numerical parameter for the numerical convergence of PDE solvers * **CI:** continuous integration, automated tests that we perform before a proposed code-change is accepted; see PR * **CPU:** `central processing unit `__, we usual mean a socket or generally the host-side of a computer (compared to the accelerator, e.g. GPU) @@ -33,6 +34,7 @@ Abbreviations * **GPU:** originally graphics processing unit, now used for fast `general purpose computing (GPGPU) `__; also called (hardware) accelerator * **IO:** input/output, usually files and/or data * **IPO:** `interprocedural optimization `__, a collection of compiler optimization techniques that analyze the whole code to avoid duplicate calculations and optimize performance +* **ISI:** Induced Spectral Incoherence (a laser pulse manipulation technique) * **LDRD:** Laboratory Directed Research and Development, a :ref:`funding program in U.S. DOE laboratories ` that kick-started ABLASTR development * **LPA:** laser-plasma acceleration, historically used for laser-electron acceleration * **LPI:** laser-plasma interaction (often for laser-solid physics) *or* laser-plasma instability (often in fusion physics), depending on context @@ -55,8 +57,11 @@ Abbreviations * **PWFA:** plasma-wakefield acceleration * **QED:** `quantum electrodynamics `__ * **RPA:** radiation-pressure acceleration (of protons/ions), e.g. hole-boring (HB) or light-sail (LS) acceleration +* **RPP:** Random Phase Plate (a laser pulse manipulation technique) * **RZ:** for the coordinate system ``r-z`` in cylindrical geometry; we use "RZ" when we refer to quasi-cylindrical geometry, decomposed in azimuthal modes (see details `here `__) * **SENSEI:** `Scalable in situ analysis and visualization `__, light weight framework for in situ data analysis offering access to multiple visualization and analysis backends +* **SEE:** secondary electron emission +* **SSD:** Smoothing by Spectral Dispersion (a laser pulse manipulation technique) * **TNSA:** target-normal sheet acceleration (of protons/ions) Terms @@ -67,7 +72,7 @@ Terms * **Ascent:** `many-core capable flyweight in situ visualization and analysis infrastructure `__, a visualization backend usable with WarpX data * **boosted frame:** a :ref:`Lorentz-boosted frame of reference ` for a simulation * **evolve:** this is a generic term to advance a quantity (same nomenclature in AMReX). - For instance, ``WarpX::EvolveE(dt)`` advances the electric field for duration ``dt``, ``PhysicalParticleContainer::Evolve(...)`` does field gather + particle push + current deposition for all particles in ``PhysicalParticleContainer``, and ``WarpX::EvolveEM`` is the central ``WarpX`` function that performs 1 PIC iteration. + For instance, ``WarpX::EvolveE(dt)`` advances the electric field for duration ``dt``, ``PhysicalParticleContainer::Evolve(...)`` does field gather + particle push + current deposition for all particles in ``PhysicalParticleContainer``, and ``WarpX::Evolve`` is the central ``WarpX`` function that performs 1 PIC iteration. * **Frontier:** an `Exascale supercomputer at OLCF `__ * **hybrid-PIC:** a plasma simulation scheme that combines fluid and kinetic approaches, with (usually) the electrons treated as a fluid and the ions as kinetic particles (see :ref:`theory-kinetic-fluid-hybrid-model`) * **laser:** most of the time, we mean a `laser pulse `__ diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index dd9f6fd3e81..98b0d85391e 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -14,21 +14,25 @@ Plasma-Based Acceleration Scientific works in laser-plasma and beam-plasma acceleration. +#. Peng, H. and Huang, T. W. and Jiang, K. and Li, R. and Wu, C. N. and Yu, M. Y. and Riconda, C. and Weber, S. and Zhou, C. T. and Ruan, S. C. + **Coherent Subcycle Optical Shock from a Superluminal Plasma Wake**. + Phys. Rev. Lett. **131**, 145003, 2023 + `DOI:10.1103/PhysRevLett.131.145003 `__ + #. Mewes SM, Boyle GJ, Ferran Pousa A, Shalloo RJ, Osterhoff J, Arran C, Corner L, Walczak R, Hooker SM, Thévenet M. **Demonstration of tunability of HOFI waveguides via start-to-end simulations**. Phys. Rev. Research **5**, 033112, 2023 `DOI:10.1103/PhysRevResearch.5.033112 `__ -#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. +#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. - 14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023. - `preprint `__, - `DOI:10.18429/JACoW-IPAC-23-WEPA101 `__ + 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. + `DOI:10.18429/JACoW-IPAC2023-WEPA101 `__ #. Wang J, Zeng M, Li D, Wang X, Gao J. **High quality beam produced by tightly focused laser driven wakefield accelerators**. - arXiv pre-print, 2023. - `DOI:10.48550/arXiv.2304.10730 `__ + Phys. Rev. Accel. Beams, **26**, 091303, 2023. + `DOI:10.1103/PhysRevAccelBeams.26.091303 `__ #. Fedeli L, Huebl A, Boillod-Cerneux F, Clark T, Gott K, Hillairet C, Jaure S, Leblanc A, Lehe R, Myers A, Piechurski C, Sato M, Zaim N, Zhang W, Vay J-L, Vincenti H. **Pushing the Frontier in the Design of Laser-Based Electron Accelerators with Groundbreaking Mesh-Refined Particle-In-Cell Simulations on Exascale-Class Supercomputers**. @@ -66,6 +70,11 @@ Laser-Plasma Interaction Scientific works in laser-ion acceleration and laser-matter interaction. +#. Knight B, Gautam C, Stoner C, Egner B, Smith J, Orban C, Manfredi J, Frische K, Dexter M, Chowdhury E, Patnaik A (2023). + **Detailed Characterization of a kHz-rate Laser-Driven Fusion at a Thin Liquid Sheet with a Neutron Detection Suite**. + High Power Laser Science and Engineering, 1-13, 2023. + `DOI:10.1017/hpl.2023.84 `__ + #. Fedeli L, Huebl A, Boillod-Cerneux F, Clark T, Gott K, Hillairet C, Jaure S, Leblanc A, Lehe R, Myers A, Piechurski C, Sato M, Zaim N, Zhang W, Vay J-L, Vincenti H. **Pushing the Frontier in the Design of Laser-Based Electron Accelerators with Groundbreaking Mesh-Refined Particle-In-Cell Simulations on Exascale-Class Supercomputers**. *SC22: International Conference for High Performance Computing, Networking, Storage and Analysis (SC)*. ISSN:2167-4337, pp. 25-36, Dallas, TX, US, 2022. @@ -136,3 +145,16 @@ High-Performance Computing and Numerics Scientific works in High-Performance Computing, applied mathematics and numerics. Please see :ref:`this section `. + +Nuclear Fusion - Magnetically Confined Plasmas +********************************************** + +#. Nicks B. S., Putvinski S. and Tajima T. + **Stabilization of the Alfvén-ion cyclotron instability through short plasmas: Fully kinetic simulations in a high-beta regime**. + Physics of Plasmas **30**, 102108, 2023. + `DOI:10.1063/5.0163889 `__ + +#. Groenewald R. E., Veksler A., Ceccherini F., Necas A., Nicks B. S., Barnes D. C., Tajima T. and Dettrick S. A. + **Accelerated kinetic model for global macro stability studies of high-beta fusion reactors**. + Physics of Plasmas **30**, 122508, 2023. + `DOI:10.1063/5.0178288 `__ diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 2209b360e26..9f2d2ff038e 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -75,11 +75,9 @@ Usage :hidden: usage/how_to_run - usage/domain_decomposition - usage/parameters - usage/python usage/examples - usage/pwfa + usage/python + usage/parameters usage/workflows usage/faq @@ -109,13 +107,14 @@ Theory :hidden: theory/intro - theory/picsar_theory + theory/pic theory/amr - theory/PML + theory/boundary_conditions theory/boosted_frame theory/input_output theory/collisions theory/kinetic_fluid_hybrid_model + theory/cold_fluid_model Development ----------- @@ -125,10 +124,10 @@ Development :hidden: developers/contributing - developers/workflows developers/developers developers/doxygen developers/gnumake + developers/workflows developers/faq .. good to have in the future: .. developers/repostructure diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 457a352b3c2..e299a60c75b 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -130,7 +130,7 @@ CMake Option Default & Values Des ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from ``WarpX_picsar_branch`` *we set and maintain a compatible commit* Repository branch for ``WarpX_picsar_repo`` ``WarpX_picsar_internal`` **ON**/OFF Needs a pre-installed PICSAR library if set to ``OFF`` -``WarpX_pyamrex_src`` *None* Path to PICSAR source directory (preferred if set) +``WarpX_pyamrex_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_pyamrex_repo`` ``https://github.com/AMReX-Codes/pyamrex.git`` Repository URI to pull and build pyAMReX from ``WarpX_pyamrex_branch`` *we set and maintain a compatible commit* Repository branch for ``WarpX_pyamrex_repo`` ``WarpX_pyamrex_internal`` **ON**/OFF Needs a pre-installed pyAMReX library if set to ``OFF`` @@ -211,7 +211,8 @@ PICMI Python Bindings .. code-block:: bash - python3 -m pip install -U pip setuptools wheel + python3 -m pip install -U pip + python3 -m pip install -U build packaging setuptools wheel python3 -m pip install -U cmake python3 -m pip install -r requirements.txt diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index a198712cc01..a939db840da 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -18,7 +18,7 @@ Optional dependencies include: - for on-node accelerated compute *one of either*: - `OpenMP 3.1+ `__: for threaded CPU execution or - - `CUDA Toolkit 11.0+ (11.3+ recommended) `__: for Nvidia GPU support (see `matching host-compilers `_) or + - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `_) or - `ROCm 5.2+ (5.5+ recommended) `__: for AMD GPU support - `FFTW3 `_: for spectral solver (PSATD) support when running on CPU or SYCL @@ -75,7 +75,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv + conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv conda activate warpx-cpu-mpich-dev # compile WarpX with -DWarpX_MPI=ON @@ -85,7 +85,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api python numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv + conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv conda activate warpx-cpu-dev # compile WarpX with -DWarpX_MPI=OFF @@ -107,6 +107,14 @@ For OpenMP support, you will further need: conda install -c conda-forge llvm-openmp +For Nvidia CUDA GPU support, you will need to have `a recent CUDA driver installed `__ or you can lower the CUDA version of `the Nvidia cuda package `__ and `conda-forge to match your drivers `__ and then add these packages: + +.. code-block:: bash + + conda install -c nvidia -c conda-forge cuda cupy + +More info for `CUDA-enabled ML packages `__. + Spack (Linux/macOS) ------------------- diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index 67468cc2f02..9617f2a7fd6 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -33,7 +33,6 @@ This section documents quick-start guides for a selection of supercomputers that :maxdepth: 1 hpc/adastra - hpc/cori hpc/crusher hpc/frontier hpc/fugaku @@ -42,6 +41,7 @@ This section documents quick-start guides for a selection of supercomputers that hpc/karolina hpc/lassen hpc/lawrencium + hpc/leonardo hpc/lumi hpc/lxplus hpc/ookami diff --git a/Docs/source/install/hpc/cori.rst b/Docs/source/install/hpc/cori.rst deleted file mode 100644 index 35421982142..00000000000 --- a/Docs/source/install/hpc/cori.rst +++ /dev/null @@ -1,412 +0,0 @@ -.. _building-cori: - -Cori (NERSC) -============ - -The `Cori cluster `_ is located at NERSC. - - -Introduction ------------- - -If you are new to this system, **please see the following resources**: - -* `GPU nodes `__ - -* `Cori user guide `__ -* Batch system: `Slurm `__ -* `Jupyter service `__ -* `Production directories `__: - - * ``$SCRATCH``: per-user production directory, purged every 30 days (20TB) - * ``/global/cscratch1/sd/m3239``: shared production directory for users in the project ``m3239``, purged every 30 days (50TB) - * ``/global/cfs/cdirs/m3239/``: community file system for users in the project ``m3239`` (100TB) - -Installation ------------- - -Use the following commands to download the WarpX source code and switch to the correct branch: - -.. code-block:: bash - - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx - -KNL -^^^ - -We use the following modules and environments on the system (``$HOME/knl_warpx.profile``). - -.. literalinclude:: ../../../../Tools/machines/cori-nersc/knl_warpx.profile.example - :language: bash - :caption: You can copy this file from ``Tools/machines/cori-nersc/knl_warpx.profile.example``. - -And install ADIOS2, BLAS++ and LAPACK++: - -.. code-block:: bash - - source $HOME/knl_warpx.profile - - # c-blosc (I/O compression) - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc - rm -rf src/c-blosc-knl-build - cmake -S src/c-blosc -B src/c-blosc-knl-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/c-blosc-1.12.1-install - cmake --build src/c-blosc-knl-build --target install --parallel 16 - - # ADIOS2 - git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2 - rm -rf src/adios2-knl-build - cmake -S src/adios2 -B src/adios2-knl-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/adios2-2.7.1-install - cmake --build src/adios2-knl-build --target install --parallel 16 - - # BLAS++ (for PSATD+RZ) - git clone https://github.com/icl-utk-edu/blaspp.git src/blaspp - rm -rf src/blaspp-knl-build - cmake -S src/blaspp -B src/blaspp-knl-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/blaspp-master-install - cmake --build src/blaspp-knl-build --target install --parallel 16 - - # LAPACK++ (for PSATD+RZ) - git clone https://github.com/icl-utk-edu/lapackpp.git src/lapackpp - rm -rf src/lapackpp-knl-build - CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-knl-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/knl/lapackpp-master-install - cmake --build src/lapackpp-knl-build --target install --parallel 16 - -For PICMI and Python workflows, also install a virtual environment: - -.. code-block:: bash - - # establish Python dependencies - python3 -m pip install --user --upgrade pip - python3 -m pip install --user virtualenv - - python3 -m venv $HOME/sw/knl/venvs/knl_warpx - source $HOME/sw/knl/venvs/knl_warpx/bin/activate - - python3 -m pip install --upgrade pip - python3 -m pip install --upgrade wheel - python3 -m pip install --upgrade cython - python3 -m pip install --upgrade numpy - python3 -m pip install --upgrade pandas - python3 -m pip install --upgrade scipy - MPICC="cc -shared" python3 -m pip install -U --no-cache-dir -v mpi4py - python3 -m pip install --upgrade openpmd-api - python3 -m pip install --upgrade matplotlib - python3 -m pip install --upgrade yt - # optional: for libEnsemble - #python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt - -Haswell -^^^^^^^ - -We use the following modules and environments on the system (``$HOME/haswell_warpx.profile``). - -.. literalinclude:: ../../../../Tools/machines/cori-nersc/haswell_warpx.profile.example - :language: bash - :caption: You can copy this file from ``Tools/machines/cori-nersc/haswell_warpx.profile.example``. - -And install ADIOS2, BLAS++ and LAPACK++: - -.. code-block:: bash - - source $HOME/haswell_warpx.profile - - # c-blosc (I/O compression) - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc - rm -rf src/c-blosc-haswell-build - cmake -S src/c-blosc -B src/c-blosc-haswell-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/haswell/c-blosc-1.12.1-install - cmake --build src/c-blosc-haswell-build --target install --parallel 16 - - # ADIOS2 - git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2 - rm -rf src/adios2-haswell-build - cmake -S src/adios2 -B src/adios2-haswell-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/haswell/adios2-2.7.1-install - cmake --build src/adios2-haswell-build --target install --parallel 16 - - # BLAS++ (for PSATD+RZ) - git clone https://github.com/icl-utk-edu/blaspp.git src/blaspp - rm -rf src/blaspp-haswell-build - cmake -S src/blaspp -B src/blaspp-haswell-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/blaspp-master-haswell-install - cmake --build src/blaspp-haswell-build --target install --parallel 16 - - # LAPACK++ (for PSATD+RZ) - git clone https://github.com/icl-utk-edu/lapackpp.git src/lapackpp - rm -rf src/lapackpp-haswell-build - CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S src/lapackpp -B src/lapackpp-haswell-build -Duse_cmake_find_lapack=ON -DBLAS_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DLAPACK_LIBRARIES=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu.a -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/haswell/lapackpp-master-install - cmake --build src/lapackpp-haswell-build --target install --parallel 16 - -For PICMI and Python workflows, also install a virtual environment: - -.. code-block:: bash - - # establish Python dependencies - python3 -m pip install --user --upgrade pip - python3 -m pip install --user virtualenv - - python3 -m venv $HOME/sw/haswell/venvs/haswell_warpx - source $HOME/sw/haswell/venvs/haswell_warpx/bin/activate - - python3 -m pip install --upgrade pip - python3 -m pip install --upgrade wheel - python3 -m pip install --upgrade cython - python3 -m pip install --upgrade numpy - python3 -m pip install --upgrade pandas - python3 -m pip install --upgrade scipy - MPICC="cc -shared" python3 -m pip install -U --no-cache-dir -v mpi4py - python3 -m pip install --upgrade openpmd-api - python3 -m pip install --upgrade matplotlib - python3 -m pip install --upgrade yt - # optional: for libEnsemble - #python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt - -GPU (V100) -^^^^^^^^^^ - -Cori provides a partition with `18 nodes that include V100 (16 GB) GPUs `__. -We use the following modules and environments on the system (``$HOME/gpu_warpx.profile``). -You can copy this file from ``Tools/machines/cori-nersc/gpu_warpx.profile.example``: - -.. literalinclude:: ../../../../Tools/machines/cori-nersc/gpu_warpx.profile.example - :language: bash - :caption: You can copy this file from ``Tools/machines/cori-nersc/gpu_warpx.profile.example``. - -And install ADIOS2: - -.. code-block:: bash - - source $HOME/gpu_warpx.profile - - # c-blosc (I/O compression) - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc - rm -rf src/c-blosc-gpu-build - cmake -S src/c-blosc -B src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/cori_gpu/c-blosc-1.12.1-install - cmake --build src/c-blosc-gpu-build --target install --parallel 16 - - git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2 - rm -rf src/adios2-gpu-build - cmake -S src/adios2 -B src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/cori_gpu/adios2-2.7.1-install - cmake --build src/adios2-gpu-build --target install --parallel 16 - -For PICMI and Python workflows, also install a virtual environment: - -.. code-block:: bash - - # establish Python dependencies - python3 -m pip install --user --upgrade pip - python3 -m pip install --user virtualenv - - python3 -m venv $HOME/sw/cori_gpu/venvs/gpu_warpx - source $HOME/sw/cori_gpu/venvs/gpu_warpx/bin/activate - - python3 -m pip install --upgrade pip - python3 -m pip install --upgrade wheel - python3 -m pip install --upgrade cython - python3 -m pip install --upgrade numpy - python3 -m pip install --upgrade pandas - python3 -m pip install --upgrade scipy - python3 -m pip install -U --no-cache-dir -v mpi4py - python3 -m pip install --upgrade openpmd-api - python3 -m pip install --upgrade matplotlib - python3 -m pip install --upgrade yt - # optional: for libEnsemble - #python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt - -Building WarpX --------------- - -We recommend to store the above lines in individual ``warpx.profile`` files, as suggested above. -If you want to run on either of the three partitions of Cori, open a new terminal, log into Cori and *source* the environment you want to work with: - -.. code-block:: bash - - # KNL: - source $HOME/knl_warpx.profile - - # Haswell: - #source $HOME/haswell_warpx.profile - - # GPU: - #source $HOME/gpu_warpx.profile - -.. warning:: - - Consider that all three Cori partitions are *incompatible*. - - Do not *source* multiple ``...warpx.profile`` files in the same terminal session. - Open a new terminal and log into Cori again, if you want to switch the targeted Cori partition. - - If you re-submit an already compiled simulation that you ran on another day or in another session, *make sure to source* the corresponding ``...warpx.profile`` again after login! - -Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile: - -.. code-block:: bash - - cd $HOME/src/warpx - rm -rf build - - # append if you target GPUs: -DWarpX_COMPUTE=CUDA - cmake -S . -B build -DWarpX_DIMS=3 - cmake --build build -j 16 - -The general :ref:`cmake compile-time options ` apply as usual. - -**That's it!** -A 3D WarpX executable is now in ``build/bin/`` and :ref:`can be run ` with a :ref:`3D example inputs file `. -Most people execute the binary directly or copy it out to a location in ``$SCRATCH``. - -The general :ref:`cmake compile-time options and instructions for Python (PICMI) bindings ` apply as usual: - -.. code-block:: bash - - # PICMI build - cd $HOME/src/warpx - - # install or update dependencies - python3 -m pip install -r requirements.txt - - # compile parallel PICMI interfaces with openPMD support and 3D, 2D, 1D and RZ - WARPX_MPI=ON BUILD_PARALLEL=16 python3 -m pip install --force-reinstall --no-deps -v . - - -.. _building-cori-tests: - -Testing -------- - -To run all tests (here on KNL), do: - -* change in ``Regressions/WarpX-tests.ini`` from ``mpiexec`` to ``srun``: ``MPIcommand = srun -n @nprocs@ @command@`` - -.. code-block:: bash - - # set test directory to a shared directory available on all nodes - # note: the tests will create the directory automatically - export WARPX_CI_TMP="$HOME/warpx-regression-tests" - - # compile with more cores - export WARPX_CI_NUM_MAKE_JOBS=16 - - # run all integration tests - # note: we set MPICC as a build-setting for mpi4py on KNL/Haswell - MPICC="cc -shared" ./run_test.sh - - -.. _running-cpp-cori: - -Running -------- - -Navigate (i.e. ``cd``) into one of the production directories (e.g. ``$SCRATCH``) before executing the instructions below. - -KNL -^^^ - -The batch script below can be used to run a WarpX simulation on 2 KNL nodes on -the supercomputer Cori at NERSC. Replace descriptions between chevrons ``<>`` -by relevant values, for instance ```` could be ``laserWakefield``. - -Do not forget to first ``source $HOME/knl_warpx.profile`` if you have not done so already for this terminal session. - -For PICMI Python runs, the ```` has to read ``python3`` and the ```` is the path to your PICMI input script. - -.. literalinclude:: ../../../../Tools/machines/cori-nersc/cori_knl.sbatch - :language: bash - :caption: You can copy this file from ``Tools/machines/cori-nersc/cori_knl.sbatch``. - -To run a simulation, copy the lines above to a file ``cori_knl.sbatch`` and run - -.. code-block:: bash - - sbatch cori_knl.sbatch - -to submit the job. - -For a 3D simulation with a few (1-4) particles per cell using FDTD Maxwell -solver on Cori KNL for a well load-balanced problem (in our case laser -wakefield acceleration simulation in a boosted frame in the quasi-linear -regime), the following set of parameters provided good performance: - -* ``amr.max_grid_size=64`` and ``amr.blocking_factor=64`` so that the size of - each grid is fixed to ``64**3`` (we are not using load-balancing here). - -* **8 MPI ranks per KNL node**, with ``OMP_NUM_THREADS=8`` (that is 64 threads - per KNL node, i.e. 1 thread per physical core, and 4 cores left to the - system). - -* **2 grids per MPI**, *i.e.*, 16 grids per KNL node. - -Haswell -^^^^^^^ - -The batch script below can be used to run a WarpX simulation on 1 `Haswell node `_ on the supercomputer Cori at NERSC. - -Do not forget to first ``source $HOME/haswell_warpx.profile`` if you have not done so already for this terminal session. - -.. literalinclude:: ../../../../Tools/machines/cori-nersc/cori_haswell.sbatch - :language: bash - :caption: You can copy this file from ``Tools/machines/cori-nersc/cori_haswell.sbatch``. - -To run a simulation, copy the lines above to a file ``cori_haswell.sbatch`` and -run - -.. code-block:: bash - - sbatch cori_haswell.sbatch - -to submit the job. - -For a 3D simulation with a few (1-4) particles per cell using FDTD Maxwell -solver on Cori Haswell for a well load-balanced problem (in our case laser -wakefield acceleration simulation in a boosted frame in the quasi-linear -regime), the following set of parameters provided good performance: - -* **4 MPI ranks per Haswell node** (2 MPI ranks per `Intel Xeon E5-2698 v3 `_), with ``OMP_NUM_THREADS=16`` (which uses `2x hyperthreading `_) - -GPU (V100) -^^^^^^^^^^ - -Do not forget to first ``source $HOME/gpu_warpx.profile`` if you have not done so already for this terminal session. - -Due to the limited amount of GPU development nodes, just request a single node with the above defined ``getNode`` function. -For single-node runs, try to run one grid per GPU. - -A multi-node batch script template can be found below: - -.. literalinclude:: ../../../../Tools/machines/cori-nersc/cori_gpu.sbatch - :language: bash - :caption: You can copy this file from ``Tools/machines/cori-nersc/cori_gpu.sbatch``. - - -.. _post-processing-cori: - -Post-Processing ---------------- - -For post-processing, most users use Python via NERSC's `Jupyter service `__ (`Docs `__). - -As a one-time preparatory setup, `create your own Conda environment as described in NERSC docs `__. -In this manual, we often use this ``conda create`` line over the officially documented one: - -.. code-block:: bash - - conda create -n myenv -c conda-forge python mamba ipykernel ipympl==0.8.6 matplotlib numpy pandas yt openpmd-viewer openpmd-api h5py fast-histogram dask dask-jobqueue pyarrow - -We then follow the `Customizing Kernels with a Helper Shell Script `__ section to finalize the setup of using this conda-environment as a custom Jupyter kernel. - -``kernel_helper.sh`` should read: - -.. code-block:: bash - - #!/bin/bash - module load python - source activate myenv - exec "$@" - -When opening a Jupyter notebook, just select the name you picked for your custom kernel on the top right of the notebook. - -Additional software can be installed later on, e.g., in a Jupyter cell using ``!mamba install -c conda-forge ...``. -Software that is not available via conda can be installed via ``!python -m pip install ...``. - -.. warning:: - - Jan 6th, 2022 (NERSC-INC0179165 and `ipympl #416 `__): - Above, we fixated the ``ipympl`` version to *not* take the latest release of `Matplotlib Jupyter Widgets `__. - This is an intentional work-around; the ``ipympl`` version needs to exactly fit the version pre-installed on the Jupyter base system. diff --git a/Docs/source/install/hpc/fugaku.rst b/Docs/source/install/hpc/fugaku.rst index 43eb414de09..e70aeefa504 100644 --- a/Docs/source/install/hpc/fugaku.rst +++ b/Docs/source/install/hpc/fugaku.rst @@ -24,7 +24,7 @@ Use the following commands to download the WarpX source code and switch to the c git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx -Compiling WarpX on Fugaku is more pratical on a compute node. Use the following commands to acquire a compute node for one hour: +Compiling WarpX on Fugaku is more practical on a compute node. Use the following commands to acquire a compute node for one hour: .. code-block:: bash diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst index 117cc32a0df..fffb917df1c 100644 --- a/Docs/source/install/hpc/karolina.rst +++ b/Docs/source/install/hpc/karolina.rst @@ -12,83 +12,60 @@ Introduction If you are new to this system, **please see the following resources**: * `IT4I user guide `__ -* Batch system: `PBS `__ +* Batch system: `SLURM `__ * Jupyter service: not provided/documented (yet) * `Filesystems `__: * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (25GB default quota) - * ``/scatch/``: `production directory `__; very fast for parallel jobs (20TB default) + * ``/scratch/``: `production directory `__; very fast for parallel jobs (10TB default) + * ``/mnt/proj/``: per-project work directory, used for long term data storage (20TB default) .. _building-karolina-preparation: -Preparation ------------ - -Use the following commands to download the WarpX source code: +Installation +------------ -.. code-block:: bash +We show how to install from scratch all the dependencies using `Spack `__. - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx +For size reasons it is not advisable to install WarpX in the ``$HOME`` directory, it should be installed in the "work directory". For this purpose we set an environment variable ``$WORK`` with the path to the "work directory". On Karolina, you can run either on GPU nodes with fast A100 GPUs (recommended) or CPU nodes. -.. tab-set:: - - .. tab-item:: A100 GPUs - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/karolina_gpu_warpx.profile``. - Create it now: - - .. code-block:: bash - - cp $HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example $HOME/karolina_gpu_warpx.profile - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example - :language: bash - - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``DD-23-83``, then run ``vi $HOME/karolina_gpu_warpx.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: - - .. code-block:: bash +Profile file +^^^^^^^^^^^^ - export proj="DD-23-83" +One can use the pre-prepared ``karolina_warpx.profile`` script below, +which you can copy to ``${HOME}/karolina_warpx.profile``, edit as required and then ``source``. - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. important:: + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_warpx.profile.example + :language: bash + :caption: Copy the contents of this file to ``${HOME}/karolina_warpx.profile``. - Now, and as the first step on future logins to Karolina, activate these environment settings: +To have the environment activated on every login, add the following line to ``${HOME}/.bashrc``: - .. code-block:: bash - - source $HOME/karolina_gpu_warpx.profile - - Finally, since Karolina does not yet provide software modules for some of our dependencies, install them once: - - .. code-block:: bash +.. code-block:: bash - bash $HOME/src/warpx/Tools/machines/karolina-it4i/install_gpu_dependencies.sh - source $HOME/sw/karolina/gpu/venvs/warpx-gpu/bin/activate + source $HOME/karolina_warpx.profile - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down +To install the ``spack`` environment and Python packages: - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_gpu_dependencies.sh - :language: bash +.. code-block:: bash + bash $WORK/src/warpx/Tools/machines/karolina-it4i/install_dependencies.sh - .. tab-item:: CPU Nodes +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - CPU usage is documentation is TODO. + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_dependencies.sh + :language: bash .. _building-karolina-compilation: @@ -98,83 +75,28 @@ Compilation Use the following :ref:`cmake commands ` to compile the application executable: -.. tab-set:: - - .. tab-item:: A100 GPUs - - .. code-block:: bash - - cd $HOME/src/warpx - rm -rf build_gpu - - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu -j 12 - - The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: - - .. code-block:: bash - - rm -rf build_gpu_py - - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu_py -j 12 --target pip_install - - .. tab-item:: CPU Nodes - - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu + cd $WORK/src/warpx + rm -rf build_gpu - cmake -S . -B build_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu -j 12 + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu -j 48 - The WarpX application executables are now in ``$HOME/src/warpx/build_cpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: +The WarpX application executables are now in ``$WORK/src/warpx/build_gpu/bin/``. +Additionally, the following commands will install WarpX as a Python module: - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu_py + cd $WORK/src/warpx + rm -rf build_gpu_py - cmake -S . -B build_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu_py -j 12 --target pip_install + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu_py -j 48 --target pip_install Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Karolina jobs (:ref:`example inputs `). -For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scatch/``. - - -.. _building-karolina-update: - -Update WarpX & Dependencies ---------------------------- - -If you already installed WarpX in the past and want to update it, start by getting the latest source code: - -.. code-block:: bash - - cd $HOME/src/warpx - - # read the output of this command - does it look ok? - git status - - # get the latest WarpX source code - git fetch - git pull - - # read the output of these commands - do they look ok? - git status - git log # press q to exit - -And, if needed, - -- :ref:`update the karolina_gpu_warpx.profile or karolina_cpu_warpx.profile files `, -- log out and into the system, activate the now updated environment profile as usual, -- :ref:`execute the dependency install scripts `. - -As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch/``. .. _running-cpp-karolina: @@ -182,33 +104,24 @@ As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and Running ------- -.. tab-set:: +The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#SBATCH --nodes=`` accordingly) on the supercomputer Karolina at IT4I. +This partition has up to `72 nodes `__. +Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. - .. tab-item:: A100 (40GB) GPUs +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. +Note that we run one MPI rank per GPU. - The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#PBS -l select=`` accordingly) on the supercomputer Karolina at IT4I. - This partition as up to `72 nodes `__. - Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. +.. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.sbatch + :language: bash + :caption: You can copy this file from ``$WORK/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.sbatch``. - Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. - Note that we run one MPI rank per GPU. - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.qsub - :language: bash - :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.qsub``. - - To run a simulation, copy the lines above to a file ``karolina_gpu.qsub`` and run - - .. code-block:: bash - - qsub karolina_gpu.qsub - - to submit the job. +To run a simulation, copy the lines above to a file ``karolina_gpu.sbatch`` and run +.. code-block:: bash - .. tab-item:: CPU Nodes + sbatch karolina_gpu.sbatch - CPU usage is documentation is TODO. +to submit the job. .. _post-processing-karolina: diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index 7b5630e0272..7285bc217f2 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -24,12 +24,27 @@ If you are new to this system, **please see the following resources**: Login ----- -.. note:: +.. tab-set:: - Lassen is currently transitioning to RHEL8. - During this transition, first SSH into lassen and then ``ssh eatoss4`` next to work with the updated RHEL8/TOSS4 nodes. + .. tab-item:: TOSS4 (RHEL8) - Approximately October 2023, the new software environment on these nodes will be the new default. + Lassen is currently transitioning to RHEL8. + During this transition, first SSH into lassen and then to the updated RHEL8/TOSS4 nodes. + + .. code-block:: bash + + ssh lassen.llnl.gov + ssh eatoss4 + + Approximately October/November 2023, the new software environment on these nodes will be the new default. + + .. tab-item:: TOSS3 (RHEL7) + + .. code-block:: bash + + ssh lassen.llnl.gov + + Approximately October/November 2023, this partition will become TOSS4 (RHEL8) as well. .. _building-lassen-preparation: @@ -41,80 +56,137 @@ Use the following commands to download the WarpX source code: .. code-block:: bash - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx -We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``. -Create it now: +.. tab-set:: -.. code-block:: bash + .. tab-item:: TOSS4 (RHEL8) - cp $HOME/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile + We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``. + Create it now: -.. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down + .. code-block:: bash - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example - :language: bash + cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile -Edit the 2nd line of this script, which sets the ``export proj=""`` variable. -For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``. -Enter the edit mode by typing ``i`` and edit line 2 to read: + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down -.. code-block:: bash + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example + :language: bash - export proj="nsldt" + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``. + Enter the edit mode by typing ``i`` and edit line 2 to read: -Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). + .. code-block:: bash -.. important:: + export proj="nsldt" - Now, and as the first step on future logins to lassen, activate these environment settings: + Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). - .. code-block:: bash + .. important:: + + Now, and as the first step on future logins to lassen, activate these environment settings: + + .. code-block:: bash + + source $HOME/lassen_v100_warpx.profile + + .. tab-item:: TOSS3 (RHEL7) + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``. + Create it now: + + .. code-block:: bash + + cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - source $HOME/lassen_v100_warpx.profile + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``. + Enter the edit mode by typing ``i`` and edit line 2 to read: + + .. code-block:: bash + + export proj="nsldt" + + Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). + + .. important:: + + Now, and as the first step on future logins to lassen, activate these environment settings: + + .. code-block:: bash + + source $HOME/lassen_v100_warpx_toss3.profile Finally, since lassen does not yet provide software modules for some of our dependencies, install them once: -.. code-block:: bash +.. tab-set:: - bash $HOME/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh - source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate + .. tab-item:: TOSS4 (RHEL8) -.. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down + .. code-block:: bash - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh - :language: bash + bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh + source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate -.. dropdown:: AI/ML Dependencies (Optional) - :animate: fade-in-slide-down + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - If you plan to run AI/ML workflows depending on pyTorch, run the next step as well. - This will take a while and should be skipped if not needed. + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh + :language: bash - .. code-block:: bash + .. dropdown:: AI/ML Dependencies (Optional) + :animate: fade-in-slide-down - runNode bash $HOME/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh + If you plan to run AI/ML workflows depending on pyTorch, run the next step as well. + This will take a while and should be skipped if not needed. - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down + .. code-block:: bash - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh - :language: bash + runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh - For `optimas dependencies `__ (incl. scikit-learn), plan another hour of build time: + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. code-block:: bash + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh + :language: bash + + For `optimas dependencies `__ (incl. scikit-learn), plan another hour of build time: + + .. code-block:: bash + + python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt + + .. tab-item:: TOSS3 (RHEL7) + + .. code-block:: bash + + bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh + source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen-toss3/bin/activate - python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh + :language: bash .. _building-lassen-compilation: @@ -125,13 +197,13 @@ Use the following :ref:`cmake commands ` to compile the applicat .. code-block:: bash - cd $HOME/src/warpx + cd /usr/workspace/${USER}/lassen/src/warpx rm -rf build_lassen cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lassen -j 8 -The WarpX application executables are now in ``$HOME/src/warpx/build_lassen/bin/``. +The WarpX application executables are now in ``/usr/workspace/${USER}/lassen/src/warpx/build_lassen/bin/``. Additionally, the following commands will install WarpX as a Python module: .. code-block:: bash @@ -155,7 +227,7 @@ If you already installed WarpX in the past and want to update it, start by getti .. code-block:: bash - cd $HOME/src/warpx + cd /usr/workspace/${USER}/lassen/src/warpx # read the output of this command - does it look ok? git status @@ -174,7 +246,7 @@ And, if needed, - log out and into the system, activate the now updated environment profile as usual, - :ref:`execute the dependency install scripts `. -As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_lassen`` and rebuild WarpX. +As a last step, clean the build directory ``rm -rf /usr/workspace/${USER}/lassen/src/warpx/build_lassen`` and rebuild WarpX. .. _running-cpp-lassen: @@ -191,15 +263,15 @@ The batch script below can be used to run a WarpX simulation on 2 nodes on the s Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. Note that the only option so far is to run with one MPI rank per GPU. -.. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen.bsub +.. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100.bsub :language: bash - :caption: You can copy this file from ``Tools/machines/lassen-llnl/lassen.bsub``. + :caption: You can copy this file from ``Tools/machines/lassen-llnl/lassen_v100.bsub``. -To run a simulation, copy the lines above to a file ``lassen.bsub`` and run +To run a simulation, copy the lines above to a file ``lassen_v100.bsub`` and run .. code-block:: bash - bsub lassen.bsub + bsub lassen_v100.bsub to submit the job. diff --git a/Docs/source/install/hpc/lawrencium.rst b/Docs/source/install/hpc/lawrencium.rst index 09eb67b2ca5..23be9b75431 100644 --- a/Docs/source/install/hpc/lawrencium.rst +++ b/Docs/source/install/hpc/lawrencium.rst @@ -80,7 +80,10 @@ Optionally, download and install Python packages for :ref:`PICMI ` python3 -m venv $HOME/sw/v100/venvs/warpx source $HOME/sw/v100/venvs/warpx/bin/activate python3 -m pip install --upgrade pip + python3 -m pip install --upgrade build + python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel + python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas diff --git a/Docs/source/install/hpc/leonardo.rst b/Docs/source/install/hpc/leonardo.rst new file mode 100644 index 00000000000..1f3c1e8fbfa --- /dev/null +++ b/Docs/source/install/hpc/leonardo.rst @@ -0,0 +1,172 @@ +.. _building-leonardo: + +Leonardo (CINECA) +================= + +The `Leonardo cluster `_ is hosted at `CINECA `_. + +On Leonardo, each one of the 3456 compute nodes features a custom Atos Bull Sequana XH21355 "Da Vinci" blade, composed of: + +* 1 x CPU Intel Ice Lake Xeon 8358 32 cores 2.60 GHz +* 512 (8 x 64) GB RAM DDR4 3200 MHz +* 4 x NVidia custom Ampere A100 GPU 64GB HBM2 +* 2 x NVidia HDR 2×100 GB/s cards + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `Leonardo website `_ +* `Leonardo user guide `_ + +Storage organization: + +* ``$HOME``: permanent, backed up, user specific (50 GB quota) +* ``$CINECA_SCRATCH``: temporary, user specific, no backup, a large disk for the storage of run time data and files, automatic cleaning procedure of data older than 40 days +* ``$PUBLIC``: permanent, no backup (50 GB quota) +* ``$WORK``: permanent, project specific, no backup + +.. _building-leonardo-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + +We use system software modules, add environment hints and further dependencies via the file ``$HOME/leonardo_gpu_warpx.profile``. +Create it now: + +.. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example $HOME/leonardo_gpu_warpx.profile + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example + :language: bash + +.. important:: + + Now, and as the first step on future logins to Leonardo, activate these environment settings: + + .. code-block:: bash + + source $HOME/leonardo_gpu_warpx.profile + +Finally, since Leonardo does not yet provide software modules for some of our dependencies, install them once: + +.. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/leonardo_cineca/install_gpu_dependencies.sh + source $HOME/sw/venvs/warpx/bin/activate + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/leonardo-cineca/install_gpu_dependencies.sh + :language: bash + + +.. _building-leonardo-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_gpu + + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu -j 16 + +The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. +Additionally, the following commands will install WarpX as a Python module: + +.. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_gpu_py + + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu_py -j 16 --target pip_install + +Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit Leonardo jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$CINECA_SCRATCH``. + +.. _building-leonardo-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd $HOME/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the leonardo_gpu_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directories ``rm -rf $HOME/src/warpx/build_gpu*`` and rebuild WarpX. + + +.. _running-leonardo: + +Running +------- +The batch script below can be used to run a WarpX simulation on multiple nodes on Leonardo. +Replace descriptions between chevrons ``<>`` by relevant values. +Note that we run one MPI rank per GPU. + +.. literalinclude:: ../../../../Tools/machines/leonardo-cineca/job.sh + :language: bash + :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/leonardo-cineca/job.sh``. + +To run a simulation, copy the lines above to a file ``job.sh`` and run + +.. code-block:: bash + + sbatch job.sh + +to submit the job. + +.. _post-processing-leonardo: + +Post-Processing +--------------- + +For post-processing, activate the environment settings: + +.. code-block:: bash + + source $HOME/leonardo_gpu_warpx.profile + +and run python scripts. diff --git a/Docs/source/install/hpc/lxplus.rst b/Docs/source/install/hpc/lxplus.rst index 413337f711d..5bd8a3c9795 100644 --- a/Docs/source/install/hpc/lxplus.rst +++ b/Docs/source/install/hpc/lxplus.rst @@ -147,7 +147,8 @@ Now, ensure Python tooling is up-to-date: .. code-block:: bash - python3 -m pip install -U pip setuptools wheel + python3 -m pip install -U pip + python3 -m pip install -U build packaging setuptools wheel python3 -m pip install -U cmake Then we compile WarpX as in the previous section (with or without CUDA) adding ``-DWarpX_PYTHON=ON`` and then we install it into our Python: diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index c5d6a9e1898..e9ebcd4e1de 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -13,7 +13,7 @@ If you are new to this system, **please see the following resources**: * `NERSC user guide `__ * Batch system: `Slurm `__ -* `Jupyter service `__ +* `Jupyter service `__ (`documentation `__) * `Filesystems `__: * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (40GB) @@ -271,11 +271,36 @@ Running Post-Processing --------------- -For post-processing, most users use Python via NERSC's `Jupyter service `__ (`Docs `__). +For post-processing, most users use Python via NERSC's `Jupyter service `__ (`documentation `__). -Please follow the same process as for :ref:`NERSC Cori post-processing `. -**Important:** The *environment + Jupyter kernel* must separate from the one you create for Cori. +As a one-time preparatory setup, log into Perlmutter via SSH and do *not* source the WarpX profile script above. +Create your own Conda environment and `Jupyter kernel `__ for post-processing: -The Perlmutter ``$PSCRATCH`` filesystem is only available on *Perlmutter* Jupyter nodes. -Likewise, Cori's ``$SCRATCH`` filesystem is only available on *Cori* Jupyter nodes. -You can use the Community FileSystem (CFS) from everywhere. +.. code-block:: bash + + module load python + + conda config --set auto_activate_base false + + # create conda environment + rm -rf $HOME/.conda/envs/warpx-pm-postproc + conda create --yes -n warpx-pm-postproc -c conda-forge mamba conda-libmamba-solver + conda activate warpx-pm-postproc + conda config --set solver libmamba + mamba install --yes -c conda-forge python ipykernel ipympl matplotlib numpy pandas yt openpmd-viewer openpmd-api h5py fast-histogram dask dask-jobqueue pyarrow + + # create Jupyter kernel + rm -rf $HOME/.local/share/jupyter/kernels/warpx-pm-postproc/ + python -m ipykernel install --user --name warpx-pm-postproc --display-name WarpX-PM-PostProcessing + echo -e '#!/bin/bash\nmodule load python\nsource activate warpx-pm-postproc\nexec "$@"' > $HOME/.local/share/jupyter/kernels/warpx-pm-postproc/kernel-helper.sh + chmod a+rx $HOME/.local/share/jupyter/kernels/warpx-pm-postproc/kernel-helper.sh + KERNEL_STR=$(jq '.argv |= ["{resource_dir}/kernel-helper.sh"] + .' $HOME/.local/share/jupyter/kernels/warpx-pm-postproc/kernel.json | jq '.argv[1] = "python"') + echo ${KERNEL_STR} | jq > $HOME/.local/share/jupyter/kernels/warpx-pm-postproc/kernel.json + + exit + + +When opening a Jupyter notebook on `https://jupyter.nersc.gov `__, just select ``WarpX-PM-PostProcessing`` from the list of available kernels on the top right of the notebook. + +Additional software can be installed later on, e.g., in a Jupyter cell using ``!mamba install -y -c conda-forge ...``. +Software that is not available via conda can be installed via ``!python -m pip install ...``. diff --git a/Docs/source/install/hpc/quartz.rst b/Docs/source/install/hpc/quartz.rst index de7c5ace848..03860687971 100644 --- a/Docs/source/install/hpc/quartz.rst +++ b/Docs/source/install/hpc/quartz.rst @@ -37,14 +37,14 @@ Create it now: .. code-block:: bash - cp $HOME/src/warpx/Tools/machines/quartz-llnl/quartz/quartz_warpx.profile.example $HOME/quartz_warpx.profile + cp $HOME/src/warpx/Tools/machines/quartz-llnl/quartz_warpx.profile.example $HOME/quartz_warpx.profile .. dropdown:: Script Details :color: light :icon: info :animate: fade-in-slide-down - .. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz/quartz_warpx.profile.example + .. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz_warpx.profile.example :language: bash Edit the 2nd line of this script, which sets the ``export proj=""`` variable. diff --git a/Docs/source/install/users.rst b/Docs/source/install/users.rst index 5174a3b5343..b7893e248c0 100644 --- a/Docs/source/install/users.rst +++ b/Docs/source/install/users.rst @@ -109,12 +109,11 @@ Given that you have the :ref:`WarpX dependencies ` install .. code-block:: bash - # optional: --user - python3 -m pip install -U pip setuptools wheel + python3 -m pip install -U pip + python3 -m pip install -U build packaging setuptools wheel python3 -m pip install -U cmake python3 -m pip wheel -v git+https://github.com/ECP-WarpX/WarpX.git - # optional: --user python3 -m pip install *whl In the future, will publish pre-compiled binary packages on `PyPI `__ for faster installs. diff --git a/Docs/source/latex_theory/AMR/AMR.tex b/Docs/source/latex_theory/AMR/AMR.tex index 948921eca5e..3ca94d95436 100644 --- a/Docs/source/latex_theory/AMR/AMR.tex +++ b/Docs/source/latex_theory/AMR/AMR.tex @@ -22,7 +22,7 @@ \section{Mesh refinement} In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss' Law around the refined patch, leading to long range errors \cite{Vaylpb2002,Colellajcp2010}. As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. \subsection{Electrostatic} -A cornerstone of the Particle-In-Cell method is that assuming a particle lying in a hypothetical infinite grid, then if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called ``momentum conserving'' gathering scheme; b) on average within one cell if using the ``energy conserving'' gathering scheme \cite{Birdsalllangdon}. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) \cite{Vaylpb2002,Colellajcp2010}. +A cornerstone of the Particle-In-Cell method is that, given a particle lying in a hypothetical infinite grid, if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called ``momentum conserving'' gathering scheme; b) on average within one cell if using the ``energy conserving'' gathering scheme \cite{Birdsalllangdon}. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) \cite{Vaylpb2002,Colellajcp2010}. A sketch of the implementation of mesh refinement in WarpX is given in Figure~\ref{fig:ESAMR} (left). Given the solution of the electric potential at a refinement level $L_n$, it is interpolated onto the boundaries of the grid patch(es) at the next refined level $L_{n+1}$. The electric potential is then computed at level $L_{n+1}$ by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. @@ -53,7 +53,7 @@ \subsection{Electrostatic} % \caption{Snapshot from a 3D self-consistent simulation of the injector in the High Current Experiment shows the beam emerging from the source at low energy (blue) and being accelerated (green-yellow-orange) and transported in a four quadrupole front end. The automatic layout of the mesh refinement patches from a 2D axisymmetric simulation of the source area shows 2 levels of refinement, concentrating the finer meshes around the emitter (white curve surface) and the beam edge (dark blue).} % \label{fig:ESHCX} %\end{figure} -%Automatic remeshing has been implemented in WarpX following the procedure described in \cite{Vaynim2005}, refining on criteria based on measures of local charge density magnitude and gradients. AMR WarpX simulations were applied to the modeling of the front end injector of the High Current Experiment (HCX) \cite{Prostprstab2005}, and provided the first numerically converged estimates of phase space beam distorsions, which directly affects beam quality \cite{Vaypop04}. Fig.~\ref{fig:ESHCX} shows snapshots from 2D axisymmetric simulation of the souce area illustrating the automatic placement of refined patches, and 3D simulation of the full injector showing the beam generation, acceleration and transport. +%Automatic remeshing has been implemented in WarpX following the procedure described in \cite{Vaynim2005}, refining on criteria based on measures of local charge density magnitude and gradients. AMR WarpX simulations were applied to the modeling of the front end injector of the High Current Experiment (HCX) \cite{Prostprstab2005}, and provided the first numerically converged estimates of phase space beam distorsions, which directly affects beam quality \cite{Vaypop04}. Fig.~\ref{fig:ESHCX} shows snapshots from 2D axisymmetric simulation of the source area illustrating the automatic placement of refined patches, and 3D simulation of the full injector showing the beam generation, acceleration and transport. \subsection{Electromagnetic} The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of \cite{Vayjcp01}, refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. diff --git a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex index 471863ac3ee..fe8365d15dd 100644 --- a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex +++ b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex @@ -175,7 +175,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} the simulation, and the artificial ``bump'' is again an arbitrary correction that departs from the underlying physics. -A new scheme was recently proposed, in \cite{KirchenARXIV2016,LeheARXIV2016}, which +A new scheme was recently proposed, in \cite{KirchenPOP2016,LehePRE2016}, which completely eliminates the NCI for a plasma drifting at a uniform relativistic velocity -- with no arbitrary correction -- by simply integrating the PIC equations in \emph{Galilean coordinates} (also known as @@ -217,7 +217,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} \emph{themselves} are not only translated but in this case, the physical equations are modified accordingly. Most importantly, the assumed time evolution of the current $\vec{J}$ within one timestep is different in a standard PSATD scheme with moving -window and in a Galilean PSATD scheme \cite{LeheARXIV2016}. +window and in a Galilean PSATD scheme \cite{LehePRE2016}. In the Galilean coordinates $\vec{x}'$, the equations of particle motion and the Maxwell equations take the form @@ -235,7 +235,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} Integrating these equations from $t=n\Delta t$ to $t=(n+1)\Delta t$ results in the following update equations (see -\cite{LeheARXIV2016} for the details of the derivation): +\cite{LehePRE2016} for the details of the derivation): % \begin{subequations} \begin{align} @@ -271,5 +271,5 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} Note that, in the limit $\vgal=\vec{0}$, (\ref{eq:disc-maxwell1}) and (\ref{eq:disc-maxwell2}) reduce to the standard PSATD equations \cite{Habericnsp73}, as expected. -As shown in \cite{KirchenARXIV2016,LeheARXIV2016}, +As shown in \cite{KirchenPOP2016,LehePRE2016}, the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. diff --git a/Docs/source/latex_theory/Makefile b/Docs/source/latex_theory/Makefile index 916bb2399d7..9b2f598f51d 100644 --- a/Docs/source/latex_theory/Makefile +++ b/Docs/source/latex_theory/Makefile @@ -7,14 +7,14 @@ all: $(SRC_FILES) clean pandoc Boosted_frame/Boosted_frame.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o boosted_frame.rst pandoc input_output/input_output.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o input_output.rst mv *.rst ../theory - cd ../../../../picsar/Doxygen/pages/latex_theory/; pandoc theory.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o picsar_theory.rst - mv ../../../../picsar/Doxygen/pages/latex_theory/picsar_theory.rst ../theory + cd ../../../../picsar/Doxygen/pages/latex_theory/; pandoc theory.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o pic.rst + mv ../../../../picsar/Doxygen/pages/latex_theory/pic.rst ../theory cp ../../../../picsar/Doxygen/images/PIC.png ../theory cp ../../../../picsar/Doxygen/images/Yee_grid.png ../theory clean: rm -f ../theory/intro.rst rm -f ../theory/warpx_theory.rst - rm -f ../theory/picsar_theory.rst + rm -f ../theory/pic.rst rm -f ../theory/PIC.png rm -f ../theory/Yee_grid.png diff --git a/Docs/source/latex_theory/allbibs.bib b/Docs/source/latex_theory/allbibs.bib index 81a02e75494..b3475c5a81b 100644 --- a/Docs/source/latex_theory/allbibs.bib +++ b/Docs/source/latex_theory/allbibs.bib @@ -4,11 +4,11 @@ BibTeX export options can be customized via Preferences -> BibTeX in Mendeley Desktop @article{QuickpicParallel, -author = {Feng, B. and Huang, C. and Decyk, V. and Mori, W.B. and Muggli, P. and Katsouleas, T.}, +author = {Feng, B. and Huang, C. and Decyk, V. and Mori, W. B. and Muggli, P. and Katsouleas, T.}, doi = {10.1016/j.jcp.2009.04.019}, issn = {00219991}, journal = {Journal of Computational Physics}, -month = {aug}, +month = {Aug}, number = {15}, pages = {5340--5348}, title = {{Enhancing parallel quasi-static particle-in-cell simulations with a pipelining algorithm}}, @@ -20,16 +20,17 @@ @book{HockneyEastwoodBook author = {Hockney, R W and Eastwood, J W}, isbn = {0-85274-392-0}, pages = {xxi+540 pp}, +publisher = {Routledge}, title = {{Computer simulation using particles}}, type = {Book}, year = {1988} } @article{Parkerjcp1991, -author = {Parker, Se and Birdsall, Ck}, +author = {Parker, S E and Birdsall, C K}, doi = {10.1016/0021-9991(91)90040-R}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {nov}, +month = {Nov}, number = {1}, pages = {91--102}, title = {{Numerical Error In Electron Orbits With Large Omega-Ce Delta-T}}, @@ -44,7 +45,7 @@ @inproceedings{Fawleyipac10 year = {2010} } @article{Godfreyjcp74, -author = {Godfrey, Bb}, +author = {Godfrey, B B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {4}, @@ -54,11 +55,11 @@ @article{Godfreyjcp74 year = {1974} } @article{Quickpic, -author = {Huang, C and Decyk, V K and Ren, C and Zhou, M and Lu, W and Mori, W B and Cooley, J H and {Antonsen Jr.}, T M and Katsouleas, T}, +author = {Huang, C and Decyk, V K and Ren, C and Zhou, M and Lu, W and Mori, W B and Cooley, J H and Antonsen, Jr, T M and Katsouleas, T}, doi = {10.1016/J.Jcp.2006.01.039}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {2}, pages = {658--679}, title = {{Quickpic: A Highly Efficient Particle-In-Cell Code For Modeling Wakefield Acceleration In Plasmas}}, @@ -70,7 +71,7 @@ @article{Lundprstab2009 doi = {10.1103/Physrevstab.12.114801}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {nov}, +month = {Nov}, number = {11}, title = {{Generation Of Initial Kinetic Distributions For Simulation Of Long-Pulse Charged Particle Beams With High Space-Charge Intensity}}, volume = {12}, @@ -81,7 +82,7 @@ @article{CowanPRSTAB13 doi = {10.1103/PhysRevSTAB.16.041303}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {apr}, +month = {Apr}, number = {4}, title = {{Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations}}, volume = {16}, @@ -102,7 +103,7 @@ @article{YuPRL2014 author = {Yu, L.-L. and Esarey, E and Schroeder, C B and Vay, J.-L. and Benedetti, C and Geddes, C G R and Chen, M and Leemans, W P}, doi = {10.1103/PhysRevLett.112.125001}, journal = {Phys. Rev. Lett.}, -month = {mar}, +month = {Mar}, number = {12}, pages = {125001}, publisher = {American Physical Society}, @@ -118,7 +119,7 @@ @article{Vaynim2007 institution = {Univ Paris Sud Xi}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {65--69}, title = {{Self-Consistent Simulations Of Heavy-Ion Beams Interacting With Electron-Clouds}}, @@ -138,10 +139,10 @@ @article{Vay2000 year = {2000} } @article{Vayjcp2011, -author = {Vay, J L and Geddes, C G R and Cormier-Michel, E and Grote, D P}, +author = {Vay, J-L and Geddes, C G R and Cormier-Michel, E and Grote, D P}, doi = {10.1016/J.Jcp.2011.04.003}, journal = {Journal of Computational Physics}, -month = {jul}, +month = {Jul}, number = {15}, pages = {5908--5929}, title = {{Numerical Methods For Instability Mitigation In The Modeling Of Laser Wakefield Accelerators In A Lorentz-Boosted Frame}}, @@ -152,21 +153,15 @@ @article{Krallpre1993 author = {Krall, J and Ting, A and Esarey, E and Sprangle, P}, doi = {10.1103/Physreve.48.2157}, journal = {Physical Review E}, -month = {sep}, +month = {Sep}, number = {3}, pages = {2157--2161}, title = {{Enhanced Acceleration In A Self-Modulated-Laser Wake-Field Accelerator}}, volume = {48}, year = {1993} } -@article{Vayarxiv10_2, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:1011.0917V2}, -title = {{Effects Of Hyperbolic Rotation In Minkowski Space On The Modeling Of Plasma Accelerators In A Lorentz Boosted Frame}}, -year = {2010} -} @article{Dennisw1997585, -author = {W., Dennis and Hewett}, +author = {Dennis W. Hewett}, doi = {10.1006/Jcph.1997.5835}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -178,12 +173,11 @@ @article{Dennisw1997585 year = {1997} } @article{Habib2016, - archivePrefix = {arXiv}, arxivId = {1603.09303}, -author = {Habib, Salman and Roser, Robert and Gerber, Richard and Antypas, Katie and Riley, Katherine and Williams, Tim and Wells, Jack and Straatsma, Tjerk and Almgren, A. and Amundson, J. and Bailey, S. and Bard, D. and Bloom, K. and Bockelman, B. and Borgland, A. and Borrill, J. and Boughezal, R. and Brower, R. and Cowan, B. and Finkel, H. and Frontiere, N. and Fuess, S. and Ge, L. and Gnedin, N. and Gottlieb, S. and Gutsche, O. and Han, T. and Heitmann, K. and Hoeche, S. and Ko, K. and Kononenko, O. and LeCompte, T. and Li, Z. and Lukic, Z. and Mori, W. and Nugent, P. and Ng, C. -K. and Oleynik, G. and O'Shea, B. and Padmanabhan, N. and Petravick, D. and Petriello, F. J. and Power, J. and Qiang, J. and Reina, L. and Rizzo, T. J. and Ryne, R. and Schram, M. and Spentzouris, P. and Toussaint, D. and Vay, J. -L. and Viren, B. and Wurthwein, F. and Xiao, L.}, +author = {Habib, Salman and Roser, Robert and Gerber, Richard and Antypas, Katie and Riley, Katherine and Williams, Tim and Wells, Jack and Straatsma, Tjerk and Almgren, A. and Amundson, J. and Bailey, S. and Bard, D. and Bloom, K. and Bockelman, B. and Borgland, A. and Borrill, J. and Boughezal, R. and Brower, R. and Cowan, B. and Finkel, H. and Frontiere, N. and Fuess, S. and Ge, L. and Gnedin, N. and Gottlieb, S. and Gutsche, O. and Han, T. and Heitmann, K. and Hoeche, S. and Ko, K. and Kononenko, O. and LeCompte, T. and Li, Z. and Lukic, Z. and Mori, W. and Nugent, P. and Ng, C.-K. and Oleynik, G. and O'Shea, B. and Padmanabhan, N. and Petravick, D. and Petriello, F. J. and Power, J. and Qiang, J. and Reina, L. and Rizzo, T. J. and Ryne, R. and Schram, M. and Spentzouris, P. and Toussaint, D. and Vay, J.-L. and Viren, B. and Wurthwein, F. and Xiao, L.}, eprint = {1603.09303}, -month = {mar}, +month = {Mar}, title = {{ASCR/HEP Exascale Requirements Review Report}}, url = {http://arxiv.org/abs/1603.09303}, year = {2016} @@ -192,8 +186,8 @@ @article{Shadwickpop09 author = {Shadwick, B A and Schroeder, C B and Esarey, E}, doi = {10.1063/1.3124185}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56704}, title = {{Nonlinear Laser Energy Depletion In Laser-Plasma Accelerators}}, @@ -206,7 +200,7 @@ @article{Leemansaac2010 journal = {Aip Conference Proceedings}, keywords = {[978-0-7354-0853-1/10/{\$}30.00]}, pages = {3--11}, -title = {{The Berkeley Lab Laser Accelerator (Bella): A 10 Gev Laser Plasma Accelerator}}, +title = {{The Berkeley Lab Laser Accelerator (Bella): A 10 GeV Laser Plasma Accelerator}}, volume = {1299}, year = {2010} } @@ -214,7 +208,7 @@ @article{Marderjcp87 author = {Marder, B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {jan}, +month = {Jan}, number = {1}, pages = {48--55}, title = {{A Method For Incorporating Gauss Law Into Electromagnetic Pic Codes}}, @@ -231,7 +225,7 @@ @article{Ohmurapiers2010 year = {2010} } @article{Adamjcp1982, -author = {Adam, Jc and Serveniere, Ag and Langdon, Ab}, +author = {Adam, J C and Serveniere, Ag and Langdon, A B}, doi = {10.1016/0021-9991(82)90076-6}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -242,7 +236,7 @@ @article{Adamjcp1982 year = {1982} } @article{Tajimaprl79, -author = {Tajima, T and Dawson, Jm}, +author = {Tajima, T and Dawson, J M}, issn = {0031-9007}, journal = {Physical Review Letters}, number = {4}, @@ -251,13 +245,19 @@ @article{Tajimaprl79 volume = {43}, year = {1979} } -@article{Benedetti08, -author = {{Benedetti et al.}, C}, -journal = {Nuclear Inst. And Methods In Physics Research A}, -pages = {94--98}, -title = {{No Title}}, +@article{Benedetti09, +author = {C. Benedetti and P. Londrillo and V. Petrillo and L. Serafini and A. Sgattoni and P. Tomassini and G. Turchetti}, +doi = {https://doi.org/10.1016/j.nima.2009.05.064}, +issn = {0168-9002}, +journal = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, +keywords = {PIC simulation, Laser–plasma interaction, FEL}, +note = {Compton sources for X/γ rays: Physics and applications}, +number = {1, Supplement }, +pages = {S94-S98}, +title = {PIC simulations of the production of high-quality electron beams via laser–plasma interaction}, +url = {https://www.sciencedirect.com/science/article/pii/S0168900209009784}, volume = {608}, -year = {2009} +year = {2009}, } @inproceedings{Godfreyicnsp80, author = {Godfrey, B B}, @@ -270,7 +270,7 @@ @article{Blumenfeld2007 author = {Blumenfeld, Ian and Clayton, Christopher E and Decker, Franz-Josef and Hogan, Mark J and Huang, Chengkun and Ischebeck, Rasmus and Iverson, Richard and Joshi, Chandrashekhar and Katsouleas, Thomas and Kirby, Neil and Lu, Wei and Marsh, Kenneth A and Mori, Warren B and Muggli, Patric and Oz, Erdem and Siemann, Robert H and Walz, Dieter and Zhou, Miaomiao}, issn = {0028-0836}, journal = {Nature}, -month = {feb}, +month = {Feb}, number = {7129}, pages = {741--744}, title = {{Energy doubling of 42[thinsp]GeV electrons in a metre-scale plasma wakefield accelerator}}, @@ -280,7 +280,7 @@ @article{Blumenfeld2007 } @inproceedings{Vayicap2002, annote = {7Th International Conference On Computational Accelerator Physics, Michigan State Univ, E Lansing, Mi, Oct 15-18, 2002}, -author = {Vay, Jl and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Friedman, A and Grote, D P}, booktitle = {Computational Accelerator Physics 2002}, editor = {Berz, M and Makino, K}, isbn = {0-7503-0939-3}, @@ -308,16 +308,15 @@ @article{VayFRACAD2014 } @article{AndriyashPoP2016, - author = "Andriyash, Igor A. and Lehe, Remi and Lifschitz, Agustin", - title = "Laser-plasma interactions with a Fourier-Bessel particle-in-cell method", - journal = "Physics of Plasmas", - year = "2016", - volume = "23", - number = "3", - eid = 033110, - pages = "", - url = "http://scitation.aip.org/content/aip/journal/pop/23/3/10.1063/1.4943281", - doi = "http://dx.doi.org/10.1063/1.4943281" +author = {Andriyash, Igor A. and Lehe, Remi and Lifschitz, Agustin}, +doi = {10.1063/1.4943281}, +eid = {033110}, +journal = {Physics of Plasmas}, +number = {3}, +pages = {}, +title = {{Laser-plasma interactions with a Fourier-Bessel particle-in-cell method}}, +volume = {23}, +year = {2016} } @article{DawsonRMP83, @@ -340,7 +339,7 @@ @inproceedings{Qiang @inproceedings{Geddespac09, address = {Vancouver, Canada}, annote = {We6Rfp075}, -author = {{Geddes et al.}, C G R}, +author = {C. G. R. Geddes and E. Cormier-Michel and E. Esarey and C. B. Schroeder and W. P. Leemans}, booktitle = {Proc. Particle Accelerator Conference}, title = {{Scaled Simulation Design Of High Quality Laser Wakefield Accelerator Stages}}, year = {2009} @@ -348,10 +347,9 @@ @inproceedings{Geddespac09 @article{LotovPRSTAB2003, author = {Lotov, K. V.}, doi = {10.1103/PhysRevSTAB.6.061301}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Lotov - 2003 - Fine wakefield structure in the blowout regime of plasma wakefield accelerators.pdf:pdf}, issn = {1098-4402}, journal = {Physical Review Special Topics - Accelerators and Beams}, -month = {jun}, +month = {Jun}, number = {6}, pages = {061301}, publisher = {American Physical Society}, @@ -361,7 +359,7 @@ @article{LotovPRSTAB2003 year = {2003} } @article{Godfreyjcp75, -author = {Godfrey, Bb}, +author = {Godfrey, B B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {1}, @@ -374,7 +372,7 @@ @article{Kishekprl2012 author = {Kishek, R A}, doi = {10.1103/Physrevlett.108.035003}, journal = {Phys. Rev. Lett.}, -month = {jan}, +month = {Jan}, number = {3}, pages = {35003}, publisher = {American Physical Society}, @@ -392,12 +390,12 @@ @article{Zhang2016 } @article{Cohennim2009, annote = {17Th International Symposium On Heavy Ion Inertial Fusion, Tokyo, Japan, Aug 04-08, 2008}, -author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J -L.}, +author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J-L}, doi = {10.1016/J.Nima.2009.03.083}, institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {53--55}, title = {{An Implicit ``Drift-Lorentz{\{}''{\}} Mover For Plasma And Beam Simulations}}, @@ -405,10 +403,11 @@ @article{Cohennim2009 year = {2009} } @article{Osiris, -author = {{Fonseca et al.}, R A}, -journal = {Lec. Notes In Comp. Sci.}, -pages = {342}, -title = {{No Title}}, +author = {Fonseca, R. A. and Silva, L. O. and Tsung, F. S. and Decyk, V. K. and Lu, W. and Ren, C. and Mori, W. B. and Deng, S. and Lee, S. and Katsouleas, T. and Adam, J. C.}, +booktitle = {Computational Science --- ICCS 2002}, +pages = {342--351}, +publisher = {Springer Berlin Heidelberg}, +title = {{OSIRIS: A Three-Dimensional, Fully Relativistic Particle in Cell Code for Modeling Plasma Based Accelerators}}, volume = {2329}, year = {2002} } @@ -439,7 +438,7 @@ @article{GeddesPRL2008 author = {Geddes, C G R and Nakamura, K and Plateau, G R and Toth, Cs. and Cormier-Michel, E and Esarey, E and Schroeder, C B and Cary, J R and Leemans, W P}, doi = {10.1103/PhysRevLett.100.215004}, journal = {Phys. Rev. Lett.}, -month = {may}, +month = {May}, number = {21}, pages = {215004}, publisher = {American Physical Society}, @@ -465,12 +464,6 @@ @misc{Huebl2015 url = {http://dx.doi.org/10.5281/zenodo.591699}, year = {2015} } -@article{Vayarxiv10_1, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:1009.2727V2}, -title = {{Modeling Laser Wakefield Accelerators In A Lorentz Boosted Frame}}, -year = {2010} -} @article{Huangscidac09, author = {Huang, C and An, W and Decyk, V K and Lu, W and Mori, W B and Tsung, F S and Tzoufras, M and Morshed, S and Antonsen, T and Feng, B and Katsouleas, T and Fonseca, R A and Martins, S F and Vieira, J and Silva, L O and Esarey, E and Geddes, C G R and Leemans, W P and Cormier-Michel, E and Vay, J.-L. and Bruhwiler, D L and Cowan, B and Cary, J R and Paul, K}, issn = {1742-6596}, @@ -494,7 +487,7 @@ @article{Cormierprstab10 title = {{Propagation Of Higher Order Modes In Plasma Channels And Shaping Of The Transverse Field In Laser Plasma Accelerators}} } @article{Yee, -author = {Yee, Ks}, +author = {Yee, K S}, issn = {0018-926X}, journal = {Ieee Transactions On Antennas And Propagation}, number = {3}, @@ -521,7 +514,7 @@ @inproceedings{Cormieraac08 booktitle = {Aip Conference Proceedings}, issn = {0094-243X}, pages = {297--302}, -title = {{Scaled Simulations Of A 10 Gev Accelerator}}, +title = {{Scaled Simulations Of A 10 GeV Accelerator}}, volume = {1086}, year = {2009} } @@ -536,11 +529,11 @@ @inproceedings{Bruhwileraac08 } @article{Vaynim2005, annote = {15Th International Symposium On Heavy Ion Inertial Fusion, Princeton, Nj, Jun 07-11, 2004}, -author = {Vay, Jl and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Friedman, A and Grote, D P}, doi = {10.1016/J.Nima.2005.01.232}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {may}, +month = {May}, number = {1-2}, pages = {347--352}, title = {{Application Of Adaptive Mesh Refinement To Pic Simulations In Heavy Ion Fusion}}, @@ -549,7 +542,7 @@ @article{Vaynim2005 } @article{BulanovSV2014, -author = {{Bulanov S V and Wilkens J J and Esirkepov T Zh and Korn G and Kraft G and Kraft S D and Molls M and Khoroshkov V S}}, +author = {Bulanov, S V and Wilkens, J J and Esirkepov, T Zh and Korn, G and Kraft, G and Kraft, S D and Molls, M and Khoroshkov, V S}, issn = {1063-7869}, journal = {Physics-Uspekhi}, number = {12}, @@ -560,11 +553,11 @@ @article{BulanovSV2014 year = {2014} } @article{Furmanprstab2002, -author = {Furman, Ma and Pivi, Mtf}, +author = {Furman, M A and Pivi, M T F}, doi = {10.1103/Physrevstab.5.124404}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {dec}, +month = {Dec}, number = {12}, title = {{Probabilistic Model For The Simulation Of Secondary Electron Emission}}, volume = {5}, @@ -573,10 +566,9 @@ @article{Furmanprstab2002 @article{Qiang2014, author = {Qiang, J. and Corlett, J. and Mitchell, C. E. and Papadopoulos, C. F. and Penn, G. and Placidi, M. and Reinsch, M. and Ryne, R. D. and Sannibale, F. and Sun, C. and Venturini, M. and Emma, P. and Reiche, S.}, doi = {10.1103/PhysRevSTAB.17.030701}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Qiang et al. - 2014 - Start-to-end simulation of x-ray radiation of a next generation light source using the real number of electrons.pdf:pdf}, issn = {1098-4402}, journal = {Physical Review Special Topics - Accelerators and Beams}, -month = {mar}, +month = {Mar}, number = {3}, pages = {030701}, publisher = {American Physical Society}, @@ -601,13 +593,12 @@ @misc{Bruhwilerpc08 year = {2008} } @article{Yu2014, - author = {Yu, Peicheng and Xu, Xinlu and Decyk, Viktor K. and An, Weiming and Vieira, Jorge and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.jcp.2014.02.016}, issn = {00219991}, journal = {Journal of Computational Physics}, keywords = {Boosted frame simulation,IN-CELL CODE,INSTABILITIES,Laser wakefield accelerator,Numerical Cerenkov instability,PARTICLE SIMULATION,PLASMAS,Particle-in-cell,Plasma simulation,Spectral solver}, -month = {jun}, +month = {Jun}, pages = {124--138}, publisher = {ACADEMIC PRESS INC ELSEVIER SCIENCE, 525 B ST, STE 1900, SAN DIEGO, CA 92101-4495 USA}, title = {{Modeling of laser wakefield acceleration in Lorentz boosted frame using EM-PIC code with spectral solver}}, @@ -616,13 +607,13 @@ @article{Yu2014 year = {2014} } @article{Vaypop2011, -author = {Vay, J -L. and Geddes, C G R and Esarey, E and Schroeder, C B and Leemans, W P and Cormier-Michel, E and Grote, D P}, +author = {Vay, J.-L. and Geddes, C G R and Esarey, E and Schroeder, C B and Leemans, W P and Cormier-Michel, E and Grote, D P}, doi = {10.1063/1.3663841}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {dec}, +journal = {Physics of Plasmas}, +month = {Dec}, number = {12}, -title = {{Modeling Of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations}}, +title = {{Modeling Of 10 GeV-1 TeV Laser-Plasma Accelerators Using Lorentz Boosted Simulations}}, volume = {18}, year = {2011} } @@ -631,7 +622,7 @@ @article{LehePRSTAB13 doi = {10.1103/PhysRevSTAB.16.021301}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {feb}, +month = {Feb}, number = {2}, title = {{Numerical growth of emittance in simulations of laser-wakefield acceleration}}, volume = {16}, @@ -641,7 +632,7 @@ @article{Langdoncpc92 author = {Langdon, A B}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {jul}, +month = {Jul}, number = {3}, pages = {447--450}, title = {{On Enforcing Gauss Law In Electromagnetic Particle-In-Cell Codes}}, @@ -653,31 +644,29 @@ @article{Colellajcp2010 doi = {10.1016/J.Jcp.2009.07.004}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {feb}, +month = {Feb}, number = {4}, pages = {947--957}, title = {{Controlling Self-Force Errors At Refinement Boundaries For Amr-Pic}}, volume = {229}, year = {2010} } - @article{Coleieee1997, author = {Cole, J. B.}, issn = {0018-9480}, journal = {Ieee Transactions On Microwave Theory And Techniques}, -month = {jun}, +month = {Jun}, number = {6}, pages = {991--996}, title = {{A High-Accuracy Realization Of The Yee Algorithm Using Non-Standard Finite Differences}}, volume = {45}, year = {1997} } - @article{Coleieee2002, author = {Cole, J. B.}, issn = {0018-926X}, journal = {Ieee Transactions On Antennas And Propagation}, -month = {sep}, +month = {Sep}, number = {9}, pages = {1185--1191}, title = {{High-Accuracy Yee Algorithm Based On Nonstandard Finite Differences: New Developments And Verifications}}, @@ -685,7 +674,6 @@ @article{Coleieee2002 year = {2002}, doi = {10.1109/Tap.2002.801268}, } - @article{HajimaNIM09, annote = {Workshop on Compton Sources for X-gamma Rays, Porto Conte, ITALY, SEP 07-12, 2008}, author = {Hajima, R and Kikuzawa, N and Nishimori, N and Hayakawa, T and Shizuma, T and Kawase, K and Kando, M and Minehara, E and Toyokawa, H and Ohgaki, H}, @@ -693,7 +681,7 @@ @article{HajimaNIM09 institution = {Ist Nazl Fis Nucl; ICFA}, issn = {0168-9002}, journal = {NUCLEAR INSTRUMENTS {\&} METHODS IN PHYSICS RESEARCH SECTION A-ACCELERATORS SPECTROMETERS DETECTORS AND ASSOCIATED EQUIPMENT}, -month = {sep}, +month = {Sep}, number = {1}, pages = {S57--S61}, title = {{Detection of radioactive isotopes by using laser Compton scattered gamma-ray beams}}, @@ -705,7 +693,7 @@ @article{MatlisJOSA11 doi = {10.1364/JOSAB.28.000023}, issn = {0740-3224}, journal = {JOURNAL OF THE OPTICAL SOCIETY OF AMERICA B-OPTICAL PHYSICS}, -month = {jan}, +month = {Jan}, number = {1}, pages = {23--27}, title = {{Single-shot spatiotemporal measurements of ultrashort THz waveforms using temporal electric-field cross correlation}}, @@ -713,7 +701,6 @@ @article{MatlisJOSA11 year = {2011} } @article{Quickpic2, - author = {An, Weiming and Decyk, Viktor K. and Mori, Warren B. and Antonsen, Thomas M.}, doi = {10.1016/j.jcp.2013.05.020}, issn = {00219991}, @@ -730,29 +717,32 @@ @misc{Cowanpriv2010 year = {2010} } @inproceedings{Geddesscidac09, -author = {{Geddes et al.}, Cgr}, +author = {Geddes, Cameron G.R. and Cormier-Michel, Estelle and Esarey, Eric H and Schroeder, Carl B and Vay, Jean-Luc and Leemans, Wim P and Bruhwiler, David L and Cary, John R and Cowan, Ben and Durant, Marc and Hamill, Paul and Messmer, Peter and Mullowney, Paul and Nieter, Chet and Paul, Kevin and Shasharina, Svetlana and Veitzer, Seth and Weber, Gunther and Rubel, Oliver and Ushizima, Daniela and Bethel, Wes and Wu, John}, booktitle = {Scidac Review 13}, -pages = {13}, +journal = {SciDAC Review}, +number = {13}, +pages = {13--21}, title = {{Laser Plasma Particle Accelerators: Large Fields For Smaller Facility Sources}}, +url = {https://www.osti.gov/biblio/971264}, year = {2009} } @article{Gomberoffpop2007, author = {Gomberoff, K and Fajans, J and Friedman, A and Grote, D and Vay, J.-L. and Wurtele, J S}, doi = {10.1063/1.2778420}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {oct}, +journal = {Physics of Plasmas}, +month = {Oct}, number = {10}, title = {{Simulations Of Plasma Confinement In An Antihydrogen Trap}}, volume = {14}, year = {2007} } @article{Morapop1997, -author = {Mora, P and Antonsen, Tm}, +author = {Mora, P and Antonsen, T M}, doi = {10.1063/1.872134}, issn = {1070-664X}, journal = {Phys. Plasmas}, -month = {jan}, +month = {Jan}, number = {1}, pages = {217--229}, title = {{Kinetic Modeling Of Intense, Short Laser Pulses Propagating In Tenuous Plasmas}}, @@ -770,12 +760,12 @@ @inproceedings{Cowanaac08 } @article{Cohenprstab2009, annote = {17Th International Symposium On Heavy Ion Inertial Fusion, Tokyo, Japan, Aug 04-08, 2008}, -author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J -L.}, +author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J-L}, doi = {10.1016/J.Nima.2009.03.083}, institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {53--55}, title = {{An Implicit ``Drift-Lorentz{\{}''{\}} Mover For Plasma And Beam Simulations}}, @@ -783,8 +773,7 @@ @article{Cohenprstab2009 year = {2009} } @article{Kaganovich2012, - -author = {Kaganovich, Igor D. and Massidda, Scott and Startsev, Edward A. and Davidson, Ronald C. and Vay, Jean Luc and Friedman, Alex}, +author = {Kaganovich, Igor D. and Massidda, Scott and Startsev, Edward A. and Davidson, Ronald C. and Vay, Jean-Luc and Friedman, Alex}, journal = {Nuclear Instruments and Methods in Physics Research, Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, keywords = {Beam dynamics,Longitudinal compression,Voltage errors}, pages = {48--63}, @@ -792,20 +781,22 @@ @article{Kaganovich2012 volume = {678}, year = {2012} } -@article{Vincenti2016, -author = {Vincenti, H. and Lehe, R. and Sasanka, R. and Vay, J-L.}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Vincenti et al. - 2016 - An efficient and portable SIMD algorithm for chargecurrent deposition in Particle-In-Cell codes.pdf:pdf}, -journal = {Computer Programs in Physics}, -pages = {To appear}, -title = {{An efficient and portable SIMD algorithm for charge/current deposition in Particle-In-Cell codes}}, -url = {https://arxiv.org/abs/1601.02056}, -year = {2016} +@article{VincentiCPC2017, +author = {H. Vincenti and M. Lobet and R. Lehe and R. Sasanka and J.-L. Vay}, +doi = {https://doi.org/10.1016/j.cpc.2016.08.023}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +pages = {145-154}, +title = {An efficient and portable SIMD algorithm for charge/current deposition in Particle-In-Cell codes}, +url = {https://www.sciencedirect.com/science/article/pii/S0010465516302764}, +volume = {210}, +year = {2017}, } @article{Vaypop2008, -author = {Vay, J L}, +author = {Vay, J-L}, doi = {10.1063/1.2837054}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56701}, title = {{Simulation Of Beams Or Plasmas Crossing At Relativistic Velocity}}, @@ -817,7 +808,7 @@ @article{Wieland2016 doi = {10.3847/0004-637X/820/1/62}, issn = {1538-4357}, journal = {The Astrophysical Journal}, -month = {mar}, +month = {Mar}, number = {1}, pages = {62}, title = {{NONRELATIVISTIC PERPENDICULAR SHOCKS MODELING YOUNG SUPERNOVA REMNANTS: NONSTATIONARY DYNAMICS AND PARTICLE ACCELERATION AT FORWARD AND REVERSE SHOCKS}}, @@ -831,11 +822,11 @@ @misc{Vay url = {http://www.nersc.gov/assets/Uploads/DOEExascaleReviewVay.pdf} } @article{Faurenature04, -author = {Faure, J and Glinec, Y and Pukhov, A and Kiselev, S and Gordienko, S and Lefebvre, E and Rousseau, Jp and Burgy, F and Malka, V}, +author = {Faure, J and Glinec, Y and Pukhov, A and Kiselev, S and Gordienko, S and Lefebvre, E and Rousseau, J P and Burgy, F and Malka, V}, doi = {10.1038/Nature02963}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {541--544}, title = {{A Laser-Plasma Accelerator Producing Monoenergetic Electron Beams}}, @@ -843,11 +834,11 @@ @article{Faurenature04 year = {2004} } @article{Greenwoodjcp04, -author = {Greenwood, Ad and Cartwright, Kl and Luginsland, Jw and Baca, Ea}, +author = {Greenwood, A D and Cartwright, K L and Luginsland, J W and Baca, E A}, doi = {10.1016/J.Jcp.2004.06.021}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {dec}, +month = {Dec}, number = {2}, pages = {665--684}, title = {{On The Elimination Of Numerical Cerenkov Radiation In Pic Simulations}}, @@ -855,8 +846,7 @@ @article{Greenwoodjcp04 year = {2004} } @article{GodfreyIEEE2014, - -author = {Godfrey, Brendan B. and Vay, Jean Luc and Haber, Irving}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {IEEE Transactions on Plasma Science}, keywords = {Accelerators,numerical stability,particle beams,particle-in-cell (PIC),relativistic effects,simulation,spectral methods}, number = {5}, @@ -874,23 +864,28 @@ @inproceedings{Cowanicap09 year = {2009} } @article{Wuprstab2011, -author = {Wu, H -C. and Meyer-Ter-Vehn, J and Hegelich, B M and Fernandez, J C}, +author = {Wu, H-C and Meyer-Ter-Vehn, J and Hegelich, B M and Fernandez, J C}, doi = {10.1103/Physrevstab.14.070702}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {jul}, +month = {Jul}, number = {7}, title = {{Nonlinear Coherent Thomson Scattering From Relativistic Electron Sheets As A Means To Produce Isolated Ultrabright Attosecond X-Ray Pulses}}, volume = {14}, year = {2011} } @article{VayAAC2010, -author = {Vay, J -. L and Geddes, C G R and Benedetti, C and Bruhwiler, D L and Cormier-Michel, E and Cowan, B M and Cary, J R and Grote, D P}, +author = {Vay, J.‐L. and Geddes, C. G. R. and Benedetti, C. and Bruhwiler, D. L. and Cormier‐Michel, E. and Cowan, B. M. and Cary, J. R. and Grote, D. P.}, doi = {10.1063/1.3520322}, -journal = {Aip Conference Proceedings}, +eprint = {https://pubs.aip.org/aip/acp/article-pdf/1299/1/244/11928106/244\_1\_online.pdf}, +issn = {0094-243X}, +journal = {AIP Conference Proceedings}, keywords = {[978-0-7354-0853-1/10/{\$}30.00]}, +month = {Nov}, +number = {1}, pages = {244--249}, title = {{Modeling Laser Wakefield Accelerators In A Lorentz Boosted Frame}}, +url = {https://doi.org/10.1063/1.3520322}, volume = {1299}, year = {2010} } @@ -905,21 +900,23 @@ @article{Vayprl07 year = {2007} } @article{VayJCP2013, - -author = {Vay, Jean Luc and Haber, Irving and Godfrey, Brendan B.}, +author = {Vay, Jean-Luc and Haber, Irving and Godfrey, Brendan B.}, +doi = {10.1016/j.jcp.2013.03.010}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, keywords = {Domain decomposition,Electromagnetic,FFT,Fast fourier transform,Parallel,Particle-In-Cell,Spectral}, +month = {Jun}, pages = {260--268}, title = {{A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas}}, volume = {243}, year = {2013} } @article{Kaganovichpop2004, -author = {Kaganovich, Id and Startsev, Ea and Davidson, Rc}, +author = {Kaganovich, I D and Startsev, E A and Davidson, R C}, doi = {10.1063/1.1758945}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {jul}, +journal = {Physics of Plasmas}, +month = {Jul}, number = {7}, pages = {3546--3552}, title = {{Nonlinear Plasma Waves Excitation By Intense Ion Beams In Background Plasma}}, @@ -928,12 +925,12 @@ @article{Kaganovichpop2004 } @article{Cohenpop2005, annote = {46Th Annual Meeting Of The Division Of Plasma Physics Of The American-Physical-Society, Savannah, Ga, Nov 15-19, 2004}, -author = {Cohen, Rh and Friedman, A and Covo, Mk and Lund, Sm and Molvik, Aw and Bieniosek, Fm and Seidl, Pa and Vay, Jl and Stoltz, P and Veitzer, S}, +author = {Cohen, R H and Friedman, A and Covo, M K and Lund, S M and Molvik, A W and Bieniosek, F M and Seidl, P A and Vay, J-L and Stoltz, P and Veitzer, S}, doi = {10.1063/1.1882292}, institution = {Amer Phys Soc}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Simulating Electron Clouds In Heavy-Ion Accelerators}}, volume = {12}, @@ -941,11 +938,11 @@ @article{Cohenpop2005 } @article{Vayfed1996, annote = {7Th International Symposium On Heavy Ion Inertial Fusion, Princeton Plasma Phys Lab, Princeton, Nj, Sep 06-09, 1995}, -author = {Vay, Jl and Deutsch, C}, +author = {Vay, J-L and Deutsch, C}, doi = {10.1016/S0920-3796(96)00502-9}, issn = {0920-3796}, journal = {Fusion Engineering And Design}, -month = {nov}, +month = {Nov}, pages = {467--476}, title = {{A Three-Dimensional Electromagnetic Particle-In-Cell Code To Simulate Heavy Ion Beam Propagation In The Reaction Chamber}}, volume = {32-33}, @@ -962,10 +959,8 @@ @article{Fengjcp09 year = {2009} } @article{BESAC2013, - author = {BESAC}, doi = {10.1037/a0034573}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/582d1799e353fccb9458ff0ccb827db4b01d7308.pdf:pdf}, issn = {1935-990X}, journal = {The American psychologist}, number = {7}, @@ -977,11 +972,11 @@ @article{BESAC2013 year = {2013} } @article{Prostprstab2005, -author = {Prost, Lr and Seidl, Pa and Bieniosek, Fm and Celata, Cm and Faltens, A and Baca, D and Henestroza, E and Kwan, Jw and Leitner, M and Waldron, Wl and Cohen, R and Friedman, A and Grote, D and Lund, Sm and Molvik, Aw and Morse, E}, +author = {Prost, L R and Seidl, P A and Bieniosek, F M and Celata, C M and Faltens, A and Baca, D and Henestroza, E and Kwan, J W and Leitner, M and Waldron, W L and Cohen, R and Friedman, A and Grote, D and Lund, S M and Molvik, A W and Morse, E}, doi = {10.1103/Physrevstab.8.020101}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {feb}, +month = {Feb}, number = {2}, title = {{High Current Transport Experiment For Heavy Ion Inertial Fusion}}, volume = {8}, @@ -1000,13 +995,12 @@ @inproceedings{Godfrey2013PPPS title = {{Numerical Stability of the Pseudo-Spectral EM PIC Algorithm}}, year = {2013} } - @article{Vaypop04, author = {Vay, J.-L. and Colella, P and Kwan, J W and Mccorquodale, P and Serafini, D B and Friedman, A and Grote, D P and Westenskow, G and Adam, J.-C. and Heron, A and Haber, I}, doi = {10.1063/1.1689669}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {2928--2934}, title = {{Application Of Adaptive Mesh Refinement To Particle-In-Cell Simulations Of Plasmas And Beams}}, @@ -1017,7 +1011,7 @@ @article{Friedmanjcp90 author = {Friedman, A}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {oct}, +month = {Oct}, number = {2}, pages = {292--312}, title = {{A 2Nd-Order Implicit Particle Mover With Adjustable Damping}}, @@ -1027,8 +1021,8 @@ @article{Friedmanjcp90 @article{Rittershoferpop2010, author = {Rittershofer, W and Schroeder, C B and Esarey, E and Gruner, F J and Leemans, W P}, doi = {10.1063/1.3430638}, -journal = {Physics Of Plasmas}, -month = {jun}, +journal = {Physics of Plasmas}, +month = {Jun}, number = {6}, pages = {63104}, title = {{Tapered Plasma Channels To Phase-Lock Accelerating And Focusing Forces In Laser-Plasma Accelerators}}, @@ -1043,16 +1037,17 @@ @article{Vorpal volume = {196}, year = {2004} } -@article{LiARXIV2016, - -archivePrefix = {arXiv}, -arxivId = {1605.01496}, -author = {Li, Fei and Yu, Peicheng and Xu, Xinlu and Fiuza, Frederico and Decyk, Viktor K. and Dalichaouch, Thamine and Davidson, Asher and Tableman, Adam and An, Weiming and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Mori, Warren B.}, -eprint = {1605.01496}, -month = {may}, -title = {{Controlling the Numerical Cerenkov Instability in PIC simulations using a customized finite difference Maxwell solver and a local FFT based current correction}}, -url = {http://arxiv.org/abs/1605.01496}, -year = {2016} +@article{LiCPC2017, +author = {Fei Li and Peicheng Yu and Xinlu Xu and Frederico Fiuza and Viktor K. Decyk and Thamine Dalichaouch and Asher Davidson and Adam Tableman and Weiming An and Frank S. Tsung and Ricardo A. Fonseca and Wei Lu and Warren B. Mori}, +doi = {https://doi.org/10.1016/j.cpc.2017.01.001}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +keywords = {PIC simulation, Hybrid Maxwell solver, Relativistic plasma drift, Numerical Cerenkov instability, Lorentz boosted frame}, +pages = {6-17}, +title = {{Controlling the numerical Cerenkov instability in PIC simulations using a customized finite difference Maxwell solver and a local FFT based current correction}}, +url = {https://www.sciencedirect.com/science/article/pii/S0010465517300012}, +volume = {214}, +year = {2017}, } @article{XuJCP2013, author = {Xu, Xinlu and Yu, Peicheng and Martins, Samual F and Tsung, Frank S and Decyk, Viktor K and Vieira, Jorge and Fonseca, Ricardo A and Lu, Wei and Silva, Luis O and Mori, Warren B}, @@ -1072,17 +1067,17 @@ @article{Cormierpre08 doi = {10.1103/Physreve.78.016404}, issn = {1539-3755}, journal = {Physical Review E}, -month = {jul}, +month = {Jul}, number = {1, Part 2}, title = {{Unphysical Kinetic Effects In Particle-In-Cell Modeling Of Laser Wakefield Accelerators}}, volume = {78}, year = {2008} } @article{Berengerjcp96, -author = {Berenger, Jp}, +author = {Berenger, J P}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {2}, pages = {363--379}, title = {{Three-Dimensional Perfectly Matched Layer For The Absorption Of Electromagnetic Waves}}, @@ -1090,16 +1085,21 @@ @article{Berengerjcp96 year = {1996} } @article{Kalmykovprl09, -author = {{Kalmykov et al.}, S}, +author = {Kalmykov, S. and Yi, S. A. and Khudik, V. and Shvets, G.}, +doi = {10.1103/PhysRevLett.103.135004}, +issue = {13}, journal = {Phys. Rev. Lett.}, +month = {Sep}, +numpages = {4}, pages = {135004}, -title = {{No Title}}, +publisher = {American Physical Society}, +title = {{Electron Self-Injection and Trapping into an Evolving Plasma Bubble}}, +url = {https://link.aps.org/doi/10.1103/PhysRevLett.103.135004}, volume = {103}, year = {2009} } @article{Geddes2015, - -author = {Geddes, Cameron G R and Rykovanov, Sergey and Matlis, Nicholas H. and Steinke, Sven and Vay, Jean Luc and Esarey, Eric H. and Ludewigt, Bernhard and Nakamura, Kei and Quiter, Brian J. and Schroeder, Carl B. and Toth, Csaba and Leemans, Wim P.}, +author = {Geddes, Cameron G R and Rykovanov, Sergey and Matlis, Nicholas H. and Steinke, Sven and Vay, Jean-Luc and Esarey, Eric H. and Ludewigt, Bernhard and Nakamura, Kei and Quiter, Brian J. and Schroeder, Carl B. and Toth, Csaba and Leemans, Wim P.}, journal = {Nuclear Instruments and Methods in Physics Research, Section B: Beam Interactions with Materials and Atoms}, keywords = {Active interrogation,Homeland security,Laser plasma accelerator,Monoenergetic photon source,Nonproliferation}, pages = {116--121}, @@ -1112,7 +1112,7 @@ @article{Antonsenprl1992 author = {Antonsen, T M and Mora, P}, doi = {10.1103/Physrevlett.69.2204}, journal = {Physical Review Letters}, -month = {oct}, +month = {Oct}, number = {15}, pages = {2204--2207}, title = {{Self-Focusing And Raman-Scattering Of Laser-Pulses In Tenuous Plasmas}}, @@ -1120,8 +1120,7 @@ @article{Antonsenprl1992 year = {1992} } @article{GodfreyCPC2015, - -author = {Godfrey, Brendan B. and Vay, Jean Luc}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Computer Physics Communications}, keywords = {Numerical stability,Particle-in-cell,Pseudo-Spectral Time-Domain,Relativistic beam}, pages = {221--225}, @@ -1142,10 +1141,10 @@ @article{VayCSD12 year = {2012} } @article{Esirkepovcpc01, -author = {Esirkepov, Tz}, +author = {Esirkepov, T Z}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {apr}, +month = {Apr}, number = {2}, pages = {144--153}, title = {{Exact Charge Conservation Scheme For Particle-In-Cell Simulation With An Arbitrary Form-Factor}}, @@ -1156,8 +1155,8 @@ @article{Martinspop10 author = {Martins, S F and Fonseca, R A and Vieira, J and Silva, L O and Lu, W and Mori, W B}, doi = {10.1063/1.3358139}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56705}, title = {{Modeling Laser Wakefield Accelerator Experiments With Ultrafast Particle-In-Cell Simulations In Boosted Frames}}, @@ -1166,7 +1165,7 @@ @article{Martinspop10 } @inproceedings{Habericnsp73, address = {Berkeley, Ca}, -author = {Haber, I and Lee, R and Klein, Hh and Boris, Jp}, +author = {Haber, I and Lee, R and Klein, H H and Boris, J P}, booktitle = {Proc. Sixth Conf. Num. Sim. Plasmas}, pages = {46--48}, title = {{Advances In Electromagnetic Simulation Techniques}}, @@ -1177,7 +1176,7 @@ @article{Martinscpc10 doi = {10.1016/J.Cpc.2009.12.023}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {may}, +month = {May}, number = {5}, pages = {869--875}, title = {{Numerical Simulations Of Laser Wakefield Accelerators In Optimal Lorentz Frames}}, @@ -1189,7 +1188,7 @@ @article{Vaycpc04 doi = {10.1016/J.Cpc.2004.06.026}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {dec}, +month = {Dec}, number = {1-3}, pages = {171--177}, title = {{Asymmetric Pml For The Absorption Of Waves. Application To Mesh Refinement In Electromagnetic Particle-In-Cell Plasma Simulations}}, @@ -1197,8 +1196,7 @@ @article{Vaycpc04 year = {2004} } @article{GodfreyJCP2014_PSATD, - -author = {Godfrey, Brendan B. and Vay, Jean Luc and Haber, Irving}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {Journal of Computational Physics}, keywords = {Numerical stability,Particle-in-cell,Pseudo-spectral,Relativistic beam}, pages = {689--704}, @@ -1211,20 +1209,19 @@ @article{PruetJAP06 doi = {10.1063/1.2202005}, issn = {0021-8979}, journal = {JOURNAL OF APPLIED PHYSICS}, -month = {jun}, +month = {Jun}, number = {12}, title = {{Detecting clandestine material with nuclear resonance fluorescence}}, volume = {99}, year = {2006} } @article{YuCPC2015-Circ, - author = {Yu, Peicheng and Xu, Xinlu and Tableman, Adam and Decyk, Viktor K. and Tsung, Frank S. and Fiuza, Frederico and Davidson, Asher and Vieira, Jorge and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.cpc.2015.08.026}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {ALGORITHM,CODES,Hybrid Maxwell solver,IN-CELL SIMULATION,LASER WAKEFIELD ACCELERATORS,LORENTZ-BOOSTED FRAME,Numerical Cerenkov instability,OSIRIS,PARTICLE SIMULATION,PIC SIMULATIONS,PIC simulation,PLASMAS,Quasi-3D algorithm,Relativistic plasma drift,STABILITY}, -month = {dec}, +month = {Dec}, pages = {144--152}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Mitigation of numerical Cerenkov radiation and instability using a hybrid finite difference-FFT Maxwell solver and a local charge conserving current deposit}}, @@ -1233,10 +1230,10 @@ @article{YuCPC2015-Circ year = {2015} } @article{Berengerjcp94, -author = {Berenger, Jp}, +author = {Berenger, J P}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {oct}, +month = {Oct}, number = {2}, pages = {185--200}, title = {{A Perfectly Matched Layer For The Absorption Of Electromagnetic-Waves}}, @@ -1253,11 +1250,11 @@ @article{Rubel2016 year = {2016} } @article{Geddesnature04, -author = {Geddes, Cgr and Toth, C and {Van Tilborg}, J and Esarey, E and Schroeder, Cb and Bruhwiler, D and Nieter, C and Cary, J and Leemans, Wp}, +author = {Geddes, C G R and Toth, C and {Van Tilborg}, J and Esarey, E and Schroeder, C B and Bruhwiler, D and Nieter, C and Cary, J and Leemans, W P}, doi = {10.1038/Nature02900}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {538--541}, title = {{High-Quality Electron Beams From A Laser Wakefield Accelerator Using Plasma-Channel Guiding}}, @@ -1265,11 +1262,11 @@ @article{Geddesnature04 year = {2004} } @article{Munzjcp2000, -author = {Munz, Cd and Omnes, P and Schneider, R and Sonnendrucker, E and Voss, U}, +author = {Munz, C D and Omnes, P and Schneider, R and Sonnendrucker, E and Voss, U}, doi = {10.1006/Jcph.2000.6507}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {jul}, +month = {Jul}, number = {2}, pages = {484--511}, title = {{Divergence Correction Techniques For Maxwell Solvers Based On A Hyperbolic Model}}, @@ -1277,10 +1274,9 @@ @article{Munzjcp2000 year = {2000} } @article{DavidsonJCP2015, - -author = {Davidson, A. and Tableman, A. and An, W. and Tsung, F.S. and Lu, W. and Vieira, J. and Fonseca, R.A. and Silva, L.O. and Mori, W.B.}, +author = {Davidson, A. and Tableman, A. and An, W. and Tsung, F. S. and Lu, W. and Vieira, J. and Fonseca, R. A. and Silva, L. O. and Mori, W. B.}, doi = {10.1016/j.jcp.2014.10.064}, -issn = {00219991}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, pages = {1063--1077}, title = {{Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in \Phi into OSIRIS}}, @@ -1301,8 +1297,7 @@ @article{GodfreyJCP2014 year = {2014} } @article{Godfrey2013, - -author = {Godfrey, Brendan B. and Vay, Jean Luc}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Journal of Computational Physics}, keywords = {Esirkepov algorithm,Numerical stability,Particle-in-cell,Relativistic beam}, pages = {33--46}, @@ -1316,8 +1311,8 @@ @article{Gilsonpop2010 doi = {10.1063/1.3354109}, institution = {Amer Phys Soc, Div Plasma Phys}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Studies Of Emittance Growth And Halo Particle Production In Intense Charged Particle Beams Using The Paul Trap Simulator Experiment}}, volume = {17}, @@ -1348,7 +1343,7 @@ @article{Steinke2016 author = {Steinke, S and van Tilborg, J and Benedetti, C and Geddes, C G R and Schroeder, C B and Daniels, J and Swanson, K K and Gonsalves, A J and Nakamura, K and Matlis, N H and Shaw, B H and Esarey, E and Leemans, W P}, issn = {0028-0836}, journal = {Nature}, -month = {feb}, +month = {Feb}, number = {7589}, pages = {190--193}, publisher = {Nature Publishing Group, a division of Macmillan Publishers Limited. All Rights Reserved.}, @@ -1376,7 +1371,7 @@ @article{Genonioppj2010 year = {2010} } @article{LewisJCP1972, -author = {Lewis, H.Ralph}, +author = {Lewis, H. Ralph}, doi = {http://dx.doi.org/10.1016/0021-9991(72)90044-7}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -1400,11 +1395,11 @@ @article{Vay2002 year = {2002} } @article{Manglesnature04, -author = {Mangles, Spd and Murphy, Cd and Najmudin, Z and Thomas, Agr and Collier, Jl and Dangor, Ae and Divall, Ej and Foster, Ps and Gallacher, Jg and Hooker, Cj and Jaroszynski, Da and Langley, Aj and Mori, Wb and Norreys, Pa and Tsung, Fs and Viskup, R and Walton, Br and Krushelnick, K}, +author = {Mangles, S P D and Murphy, C D and Najmudin, Z and Thomas, A G R and Collier, J L and Dangor, A E and Divall, E J and Foster, P S and Gallacher, J G and Hooker, C J and Jaroszynski, D A and Langley, A J and Mori, W B and Norreys, P A and Tsung, F S and Viskup, R and Walton, B R and Krushelnick, K}, doi = {10.1038/Nature02939}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {535--538}, title = {{Monoenergetic Beams Of Relativistic Electrons From Intense Laser-Plasma Interactions}}, @@ -1423,8 +1418,8 @@ @inproceedings{Schroederaac08 @article{Vaypop98, author = {Vay, J.-L. and Deutsch, C}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {apr}, +journal = {Physics of Plasmas}, +month = {Apr}, number = {4}, pages = {1190--1197}, title = {{Charge Compensated Ion Beam Propagation In A Reactor Sized Chamber}}, @@ -1432,11 +1427,11 @@ @article{Vaypop98 year = {1998} } @article{Friedmanjcp1991, -author = {Friedman, A and Parker, Se and Ray, Sl and Birdsall, Ck}, +author = {Friedman, A and Parker, S E and Ray, S L and Birdsall, C K}, doi = {10.1016/0021-9991(91)90265-M}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {1}, pages = {54--70}, title = {{Multiscale Particle-In-Cell Plasma Simulation}}, @@ -1444,11 +1439,11 @@ @article{Friedmanjcp1991 year = {1991} } @article{Liumotl1997, -author = {Liu, Qh}, +author = {Liu, Q H}, doi = {10.1002/(Sici)1098-2760(19970620)15:3<158::Aid-Mop11>3.3.Co;2-T}, issn = {0895-2477}, journal = {Microwave And Optical Technology Letters}, -month = {jun}, +month = {Jun}, number = {3}, pages = {158--165}, title = {{The PSTD Algorithm: A Time-Domain Method Requiring Only Two Cells Per Wavelength}}, @@ -1475,45 +1470,19 @@ @article{Leemansphysicstoday10 author = {Leemans, Wim and Esarey, Eric}, issn = {0031-9228}, journal = {Physics Today}, -month = {mar}, +month = {Mar}, number = {3}, pages = {44--49}, title = {{Laser-Driven Plasma-Wave Electron Accelerators}}, volume = {62}, year = {2009} } -@article{Lehe2015, - -archivePrefix = {arXiv}, -arxivId = {1507.04790}, -author = {Lehe, Remi and Kirchen, Manuel and Andriyash, Igor a. and Godfrey, Brendan B. and Vay, Jean-Luc}, -eprint = {1507.04790}, -isbn = {5104866785}, -journal = {arXiv.org}, -keywords = {cylindrical geometry,hankel transform,particle-in-cell,pseudo-spectral}, -pages = {1507.04790v1}, -title = {{A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}}, -url = {http://arxiv.org/abs/1507.04790}, -volume = {physics.pl}, -year = {2015} -} -@article{VayJCP13, -author = {Vay, Jean-Luc and Haber, Irving and Godfrey, Brendan B}, -doi = {10.1016/j.jcp.2013.03.010}, -issn = {0021-9991}, -journal = {Journal of Computational Physics}, -month = {jun}, -pages = {260--268}, -title = {{A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas}}, -volume = {243}, -year = {2013} -} @inproceedings{Martinspac09, address = {Vancouver, Canada}, annote = {Th4Gbc05}, -author = {{Martins et al.}, S F}, +author = {Martins, S F and Fonseca, R A and Silva, L O and Mori, W B}, booktitle = {Proc. Particle Accelerator Conference}, -title = {{Boosted Frame Pic Simulations Of Lwfa: Towards The Energy Frontier}}, +title = {{Boosted Frame PIC Simulations of LWFA: Towards the Energy Frontier}}, year = {2009} } @article{Vayscidac09, @@ -1535,32 +1504,32 @@ @article{Sprangleprl90 author = {Sprangle, P and Esarey, E and Ting, A}, issn = {0031-9007}, journal = {Physical Review Letters}, -month = {apr}, +month = {Apr}, number = {17}, pages = {2011--2014}, -title = {{Nonlinear-Theory Of Intense Laser-Plasma Interactions}}, +title = {{Nonlinear theory of intense laser-plasma interactions}}, volume = {64}, year = {1990} } @article{Molvikpop2007, annote = {48Th Annual Meeting Of The Division Of Plasma Physics Of The Aps, Philadelphia, Pa, Jan 30-Nov 03, 2006}, -author = {Molvik, A W and Covo, M Kireeff and Cohen, R and Friedman, A and Lund, S M and Sharp, W and Vay, J-L. and Baca, D and Bieniosek, F and Leister, C and Seidl, P}, +author = {Molvik, A W and Covo, M Kireeff and Cohen, R and Friedman, A and Lund, S M and Sharp, W and Vay, J-L and Baca, D and Bieniosek, F and Leister, C and Seidl, P}, doi = {10.1063/1.2436850}, institution = {Aps, Div Plasma Phys}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Quantitative Experiments With Electrons In A Positively Charged Beam}}, volume = {14}, year = {2007} } @article{QuiterJAP08, -author = {Quiter, B J and Prussin, S G and Pohl, B and Hall, J and Trebes, J and Stone, G and Descalle, M -A.}, +author = {Quiter, B J and Prussin, S G and Pohl, B and Hall, J and Trebes, J and Stone, G and Descalle, M-A}, doi = {10.1063/1.2876028}, issn = {0021-8979}, journal = {JOURNAL OF APPLIED PHYSICS}, -month = {mar}, +month = {Mar}, number = {6}, title = {{A method for high-resolution x-ray imaging of intermodal cargo containers for fissionable materials}}, volume = {103}, @@ -1568,7 +1537,7 @@ @article{QuiterJAP08 } @book{Taflove2000, address = {Norwood}, -author = {Taflove and Hagness}, +author = {Allen Taflove and Susan C. Hagness}, edition = {2Nd}, publisher = {Ma: Artech House}, title = {{Computational Electrodynamics: The Finite-Difference Time-Domain Method}}, @@ -1578,7 +1547,7 @@ @article{Schroederprl2011 author = {Schroeder, C B and Benedetti, C and Esarey, E and Leemans, W P}, doi = {10.1103/Physrevlett.106.135002}, journal = {Physical Review Letters}, -month = {mar}, +month = {Mar}, number = {13}, pages = {135002}, title = {{Nonlinear Pulse Propagation And Phase Velocity Of Laser-Driven Plasma Waves}}, @@ -1600,10 +1569,8 @@ @article{Logannim2007 year = {2007} } @article{Yu2016, - author = {Yu, Peicheng and Xu, Xinlu and Davidson, Asher and Tableman, Adam and Dalichaouch, Thamine and Li, Fei and Meyers, Michael D. and An, Weiming and Tsung, Frank S. and Decyk, Viktor K. and Fiuza, Frederico and Vieira, Jorge and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.jcp.2016.04.014}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Yu et al. - 2016 - Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry.pdf:pdf}, issn = {00219991}, journal = {Journal of Computational Physics}, title = {{Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry}}, @@ -1611,7 +1578,7 @@ @article{Yu2016 } @article{Borisjcp73, address = {525 B St, Ste 1900, San Diego, Ca 92101-4495}, -author = {Boris, Jp and Lee, R}, +author = {Boris, J P and Lee, R}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {1}, @@ -1624,7 +1591,7 @@ @article{Borisjcp73 } @inproceedings{BorisICNSP70, address = {Naval Res. Lab., Wash., D. C.}, -author = {Boris, Jp}, +author = {Boris, J P}, booktitle = {Proc. Fourth Conf. Num. Sim. Plasmas}, pages = {3--67}, title = {{Relativistic Plasma Simulation-Optimization of a Hybrid Code}}, @@ -1638,7 +1605,7 @@ @article{Vayjcp01 author = {Vay, J.-L.}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {feb}, +month = {Feb}, number = {1}, pages = {72--98}, title = {{An Extended Fdtd Scheme For The Wave Equation: Application To Multiscale Electromagnetic Simulation}}, @@ -1663,10 +1630,10 @@ @article{Leemansnature06 doi = {10.1038/Nphys418}, issn = {1745-2473}, journal = {Nature Physics}, -month = {oct}, +month = {Oct}, number = {10}, pages = {696--699}, -title = {{Gev Electron Beams From A Centimetre-Scale Accelerator}}, +title = {{GeV electron beams from a centimetre-scale accelerator}}, volume = {2}, year = {2006} } @@ -1675,7 +1642,7 @@ @article{ChenPRSTAB13 doi = {10.1103/PhysRevSTAB.16.030701}, issn = {1098-4402}, journal = {PHYSICAL REVIEW SPECIAL TOPICS-ACCELERATORS AND BEAMS}, -month = {mar}, +month = {Mar}, number = {3}, title = {{Modeling classical and quantum radiation from laser-plasma accelerators}}, volume = {16}, @@ -1686,7 +1653,7 @@ @article{Hipace doi = {10.1088/0741-3335/56/8/084012}, issn = {0741-3335}, journal = {Plasma Physics and Controlled Fusion}, -month = {aug}, +month = {Aug}, number = {8}, pages = {084012}, publisher = {IOP Publishing}, @@ -1696,7 +1663,7 @@ @article{Hipace year = {2014} } @article{Morsenielson1971, -author = {Morse, Rl and Nielson, Cw}, +author = {Morse, R L and Nielson, C W}, doi = {10.1063/1.1693518}, issn = {1070-6631}, journal = {Phys. Fluids}, @@ -1706,15 +1673,18 @@ @article{Morsenielson1971 volume = {14}, year = {1971} } -@article{Vaydpf09, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:0909.5603}, -title = {{Speeding Up Simulations Of Relativistic Systems Using An Optimal Boosted Frame}}, +@inproceedings{Vaydpf09, +archivePrefix = {arXiv}, +author = {Vay, J.-L. and Fawley, W. M. and Geddes, C. G. R. and Cormier-Michel, E. and Grote, D. P.}, +booktitle = {{Meeting of the Division of Particles and Fields of the American Physical Society (DPF 2009)}}, +eprint = {0909.5603}, +month = {Sep}, +primaryClass = {physics.acc-ph}, +title = {{Speeding up simulations of relativistic systems using an optimal boosted frame}}, year = {2009} } @misc{Vay2014, - -author = {Vay, Jean Luc and Godfrey, Brendan B.}, +author = {Vay, Jean-Luc and Godfrey, Brendan B.}, booktitle = {Comptes Rendus - Mecanique}, keywords = {Numerical instability,Particle-In-Cell,Plasma simulation,Special relativity}, number = {10-11}, @@ -1724,15 +1694,8 @@ @misc{Vay2014 volume = {342}, year = {2014} } -@article{Lehearxiv2015, -author = {Lehe, R and Kirchen, M and Andriyash, I.{\~{}}A. and Godfrey, B.{\~{}}B. and Vay, J.-L.}, -journal = {arXiv:1507.04790}, -title = {{A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}}, -year = {2015} -} @article{Friedman2014, - -author = {Friedman, Alex and Cohen, Ronald H. and Grote, David P. and Lund, Steven M. and Sharp, William M. and Vay, Jean Luc and Haber, Irving and Kishek, Rami A.}, +author = {Friedman, Alex and Cohen, Ronald H. and Grote, David P. and Lund, Steven M. and Sharp, William M. and Vay, Jean-Luc and Haber, Irving and Kishek, Rami A.}, journal = {IEEE Transactions on Plasma Science}, keywords = {Algorithms,Maxwell,Ned Birdsall,computer,laser,numerical simulation,particle beam,particle-in-cell,plasma}, number = {5}, @@ -1743,7 +1706,6 @@ @article{Friedman2014 year = {2014} } @article{Lcode, - author = {Lotov, K. V.}, doi = {10.1063/1.872765}, issn = {1070664X}, @@ -1770,9 +1732,7 @@ @article{GodfreyJCP2014_2 year = {2014} } @article{Turbowave, - author = {Gordon, Daniel F and Mori, W B and Antonsen, Thomas M}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Gordon, Mori, Antonsen - 2000 - A Ponderomotive Guiding Center Particle-in-Cell Code for Efficient Modeling of Laser–Plasma Interactio.pdf:pdf}, journal = {IEEE TRANSACTIONS ON PLASMA SCIENCE}, keywords = {Index Terms—Particle code,plasma simulation}, number = {4}, @@ -1787,7 +1747,7 @@ @article{Habernim2009 institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {64--68}, title = {{Scaled Electron Studies At The University Of Maryland}}, @@ -1805,13 +1765,12 @@ @article{Folegatijpcs2011 year = {2011} } @article{YuCPC2015, - author = {Yu, Peicheng and Xu, Xinlu and Decyk, Viktor K. and Fiuza, Frederico and Vieira, Jorge and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.cpc.2015.02.018}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {ALGORITHM,LASER WAKEFIELD ACCELERATORS,LORENTZ-BOOSTED FRAME,Numerical Cerenkov instability,Numerical dispersion relation,PARTICLE SIMULATION,PLASMA,Particle-in-cell,Plasma simulation,Relativistic drifting plasma,SHOCKS,STABILITY,Spectral solver,WAVES}, -month = {jul}, +month = {Jul}, pages = {32--47}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Elimination of the numerical Cerenkov instability for spectral EM-PIC codes}}, @@ -1824,19 +1783,13 @@ @article{PukhovJPP99 doi = {10.1017/S0022377899007515}, issn = {0022-3778}, journal = {Journal of Plasma Physics}, -month = {apr}, +month = {Apr}, number = {3}, pages = {425--433}, title = {{Three-dimensional electromagnetic relativistic particle-in-cell code VLPL (Virtual Laser Plasma Lab)}}, volume = {61}, year = {1999} } -@article{Vincentiarxiv2015, -author = {Vincenti, H and Vay, J.-L.}, -journal = {arXiv:1507.05572}, -title = {{Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver}}, -year = {2015} -} @inproceedings{Vayipac10, address = {Tokyo, Japan}, annote = {Weobra02}, @@ -1849,7 +1802,7 @@ @article{Cormierprstab2011 author = {Cormier-Michel, E and Esarey, E and Geddes, C G R and Schroeder, C B and Paul, K and Mullowney, P J and Cary, J R and Leemans, W P}, doi = {10.1103/Physrevstab.14.031303}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {mar}, +month = {Mar}, number = {3}, pages = {31303}, title = {{Control Of Focusing Fields In Laser-Plasma Accelerators Using Higher-Order Modes}}, @@ -1857,10 +1810,8 @@ @article{Cormierprstab2011 year = {2011} } @article{Lehe2016, - author = {Lehe, R{\'{e}}mi and Kirchen, Manuel and Andriyash, Igor A. and Godfrey, Brendan B. and Vay, Jean-Luc}, doi = {10.1016/j.cpc.2016.02.007}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Lehe et al. - 2016 - A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm.pdf:pdf}, issn = {00104655}, journal = {Computer Physics Communications}, pages = {66--82}, @@ -1877,8 +1828,7 @@ @book{Birdsalllangdon year = {1991} } @inproceedings{Grote2005, - -author = {Grote, David P. and Friedman, Alex and Vay, Jean Luc and Haber, Irving}, +author = {Grote, David P. and Friedman, Alex and Vay, Jean-Luc and Haber, Irving}, booktitle = {AIP Conference Proceedings}, pages = {55--58}, title = {{The WARP code: Modeling high intensity ion beams}}, @@ -1905,7 +1855,7 @@ @article{LeemansPRL2014 author = {Leemans, W P and Gonsalves, A J and Mao, H.-S. and Nakamura, K and Benedetti, C and Schroeder, C B and T{\'{o}}th, Cs. and Daniels, J and Mittelberger, D E and Bulanov, S S and Vay, J.-L. and Geddes, C G R and Esarey, E}, doi = {10.1103/PhysRevLett.113.245002}, journal = {Phys. Rev. Lett.}, -month = {dec}, +month = {Dec}, number = {24}, pages = {245002}, publisher = {American Physical Society}, @@ -1918,7 +1868,7 @@ @article{Abejcp86 author = {Abe, H and Sakairi, N and Itatani, R and Okuda, H}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {apr}, +month = {Apr}, number = {2}, pages = {247--267}, title = {{High-Order Spline Interpolations In The Particle Simulation}}, @@ -1938,11 +1888,11 @@ @article{LeeCPC2015 year = {2015} } @article{Vaylpb2002, -author = {Vay, Jl and Colella, P and Mccorquodale, P and {Van Straalen}, B and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Colella, P and Mccorquodale, P and {Van Straalen}, B and Friedman, A and Grote, D P}, doi = {10.1017/S0263034602204139}, issn = {0263-0346}, journal = {Laser And Particle Beams}, -month = {dec}, +month = {Dec}, number = {4}, pages = {569--575}, title = {{Mesh Refinement For Particle-In-Cell Plasma Simulations: Applications To And Benefits For Heavy Ion Fusion}}, @@ -1962,7 +1912,7 @@ @article{Tzoufrasprl2008 author = {Tzoufras, M and Lu, W and Tsung, F S and Huang, C and Mori, W B and Katsouleas, T and Vieira, J and Fonseca, R A and Silva, L O}, doi = {10.1103/Physrevlett.101.145002}, journal = {Physical Review Letters}, -month = {oct}, +month = {Oct}, number = {14}, pages = {145002}, title = {{Beam Loading In The Nonlinear Regime Of Plasma-Based Acceleration Rid C-6436-2009 Rid B-7680-2009 Rid C-3169-2009}}, @@ -1973,7 +1923,7 @@ @article{Bulanovphysfluid1992 author = {Bulanov, S V and Inovenkov, I N and Kirsanov, V I and Naumova, N M and Sakharov, A S}, doi = {10.1063/1.860046}, journal = {Physics Of Fluids B-Plasma Physics}, -month = {jul}, +month = {Jul}, number = {7}, pages = {1935--1942}, title = {{Nonlinear Depletion Of Ultrashort And Relativistically Strong Laser-Pulses In An Underdense Plasma}}, @@ -1985,7 +1935,7 @@ @article{Martinsnaturephysics10 doi = {10.1038/Nphys1538}, issn = {1745-2473}, journal = {Nature Physics}, -month = {apr}, +month = {Apr}, number = {4}, pages = {311--316}, title = {{Exploring Laser-Wakefield-Accelerator Regimes For Near-Term Lasers Using Particle-In-Cell Simulation In Lorentz-Boosted Frames}}, @@ -1995,8 +1945,8 @@ @article{Martinsnaturephysics10 @article{Friedmanpop10, author = {Friedman, A and Barnard, J J and Cohen, R H and Grote, D P and Lund, S M and Sharp, W M and Faltens, A and Henestroza, E and Jung, J.-Y. and Kwan, J W and Lee, E P and Leitner, M A and Logan, B G and Vay, J.-L. and Waldron, W L and Davidson, R C and Dorf, M and Gilson, E P and Kaganovich, I D}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {056704 (9 Pp.)}, title = {{Beam Dynamics Of The Neutralized Drift Compression Experiment-Ii, A Novel Pulse-Compressing Ion Accelerator}}, @@ -2006,8 +1956,8 @@ @article{Friedmanpop10 @article{Schroederpop2006, author = {Schroeder, C B and Esarey, E and Shadwick, B A and Leemans, W P}, doi = {10.1063/1.2173960}, -journal = {Physics Of Plasmas}, -month = {mar}, +journal = {Physics of Plasmas}, +month = {Mar}, number = {3}, pages = {33103}, title = {{Trapping, Dark Current, And Wave Breaking In Nonlinear Plasma Waves}}, @@ -2016,12 +1966,12 @@ @article{Schroederpop2006 } @article{Friedmanpfb92, annote = {33Rd Annual Meeting Of The Division Of Plasma Physics Of The American Physical Soc, Tampa, Fl, Nov 04-08, 1991}, -author = {Friedman, A and Grote, Dp and Haber, I}, +author = {Friedman, A and Grote, D P and Haber, I}, doi = {10.1063/1.860024}, institution = {Amer Phys Soc, Div Plasma Phys}, issn = {0899-8221}, journal = {Physics Of Fluids B-Plasma Physics}, -month = {jul}, +month = {Jul}, number = {7, Part 2}, pages = {2203--2210}, title = {{3-Dimensional Particle Simulation Of Heavy-Ion Fusion Beams}}, @@ -2039,7 +1989,7 @@ @article{Winklehnerji2010 doi = {10.1088/1748-0221/5/12/P12001}, issn = {1748-0221}, journal = {Journal of Instrumentation}, -month = {dec}, +month = {Dec}, title = {{Comparison Of Extraction And Beam Transport Simulations With Emittance Measurements From The Ecr Ion Source Venus}}, volume = {5}, year = {2010} @@ -2047,19 +1997,18 @@ @article{Winklehnerji2010 @inproceedings{Vaypac09, address = {Vancouver, Canada}, annote = {Tu1Pbi04}, -author = {{Vay et al.}, J.-L.}, +author = {Vay, J-L and Fawley, W M and Geddes, C G R and Cormier-Michel, E and Grote, D P}, booktitle = {Proc. Particle Accelerator Conference}, -title = {{Application Of The Reduction Of Scale Range In A Lorentz Boosted Frame To The Numerical Simulation Of Particle Acceleration Devices}}, +title = {{Application of the reduction of scale range in a Lorentz boosted frame to the numerical simulation of particle acceleration devices}}, year = {2009} } -@article{Vincenti2016a, - +@article{VincentiCPC2017a, author = {Vincenti, H. and Vay, J.-L.}, doi = {10.1016/j.cpc.2015.11.009}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {3D electromagnetic simulations,ABSORPTION,ALGORITHM,APPROXIMATIONS,CLOSED-FORM EXPRESSIONS,Domain decomposition technique,Effects of stencil truncation errors,PERFECTLY MATCHED LAYER,Perfectly Matched Layers,Pseudo-spectral Maxwell solver,SIMULATIONS,TAYLOR-SERIES,Very high-order Maxwell solver,WAVES}, -month = {mar}, +month = {Mar}, pages = {147--167}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver}}, @@ -2072,7 +2021,7 @@ @article{Vayjcp02 doi = {10.1006/Jcph.2002.7175}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {dec}, +month = {Dec}, number = {2}, pages = {367--399}, title = {{Asymmetric Perfectly Matched Layer For The Absorption Of Waves}}, @@ -2093,14 +2042,14 @@ @book{Geddesdissertation05 year = {2005} } @article{Tsungpop06, -author = {Tsung, Fs and Lu, W and Tzoufras, M and Mori, Wb and Joshi, C and Vieira, Jm and Silva, Lo and Fonseca, Ra}, +author = {Tsung, F. S. and Lu, W. and Tzoufras, M. and Mori, W. B. and Joshi, C. and Vieira, J. M. and Silva, L. O. and Fonseca, R. A.}, doi = {10.1063/1.2198535}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56708}, -title = {{Simulation Of Monoenergetic Electron Generation Via Laser Wakefield Accelerators For 5-25 Tw Lasers}}, +title = {{Simulation Of Monoenergetic Electron Generation Via Laser Wakefield Accelerators For 5-25 TW Lasers}}, volume = {13}, year = {2006} } @@ -2108,7 +2057,6 @@ @inproceedings{INFERNO address = {Rostock-Warnemünde, Germany}, author = {Benedetti, Carlo and Schroeder, Carl B. and Esarey, Eric and Leemans, Wim P.}, booktitle = {ICAP}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Benedetti et al. - 2012 - Efficient Modeling of Laser-plasma Accelerators Using the Ponderomotive-based Code INF{\&}ampRNO.pdf:pdf}, pages = {THAAI2}, publisher = {Jacow}, title = {{Efficient Modeling of Laser-plasma Accelerators Using the Ponderomotive-based Code INF{\&}RNO}}, @@ -2120,7 +2068,7 @@ @article{GonsalvesNP2011 doi = {10.1038/NPHYS2071}, issn = {1745-2473}, journal = {NATURE PHYSICS}, -month = {nov}, +month = {Nov}, number = {11}, pages = {862--866}, title = {{Tunable laser plasma accelerator based on longitudinal density tailoring}}, @@ -2132,14 +2080,14 @@ @article{Ohtsuboprstab2010 doi = {10.1103/Physrevstab.13.044201}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {apr}, +month = {Apr}, number = {4}, title = {{Experimental Study Of Coherent Betatron Resonances With A Paul Trap}}, volume = {13}, year = {2010} } @inproceedings{Warp, -author = {Grote, D P and Friedman, A and Vay, J.-L. and Haber, I}, +author = {Grote, D P and Friedman, A and Vay, J-L and Haber, I}, booktitle = {Aip Conference Proceedings}, issn = {0094-243X}, number = {749}, @@ -2148,11 +2096,11 @@ @inproceedings{Warp year = {2005} } @article{Mccorquodalejcp2004, -author = {Mccorquodale, P and Colella, P and Grote, Dp and Vay, Jl}, +author = {Mccorquodale, P and Colella, P and Grote, D P and Vay, J-L}, doi = {10.1016/J.Jcp.2004.04.022}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {nov}, +month = {Nov}, number = {1}, pages = {34--60}, title = {{A Node-Centered Local Refinement Algorithm For Poisson's Equation In Complex Geometries}}, @@ -2162,7 +2110,6 @@ @article{Mccorquodalejcp2004 @article{Londrillo2010, author = {Londrillo, P. and Benedetti, C. and Sgattoni, A.}, doi = {10.1016/j.nima.2010.01.055}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Londrillo, Benedetti, Sgattoni - 2010 - Charge preserving high order PIC schemes.pdf:pdf}, issn = {01689002}, journal = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, number = {1}, @@ -2172,7 +2119,7 @@ @article{Londrillo2010 year = {2010} } @article{GodfreyJCP2014_FDTD, -author = {Godfrey, Brendan B. and Vay, Jean Luc}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Journal of Computational Physics}, keywords = {Finite difference time-domain,Numerical stability,Particle-in-cell,Relativistic beam}, pages = {1--6}, @@ -2181,11 +2128,11 @@ @article{GodfreyJCP2014_FDTD year = {2014} } @article{Shortley-Weller, -author = {Shortley, Gh and Weller, R}, +author = {Shortley, G H and Weller, R}, doi = {10.1063/1.1710426}, issn = {0021-8979}, journal = {Journal of Applied Physics}, -month = {may}, +month = {May}, number = {5}, pages = {334--348}, title = {{The Numerical Solution Of Laplace's Equation}}, @@ -2193,35 +2140,50 @@ @article{Shortley-Weller year = {1938} } @article{VayPOPL2011, -author = {Vay, Jl and Geddes, C G R and Cormier-Michel, E and Grote, D P}, +author = {Vay, J.-L. and Geddes, C. G. R. and Cormier-Michel, E. and Grote, D. P.}, doi = {10.1063/1.3559483}, -journal = {Physics Of Plasmas}, -month = {mar}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.3559483/16019930/030701\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Mar}, number = {3}, -pages = {30701}, -title = {{Effects Of Hyperbolic Rotation In Minkowski Space On The Modeling Of Plasma Accelerators In A Lorentz Boosted Frame}}, +pages = {030701}, +title = {{Effects of hyperbolic rotation in Minkowski space on the modeling of plasma accelerators in a Lorentz boosted frame}}, +url = {https://doi.org/10.1063/1.3559483}, volume = {18}, year = {2011} } - -@article{KirchenARXIV2016, -author = {Kirchen, M. and Lehe, R. and Godfrey, B.~B. and Dornmair, I. and Jalas, S. and Peters, K. and Vay, J.-L. and Maier, A.~R.}, -journal = {arXiv:1608.00215}, +@article{KirchenPOP2016, +author = {Kirchen, M. and Lehe, R. and Godfrey, B. B. and Dornmair, I. and Jalas, S. and Peters, K. and Vay, J.-L. and Maier, A. R.}, +doi = {10.1063/1.4964770}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4964770/14024121/100704\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Oct}, +number = {10}, +pages = {100704}, title = {{Stable discrete representation of relativistically drifting plasmas}}, +url = {https://doi.org/10.1063/1.4964770}, +volume = {23}, year = {2016} } - -@article{LeheARXIV2016, -author = {Lehe, R. and Kirchen, M. and Godfrey, B.~B. and Maier, A.~R. and Vay, J.-L.}, -journal = {arXiv:1608.00227}, -title = {{Elimination of Numerical Cherenkov Instability in flowing-plasma Particle-In-Cell simulations by using Galilean coordinates}}, +@article{LehePRE2016, +author = {Lehe, Remi and Kirchen, Manuel and Godfrey, Brendan B. and Maier, Andreas R. and Vay, Jean-Luc}, +doi = {10.1103/PhysRevE.94.053305}, +issue = {5}, +journal = {Phys. Rev. E}, +month = {Nov}, +numpages = {16}, +pages = {053305}, +publisher = {American Physical Society}, +title = {{Elimination of numerical Cherenkov instability in flowing-plasma particle-in-cell simulations by using Galilean coordinates}}, +url = {https://link.aps.org/doi/10.1103/PhysRevE.94.053305}, +volume = {94}, year = {2016} } - @book{godfrey1985iprop, - title={The IPROP Three-Dimensional Beam Propagation Code}, - author={Godfrey, B.B.}, - url={https://books.google.com/books?id=hos\_OAAACAAJ}, - year={1985}, - publisher={Defense Technical Information Center} +author = {Godfrey, B. B.}, +publisher = {Defense Technical Information Center}, +title = {{The IPROP Three-Dimensional Beam Propagation Code}}, +year = {1985} } diff --git a/Docs/source/latex_theory/input_output/input_output.tex b/Docs/source/latex_theory/input_output/input_output.tex index e013e236445..446f3f2b5d6 100644 --- a/Docs/source/latex_theory/input_output/input_output.tex +++ b/Docs/source/latex_theory/input_output/input_output.tex @@ -30,7 +30,7 @@ \subsection{Inputs and outputs in a boosted frame simulation} \subsubsection{Input in a boosted frame simulation} \paragraph{Particles - } -Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse ``electron cloud'' instability \cite{Vayprl07}. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity $\{x,y,z,v_x,v_y,v_z\}$ for each beam macroparticle at time $t=0$ for a beam moving at the average velocity $v_b=\beta_b c$ (where $c$ is the speed of light) in the laboratory, and using the standard synchronization ($z=z'=0$ at $t=t'=0$) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity $\beta c$ in the laboratory is as follows (the superscript $'$ relates to quantities known in the boosted frame while the superscript $^*$ relates to quantities that are know at a given longitudinal position $z^*$ but different times of arrival): +Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse ``electron cloud'' instability \cite{Vayprl07}. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity $\{x,y,z,v_x,v_y,v_z\}$ for each beam macroparticle at time $t=0$ for a beam moving at the average velocity $v_b=\beta_b c$ (where $c$ is the speed of light) in the laboratory, and using the standard synchronization ($z=z'=0$ at $t=t'=0$) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity $\beta c$ in the laboratory is as follows (the superscript $'$ relates to quantities known in the boosted frame while the superscript $^*$ relates to quantities that are know at a given longitudinal position $z^*$ but different times of arrival): \begin{enumerate} \item project positions at $z^*=0$ assuming ballistic propagation diff --git a/Docs/source/maintenance/performance_tests.rst b/Docs/source/maintenance/performance_tests.rst index 13b1f6e3acb..8b902297ccb 100644 --- a/Docs/source/maintenance/performance_tests.rst +++ b/Docs/source/maintenance/performance_tests.rst @@ -75,6 +75,7 @@ Then, in ``$AUTOMATED_PERF_TESTS``, create a file ``run_automated_performance_te # lines below this comment once, before submission. # The commented lines take too long for the job script. #python3 -m pip install --upgrade pip + #python3 -m pip install --upgrade build packaging setuptools wheel #python3 -m pip install --upgrade cython #python3 -m pip install --upgrade numpy #python3 -m pip install --upgrade markupsafe diff --git a/Docs/source/maintenance/release.rst b/Docs/source/maintenance/release.rst index e4047563f76..f25b8c313e4 100644 --- a/Docs/source/maintenance/release.rst +++ b/Docs/source/maintenance/release.rst @@ -13,6 +13,7 @@ The following scripts automate this workflow, in case one needs a newer commit o .. code-block:: sh ./Tools/Release/updateAMReX.py + ./Tools/Release/updatepyAMReX.py ./Tools/Release/updatePICSAR.py @@ -32,6 +33,7 @@ In order to create a GitHub release, you need to: .. code-block:: sh ./Tools/Release/updateAMReX.py + ./Tools/Release/updatepyAMReX.py ./Tools/Release/updatePICSAR.py ./Tools/Release/newVersion.sh diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 72d54456a89..d94938775c6 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -1,398 +1,38 @@ -@article{Tsung2006, -author = {Tsung,F. S. and Lu,W. and Tzoufras,M. and Mori,W. B. and Joshi,C. and Vieira,J. M. and Silva,L. O. and Fonseca,R. A. }, -title = {Simulation of monoenergetic electron generation via laser wakefield accelerators for 5–25TW lasers}, -journal = {Physics of Plasmas}, -volume = {13}, -number = {5}, -pages = {056708}, -year = {2006}, -doi = {10.1063/1.2198535} -} - -@article{Geddes2008, -doi = {10.1088/1742-6596/125/1/012002}, -year = {2008}, -volume = {125}, -number = {1}, -pages = {012002}, -author = {C G R Geddes and D L Bruhwiler and J R Cary and W B Mori and J-L Vay and S F Martins and T Katsouleas and E Cormier-Michel and W M Fawley and C Huang and X Wang and B Cowan and V K Decyk and E Esarey and R A Fonseca and W Lu and P Messmer and P Mullowney and K Nakamura and K Paul and G R Plateau and C B Schroeder and L O Silva and C Toth and F S Tsung and M Tzoufras and T Antonsen and J Vieira and W P Leemans}, -title = {Computational studies and optimization of wakefield accelerators}, -journal = {Journal of Physics: Conference Series} -} - -@techreport{Geddes2009, -title={Laser plasma particle accelerators: Large fields for smaller facility sources}, -author={Geddes, Cameron GR and Cormier-Michel, Estelle and Esarey, Eric H and Schroeder, Carl B and Vay, Jean-Luc and Leemans, Wim P and Bruhwiler, David L and Cary, John R and Cowan, Ben and Durant, Marc and others}, -year={2009}, -institution={Lawrence Berkeley National Laboratory (LBNL), Berkeley, CA (United States)} -} - -@article{Geddes2010, -title={Scaled simulation design of high quality laser wakefield accelerator stages}, -author={Geddes, CGR}, -journal={}, -year={2010} -} - -@article{Huang2009, -doi = {10.1088/1742-6596/180/1/012005}, -year = {2009}, -volume = {180}, -number = {1}, -pages = {012005}, -author = {C Huang and W An and V K Decyk and W Lu and W B Mori and F S Tsung and M Tzoufras and S Morshed and T Antonsen and B Feng and T Katsouleas and R A Fonseca and S F Martins and J Vieira and L O Silva and E Esarey and C G R Geddes and W P Leemans and E Cormier-Michel and J-L Vay and D L Bruhwiler and B Cowan and J R Cary and K Paul}, -title = {Recent results and future challenges for large scale particle-in-cell simulations of plasma-based accelerator concepts}, -journal = {Journal of Physics: Conference Series} -} - -@article{Leemans2014, -title = {Multi-GeV Electron Beams from Capillary-Discharge-Guided Subpetawatt Laser Pulses in the Self-Trapping Regime}, -author = {Leemans, W. P. and Gonsalves, A. J. and Mao, H.-S. and Nakamura, K. and Benedetti, C. and Schroeder, C. B. and T\'oth, Cs. and Daniels, J. and Mittelberger, D. E. and Bulanov, S. S. and Vay, J.-L. and Geddes, C. G. R. and Esarey, E.}, -journal = {Phys. Rev. Lett.}, -volume = {113}, -issue = {24}, -pages = {245002}, -numpages = {5}, -year = {2014}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.113.245002} -} - -@article{Blumenfeld2007, -title={Energy doubling of 42 GeV electrons in a metre-scale plasma wakefield accelerator}, -author={Blumenfeld, Ian and Clayton, Christopher E and Decker, Franz-Josef and Hogan, Mark J and Huang, Chengkun and Ischebeck, Rasmus and Iverson, Richard and Joshi, Chandrashekhar and Katsouleas, Thomas and Kirby, Neil and others}, -journal={Nature}, -volume={445}, -number={7129}, -pages={741--744}, -year={2007}, -publisher={Nature Publishing Group UK London}, -doi={10.1038/nature05538} -} - -@article{Bulanov2014, -doi = {10.3367/UFNe.0184.201412a.1265}, -year = {2014}, -publisher = {Turpion Ltd and the Russian Academy of Sciences}, -volume = {57}, -number = {12}, -pages = {1149}, -author = {S V Bulanov and J J Wilkens and T Zh Esirkepov and G Korn and G Kraft and S D Kraft and M Molls and V S Khoroshkov}, -title = {Laser ion acceleration for hadron therapy}, -journal = {Physics-Uspekhi} -} - -@article{Steinke2016, -title={Multistage coupling of independent laser-plasma accelerators}, -author={Steinke, S and Van Tilborg, J and Benedetti, C and Geddes, CGR and Schroeder, CB and Daniels, J and Swanson, KK and Gonsalves, AJ and Nakamura, K and Matlis, NH and others}, -journal={Nature}, -volume={530}, -number={7589}, -pages={190--193}, -year={2016}, -publisher={Nature Publishing Group UK London}, -doi={10.1038/nature16525} -} - -@article{Sprangle1990, -title = {Nonlinear theory of intense laser-plasma interactions}, -author = {Sprangle, P. and Esarey, E. and Ting, A.}, -journal = {Phys. Rev. Lett.}, -volume = {64}, -issue = {17}, -pages = {2011--2014}, -year = {1990}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.64.2011} -} - -@article{Antonsen1992, -title = {Self-focusing and Raman scattering of laser pulses in tenuous plasmas}, -author = {Antonsen, T. M. and Mora, P.}, -journal = {Phys. Rev. Lett.}, -volume = {69}, -issue = {15}, -pages = {2204--2207}, -year = {1992}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.69.2204} -} - -@article{Krall1993, -title = {Enhanced acceleration in a self-modulated-laser wake-field accelerator}, -author = {Krall, J. and Ting, A. and Esarey, E. and Sprangle, P.}, -journal = {Phys. Rev. E}, -volume = {48}, -issue = {3}, -pages = {2157--2161}, -year = {1993}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevE.48.2157} -} - -@article{Mora1997, -author = {Mora,Patrick and Antonsen, Jr.,Thomas M. }, -title = {Kinetic modeling of intense, short laser pulses propagating in tenuous plasmas}, -journal = {Physics of Plasmas}, -volume = {4}, -number = {1}, -pages = {217-229}, -year = {1997}, -doi = {10.1063/1.872134} -} - -@article{Huang2006, -title = {QUICKPIC: A highly efficient particle-in-cell code for modeling wakefield acceleration in plasmas}, -journal = {Journal of Computational Physics}, -volume = {217}, -number = {2}, -pages = {658-679}, -year = {2006}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2006.01.039}, -author = {C. Huang and V.K. Decyk and C. Ren and M. Zhou and W. Lu and W.B. Mori and J.H. Cooley and T.M. Antonsen and T. Katsouleas} -} - -@article{Benedetti2010, -author = {Benedetti,C. and Schroeder,C. B. and Esarey,E. and Geddes,C. G. R. and Leemans,W. P. }, -title = {Efficient Modeling of Laser‐Plasma Accelerators with INF\&RNO}, +@article{TajimaDawson1982, +abstract = "{Parallel intense laser beam ω0, k0 and ω1, k1 shone on a plasma with frequency separation equal to the plasma frequency ωp is capable of creating a coherent large electrostatic field and accelerating particles to high energies in large flux. The photon beam excites through the forward Raman scattering large amplitude plasmons whose phase velocity is equal to (ω−ω1)/(k0−k1), close to c in an underdense plasma. The plasmon traps electrons with electrostatic field EL=γ1/2⊥ mcωp/c, of the order of a few GeV/cm for plasma density to 1018 cm−3. Because of the phase velocity of the field close to c this field carries trapped electrons to high energies: W=2mc2(ω0/ωp)2. Preaccelerated particles (ions, for examples) coherent with the plasmon fields can also be accelerated. The (multiple) forward Raman instability saturates only when a sizable electron population is trapped and most of the electromagnetic energy is cascaded down to the frequency close to the cut‐off (ωp).}", +author = {Tajima, T. and Dawson, J. M.}, +doi = {10.1063/1.33805}, +issn = {0094-243X}, journal = {AIP Conference Proceedings}, -volume = {1299}, +month = {Sep}, number = {1}, -pages = {250-255}, -year = {2010}, -doi = {10.1063/1.3520323} -} - -@article{Cowan2011, -title = {Characteristics of an envelope model for laser–plasma accelerator simulation}, -journal = {Journal of Computational Physics}, -volume = {230}, -number = {1}, -pages = {61-86}, -year = {2011}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2010.09.009}, -author = {Benjamin M. Cowan and David L. Bruhwiler and Estelle Cormier-Michel and Eric Esarey and Cameron G.R. Geddes and Peter Messmer and Kevin M. Paul} -} - -@article{Vay2007, -title = {Noninvariance of Space- and Time-Scale Ranges under a Lorentz Transformation and the Implications for the Study of Relativistic Interactions}, -author = {Vay, J.-L.}, -journal = {Phys. Rev. Lett.}, -volume = {98}, -issue = {13}, -pages = {130405}, -year = {2007}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.98.130405} -} - -@article{Vay2009a, -doi = {10.1088/1742-6596/180/1/012006}, -year = {2009}, -volume = {180}, -number = {1}, -pages = {012006}, -author = {J-L Vay and D L Bruhwiler and C G R Geddes and W M Fawley and S F Martins and J R Cary and E Cormier-Michel and B Cowan and R A Fonseca and M A Furman and W Lu and W B Mori and L O Silva}, -title = {Simulating relativistic beam and plasma systems using an optimal boosted frame}, -journal = {Journal of Physics: Conference Series} -} - -@article{Vay2009b, -title = {Application of the reduction of scale range in a Lorentz boosted frame to the numerical simulation of particle acceleration devices.}, -author = {Vay, J and Fawley, W M and Geddes, C G and Cormier-Michel, E and Grote, D P}, -url = {https://www.osti.gov/biblio/952754}, -journal = {}, -place = {United States}, -year = {2009}, -} - -@article{Vay2010, -author = {Vay,J.‐L. and Geddes,C. G. R. and Benedetti,C. and Bruhwiler,D. L. and Cormier‐Michel,E. and Cowan,B. M. and Cary,J. R. and Grote,D. P. }, -title = {Modeling Laser Wakefield Accelerators in a Lorentz Boosted Frame}, -journal = {AIP Conference Proceedings}, -volume = {1299}, -number = {1}, -pages = {244-249}, -year = {2010}, -doi = {10.1063/1.3520322} -} - -@article{Vay2011a, -title = {Numerical methods for instability mitigation in the modeling of laser wakefield accelerators in a Lorentz-boosted frame}, -journal = {Journal of Computational Physics}, -volume = {230}, -number = {15}, -pages = {5908-5929}, -year = {2011}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2011.04.003}, -author = {J.-L. Vay and C.G.R. Geddes and E. Cormier-Michel and D.P. Grote} -} - -@article{Vay2011b, -author = {Vay,J.-L. and Geddes,C. G. R. and Cormier-Michel,E. and Grote,D. P. }, -title = {Effects of hyperbolic rotation in Minkowski space on the modeling of plasma accelerators in a Lorentz boosted frame}, -journal = {Physics of Plasmas}, -volume = {18}, -number = {3}, -pages = {030701}, -year = {2011}, -doi = {10.1063/1.3559483} -} - -@article{Vay2011c, -author = {Vay,J.-L. and Geddes,C. G. R. and Esarey,E. and Schroeder,C. B. and Leemans,W. P. and Cormier-Michel,E. and Grote,D. P. }, -title = {Modeling of 10 GeV-1 TeV laser-plasma accelerators using Lorentz boosted simulations}, -journal = {Physics of Plasmas}, -volume = {18}, -number = {12}, -pages = {123103}, -year = {2011}, -doi = {10.1063/1.3663841} -} - -@article{Martins2010a, -title = {Numerical simulations of laser wakefield accelerators in optimal Lorentz frames}, -journal = {Computer Physics Communications}, -volume = {181}, -number = {5}, -pages = {869-875}, -year = {2010}, -issn = {0010-4655}, -doi = {10.1016/j.cpc.2009.12.023}, -author = {Samuel F. Martins and Ricardo A. Fonseca and Luís O. Silva and Wei Lu and Warren B. Mori} -} - -@article{Martins2010b, -title={Exploring laser-wakefield-accelerator regimes for near-term lasers using particle-in-cell simulation in Lorentz-boosted frames}, -author={Martins, Samuel F and Fonseca, RA and Lu, Wei and Mori, Warren B and Silva, LO}, -journal={Nature Physics}, -volume={6}, -number={4}, -pages={311--316}, -year={2010}, -publisher={Nature Publishing Group UK London}, -doi={10.1038/nphys1538} -} - -@article{Martins2010c, -author = {Martins,S. F. and Fonseca,R. A. and Vieira,J. and Silva,L. O. and Lu,W. and Mori,W. B. }, -title = {Modeling laser wakefield accelerator experiments with ultrafast particle-in-cell simulations in boosted frames}, -journal = {Physics of Plasmas}, -volume = {17}, -number = {5}, -pages = {056705}, -year = {2010}, -doi = {10.1063/1.3358139} -} - -@article{Bruhwiler2009, -author = {Bruhwiler,David L. and Cary,John R. and Cowan,Benjamin M. and Paul,Kevin and Geddes,Cameron G. R. and Mullowney,Paul J. and Messmer,Peter and Esarey,Eric and Cormier‐Michel,Estelle and Leemans,Wim and Vay,Jean‐Luc }, -title = {New Developments in the Simulation of Advanced Accelerator Concepts}, -journal = {AIP Conference Proceedings}, -volume = {1086}, -number = {1}, -pages = {29-37}, -year = {2009}, -doi = {10.1063/1.3080922} -} - -@article{Yu2016, -title = {Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry}, -journal = {Journal of Computational Physics}, -volume = {316}, -pages = {747-759}, -year = {2016}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2016.04.014}, -author = {Peicheng Yu and Xinlu Xu and Asher Davidson and Adam Tableman and Thamine Dalichaouch and Fei Li and Michael D. Meyers and Weiming An and Frank S. Tsung and Viktor K. Decyk and Frederico Fiuza and Jorge Vieira and Ricardo A. Fonseca and Wei Lu and Luis O. Silva and Warren B. Mori} -} - -@inproceedings{Godfrey1985, -title={The IPROP Three-Dimensional Beam Propagation Code}, -booktitle={}, -author={B. B. Godfrey}, -year={1985} -} - -@article{Lifschitz2009, -title = {Particle-in-Cell modelling of laser–plasma interaction using Fourier decomposition}, -journal = {Journal of Computational Physics}, -volume = {228}, -number = {5}, -pages = {1803-1814}, -year = {2009}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2008.11.017}, -author = {A.F. Lifschitz and X. Davoine and E. Lefebvre and J. Faure and C. Rechatin and V. Malka} -} - -@article{Davidson2015, -title = {Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in ϕ into OSIRIS}, -journal = {Journal of Computational Physics}, -volume = {281}, -pages = {1063-1077}, -year = {2015}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2014.10.064}, -author = {A. Davidson and A. Tableman and W. An and F.S. Tsung and W. Lu and J. Vieira and R.A. Fonseca and L.O. Silva and W.B. Mori} -} - -@article{Lehe2016, -title = {A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}, -journal = {Computer Physics Communications}, -volume = {203}, -pages = {66-82}, -year = {2016}, -issn = {0010-4655}, -doi = {10.1016/j.cpc.2016.02.007}, -author = {Rémi Lehe and Manuel Kirchen and Igor A. Andriyash and Brendan B. Godfrey and Jean-Luc Vay} -} - -@article{Andriyash2016, -author = {Andriyash,Igor A. and Lehe,Remi and Lifschitz,Agustin }, -title = {Laser-plasma interactions with a Fourier-Bessel particle-in-cell method}, -journal = {Physics of Plasmas}, -volume = {23}, -number = {3}, -pages = {033110}, -year = {2016}, -doi = {10.1063/1.4943281} -} - -@article{Shadwick2009, -author = {Shadwick,B. A. and Schroeder,C. B. and Esarey,E. }, -title = {Nonlinear laser energy depletion in laser-plasma accelerators}, -journal = {Physics of Plasmas}, -volume = {16}, -number = {5}, -pages = {056704}, -year = {2009}, -doi = {10.1063/1.3124185} +pages = {69--93}, +title = {{Laser accelerator by plasma waves}}, +url = {https://doi.org/10.1063/1.33805}, +volume = {91}, +year = {1982} } -@article{CormierMichel2009, -author = {Cormier‐Michel,Estelle and Geddes,C. G. R. and Esarey,E. and Schroeder,C. B. and Bruhwiler,D. L. and Paul,K. and Cowan,B. and Leemans,W. P. }, -title = {Scaled simulations of a 10 GeV accelerator}, -journal = {AIP Conference Proceedings}, -volume = {1086}, -number = {1}, -pages = {297-302}, -year = {2009}, -doi = {10.1063/1.3080921} +@article{Esarey1996, +author = {Esarey, E. and Sprangle, P. and Krall, J. and Ting, A.}, +doi = {10.1109/27.509991}, +journal = {IEEE Transactions on Plasma Science}, +number = {2}, +pages = {252--288}, +title = {{Overview of plasma-based accelerator concepts}}, +volume = {24}, +year = {1996} } @ARTICLE{Birdsall1991, - author = {Birdsall, C.K.}, - journal = {IEEE Transactions on Plasma Science}, - title = {Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}, - year = {1991}, - volume = {19}, - number = {2}, - pages = {65-85}, - doi = {10.1109/27.106800} +author = {Birdsall, C. K.}, +doi = {10.1109/27.106800}, +journal = {IEEE Transactions on Plasma Science}, +number = {2}, +pages = {65--85}, +title = {{Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}}, +volume = {19}, +year = {1991} } @misc{Lim2007, @@ -400,86 +40,295 @@ @misc{Lim2007 issn = {0419-4217}, language = {eng}, number = {3}, -title = {The interaction of energetic charged particles with gas and boundaries in the particle simulation of plasmas}, +title = {{The interaction of energetic charged particles with gas and boundaries in the particle simulation of plasmas}}, +url = {https://search.library.berkeley.edu/permalink/01UCS_BER/s4lks2/cdi_proquest_miscellaneous_35689087}, volume = {69}, -year = {2007}, -url = {https://search.library.berkeley.edu/permalink/01UCS_BER/s4lks2/cdi_proquest_miscellaneous_35689087} +year = {2007} } @article{Turner2013, author = {Turner, M. M. and Derzsi, A. and Donkó, Z. and Eremin, D. and Kelly, S. J. and Lafleur, T. and Mussenbrock, T.}, -title = "{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}", +doi = {10.1063/1.4775084}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {20}, +month = {Jan}, +note = {013507}, number = {1}, -year = {2013}, -month = {01}, -issn = {1070-664X}, -doi = {10.1063/1.4775084}, +title = {{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}}, url = {https://doi.org/10.1063/1.4775084}, -note = {013507} -} - -@misc{winske2022hybrid, -title={Hybrid codes (massless electron fluid)}, -author={D. Winske and Homa Karimabadi and Ari Le and N. Omidi and Vadim Roytershteyn and Adam Stanier}, -year={2022}, -eprint={2204.01676}, -archivePrefix={arXiv}, -primaryClass={physics.plasm-ph} +volume = {20}, +year = {2013} } -@incollection{NIELSON1976, -title = {Particle-Code Models in the Nonradiative Limit}, -editor = {JOHN KILLEEN}, -series = {Methods in Computational Physics: Advances in Research and Applications}, -publisher = {Elsevier}, -volume = {16}, -pages = {367-388}, -year = {1976}, +@incollection{Nielson1976, +author = {Clair W. Nielson and H. Ralph Lewis}, booktitle = {Controlled Fusion}, +doi = {10.1016/B978-0-12-460816-0.50015-4}, +editor = {John Killeen}, issn = {0076-6860}, -doi = {https://doi.org/10.1016/B978-0-12-460816-0.50015-4}, -author = {CLAIR W. NIELSON and H. RALPH LEWIS} +pages = {367--388}, +publisher = {Elsevier}, +series = {Methods in Computational Physics: Advances in Research and Applications}, +title = {{Particle-Code Models in the Nonradiative Limit}}, +volume = {16}, +year = {1976} } @article{MUNOZ2018, -title = {A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}, +author = {P. A. Muñoz and N. Jain and P. Kilian and J. Büchner}, +doi = {https://doi.org/10.1016/j.cpc.2017.10.012}, +issn = {0010-4655}, journal = {Computer Physics Communications}, -volume = {224}, +keywords = {Plasma simulation, Hybrid methods, Particle-in-cell method}, pages = {245-264}, -year = {2018}, -issn = {0010-4655}, -doi = {https://doi.org/10.1016/j.cpc.2017.10.012}, +title = {{A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}}, url = {https://www.sciencedirect.com/science/article/pii/S0010465517303521}, -author = {P.A. Muñoz and N. Jain and P. Kilian and J. Büchner}, -keywords = {Plasma simulation, Hybrid methods, Particle-in-cell method} +volume = {224}, +year = {2018} } @article{Le2016, +abstract = "{We present the first hybrid simulations with kinetic ions and recently developed equations of state for the electron fluid appropriate for reconnection with a guide field. The equations of state account for the main anisotropy of the electron pressure tensor. Magnetic reconnection is studied in two systems, an initially force-free current sheet and a Harris sheet. The hybrid model with the equations of state is compared to two other models, hybrid simulations with isothermal electrons and fully kinetic simulations. Including the anisotropic equations of state in the hybrid model provides a better match to the fully kinetic model. In agreement with fully kinetic results, the main feature captured is the formation of an electron current sheet that extends several ion inertial lengths. This electron current sheet modifies the Hall magnetic field structure near the X-line, and it is not observed in the standard hybrid model with isotropic electrons. The saturated reconnection rate in this regime nevertheless remains similar in all three models. Implications for global modeling are discussed.}", author = {Le, A. and Daughton, W. and Karimabadi, H. and Egedal, J.}, -title = "{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}", +doi = {10.1063/1.4943893}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {23}, +month = {Mar}, +note = {032114}, number = {3}, -year = {2016}, -month = {03}, -abstract = "{We present the first hybrid simulations with kinetic ions and recently developed equations of state for the electron fluid appropriate for reconnection with a guide field. The equations of state account for the main anisotropy of the electron pressure tensor. Magnetic reconnection is studied in two systems, an initially force-free current sheet and a Harris sheet. The hybrid model with the equations of state is compared to two other models, hybrid simulations with isothermal electrons and fully kinetic simulations. Including the anisotropic equations of state in the hybrid model provides a better match to the fully kinetic model. In agreement with fully kinetic results, the main feature captured is the formation of an electron current sheet that extends several ion inertial lengths. This electron current sheet modifies the Hall magnetic field structure near the X-line, and it is not observed in the standard hybrid model with isotropic electrons. The saturated reconnection rate in this regime nevertheless remains similar in all three models. Implications for global modeling are discussed.}", -issn = {1070-664X}, -doi = {10.1063/1.4943893}, +title = {{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}}, url = {https://doi.org/10.1063/1.4943893}, -note = {032114} +volume = {23}, +year = {2016} } @article{Stanier2020, -title = {A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}, +author = {A. Stanier and L. Chacón and A. Le}, +doi = {https://doi.org/10.1016/j.jcp.2020.109705}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, -volume = {420}, +keywords = {Hybrid, Particle-in-cell, Plasma, Asymptotic-preserving, Cancellation problem, Space weather}, pages = {109705}, -year = {2020}, -issn = {0021-9991}, -doi = {https://doi.org/10.1016/j.jcp.2020.109705}, +title = {{A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}}, url = {https://www.sciencedirect.com/science/article/pii/S0021999120304794}, -author = {A. Stanier and L. Chacón and A. Le}, -keywords = {Hybrid, Particle-in-cell, Plasma, Asymptotic-preserving, Cancellation problem, Space weather}, +volume = {420}, +year = {2020} +} + +@book{Stix1992, +author = {Stix, T. H.}, +bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC}, +date-added = {2023-06-29 13:51:16 -0700}, +date-modified = {2023-06-29 13:51:16 -0700}, +isbn = {978-0-88318-859-0}, +lccn = {lc91033341}, +publisher = {American Inst. of Physics}, +title = {{Waves in Plasmas}}, +url = {https://books.google.com/books?id=OsOWJ8iHpmMC}, +year = {1992} +} + +@article{Macchi2013, +author = {Macchi, Andrea and Borghesi, Marco and Passoni, Matteo}, +doi = {10.1103/RevModPhys.85.751}, +issue = {2}, +journal = {Rev. Mod. Phys.}, +month = {May}, +numpages = {0}, +pages = {751--793}, +publisher = {American Physical Society}, +title = {{Ion acceleration by superintense laser-plasma interaction}}, +url = {https://link.aps.org/doi/10.1103/RevModPhys.85.751}, +volume = {85}, +year = {2013} +} + +@article{Wilks2001, +abstract = "{An explanation for the energetic ions observed in the PetaWatt experiments is presented. In solid target experiments with focused intensities exceeding 1020 W/cm2, high-energy electron generation, hard bremsstrahlung, and energetic protons have been observed on the backside of the target. In this report, an attempt is made to explain the physical process present that will explain the presence of these energetic protons, as well as explain the number, energy, and angular spread of the protons observed in experiment. In particular, we hypothesize that hot electrons produced on the front of the target are sent through to the back off the target, where they ionize the hydrogen layer there. These ions are then accelerated by the hot electron cloud, to tens of MeV energies in distances of order tens of μm, whereupon they end up being detected in the radiographic and spectrographic detectors.}", +author = {Wilks, S. C. and Langdon, A. B. and Cowan, T. E. and Roth, M. and Singh, M. and Hatchett, S. and Key, M. H. and Pennington, D. and MacKinnon, A. and Snavely, R. A.}, +doi = {10.1063/1.1333697}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/8/2/542/12669088/542\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Feb}, +number = {2}, +pages = {542-549}, +title = {{Energetic proton generation in ultra-intense laser–solid interactions}}, +url = {https://doi.org/10.1063/1.1333697}, +volume = {8}, +year = {2001} +} + +@article{Bulanov2008, +author = {Bulanov, S. S. and Brantov, A. and Bychenkov, V. Yu. and Chvykov, V. and Kalinchenko, G. and Matsuoka, T. and Rousseau, P. and Reed, S. and Yanovsky, V. and Litzenberg, D. W. and Krushelnick, K. and Maksimchuk, A.}, +doi = {10.1103/PhysRevE.78.026412}, +issue = {2}, +journal = {Phys. Rev. E}, +month = {Aug}, +numpages = {6}, +pages = {026412}, +publisher = {American Physical Society}, +title = {{Accelerating monoenergetic protons from ultrathin foils by flat-top laser pulses in the directed-Coulomb-explosion regime}}, +url = {https://link.aps.org/doi/10.1103/PhysRevE.78.026412}, +volume = {78}, +year = {2008} +} + +@article{Dromey2004, +abstract = "{Plasma mirrors are devices capable of switching very high laser powers on subpicosecond time scales with a dynamic range of 20–30 dB. A detailed study of their performance in the near-field of the laser beam is presented, a setup relevant to improving the pulse contrast of modern ultrahigh power lasers (TW–PW). The conditions under which high reflectivity can be achieved and focusability of the reflected beam retained are identified. At higher intensities a region of high specular reflectivity with rapidly decreasing focusability was observed, suggesting that specular reflectivity alone is not an adequate guide to the ideal range of plasma mirror operation. It was found that to achieve high reflectivity with negligible phasefront distortion of the reflected beam the inequality csΔt\\<λLaser must be met (cs: sound speed, Δt: time from plasma formation to the peak of the pulse). The achievable contrast enhancement is given by the ratio of plasma mirror reflectivity to cold reflectivity.}", +author = {Dromey, B. and Kar, S. and Zepf, M. and Foster, P.}, +doi = {10.1063/1.1646737}, +eprint = {https://pubs.aip.org/aip/rsi/article-pdf/75/3/645/8814694/645\_1\_online.pdf}, +issn = {0034-6748}, +journal = {Review of Scientific Instruments}, +month = {Feb}, +number = {3}, +pages = {645-649}, +title = {{The plasma mirror—A subpicosecond optical switch for ultrahigh power lasers}}, +url = {https://doi.org/10.1063/1.1646737}, +volume = {75}, +year = {2004} +} + +@article{Roedel2010, +author = {R\"{o}del,  C. and Heyer,  M. and Behmke,  M. and K\"{u}bel,  M. and J\"{a}ckel,  O. and Ziegler,  W. and Ehrt,  D. and Kaluza,  M. C. and Paulus,  G. G.}, +DOI = {10.1007/s00340-010-4329-7}, +ISSN = {1432-0649}, +journal = {Applied Physics B}, +month = {Nov}, +number = {2}, +pages = {295–302}, +publisher = {Springer Science and Business Media LLC}, +title = {{High repetition rate plasma mirror for temporal contrast enhancement of terawatt femtosecond laser pulses by three orders of magnitude}}, +url = {http://dx.doi.org/10.1007/s00340-010-4329-7}, +volume = {103}, +year = {2010} +} + +@misc{SandbergPASC24, +address = {Zuerich, Switzerland}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +booktitle = {Proc. of PASC24}, +note = {submitted}, +series = {PASC'24 - Platform for Advanced Scientific Computing}, +title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, +venue = {Zuerich, Switzerland}, +year = {2024} +} + +@inproceedings{SandbergIPAC23, +address = {Venice, Italy}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +booktitle = {Proc. 14th International Particle Accelerator Conference}, +doi = {10.18429/JACoW-IPAC2023-WEPA101}, +isbn = {978-3-95450-231-8}, +issn = {2673-5490}, +language = {English}, +month = {May}, +number = {14}, +pages = {2885-2888}, +paper = {WEPA101}, +publisher = {JACoW Publishing, Geneva, Switzerland}, +series = {IPAC'23 - 14th International Particle Accelerator Conference}, +title = {{Hybrid beamline element ML-training for surrogates in the ImpactX beam-dynamics code}}, +url = {https://indico.jacow.org/event/41/contributions/2276}, +venue = {Venice, Italy}, +year = {2023} +} + +@article{HigueraPOP2017, +author = {Higuera, A. V. and Cary, J. R.}, +doi = {10.1063/1.4979989}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4979989/15988441/052104\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {04}, +number = {5}, +pages = {052104}, +title = {{Structure-preserving second-order integration of relativistic charged particle trajectories in electromagnetic fields}}, +url = {https://doi.org/10.1063/1.4979989}, +volume = {24}, +year = {2017} +} + +@article{ShuJCP1988, +author = {Chi-Wang Shu and Stanley Osher}, +doi = {https://doi.org/10.1016/0021-9991(88)90177-5}, +issn = {0021-9991}, +journal = {Journal of Computational Physics}, +number = {2}, +pages = {439-471}, +title = {Efficient implementation of essentially non-oscillatory shock-capturing schemes}, +url = {https://www.sciencedirect.com/science/article/pii/0021999188901775}, +volume = {77}, +year = {1988} +} + +@Inbook{VanLeerBookChapter1997, +author = {Van Leer, Bram}, +bookTitle = {Upwind and High-Resolution Schemes}, +doi = {10.1007/978-3-642-60543-7_3}, +editor = {Hussaini, M. Yousuff and {van Leer}, Bram and {Van Rosendale}, John}, +isbn = {978-3-642-60543-7}, +pages = {33--52}, +publisher = {Springer Berlin Heidelberg}, +title = {On The Relation Between The Upwind-Differencing Schemes Of Godunov, Engquist---Osher and Roe}, +url = {https://doi.org/10.1007/978-3-642-60543-7_3}, +year = {1997} +} + +@article{Yakimenko2019, + title = {Prospect of Studying Nonperturbative QED with Beam-Beam Collisions}, + author = {Yakimenko, V. and Meuren, S. and Del Gaudio, F. and Baumann, C. and Fedotov, A. and Fiuza, F. and Grismayer, T. and Hogan, M. J. and Pukhov, A. and Silva, L. O. and White, G.}, + journal = {Phys. Rev. Lett.}, + volume = {122}, + issue = {19}, + pages = {190404}, + numpages = {7}, + year = {2019}, + month = {May}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevLett.122.190404}, + url = {https://link.aps.org/doi/10.1103/PhysRevLett.122.190404} +} + +@article{Groenewald2023, +author = {Groenewald, R. E. and Veksler, A. and Ceccherini, F. and Necas, A. and Nicks, B. S. and Barnes, D. C. and Tajima, T. and Dettrick, S. A.}, +title = "{Accelerated kinetic model for global macro stability studies of high-beta fusion reactors}", +journal = {Physics of Plasmas}, +volume = {30}, +number = {12}, +pages = {122508}, +year = {2023}, +month = {12}, +issn = {1070-664X}, +doi = {10.1063/5.0178288}, +} + +@article{Perez2012, + author = {Pérez, F. and Gremillet, L. and Decoster, A. and Drouin, M. and Lefebvre, E.}, + title = "{Improved modeling of relativistic collisions and collisional ionization in particle-in-cell codes}", + journal = {Physics of Plasmas}, + volume = {19}, + number = {8}, + pages = {083104}, + year = {2012}, + month = {08}, + issn = {1070-664X}, + doi = {10.1063/1.4742167}, + url = {https://doi.org/10.1063/1.4742167}, + eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4742167/13891570/083104\_1\_online.pdf}, +} + +@article{Higginson2019, + doi = {10.1016/j.jcp.2019.03.020}, + url = {https://doi.org/10.1016/j.jcp.2019.03.020}, + year = {2019}, + month = jul, + publisher = {Elsevier {BV}}, + volume = {388}, + pages = {439--453}, + author = {Drew Pitney Higginson and Anthony Link and Andrea Schmidt}, + title = {A pairwise nuclear fusion algorithm for weighted particle-in-cell plasma simulations}, + journal = {Journal of Computational Physics} } diff --git a/Docs/source/theory/PML.rst b/Docs/source/theory/PML.rst deleted file mode 100644 index d60be931c5f..00000000000 --- a/Docs/source/theory/PML.rst +++ /dev/null @@ -1,242 +0,0 @@ -.. _theory-bc: - -Boundary conditions -=================== - -Open boundary condition for electromagnetic waves -------------------------------------------------- - -For the TE case, the original Berenger’s Perfectly Matched Layer (PML) writes - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{\partial H_{z}}{\partial y}\label{PML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{\partial H_{z}}{\partial x}\label{PML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{\partial E_{y}}{\partial x}\label{PML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{\partial E_{x}}{\partial y}\label{PML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{PML_def_5}\end{aligned} - -This can be generalized to - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z}\label{APML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z}\label{APML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y}\label{APML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x}\label{APML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{APML_def_5}\end{aligned} - -For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, -this system reduces to the Berenger PML medium, while adding the additional -constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` -leads to the system of Maxwell equations in vacuum. - -.. _theory-bc-propa-plane-wave: - -Propagation of a Plane Wave in an APML Medium -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) -and pulsation :math:`\omega` propagating in the APML medium with an -angle :math:`\varphi` relative to the x axis - -.. math:: - - \begin{aligned} - E_{x} = & -E_{0}\sin \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_1} \\ - E_{y} = & E_{0}\cos \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_2} \\ - H_{zx} = & H_{zx0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_AMPL_def_3} \\ - H_{zy} = & H_{zy0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_4}\end{aligned} - -where :math:`\alpha` and\ :math:`\beta` are two complex constants to -be determined. - -Introducing (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_def_2] <#Plane_wave_APML_def_2>`__), -(`[Plane_wave_AMPL_def_3] <#Plane_wave_AMPL_def_3>`__) and (`[Plane_wave_APML_def_4] <#Plane_wave_APML_def_4>`__) -into (`[APML_def_1] <#APML_def_1>`__), (`[APML_def_2] <#APML_def_2>`__), (`[APML_def_3] <#APML_def_3>`__) -and (`[APML_def_4] <#APML_def_4>`__) gives - -.. math:: - - \begin{aligned} - \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = & \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_1} \\ - \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = & \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_2} \\ - \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = & \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi \label{Plane_wave_APML_1_3} \\ - \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = & \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi \label{Plane_wave_APML_1_4}\end{aligned} - -Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using (`[Plane_wave_APML_1_1] <#Plane_wave_APML_1_1>`__) -and (`[Plane_wave_APML_1_2] <#Plane_wave_APML_1_2>`__), we get - -.. math:: - - \begin{aligned} - \beta = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}}\label{Plane_wave_APML_beta_of_g} \\ - \alpha = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}}\label{Plane_wave_APML_alpha_of_g}\end{aligned} - -Adding :math:`H_{zx0}` and :math:`H_{zy0}` from (`[Plane_wave_APML_1_3] <#Plane_wave_APML_1_3>`__) -and (`[Plane_wave_APML_1_4] <#Plane_wave_APML_1_4>`__) and substituting the expressions -for :math:`\alpha` and :math:`\beta` from (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__) -and (`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) yields - -.. math:: - - \begin{aligned} - \frac{1}{Z} = & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber \\ - + & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi\end{aligned} - -If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then - -.. math:: - - \begin{aligned} - Z = & \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}}\label{APML_impedance}\end{aligned} - -which is the impedance of vacuum. Hence, like the PML, given some -restrictions on the parameters, the APML does not generate any reflection -at any angle and any frequency. As for the PML, this property is not -retained after discretization, as shown subsequently in this paper. - -Calling :math:`\psi` any component of the field and :math:`\psi _{0}` -its magnitude, we get from (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__), -(`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) and (`[APML_impedance] <#APML_impedance>`__) that - -.. math:: - - \label{Plane_wave_absorption} - \psi =\psi _{0}e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x}e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y} - -We assume that we have an APML layer of thickness :math:`\delta` (measured -along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` -and :math:`c_{y}=c.` Using (`[Plane_wave_absorption] <#Plane_wave_absorption>`__), we determine -that the coefficient of reflection given by this layer is - -.. math:: - - \begin{aligned} - R_{APML}\left( \theta \right) = & e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta }\nonumber \\ - = & e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }\end{aligned} - -which happens to be the same as the PML theoretical coefficient of -reflection if we assume :math:`c_{x}=c`. Hence, it follows that for -the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` -seems to be of no interest. However, although this conclusion is true -at the infinitesimal limit, it does not hold for the discretized counterpart. - -Discretization -~~~~~~~~~~~~~~ - -.. math:: - - \begin{aligned} - \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = & \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} \\ - % - \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = & - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n}_{j+1/2,k+1/2,l}}{2} = & - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n}_{j+1/2,k+1/2,l}}{2} = & \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - c_x = & c e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta x}{1-e^{-\sigma_x\Delta t}} \\ - c_y = & c e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta y}{1-e^{-\sigma_y\Delta t}} \\ - c^*_x = & c e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta x}{1-e^{-\sigma^*_x\Delta t}} \\ - c^*_y = & c e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta y}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. _theory-bc-pec: - -Perfect Electrical Conductor ----------------------------- - -This boundary can be used to model a dielectric or metallic surface. -For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic -field are set to 0. In the guard-cell region, the tangential electric field is set equal and -opposite to the respective field component in the mirror location across the PEC -boundary, and the normal electric field is set equal to the field component in the -mirror location in the domain across the PEC boundary. Similarly, the tangential -(and normal) magnetic field components are set equal (and opposite) to the respective -magnetic field components in the mirror locations across the PEC boundary. - -The PEC boundary condition also impacts the deposition of charge and current density. -On the boundary the charge density and parallel current density is set to zero. If -a reflecting boundary condition is used for the particles, density overlapping -with the PEC will be reflected back into the domain (for both charge and current -density). If absorbing boundaries are used, an image charge (equal weight but -opposite charge) is considered in the mirror location accross the boundary, and -the density from that charge is also deposited in the simulation domain. The -figure below shows the effect of this. The left boundary is absorbing while -the right boundary is reflecting. - -.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png - :alt: PEC boundary deposition - :width: 80% diff --git a/Docs/source/theory/amr.rst b/Docs/source/theory/amr.rst index 730593beea4..d83b1d7db9d 100644 --- a/Docs/source/theory/amr.rst +++ b/Docs/source/theory/amr.rst @@ -3,158 +3,64 @@ Mesh refinement =============== -.. raw:: latex - - \centering +.. _fig_ESAMR: .. figure:: ICNSP_2011_Vay_fig1.png :alt: Sketches of the implementation of mesh refinement in WarpX with the electrostatic (left) and electromagnetic (right) solvers. In both cases, the charge/current from particles are deposited at the finest levels first, then interpolated recursively to coarser levels. In the electrostatic case, the potential is calculated first at the coarsest level :math:`L_0`, the solution interpolated to the boundaries of the refined patch :math:`r` at the next level :math:`L_{1}` and the potential calculated at :math:`L_1`. The procedure is repeated iteratively up to the highest level. In the electromagnetic case, the fields are computed independently on each grid and patch without interpolation at boundaries. Patches are terminated by absorbing layers (PML) to prevent the reflection of electromagnetic waves. Additional coarse patch :math:`c` and fine grid :math:`a` are needed so that the full solution is obtained by substitution on :math:`a` as :math:`F_{n+1}(a)=F_{n+1}(r)+I[F_n( s )-F_{n+1}( c )]` where :math:`F` is the field, and :math:`I` is a coarse-to-fine interpolation operator. In both cases, the field solution at a given level :math:`L_n` is unaffected by the solution at higher levels :math:`L_{n+1}` and up, allowing for mitigation of some spurious effects (see text) by providing a transition zone via extension of the patches by a few cells beyond the desired refined area (red & orange rectangles) in which the field is interpolated onto particles from the coarser parent level only. - :name: fig:ESAMR - :width: 15cm + :width: 95% Sketches of the implementation of mesh refinement in WarpX with the electrostatic (left) and electromagnetic (right) solvers. In both cases, the charge/current from particles are deposited at the finest levels first, then interpolated recursively to coarser levels. In the electrostatic case, the potential is calculated first at the coarsest level :math:`L_0`, the solution interpolated to the boundaries of the refined patch :math:`r` at the next level :math:`L_{1}` and the potential calculated at :math:`L_1`. The procedure is repeated iteratively up to the highest level. In the electromagnetic case, the fields are computed independently on each grid and patch without interpolation at boundaries. Patches are terminated by absorbing layers (PML) to prevent the reflection of electromagnetic waves. Additional coarse patch :math:`c` and fine grid :math:`a` are needed so that the full solution is obtained by substitution on :math:`a` as :math:`F_{n+1}(a)=F_{n+1}(r)+I[F_n( s )-F_{n+1}( c )]` where :math:`F` is the field, and :math:`I` is a coarse-to-fine interpolation operator. In both cases, the field solution at a given level :math:`L_n` is unaffected by the solution at higher levels :math:`L_{n+1}` and up, allowing for mitigation of some spurious effects (see text) by providing a transition zone via extension of the patches by a few cells beyond the desired refined area (red & orange rectangles) in which the field is interpolated onto particles from the coarser parent level only. -The mesh refinement methods that have been implemented in WarpX were developed following the following principles: i) avoidance of spurious effects from mesh refinement, or minimization of such effects; ii) user controllability of the spurious effects’ relative magnitude; iii) simplicity of implementation. The two main generic issues that were identified are: a) spurious self-force on macroparticles close to the mesh refinement interface (J. Vay et al. 2002; Colella and Norgaard 2010); b) reflection (and possible amplification) of short wavelength electromagnetic waves at the mesh refinement interface (Vay 2001). The two effects are due to the loss of translation invariance introduced by the asymmetry of the grid on each side of the mesh refinement interface. +The mesh refinement methods that have been implemented in WarpX were developed following the following principles: i) avoidance of spurious effects from mesh refinement, or minimization of such effects; ii) user controllability of the spurious effects’ relative magnitude; iii) simplicity of implementation. The two main generic issues that were identified are: a) spurious self-force on macroparticles close to the mesh refinement interface :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`; b) reflection (and possible amplification) of short wavelength electromagnetic waves at the mesh refinement interface :cite:p:`amr-Vayjcp01`. The two effects are due to the loss of translation invariance introduced by the asymmetry of the grid on each side of the mesh refinement interface. -In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss’ Law around the refined patch, leading to long range errors (J. Vay et al. 2002; Colella and Norgaard 2010). As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. +In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss’ Law around the refined patch, leading to long range errors :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`. As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. Electrostatic ------------- -A cornerstone of the Particle-In-Cell method is that assuming a particle lying in a hypothetical infinite grid, then if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called “momentum conserving” gathering scheme; b) on average within one cell if using the “energy conserving” gathering scheme (Birdsall and Langdon 1991). A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) (J. Vay et al. 2002; Colella and Norgaard 2010). - -A sketch of the implementation of mesh refinement in WarpX is given in Figure \ `[fig:ESAMR] <#fig:ESAMR>`__ (left). Given the solution of the electric potential at a refinement level :math:`L_n`, it is interpolated onto the boundaries of the grid patch(es) at the next refined level :math:`L_{n+1}`. The electric potential is then computed at level :math:`L_{n+1}` by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. +A cornerstone of the Particle-In-Cell method is that given a particle lying in a hypothetical infinite grid, if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called “momentum conserving” gathering scheme; b) on average within one cell if using the “energy conserving” gathering scheme :cite:p:`amr-Birdsalllangdon`. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`. -.. raw:: latex +A sketch of the implementation of mesh refinement in WarpX is given in :numref:`fig_ESAMR`. Given the solution of the electric potential at a refinement level :math:`L_n`, it is interpolated onto the boundaries of the grid patch(es) at the next refined level :math:`L_{n+1}`. The electric potential is then computed at level :math:`L_{n+1}` by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. - \centering +.. _fig_ESselfforce: .. figure:: ICNSP_2011_Vay_fig2.png :alt: Position history of one charged particle attracted by its image induced by a nearby metallic (dirichlet) boundary. The particle is initialized at rest. Without refinement patch (reference case), the particle is accelerated by its image, is reflected specularly at the wall, then decelerates until it reaches its initial position at rest. If the particle is initialized inside a refinement patch, the particle is initially accelerated toward the wall but is spuriously reflected before it reaches the boundary of the patch whether using the method implemented in WarpX or the MC method. Providing a surrounding transition region 2 or 4 cells wide in which the potential is interpolated from the parent coarse solution reduces significantly the effect of the spurious self-force. - :name: fig:ESselfforce - :width: 15cm + :width: 95% Position history of one charged particle attracted by its image induced by a nearby metallic (dirichlet) boundary. The particle is initialized at rest. Without refinement patch (reference case), the particle is accelerated by its image, is reflected specularly at the wall, then decelerates until it reaches its initial position at rest. If the particle is initialized inside a refinement patch, the particle is initially accelerated toward the wall but is spuriously reflected before it reaches the boundary of the patch whether using the method implemented in WarpX or the MC method. Providing a surrounding transition region 2 or 4 cells wide in which the potential is interpolated from the parent coarse solution reduces significantly the effect of the spurious self-force. -The presence of the self-force is illustrated on a simple test case that was introduced in (J. Vay et al. 2002) and also used in (Colella and Norgaard 2010): a single macroparticle is initialized at rest within a single refinement patch four cells away from the patch refinement boundary. The patch at level :math:`L_1` has :math:`32\times32` cells and is centered relative to the lowest :math:`64\times64` grid at level :math:`L_0` (“main grid”), while the macroparticle is centered in one direction but not in the other. The boundaries of the main grid are perfectly conducting, so that the macroparticle is attracted to the closest wall by its image. Specular reflection is applied when the particle reaches the boundary so that the motion is cyclic. The test was performed with WarpX using either linear or quadratic interpolation when gathering the main grid solution onto the refined patch boundary. It was also performed using another method from P. McCorquodale et al (labeled “MC” in this paper) based on the algorithm given in (Mccorquodale et al. 2004), which employs a more elaborate procedure involving two-ways interpolations between the main grid and the refined patch. A reference case was also run using a single :math:`128\times128` grid with no refined patch, in which it is observed that the particle propagates toward the closest boundary at an accelerated pace, is reflected specularly at the boundary, then slows down until it reaches its initial position at zero velocity. The particle position histories are shown for the various cases in Fig. `[fig:ESselfforce] <#fig:ESselfforce>`__. In all the cases using the refinement patch, the particle was spuriously reflected near the patch boundary and was effectively trapped in the patch. We notice that linear interpolation performs better than quadratic, and that the simple method implemented in WarpX performs better than the other proposed method for this test (see discussion below). - -.. raw:: latex +The presence of the self-force is illustrated on a simple test case that was introduced in :cite:t:`amr-Vaylpb2002` and also used in :cite:t:`amr-Colellajcp2010`: a single macroparticle is initialized at rest within a single refinement patch four cells away from the patch refinement boundary. The patch at level :math:`L_1` has :math:`32\times32` cells and is centered relative to the lowest :math:`64\times64` grid at level :math:`L_0` ("main grid"), while the macroparticle is centered in one direction but not in the other. The boundaries of the main grid are perfectly conducting, so that the macroparticle is attracted to the closest wall by its image. Specular reflection is applied when the particle reaches the boundary so that the motion is cyclic. The test was performed with WarpX using either linear or quadratic interpolation when gathering the main grid solution onto the refined patch boundary. It was also performed using another method from P. McCorquodale et al (labeled "MC" in this paper) based on the algorithm given in :cite:t:`amr-Mccorquodalejcp2004`, which employs a more elaborate procedure involving two-ways interpolations between the main grid and the refined patch. A reference case was also run using a single :math:`128\times128` grid with no refined patch, in which it is observed that the particle propagates toward the closest boundary at an accelerated pace, is reflected specularly at the boundary, then slows down until it reaches its initial position at zero velocity. The particle position histories are shown for the various cases in :numref:`fig_ESselfforce`. In all the cases using the refinement patch, the particle was spuriously reflected near the patch boundary and was effectively trapped in the patch. We notice that linear interpolation performs better than quadratic, and that the simple method implemented in WarpX performs better than the other proposed method for this test (see discussion below). - \centering +.. _fig_ESselfforcemap: .. figure:: ICNSP_2011_Vay_fig3.png :alt: (left) Maps of the magnitude of the spurious self-force :math:`\epsilon` in arbitrary units within one quarter of the refined patch, defined as :math:`\epsilon=\sqrt{(E_x-E_x^{ref})^2+(E_y-E_y^{ref})^2}`, where :math:`E_x` and :math:`E_y` are the electric field components within the patch experienced by one particle at a given location and :math:`E_x^{ref}` and :math:`E_y^{ref}` are the electric field from a reference solution. The map is given for the WarpX and the MC mesh refinement algorithms and for linear and quadratic interpolation at the patch refinement boundary. (right) Lineouts of the maximum (taken over neighboring cells) of the spurious self-force. Close to the interface boundary (x=0), the spurious self-force decreases at a rate close to one order of magnitude per cell (red line), then at about one order of magnitude per six cells (green line). - :name: fig:ESselfforcemap - :width: 15cm + :width: 95% (left) Maps of the magnitude of the spurious self-force :math:`\epsilon` in arbitrary units within one quarter of the refined patch, defined as :math:`\epsilon=\sqrt{(E_x-E_x^{ref})^2+(E_y-E_y^{ref})^2}`, where :math:`E_x` and :math:`E_y` are the electric field components within the patch experienced by one particle at a given location and :math:`E_x^{ref}` and :math:`E_y^{ref}` are the electric field from a reference solution. The map is given for the WarpX and the MC mesh refinement algorithms and for linear and quadratic interpolation at the patch refinement boundary. (right) Lineouts of the maximum (taken over neighboring cells) of the spurious self-force. Close to the interface boundary (x=0), the spurious self-force decreases at a rate close to one order of magnitude per cell (red line), then at about one order of magnitude per six cells (green line). -The magnitude of the spurious self-force as a function of the macroparticle position was mapped and is shown in Fig. `[fig:ESselfforcemap] <#fig:ESselfforcemap>`__ for the WarpX and MC algorithms using linear or quadratic interpolations between grid levels. It is observed that the magnitude of the spurious self-force decreases rapidly with the distance between the particle and the refined patch boundary, at a rate approaching one order of magnitude per cell for the four cells closest to the boundary and about one order of magnitude per six cells beyond. The method implemented in WarpX offers a weaker spurious force on average and especially at the cells that are the closest to the coarse-fine interface where it is the largest and thus matters most. +The magnitude of the spurious self-force as a function of the macroparticle position was mapped and is shown in :numref:`fig_ESselfforcemap` for the WarpX and MC algorithms using linear or quadratic interpolations between grid levels. It is observed that the magnitude of the spurious self-force decreases rapidly with the distance between the particle and the refined patch boundary, at a rate approaching one order of magnitude per cell for the four cells closest to the boundary and about one order of magnitude per six cells beyond. The method implemented in WarpX offers a weaker spurious force on average and especially at the cells that are the closest to the coarse-fine interface where it is the largest and thus matters most. We notice that the magnitude of the spurious self-force depends strongly on the distance to the edge of the patch and to the nodes of the underlying coarse grid, but weakly on the order of deposition and size of the patch. -A method was devised and implemented in WarpX for reducing the magnitude of spurious self-forces near the coarse-fine boundaries as follows. Noting that the coarse grid solution is unaffected by the presence of the patch and is thus free of self-force, extra “transition” cells are added around the “effective” refined area. -Within the effective area, the particles gather the potential in the fine grid. In the extra transition cells surrounding the refinement patch, the force is gathered directly from the coarse grid (an option, which has not yet been implemented, would be to interpolate between the coarse and fine grid field solutions within the transition zone so as to provide continuity of the force experienced by the particles at the interface). The number of cells allocated in the transition zones is controllable by the user in WarpX, giving the opportunity to check whether the spurious self-force is affecting the calculation by repeating it using different thicknesses of the transition zones. The control of the spurious force using the transition zone is illustrated in Fig. \ `[fig:ESselfforce] <#fig:ESselfforce>`__, where the calculation with WarpX using linear interpolation at the patch interface was repeated using either two or four cells transition regions (measured in refined patch cell units). Using two extra cells allowed for the particle to be free of spurious trapping within the refined area and follow a trajectory that is close to the reference one, and using four extra cells improved further to the point where the resulting trajectory becomes indistinguishable from the reference one. -We note that an alternative method was devised for reducing the magnitude of self-force near the coarse-fine boundaries for the MC method, by using a special deposition procedure near the interface (Colella and Norgaard 2010). +A method was devised and implemented in WarpX for reducing the magnitude of spurious self-forces near the coarse-fine boundaries as follows. Noting that the coarse grid solution is unaffected by the presence of the patch and is thus free of self-force, extra "transition" cells are added around the "effective" refined area. +Within the effective area, the particles gather the potential in the fine grid. In the extra transition cells surrounding the refinement patch, the force is gathered directly from the coarse grid (an option, which has not yet been implemented, would be to interpolate between the coarse and fine grid field solutions within the transition zone so as to provide continuity of the force experienced by the particles at the interface). The number of cells allocated in the transition zones is controllable by the user in WarpX, giving the opportunity to check whether the spurious self-force is affecting the calculation by repeating it using different thicknesses of the transition zones. The control of the spurious force using the transition zone is illustrated in :numref:`fig_ESselfforce`, where the calculation with WarpX using linear interpolation at the patch interface was repeated using either two or four cells transition regions (measured in refined patch cell units). Using two extra cells allowed for the particle to be free of spurious trapping within the refined area and follow a trajectory that is close to the reference one, and using four extra cells improved further to the point where the resulting trajectory becomes indistinguishable from the reference one. +We note that an alternative method was devised for reducing the magnitude of self-force near the coarse-fine boundaries for the MC method, by using a special deposition procedure near the interface :cite:p:`amr-Colellajcp2010`. Electromagnetic --------------- -The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of (Vay 2001), refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. +The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of :cite:t:`amr-Vayjcp01`, refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. -To circumvent this issue, an additional coarse patch (with the same resolution as the parent grid) is added, as shown in Fig. \ `[fig:ESAMR] <#fig:ESAMR>`__-right and described in (Vay, Adam, and Heron 2004). Both the fine and the coarse grid patches are terminated by Perfectly Matched Layers, reducing wave reflection by orders of magnitude, controllable by the user (Berenger 1996; J.-L. Vay 2002). The source current resulting from the motion of charged macroparticles within the refined region is accumulated on the fine patch and is then interpolated onto the coarse patch and added onto the parent grid. The process is repeated recursively from the finest level down to the coarsest. The Maxwell equations are then solved for one time interval on the entire set of grids, by default for one time step using the time step of the finest grid. The field on the coarse and fine patches only contain the contributions from the particles that have evolved within the refined area but not from the current sources outside the area. The total contribution of the field from sources within and outside the refined area is obtained by adding the field from the refined grid :math:`F(r)`, and adding an interpolation :math:`I` of the difference between the relevant subset :math:`s` of the field in the parent grid :math:`F(s)` and the field of the coarse grid :math:`F( c )`, on an auxiliary grid :math:`a`, i.e. :math:`F(a)=F(r)+I[F(s)-F( c )]`. The field on the parent grid subset :math:`F(s)` contains contributions from sources from both within and outside of the refined area. Thus, in effect, there is substitution of the coarse field resulting from sources within the patch area by its fine resolution counterpart. The operation is carried out recursively starting at the coarsest level up to the finest. +To circumvent this issue, an additional coarse patch (with the same resolution as the parent grid) is added, as shown in :numref:`fig_ESAMR` and described in :cite:t:`amr-Vaycpc04`. Both the fine and the coarse grid patches are terminated by Perfectly Matched Layers, reducing wave reflection by orders of magnitude, controllable by the user :cite:p:`amr-Berengerjcp96,amr-Vayjcp02`. The source current resulting from the motion of charged macroparticles within the refined region is accumulated on the fine patch and is then interpolated onto the coarse patch and added onto the parent grid. The process is repeated recursively from the finest level down to the coarsest. The Maxwell equations are then solved for one time interval on the entire set of grids, by default for one time step using the time step of the finest grid. The field on the coarse and fine patches only contain the contributions from the particles that have evolved within the refined area but not from the current sources outside the area. The total contribution of the field from sources within and outside the refined area is obtained by adding the field from the refined grid :math:`F(r)`, and adding an interpolation :math:`I` of the difference between the relevant subset :math:`s` of the field in the parent grid :math:`F(s)` and the field of the coarse grid :math:`F( c )`, on an auxiliary grid :math:`a`, i.e. :math:`F(a)=F(r)+I[F(s)-F( c )]`. The field on the parent grid subset :math:`F(s)` contains contributions from sources from both within and outside of the refined area. Thus, in effect, there is substitution of the coarse field resulting from sources within the patch area by its fine resolution counterpart. The operation is carried out recursively starting at the coarsest level up to the finest. An option has been implemented in which various grid levels are pushed with different time steps, given as a fixed fraction of the individual grid Courant conditions (assuming same cell aspect ratio for all grids and refinement by integer factors). In this case, the fields from the coarse levels, which are advanced less often, are interpolated in time. The substitution method has two potential drawbacks due to the inexact cancellation between the coarse and fine patches of : (i) the remnants of ghost fixed charges created by the particles entering and leaving the patches (this effect is due to the use of the electromagnetic solver and is different from the spurious self-force that was described for the electrostatic case); (ii) if using a Maxwell solver with a low-order stencil, the electromagnetic waves traveling on each patch at slightly different velocity due to numerical dispersion. The first issue results in an effective spurious multipole field whose magnitude decreases very rapidly with the distance to the patch boundary, similarly to the spurious self-force in the electrostatic case. Hence, adding a few extra transition cells surrounding the patches mitigates this effect very effectively. -The tunability of WarpX’s electromagnetic finite-difference and pseudo-spectral solvers provides the means to optimize the numerical dispersion so as to minimize the second effect for a given application, which has been demonstrated on the laser-plasma interaction test case presented in (Vay, Adam, and Heron 2004). -Both effects and their mitigation are described in more detail in (Vay, Adam, and Heron 2004). +The tunability of WarpX’s electromagnetic finite-difference and pseudo-spectral solvers provides the means to optimize the numerical dispersion so as to minimize the second effect for a given application, which has been demonstrated on the laser-plasma interaction test case presented in :cite:t:`amr-Vaycpc04`. +Both effects and their mitigation are described in more detail in :cite:t:`amr-Vaycpc04`. Caustics are supported anywhere on the grid with an accuracy that is set by the local resolution, and will be adequately resolved if the grid resolution supports the necessary modes from their sources to the points of wavefront crossing. The mesh refinement method that is implemented in WarpX has the potential to provide higher efficiency than the standard use of fixed gridding, by offering a path toward adaptive gridding following wavefronts. -.. raw:: html - -
- -.. raw:: html - -
- -Berenger, Jp. 1996. “Three-Dimensional Perfectly Matched Layer for the Absorption of Electromagnetic Waves.” *Journal of Computational Physics* 127 (2): 363–79. - -.. raw:: html - -
- -.. raw:: html - -
- -Birdsall, C K, and A B Langdon. 1991. *Plasma Physics via Computer Simulation*. Adam-Hilger. - -.. raw:: html - -
- -.. raw:: html - -
- -Colella, Phillip, and Peter C Norgaard. 2010. “Controlling Self-Force Errors at Refinement Boundaries for Amr-Pic.” *Journal of Computational Physics* 229 (4): 947–57. https://doi.org/10.1016/J.Jcp.2009.07.004. - -.. raw:: html - -
- -.. raw:: html - -
- -Mccorquodale, P, P Colella, Dp Grote, and Jl Vay. 2004. “A Node-Centered Local Refinement Algorithm For Poisson’s Equation In Complex Geometries.” *Journal of Computational Physics* 201 (1): 34–60. https://doi.org/10.1016/J.Jcp.2004.04.022. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2001. “An Extended Fdtd Scheme for the Wave Equation: Application to Multiscale Electromagnetic Simulation.” *Journal of Computational Physics* 167 (1): 72–98. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2002. “Asymmetric Perfectly Matched Layer for the Absorption of Waves.” *Journal of Computational Physics* 183 (2): 367–99. https://doi.org/10.1006/Jcph.2002.7175. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., J.-C. Adam, and A Heron. 2004. “Asymmetric Pml for the Absorption of Waves. Application to Mesh Refinement in Electromagnetic Particle-in-Cell Plasma Simulations.” *Computer Physics Communications* 164 (1-3): 171–77. https://doi.org/10.1016/J.Cpc.2004.06.026. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jl, P Colella, P Mccorquodale, B Van Straalen, A Friedman, and Dp Grote. 2002. “Mesh Refinement for Particle-in-Cell Plasma Simulations: Applications to and Benefits for Heavy Ion Fusion.” *Laser and Particle Beams* 20 (4): 569–75. https://doi.org/10.1017/S0263034602204139. - -.. raw:: html - -
- -.. raw:: html - -
+.. bibliography:: + :keyprefix: amr- diff --git a/Docs/source/theory/boosted_frame.rst b/Docs/source/theory/boosted_frame.rst index 4b80c3d2ccb..ea1f662bd30 100644 --- a/Docs/source/theory/boosted_frame.rst +++ b/Docs/source/theory/boosted_frame.rst @@ -5,16 +5,18 @@ Moving window and optimal Lorentz boosted frame The simulations of plasma accelerators from first principles are extremely computationally intensive, due to the need to resolve the evolution of a driver (laser or particle beam) and an accelerated particle beam into a plasma structure that is orders of magnitude longer and wider than the accelerated beam. As is customary in the modeling of particle beam dynamics in standard particle accelerators, a moving window is commonly used to follow the driver, the wake and the accelerated beam. This results in huge savings, by avoiding the meshing of the entire plasma that is orders of magnitude longer than the other length scales of interest. +.. _fig_Boosted_frame: + .. figure:: Boosted_frame.png - :alt: [fig:PIC] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. + :alt: [fig:Boosted-frame] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. - [fig:PIC] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. + A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. -Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude (Vay 2007; J -L. Vay et al. 2011). This is a result of the properties of Lorentz contraction and dilation of space and time. In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. As sketched in Fig. `[fig:PIC] <#fig:PIC>`__, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity). The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). +Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude :cite:p:`bf-Vayprl07,bf-Vaypop2011`. This is a result of the properties of Lorentz contraction and dilation of space and time. In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. As sketched in :numref:`fig_Boosted_frame`, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity). The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). The modeling of a plasma acceleration stage in a boosted frame involves the fully electromagnetic modeling of a plasma propagating at near the speed of light, for which Numerical Cerenkov -(Boris and Lee 1973; Haber et al. 1973) is a potential issue, as explained in more details below. +:cite:p:`bf-Borisjcp73,bf-Habericnsp73` is a potential issue, as explained in more details below. In addition, for a frame of reference moving in the direction of the accelerated beam (or equivalently the wake of the laser), waves emitted by the plasma in the forward direction expand while the ones emitted in the backward direction contract, following the properties of the Lorentz transformation. @@ -25,16 +27,15 @@ Backscatter is weak in the short-pulse regime, and does not interact as strongly with the beam as do the forward propagating waves which stay in phase for a long period. It is thus often assumed that the backward propagating waves can be neglected in the modeling of plasma accelerator stages. The accuracy of this assumption has been demonstrated by -comparison between explicit codes which include both forward and backward waves and envelope or quasistatic codes which neglect backward waves -(Geddes et al. 2008; Geddes et al. 2009; Cowan et al. 2009). +comparison between explicit codes which include both forward and backward waves and envelope or quasistatic codes which neglect backward waves :cite:p:`bf-Geddesjp08,bf-Geddespac09,bf-Cowanaac08`. Theoretical speedup dependency with the frame boost --------------------------------------------------- -The derivation that is given here reproduces the one given in (J -L. Vay et al. 2011), where the obtainable speedup is derived as an extension of the formula that was derived earlier(Vay 2007), taking in addition into account the group velocity of the laser as it traverses the plasma. +The derivation that is given here reproduces the one given in :cite:t:`bf-Vaypop2011`, where the obtainable speedup is derived as an extension of the formula that was derived earlier :cite:p:`bf-Vayprl07`, taking in addition into account the group velocity of the laser as it traverses the plasma. Assuming that the simulation box is a fixed number of plasma periods long, which implies the use (which is standard) of a moving window following -the wake and accelerated beam, the speedup is given by the ratio of the time taken by the laser pulse and the plasma to cross each other, divided by the shortest time scale of interest, that is the laser period. To first order, the wake velocity :math:`v_w` is set by the 1D group velocity of the laser driver, which in the linear (low intensity) limit, is given by (Esarey, Schroeder, and Leemans 2009): +the wake and accelerated beam, the speedup is given by the ratio of the time taken by the laser pulse and the plasma to cross each other, divided by the shortest time scale of interest, that is the laser period. To first order, the wake velocity :math:`v_w` is set by the 1D group velocity of the laser driver, which in the linear (low intensity) limit, is given by :cite:p:`bf-Esareyrmp09`: .. math:: v_w/c=\beta_w=\left(1-\frac{\omega_p^2}{\omega^2}\right)^{1/2} @@ -58,76 +59,78 @@ In a frame moving at :math:`\beta c`, the quantities become .. math:: \begin{aligned} - \lambda_p^*&=&\lambda_p/\left[\gamma \left(1-\beta_w \beta\right)\right] \\ - L^*&=&L/\gamma \\ - \lambda^*&=& \gamma\left(1+\beta\right) \lambda\\ - \beta_w^*&=&\left(\beta_w-\beta\right)/\left(1-\beta_w\beta\right) \\ - v_p^*&=&-\beta c \\ - T^*&=&\frac{L^*+\eta \lambda_p^*}{v_w^*-v_p^*} \\ - R_t^*&=&\frac{T^* c}{\lambda^*} = \frac{\left(L^*+\eta \lambda_p^*\right)}{\left(\beta_w^*+\beta\right) \lambda^*}\end{aligned} + \lambda_p^* & = \lambda_p/\left[\gamma \left(1-\beta_w \beta\right)\right] + \\ + L^* & = L/\gamma + \\ + \lambda^* & = \gamma\left(1+\beta\right) \lambda + \\ + \beta_w^* & = \left(\beta_w-\beta\right)/\left(1-\beta_w\beta\right) + \\ + v_p^* & = -\beta c + \\ + T^* & = \frac{L^*+\eta \lambda_p^*}{v_w^*-v_p^*} + \\ + R_t^* & = \frac{T^* c}{\lambda^*} = \frac{\left(L^*+\eta \lambda_p^*\right)}{\left(\beta_w^*+\beta\right) \lambda^*} + \end{aligned} where :math:`\gamma=1/\sqrt{1-\beta^2}`. The expected speedup from performing the simulation in a boosted frame is given by the ratio of :math:`R_{lab}` and :math:`R_t^*` .. math:: - S=\frac{R_{lab}}{R_t^*}=\frac{\left(1+\beta\right)\left(L+\eta \lambda_p\right)}{\left(1-\beta\beta_w\right)L+\eta \lambda_p} - \label{Eq_scaling1d0} + :label: Eq_scaling1d0 -We note that assuming that :math:`\beta_w\approx1` (which is a valid approximation for most practical cases of interest) and that :math:`\gamma<<\gamma_w`, this expression is consistent with the expression derived earlier (Vay 2007) for the laser-plasma acceleration case, which states that :math:`R_t^*=\alpha R_t/\left(1+\beta\right)` with :math:`\alpha=\left(1-\beta+l/L\right)/\left(1+l/L\right)`, where :math:`l` is the laser length which is generally proportional to :math:`\eta \lambda_p`, and :math:`S=R_t/R_T^*`. However, higher values of :math:`\gamma` are of interest for maximum speedup, as shown below. +We note that assuming that :math:`\beta_w\approx1` (which is a valid approximation for most practical cases of interest) and that :math:`\gamma<<\gamma_w`, this expression is consistent with the expression derived earlier :cite:p:`bf-Vayprl07` for the laser-plasma acceleration case, which states that :math:`R_t^*=\alpha R_t/\left(1+\beta\right)` with :math:`\alpha=\left(1-\beta+l/L\right)/\left(1+l/L\right)`, where :math:`l` is the laser length which is generally proportional to :math:`\eta \lambda_p`, and :math:`S=R_t/R_T^*`. However, higher values of :math:`\gamma` are of interest for maximum speedup, as shown below. -For intense lasers (:math:`a\sim 1`) typically used for acceleration, the energy gain is limited by dephasing (Schroeder et al. 2011), which occurs over a scale length :math:`L_d \sim \lambda_p^3/2\lambda^2`. -Acceleration is compromised beyond :math:`L_d` and in practice, the plasma length is proportional to the dephasing length, i.e. :math:`L= \xi L_d`. In most cases, :math:`\gamma_w^2>>1`, which allows the approximations :math:`\beta_w\approx1-\lambda^2/2\lambda_p^2`, and :math:`L=\xi \lambda_p^3/2\lambda^2\approx \xi \gamma_w^2 \lambda_p/2>>\eta \lambda_p`, so that Eq.(\ `[Eq_scaling1d0] <#Eq_scaling1d0>`__) becomes +For intense lasers (:math:`a\sim 1`) typically used for acceleration, the energy gain is limited by dephasing :cite:p:`bf-Schroederprl2011`, which occurs over a scale length :math:`L_d \sim \lambda_p^3/2\lambda^2`. +Acceleration is compromised beyond :math:`L_d` and in practice, the plasma length is proportional to the dephasing length, i.e. :math:`L= \xi L_d`. In most cases, :math:`\gamma_w^2>>1`, which allows the approximations :math:`\beta_w\approx1-\lambda^2/2\lambda_p^2`, and :math:`L=\xi \lambda_p^3/2\lambda^2\approx \xi \gamma_w^2 \lambda_p/2>>\eta \lambda_p`, so that Eq.(:eq:`Eq_scaling1d0`) becomes .. math:: - S=\left(1+\beta\right)^2\gamma^2\frac{\xi\gamma_w^2}{\xi\gamma_w^2+\left(1+\beta\right)\gamma^2\left(\xi\beta/2+2\eta\right)} - \label{Eq_scaling1d} + :label: Eq_scaling1d -For low values of :math:`\gamma`, i.e. when :math:`\gamma<<\gamma_w`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) reduces to +For low values of :math:`\gamma`, i.e. when :math:`\gamma<<\gamma_w`, Eq.(:eq:`Eq_scaling1d`) reduces to .. math:: - S_{\gamma<<\gamma_w}=\left(1+\beta\right)^2\gamma^2 - \label{Eq_scaling1d_simpl2} + :label: Eq_scaling1d_simpl2 -Conversely, if :math:`\gamma\rightarrow\infty`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) becomes +Conversely, if :math:`\gamma\rightarrow\infty`, Eq.(`Eq_scaling1d`) becomes .. math:: - S_{\gamma\rightarrow\infty}=\frac{4}{1+4\eta/\xi}\gamma_w^2 - \label{Eq_scaling_gamma_inf} + :label: Eq_scaling_gamma_inf -Finally, in the frame of the wake, i.e. when :math:`\gamma=\gamma_w`, assuming that :math:`\beta_w\approx1`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) gives +Finally, in the frame of the wake, i.e. when :math:`\gamma=\gamma_w`, assuming that :math:`\beta_w\approx1`, Eq.(:eq:`Eq_scaling1d`) gives .. math:: - S_{\gamma=\gamma_w}\approx\frac{2}{1+2\eta/\xi}\gamma_w^2 - \label{Eq_scaling_gamma_wake} + :label: Eq_scaling_gamma_wake -Since :math:`\eta` and :math:`\xi` are of order unity, and the practical regimes of most interest satisfy :math:`\gamma_w^2>>1`, the speedup that is obtained by using the frame of the wake will be near the maximum obtainable value given by Eq.(\ `[Eq_scaling_gamma_inf] <#Eq_scaling_gamma_inf>`__). +Since :math:`\eta` and :math:`\xi` are of order unity, and the practical regimes of most interest satisfy :math:`\gamma_w^2>>1`, the speedup that is obtained by using the frame of the wake will be near the maximum obtainable value given by Eq.(:eq:`Eq_scaling_gamma_inf`). -Note that without the use of a moving window, the relativistic effects that are at play in the time domain would also be at play in the spatial domain (Vay 2007), and the :math:`\gamma^2` scaling would transform to :math:`\gamma^4`. Hence, it is important to use a moving window even in simulations in a Lorentz boosted frame. For very high values of the boosted frame, the optimal velocity of the moving window may vanish (i.e. no moving window) or even reverse. +Note that without the use of a moving window, the relativistic effects that are at play in the time domain would also be at play in the spatial domain :cite:p:`bf-Vayprl07`, and the :math:`\gamma^2` scaling would transform to :math:`\gamma^4`. Hence, it is important to use a moving window even in simulations in a Lorentz boosted frame. For very high values of the boosted frame, the optimal velocity of the moving window may vanish (i.e. no moving window) or even reverse. .. _theory-boostedframe-galilean: Numerical Stability and alternate formulation in a Galilean frame ----------------------------------------------------------------- -The numerical Cherenkov instability (NCI) (Godfrey 1974) +The numerical Cherenkov instability (NCI) :cite:p:`bf-Godfreyjcp74` is the most serious numerical instability affecting multidimensional PIC simulations of relativistic particle beams and streaming plasmas -(Martins et al. 2010; Vay et al. 2010; J L Vay et al. 2011; Sironi and Spitkovsky 2011; Godfrey and Vay 2013; Xu et al. 2013). +:cite:p:`bf-Martinscpc10,bf-VayAAC2010,bf-Vayjcp2011,bf-Spitkovsky:Icnsp2011,bf-GodfreyJCP2013,bf-XuJCP2013`. It arises from coupling between possibly numerically distorted electromagnetic modes and spurious beam modes, the latter due to the mismatch between the Lagrangian -treatment of particles and the Eulerian treatment of fields (Godfrey 1975). +treatment of particles and the Eulerian treatment of fields :cite:p:`bf-Godfreyjcp75`. In recent papers the electromagnetic dispersion -relations for the numerical Cherenkov instability were derived and solved for both FDTD (Godfrey and Vay 2013; Brendan B. Godfrey and Vay 2014) -and PSATD (Brendan B. Godfrey, Vay, and Haber 2014a, 2014b) algorithms. +relations for the numerical Cherenkov instability were derived and solved for both FDTD :cite:p:`bf-GodfreyJCP2013,bf-GodfreyJCP2014_FDTD` +and PSATD :cite:p:`bf-GodfreyJCP2014_PSATD,bf-GodfreyIEEE2014` algorithms. -Several solutions have been proposed to mitigate the NCI (Brendan B Godfrey, Vay, and Haber 2014; Brendan B. Godfrey, Vay, and Haber 2014b, 2014a; Godfrey and Vay 2015; Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015). Although +Several solutions have been proposed to mitigate the NCI :cite:p:`bf-GodfreyJCP2014,bf-GodfreyIEEE2014,bf-GodfreyJCP2014_PSATD,bf-GodfreyCPC2015,bf-YuCPC2015,bf-YuCPC2015-Circ`. Although these solutions efficiently reduce the numerical instability, they typically introduce either strong smoothing of the currents and fields, or arbitrary numerical corrections, which are @@ -137,47 +140,46 @@ it is sometimes unclear to what extent these added corrections could impact the physics at stake for a given resolution. For instance, NCI-specific corrections include periodically smoothing -the electromagnetic field components (Martins et al. 2010), -using a special time step (Vay et al. 2010; J L Vay et al. 2011) or -applying a wide-band smoothing of the current components (Vay et al. 2010; J L Vay et al. 2011; J. Vay et al. 2011). Another set of mitigation methods +the electromagnetic field components :cite:p:`bf-Martinscpc10`, +using a special time step :cite:p:`bf-VayAAC2010,bf-Vayjcp2011` or +applying a wide-band smoothing of the current components :cite:p:`bf-VayAAC2010,bf-Vayjcp2011,bf-VayPOPL2011`. Another set of mitigation methods involve scaling the deposited currents by a carefully-designed wavenumber-dependent factor -(Brendan B. Godfrey and Vay 2014; Brendan B. Godfrey, Vay, and Haber 2014b) or slightly modifying the +:cite:p:`bf-GodfreyJCP2014_FDTD,bf-GodfreyIEEE2014` or slightly modifying the ratio of electric and magnetic fields (:math:`E/B`) before gathering their value onto the macroparticles -(Brendan B. Godfrey, Vay, and Haber 2014a; Godfrey and Vay 2015). +:cite:p:`bf-GodfreyJCP2014_PSATD,bf-GodfreyCPC2015`. Yet another set of NCI-specific corrections -(Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015) consists +:cite:p:`bf-YuCPC2015,bf-YuCPC2015-Circ` consists in combining a small timestep :math:`\Delta t`, a sharp low-pass spatial filter, and a spectral or high-order scheme that is tuned so as to create a small, artificial “bump” in the dispersion relation -(Yu, Xu, Decyk, et al. 2015). While most mitigation methods have only been applied +:cite:p:`bf-YuCPC2015`. While most mitigation methods have only been applied to Cartesian geometry, this last -set of methods ((Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015)) +set of methods :cite:p:`bf-YuCPC2015,bf-YuCPC2015-Circ` has the remarkable property that it can be applied -(Yu, Xu, Tableman, et al. 2015) to both Cartesian geometry and +:cite:p:`bf-YuCPC2015-Circ` to both Cartesian geometry and quasi-cylindrical geometry (i.e. cylindrical geometry with -azimuthal Fourier decomposition (Lifschitz et al. 2009; Davidson et al. 2015; R. Lehe et al. 2016)). However, +azimuthal Fourier decomposition :cite:p:`bf-LifschitzJCP2009,bf-DavidsonJCP2015,bf-Lehe2016`). However, the use of a small timestep proportionally slows down the progress of the simulation, and the artificial “bump” is again an arbitrary correction that departs from the underlying physics. -A new scheme was recently proposed, in (Kirchen et al. 2016; Lehe et al. 2016), which +A new scheme was recently proposed, in :cite:t:`bf-KirchenPOP2016,bf-LehePRE2016`, which completely eliminates the NCI for a plasma drifting at a uniform relativistic velocity – with no arbitrary correction – by simply integrating the PIC equations in *Galilean coordinates* (also known as *comoving coordinates*). More precisely, in the new method, the Maxwell equations *in Galilean coordinates* are integrated analytically, using only natural hypotheses, within the PSATD -framework (Pseudo-Spectral-Analytical-Time-Domain (Haber et al. 1973; Vay, Haber, and Godfrey 2013)). +framework (Pseudo-Spectral-Analytical-Time-Domain :cite:p:`bf-Habericnsp73,bf-VayJCP2013`). The idea of the proposed scheme is to perform a Galilean change of coordinates, and to carry out the simulation in the new coordinates: .. math:: - - \label{eq:change-var} \boldsymbol{x}' = \boldsymbol{x} - \boldsymbol{v}_{gal}t + :label: change-var where :math:`\boldsymbol{x} = x\,\boldsymbol{u}_x + y\,\boldsymbol{u}_y + z\,\boldsymbol{u}_z` and :math:`\boldsymbol{x}' = x'\,\boldsymbol{u}_x + y'\,\boldsymbol{u}_y + z'\,\boldsymbol{u}_z` are the @@ -190,10 +192,10 @@ plasma, the plasma does not move with respect to the grid in the Galilean coordinates :math:`\boldsymbol{x}'` – or, equivalently, in the standard coordinates :math:`\boldsymbol{x}`, the grid moves along with the plasma. The heuristic intuition behind this scheme is that these coordinates should prevent the discrepancy between the Lagrangian and -Eulerian point of view, which gives rise to the NCI (Godfrey 1975). +Eulerian point of view, which gives rise to the NCI :cite:p:`bf-Godfreyjcp75`. An important remark is that the Galilean change of -coordinates (`[eq:change-var] <#eq:change-var>`__) is a simple translation. Thus, when used in +coordinates in Eq. (:eq:`change-var`) is a simple translation. Thus, when used in the context of Lorentz-boosted simulations, it does of course preserve the relativistic dilatation of space and time which gives rise to the characteristic computational speedup of the boosted-frame technique. @@ -206,378 +208,81 @@ translate the boundaries, in the Galilean scheme the gridpoints *themselves* are not only translated but in this case, the physical equations are modified accordingly. Most importantly, the assumed time evolution of the current :math:`\boldsymbol{J}` within one timestep is different in a standard PSATD scheme with moving -window and in a Galilean PSATD scheme (Lehe et al. 2016). +window and in a Galilean PSATD scheme :cite:p:`bf-LehePRE2016`. In the Galilean coordinates :math:`\boldsymbol{x}'`, the equations of particle motion and the Maxwell equations take the form .. math:: + \frac{d\boldsymbol{x}'}{dt} = \frac{\boldsymbol{p}}{\gamma m} - \boldsymbol{v}_{gal} + :label: motion1 - \begin{aligned} - \frac{d\boldsymbol{x}'}{dt} &= \frac{\boldsymbol{p}}{\gamma m} - \boldsymbol{v}_{gal}\label{eq:motion1} \\ - \frac{d\boldsymbol{p}}{dt} &= q \left( \boldsymbol{E} + - \frac{\boldsymbol{p}}{\gamma m} \times \boldsymbol{B} \right) \label{eq:motion2}\\ - \left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{B} &= -\boldsymbol{\nabla'}\times\boldsymbol{E} \label{eq:maxwell1}\\ - \frac{1}{c^2}\left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{E} &= \boldsymbol{\nabla'}\times\boldsymbol{B} - \mu_0\boldsymbol{J} \label{eq:maxwell2}\end{aligned} +.. math:: + \frac{d\boldsymbol{p}}{dt} = q \left( \boldsymbol{E} + \frac{\boldsymbol{p}}{\gamma m} \times \boldsymbol{B} \right) + :label: motion2 + +.. math:: + \left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{B} = -\boldsymbol{\nabla'}\times\boldsymbol{E} + :label: maxwell1 + +.. math:: + \frac{1}{c^2}\left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{E} = \boldsymbol{\nabla'}\times\boldsymbol{B} - \mu_0\boldsymbol{J} + :label: maxwell2 where :math:`\boldsymbol{\nabla'}` denotes a spatial derivative with respect to the Galilean coordinates :math:`\boldsymbol{x}'`. Integrating these equations from :math:`t=n\Delta t` to :math:`t=(n+1)\Delta t` results in the following update equations (see -(Lehe et al. 2016) for the details of the derivation): +:cite:t:`bf-LehePRE2016` for the details of the derivation): .. math:: - \begin{aligned} - \mathbf{\tilde{B}}^{n+1} &= \theta^2 C \mathbf{\tilde{B}}^n - -\frac{\theta^2 S}{ck}i\boldsymbol{k}\times \mathbf{\tilde{E}}^n \nonumber \\ - & + \;\frac{\theta \chi_1}{\epsilon_0c^2k^2}\;i\boldsymbol{k} \times - \mathbf{\tilde{J}}^{n+1/2} \label{eq:disc-maxwell1}\\ - \mathbf{\tilde{E}}^{n+1} &= \theta^2 C \mathbf{\tilde{E}}^n - +\frac{\theta^2 S}{k} \,c i\boldsymbol{k}\times \mathbf{\tilde{B}}^n \nonumber \\ - & +\frac{i\nu \theta \chi_1 - \theta^2S}{\epsilon_0 ck} \; \mathbf{\tilde{J}}^{n+1/2}\nonumber \\ - & - \frac{1}{\epsilon_0k^2}\left(\; \chi_2\;\hat{\mathcal{\rho}}^{n+1} - - \theta^2\chi_3\;\hat{\mathcal{\rho}}^{n} \;\right) i\boldsymbol{k} \label{eq:disc-maxwell2}\end{aligned} - -where we used the short-hand notations :math:`\mathbf{\tilde{E}}^n \equiv -% -\mathbf{\tilde{E}}(\boldsymbol{k}, n\Delta t)`, :math:`\mathbf{\tilde{B}}^n \equiv -\mathbf{\tilde{B}}(\boldsymbol{k}, n\Delta t)` as well as: + \mathbf{\tilde{B}}^{n+1} & = \theta^2 C \mathbf{\tilde{B}}^n -\frac{\theta^2 S}{ck}i\boldsymbol{k}\times \mathbf{\tilde{E}}^n \nonumber + \\ + & + \;\frac{\theta \chi_1}{\epsilon_0c^2k^2}\;i\boldsymbol{k} \times \mathbf{\tilde{J}}^{n+1/2} + \end{aligned} + :label: disc-maxwell1 .. math:: - \begin{aligned} - &C = \cos(ck\Delta t) \quad S = \sin(ck\Delta t) \quad k - = |\boldsymbol{k}| \label{eq:def-C-S}\\& - \nu = \frac{\boldsymbol{k}\cdot\boldsymbol{v}_{gal}}{ck} \quad \theta = - e^{i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2} \quad \theta^* = - e^{-i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2} \label{eq:def-nu-theta}\\& - \chi_1 = \frac{1}{1 -\nu^2} \left( \theta^* - C \theta + i - \nu \theta S \right) \label{eq:def-chi1}\\& - \chi_2 = \frac{\chi_1 - \theta(1-C)}{\theta^*-\theta} \quad - \chi_3 = \frac{\chi_1-\theta^*(1-C)}{\theta^*-\theta} \label{eq:def-chi23}\end{aligned} - -Note that, in the limit :math:`\boldsymbol{v}_{gal}=\boldsymbol{0}`, -(`[eq:disc-maxwell1] <#eq:disc-maxwell1>`__) and (`[eq:disc-maxwell2] <#eq:disc-maxwell2>`__) reduce to the standard PSATD -equations (Haber et al. 1973), as expected. -As shown in (Kirchen et al. 2016; Lehe et al. 2016), -the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. - -.. raw:: html - -
- -.. raw:: html - -
- -Boris, Jp, and R Lee. 1973. “Nonphysical Self Forces in Some Electromagnetic Plasma-Simulation Algorithms.” Note. *Journal of Computational Physics* 12 (1). 525 B St, Ste 1900, San Diego, Ca 92101-4495: Academic Press Inc Jnl-Comp Subscriptions: 131–36. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, B, D Bruhwiler, E Cormier-Michel, E Esarey, C G R Geddes, P Messmer, and K Paul. 2009. “Laser Wakefield Simulation Using A Speed-of-Light Frame Envelope Model.” In *Aip Conference Proceedings*, 1086:309–14. - -.. raw:: html - -
- -.. raw:: html - -
- -Davidson, A., A. Tableman, W. An, F.S. Tsung, W. Lu, J. Vieira, R.A. Fonseca, L.O. Silva, and W.B. Mori. 2015. “Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in :math:`\Phi` into OSIRIS.” *Journal of Computational Physics* 281: 1063–77. https://doi.org/10.1016/j.jcp.2014.10.064. - -.. raw:: html - -
- -.. raw:: html - -
- -Esarey, E, C B Schroeder, and W P Leemans. 2009. “Physics of Laser-Driven Plasma-Based Electron Accelerators.” *Rev. Mod. Phys.* 81 (3): 1229–85. https://doi.org/10.1103/Revmodphys.81.1229. - -.. raw:: html - -
- -.. raw:: html - -
- -Geddes, C G R, D L Bruhwiler, J R Cary, W B Mori, J.-L. Vay, S F Martins, T Katsouleas, et al. 2008. “Computational Studies and Optimization of Wakefield Accelerators.” In *Journal of Physics: Conference Series*, 125:012002 (11 Pp.). - -.. raw:: html - -
- -.. raw:: html - -
- -Geddes et al., C G R. 2009. “Scaled Simulation Design of High Quality Laser Wakefield Accelerator Stages.” In *Proc. Particle Accelerator Conference*. Vancouver, Canada. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Bb. 1974. “Numerical Cherenkov Instabilities in Electromagnetic Particle Codes.” *Journal of Computational Physics* 15 (4): 504–21. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 1975. “Canonical Momenta and Numerical Instabilities in Particle Codes.” *Journal of Computational Physics* 19 (1): 58–76. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B, and Jean-Luc Vay. 2013. “Numerical stability of relativistic beam multidimensional {PIC} simulations employing the Esirkepov algorithm.” *Journal of Computational Physics* 248 (0): 33–46. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2013.04.006. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B., and Jean Luc Vay. 2014. “Suppressing the numerical Cherenkov instability in FDTD PIC codes.” *Journal of Computational Physics* 267: 1–6. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2015. “Improved numerical Cherenkov instability suppression in the generalized PSTD PIC algorithm.” *Computer Physics Communications* 196. Elsevier: 221–25. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B., Jean Luc Vay, and Irving Haber. 2014a. “Numerical stability analysis of the pseudo-spectral analytical time-domain PIC algorithm.” *Journal of Computational Physics* 258: 689–704. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2014b. “Numerical stability improvements for the pseudospectral EM PIC algorithm.” *IEEE Transactions on Plasma Science* 42 (5). Institute of Electrical; Electronics Engineers Inc.: 1339–44. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B, Jean-Luc Vay, and Irving Haber. 2014. “Numerical stability analysis of the pseudo-spectral analytical time-domain {PIC} algorithm.” *Journal of Computational Physics* 258 (0): 689–704. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2013.10.053. - -.. raw:: html + \mathbf{\tilde{E}}^{n+1} & = \theta^2 C \mathbf{\tilde{E}}^n +\frac{\theta^2 S}{k} \,c i\boldsymbol{k}\times \mathbf{\tilde{B}}^n \nonumber + \\ + & + \frac{i\nu \theta \chi_1 - \theta^2S}{\epsilon_0 ck} \; \mathbf{\tilde{J}}^{n+1/2}\nonumber + \\ + & - \frac{1}{\epsilon_0k^2}\left(\; \chi_2\;\hat{\mathcal{\rho}}^{n+1} - \theta^2\chi_3\;\hat{\mathcal{\rho}}^{n} \;\right) i\boldsymbol{k} + \end{aligned} + :label: disc-maxwell2 + +where we used the short-hand notations +:math:`\mathbf{\tilde{E}}^n \equiv \mathbf{\tilde{E}}(\boldsymbol{k}, n\Delta t)`, +:math:`\mathbf{\tilde{B}}^n \equiv \mathbf{\tilde{B}}(\boldsymbol{k}, n\Delta t)` as well as: -
- -.. raw:: html - -
- -Haber, I, R Lee, Hh Klein, and Jp Boris. 1973. “Advances in Electromagnetic Simulation Techniques.” In *Proc. Sixth Conf. Num. Sim. Plasmas*, 46–48. Berkeley, Ca. - -.. raw:: html - -
- -.. raw:: html - -
- -Kirchen, M., R. Lehe, B. B. Godfrey, I. Dornmair, S. Jalas, K. Peters, J.-L. Vay, and A. R. Maier. 2016. “Stable discrete representation of relativistically drifting plasmas.” *arXiv:1608.00215*. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, Rémi, Manuel Kirchen, Igor A. Andriyash, Brendan B. Godfrey, and Jean-Luc Vay. 2016. “A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm.” *Computer Physics Communications* 203: 66–82. https://doi.org/10.1016/j.cpc.2016.02.007. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, R., M. Kirchen, B. B. Godfrey, A. R. Maier, and J.-L. Vay. 2016. “Elimination of Numerical Cherenkov Instability in flowing-plasma Particle-In-Cell simulations by using Galilean coordinates.” *arXiv:1608.00227*. - -.. raw:: html - -
- -.. raw:: html - -
- -Lifschitz, A F, X Davoine, E Lefebvre, J Faure, C Rechatin, and V Malka. 2009. “Particle-in-Cell modelling of laser-plasma interaction using Fourier decomposition.” *Journal of Computational Physics* 228 (5): 1803–14. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2008.11.017. - -.. raw:: html - -
- -.. raw:: html - -
- -Martins, Samuel F, Ricardo A Fonseca, Luis O Silva, Wei Lu, and Warren B Mori. 2010. “Numerical Simulations of Laser Wakefield Accelerators in Optimal Lorentz Frames.” *Computer Physics Communications* 181 (5): 869–75. https://doi.org/10.1016/J.Cpc.2009.12.023. - -.. raw:: html - -
- -.. raw:: html - -
- -Schroeder, C B, C Benedetti, E Esarey, and W P Leemans. 2011. “Nonlinear Pulse Propagation and Phase Velocity of Laser-Driven Plasma Waves.” *Physical Review Letters* 106 (13): 135002. https://doi.org/10.1103/Physrevlett.106.135002. - -.. raw:: html - -
- -.. raw:: html - -
- -Sironi, L, and A Spitkovsky. 2011. “No Title.” - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean Luc, Irving Haber, and Brendan B. Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243: 260–68. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2007. “Noninvariance of Space- and Time-Scale Ranges Under A Lorentz Transformation and the Implications for the Study of Relativistic Interactions.” *Physical Review Letters* 98 (13): 130405/1–4. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -. L, C G R Geddes, C Benedetti, D L Bruhwiler, E Cormier-Michel, B M Cowan, J R Cary, and D P Grote. 2010. “Modeling Laser Wakefield Accelerators in A Lorentz Boosted Frame.” *Aip Conference Proceedings* 1299: 244–49. https://doi.org/10.1063/1.3520322. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Numerical Methods for Instability Mitigation in the Modeling of Laser Wakefield Accelerators in A Lorentz-Boosted Frame.” *Journal of Computational Physics* 230 (15): 5908–29. https://doi.org/10.1016/J.Jcp.2011.04.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jl, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Effects of Hyperbolic Rotation in Minkowski Space on the Modeling of Plasma Accelerators in A Lorentz Boosted Frame.” *Physics of Plasmas* 18 (3): 30701. https://doi.org/10.1063/1.3559483. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -L., C G R Geddes, E Esarey, C B Schroeder, W P Leemans, E Cormier-Michel, and D P Grote. 2011. “Modeling of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations.” *Physics of Plasmas* 18 (12). https://doi.org/10.1063/1.3663841. - -.. raw:: html - -
- -.. raw:: html - -
- -Xu, Xinlu, Peicheng Yu, Samual F Martins, Frank S Tsung, Viktor K Decyk, Jorge Vieira, Ricardo A Fonseca, Wei Lu, Luis O Silva, and Warren B Mori. 2013. “Numerical instability due to relativistic plasma drift in EM-PIC simulations.” *Computer Physics Communications* 184 (11): 2503–14. https://doi.org/http://dx.doi.org/10.1016/j.cpc.2013.07.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Yu, Peicheng, Xinlu Xu, Viktor K. Decyk, Frederico Fiuza, Jorge Vieira, Frank S. Tsung, Ricardo A. Fonseca, Wei Lu, Luis O. Silva, and Warren B. Mori. 2015. “Elimination of the numerical Cerenkov instability for spectral EM-PIC codes.” *Computer Physics Communications* 192 (July). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 32–47. https://doi.org/10.1016/j.cpc.2015.02.018. - -.. raw:: html - -
- -.. raw:: html +.. math:: + C = \cos(ck\Delta t), \quad S = \sin(ck\Delta t), \quad k = |\boldsymbol{k}|, + :label: def-C-S -
+.. math:: + \nu = \frac{\boldsymbol{k}\cdot\boldsymbol{v}_{gal}}{ck}, \quad \theta = e^{i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2}, + :label: def-nu-theta -Yu, Peicheng, Xinlu Xu, Adam Tableman, Viktor K. Decyk, Frank S. Tsung, Frederico Fiuza, Asher Davidson, et al. 2015. “Mitigation of numerical Cerenkov radiation and instability using a hybrid finite difference-FFT Maxwell solver and a local charge conserving current deposit.” *Computer Physics Communications* 197 (December). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 144–52. https://doi.org/10.1016/j.cpc.2015.08.026. +.. math:: + \chi_1 = \frac{1}{1 -\nu^2} \left( \theta^* - C \theta + i \nu \theta S \right), + :label: def-chi1 -.. raw:: html +.. math:: + \chi_2 = \frac{\chi_1 - \theta(1-C)}{\theta^*-\theta} + :label: def-chi2 -
+.. math:: + \chi_3 = \frac{\chi_1-\theta^*(1-C)}{\theta^*-\theta} + :label: def-chi3 -.. raw:: html +Note that, in the limit :math:`\boldsymbol{v}_{gal}=\boldsymbol{0}`, +Eqs. (:eq:`disc-maxwell1`) and (:eq:`disc-maxwell2`) reduce to the standard PSATD +equations :cite:p:`bf-Habericnsp73`, as expected. +As shown in :cite:t:`bf-KirchenPOP2016,bf-LehePRE2016`, +the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. -
+.. bibliography:: + :keyprefix: bf- diff --git a/Docs/source/theory/boundary_conditions.rst b/Docs/source/theory/boundary_conditions.rst new file mode 100644 index 00000000000..395b072ccbe --- /dev/null +++ b/Docs/source/theory/boundary_conditions.rst @@ -0,0 +1,303 @@ +.. _theory-bc: + +Boundary conditions +=================== + +.. _theory-bc-PML: + +Perfectly Matched Layer: open boundary condition for electromagnetic waves +-------------------------------------------------------------------------- + +For the transverse electric (TE) case, the original Berenger’s Perfectly Matched Layer (PML) paper :cite:p:`bc-Berengerjcp94` writes + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{\partial H_{z}}{\partial y} + :label: PML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{\partial H_{z}}{\partial x} + :label: PML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{\partial E_{y}}{\partial x} + :label: PML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{\partial E_{x}}{\partial y} + :label: PML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: PML_def_5 + +This can be generalized to + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z} + :label: APML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z} + :label: APML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y} + :label: APML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x} + :label: APML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: APML_def_5 + +For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, +this system reduces to the Berenger PML medium, while adding the additional +constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` +leads to the system of Maxwell equations in vacuum. + +.. _theory-bc-propa-plane-wave: + +Propagation of a Plane Wave in an APML Medium +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) +and pulsation :math:`\omega` propagating in the APML medium with an +angle :math:`\varphi` relative to the x axis + +.. math:: + E_{x} = -E_{0}\sin \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_1 + +.. math:: + E_{y} = E_{0}\cos \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_2 + +.. math:: + H_{zx} = H_{zx0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_AMPL_def_3 + +.. math:: + H_{zy} = H_{zy0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_4 + +where :math:`\alpha` and :math:`\beta` are two complex constants to +be determined. + +Introducing Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_def_2`), +(:eq:`Plane_wave_AMPL_def_3`) and (:eq:`Plane_wave_APML_def_4`) +into Eqs. (:eq:`APML_def_1`), (:eq:`APML_def_2`), (:eq:`APML_def_3`) +and (:eq:`APML_def_4`) gives + +.. math:: + \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_1 + +.. math:: + \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_2 + +.. math:: + \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi + :label: Plane_wave_APML_1_3 + +.. math:: + \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi + :label: Plane_wave_APML_1_4 + +Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using Eqs. (:eq:`Plane_wave_APML_1_1`) +and (:eq:`Plane_wave_APML_1_2`), we get + +.. math:: + \beta = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}} + :label: Plane_wave_APML_beta_of_g + +.. math:: + \alpha = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}} + :label: Plane_wave_APML_alpha_of_g + +Adding :math:`H_{zx0}` and :math:`H_{zy0}` from Eqs. (:eq:`Plane_wave_APML_1_3`) +and (:eq:`Plane_wave_APML_1_4`) and substituting the expressions +for :math:`\alpha` and :math:`\beta` from Eqs. (:eq:`Plane_wave_APML_beta_of_g`) +and (:eq:`Plane_wave_APML_alpha_of_g`) yields + +.. math:: + + \begin{aligned} + \frac{1}{Z} & = \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber + \\ + & + \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi + \end{aligned} + +If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then + +.. math:: + Z = \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}} + :label: APML_impedance + +which is the impedance of vacuum. Hence, like the PML, given some +restrictions on the parameters, the APML does not generate any reflection +at any angle and any frequency. As for the PML, this property is not +retained after discretization, as shown subsequently. + +Calling :math:`\psi` any component of the field and :math:`\psi _{0}` +its magnitude, we get from Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_beta_of_g`), +(:eq:`Plane_wave_APML_alpha_of_g`) and (:eq:`APML_impedance`) that + +.. math:: + \psi =\psi _{0} \: e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x} e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y}. + :label: Plane_wave_absorption + +We assume that we have an APML layer of thickness :math:`\delta` (measured +along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` +and :math:`c_{y}=c.` Using (:eq:`Plane_wave_absorption`), we determine +that the coefficient of reflection given by this layer is + +.. math:: + + \begin{aligned} + R_{\mathrm{APML}}\left( \theta \right) & = e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta },\nonumber + \\ + & = e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }, + \end{aligned} + +which happens to be the same as the PML theoretical coefficient of +reflection if we assume :math:`c_{x}=c`. Hence, it follows that for +the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` +seems to be of no interest. However, although this conclusion is true +at the infinitesimal limit, it does not hold for the discretized counterpart. + +Discretization +~~~~~~~~~~~~~~ + +In the following we set :math:`\varepsilon_0 = \mu_0 = 1`. We discretize Eqs. (:eq:`PML_def_1`), (:eq:`PML_def_2`), (:eq:`PML_def_3`), and (:eq:`PML_def_4`) to obtain + +.. math:: + \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} + +.. math:: + \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} + +and this can be solved to obtain the following leapfrog integration equations + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we account for higher order :math:`\Delta t` terms, a better approximation is given by + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +More generally, this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we set + +.. math:: + + \begin{aligned} + c_x & = c \: e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta t}{1-e^{-\sigma_x\Delta t}} \\ + c_y & = c \: e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta t}{1-e^{-\sigma_y\Delta t}} \\ + c^*_x & = c \: e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta t}{1-e^{-\sigma^*_x\Delta t}} \\ + c^*_y & = c \: e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta t}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} + +then this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] + \end{aligned} + +When the generalized conductivities are zero, the update equations are + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +as expected. + +.. _theory-bc-pec: + +Perfect Electrical Conductor +---------------------------- + +This boundary can be used to model a dielectric or metallic surface. +For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic +field are set to 0. In the guard-cell region, the tangential electric field is set equal and +opposite to the respective field component in the mirror location across the PEC +boundary, and the normal electric field is set equal to the field component in the +mirror location in the domain across the PEC boundary. Similarly, the tangential +(and normal) magnetic field components are set equal (and opposite) to the respective +magnetic field components in the mirror locations across the PEC boundary. + +The PEC boundary condition also impacts the deposition of charge and current density. +On the boundary the charge density and parallel current density is set to zero. If +a reflecting boundary condition is used for the particles, density overlapping +with the PEC will be reflected back into the domain (for both charge and current +density). If absorbing boundaries are used, an image charge (equal weight but +opposite charge) is considered in the mirror location accross the boundary, and +the density from that charge is also deposited in the simulation domain. :numref:`fig_PEC_boundary_deposition` +shows the effect of this. The left boundary is absorbing while +the right boundary is reflecting. + +.. _fig_PEC_boundary_deposition: + +.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png + :alt: Plot of PEC boundary current deposition showing current vs position along the ``x``-axis. + :width: 100% + + PEC boundary current deposition along the ``x``-axis. The left boundary is absorbing while the right boundary is reflecting. + +.. bibliography:: + :keyprefix: bc- diff --git a/Docs/source/theory/cold_fluid_model.rst b/Docs/source/theory/cold_fluid_model.rst new file mode 100644 index 00000000000..813a568c540 --- /dev/null +++ b/Docs/source/theory/cold_fluid_model.rst @@ -0,0 +1,112 @@ +.. _theory-cold-fluid-model: + +Cold Relativistic Fluid Model +============================= + +An alternate to the representation of the plasma as macroparticles, is the cold relativistic fluid model. +The cold relativistic fluid model is typically faster to compute than +particles and useful to replace particles when kinetic effects are negligible. This +can be done for certain parts of the plasma, such as the background plasma, while still +representing particle beams as a group of macroparticles. The two models then couple through +Maxwell's equations. + +In the cold limit (zero internal temperature and pressure) of a relativistic plasma, the Maxwell-Fluid +equations govern the plasma evolution. The fluid equations per species, ``s``, are given by, + +.. math:: + + \frac{\partial N_s}{\partial t} + \nabla \cdot (N_s\mathbf{V}_s) &= 0 \\ + \frac{\partial (N\mathbf{U})_s}{\partial t} + \nabla \cdot ((N\mathbf{U})_s\mathbf{V}_s) &= \frac{q_sN_s}{m_s}(\mathbf{E}_s + \mathbf{V}_s \times \mathbf{B}_s). + +Where the fields are updated via Maxwell's equations, + +.. math:: + + \nabla \cdot \mathbf{E} &= \frac{\rho}{\varepsilon_0} \\ + \nabla \cdot \mathbf{B} &= 0 \\ + \nabla \times \mathbf{E} &= -\frac{\partial \mathbf{B}}{\partial t} \\ + \nabla \times \mathbf{B} &= \mu_0 \mathbf{J} + \mu_0 \varepsilon_0 \frac{\partial \mathbf{E}}{\partial t}. + +The fluids are coupled to the fields through, + +.. math:: + + \rho &= \rho_{ptcl}+\sum_s q_sN_s \\ + \mathbf{J} &= \mathbf{J}_{ptcl}+\sum_s q_sN_s\mathbf{V}_s \\ + \mathbf{V}_s &= \frac{ \mathbf{U}_s }{ \sqrt{ 1 + \mathbf{U}_s^2/c^2} } \\ + (N\mathbf{U})_s &= N_s\mathbf{U}_s + +where the particle quantities are calculated by the PIC algorithm. + + +Implementation details +---------------------- + +.. _fig_fluid_loop: + +.. figure:: https://github.com/ECP-WarpX/WarpX/assets/69021085/dcbcc0e4-7899-43e4-b580-f57eb359b457 + :alt: Figure showing fluid Loop embedded within the overall PIC loop. + + Fluid Loop embedded within the overall PIC loop. + +The fluid timeloop is embedded inside the standard PIC timeloop and consists of +the following steps: 1. Higuera and Cary push of the momentum 2. Non-inertial (momentum source) +terms (only in cylindrical geometry) 3. boundary conditions and MPI Communications 4. MUSCL +scheme for advection terms 5. Current and Charge Deposition. :numref:`fig_fluid_loop` gives +a visual representation of these steps, and we describe each of these in more detail. + +Step 0: **Preparation** + Before the fluid loop begins, it is assumed that the program is in the state where fields :math:`\mathbf{E}` + and :math:`\mathbf{B}` are available integer timestep. The + fluids themselves are represented by arrays of fluid quantities (density and + momentum density, :math:`\mathbf{Q} \equiv \{ N, NU_x, NU_y, NU_z \}`) known + on a nodal grid and at half-integer timestep. + +Step 1: **Higuera and Cary Push** + The time staggering of the fields is used by the momentum source term, which is solved with a + Higuera and Cary push :cite:p:`cfm-HigueraPOP2017`. We do not adopt spatial + grid staggering, all discretized fluid quantities exist on the nodal grid. External fields + can be included at this step. + +Step 2: **Non-inertial Terms** + In RZ, the divergence of the flux terms has additional non-zero elements outside of the + derivatives. These terms are Strang split and are time integrated via equation 2.18 from :cite:t:`cfm-ShuJCP1988`, + which is the SSP-RK3 integrator. + +Step 3: **Boundary Conditions and Communications** + At this point, the code applies boundary conditions (assuming Neumann boundary conditions + for the fluid quantities) and exchanges guard cells between + MPI ranks in preparation of derivative terms in the next step. + +Step 4: **Advective Push** + For the advective term, a MUSCL scheme with a low-diffusion minmod slope + limiting is used. We further simplify the conservative equations in terms of primitive + variables, :math:`\{ N, U_x, U_y, U_z \}`. Which we found to be + more stable than conservative variables for the MUSCL reconstruction. Details of + the scheme can be found in :cite:t:`cfm-VanLeerBookChapter1997`. + +Step 5: **Current and Charge Deposition** + Once this series of steps is complete and the fluids have been evolved by an entire + timestep, the current and charge is deposited onto the grid and added to the total current and charge + densities. + +.. note:: + The algorithm is safe with zero fluid density. + + It also implements a positivity limiter on the density to prevent negative density regions from forming. + + There is currently no ability to perform azimuthal mode decomposition in RZ. + + Mesh refinement is not supported for the fluids. + + The implemented MUSCL scheme has a simplified slope averaging, see the extended writeup for details. + + More details on the precise implementation are available here, `WarpX_Cold_Rel_Fluids.pdf`_. +.. _WarpX_Cold_Rel_Fluids.pdf: https://github.com/ECP-WarpX/WarpX/files/12886437/WarpX_Cold_Rel_Fluids.pdf + +.. warning:: + If using the fluid model with the Kinetic-Fluid Hybrid model or the electrostatic solver, there is a known + issue that the fluids deposit at a half-timestep offset in the charge-density. + +.. bibliography:: + :keyprefix: cfm- diff --git a/Docs/source/theory/collisions.rst b/Docs/source/theory/collisions.rst index 3f24cd8f331..52e36521125 100644 --- a/Docs/source/theory/collisions.rst +++ b/Docs/source/theory/collisions.rst @@ -3,18 +3,36 @@ Collisions ========== -Monte Carlo Collisions ----------------------- - -The Monte Carlo collisions (MCC) module can be found in *Source/Particles/Collisions/BackgroundMCC/*. -Several types of collisions between simulation particles and a neutral background gas are supported including elastic scattering, back scattering, charge exchange, excitation collisions and impact ionization. -An instance of the class :cpp:class:`MCCProcess` is created for each type of collision included in a simulation. This class saves information about the type of collision and the collision cross-section as a function of energy. - -The so-called null collision strategy is used in order to minimize the computational burden of the MCC module. -This strategy is standard in PIC-MCC and a detailed description can be found elsewhere, for example in :cite:t:`b-Birdsall1991`. -In short the maximum collision probability is found over a sensible range of energies and is used to pre-select the appropriate number of macroparticles for collision consideration. Only these pre-selected particles are then individually considered for a collision based on their energy and the cross-sections of all the different collisional processes included. - -The MCC implementation assumes that the background neutral particles are **thermal**, and are moving at non-relativistic velocities in the lab frame. For each simulation particle considered for a collision, a velocity vector for a neutral particle is randomly chosen. The particle velocity is then boosted to the stationary frame of the neutral through a Galilean transformation. The energy of the collision is calculated using the particle utility function, ``ParticleUtils::getCollisionEnergy()``, as +WarpX includes several different models to capture collisional processes +including collisions between kinetic particles (Coulomb collisions, DSMC, +nuclear fusion) as well as collisions between kinetic particles and a fixed +(i.e. non-evolving) background species (MCC, background stopping). + +.. _theory-collisions-mcc: + +Background Monte Carlo Collisions (MCC) +--------------------------------------- + +Several types of collisions between simulation particles and a neutral +background gas are supported including elastic scattering, back scattering, +charge exchange, excitation collisions and impact ionization. + +The so-called null collision strategy is used in order to minimize the +computational burden of the MCC module. This strategy is standard in PIC-MCC and +a detailed description can be found elsewhere, for example in :cite:t:`b-Birdsall1991`. +In short the maximum collision probability is found over a sensible range of +energies and is used to pre-select the appropriate number of macroparticles for +collision consideration. Only these pre-selected particles are then individually +considered for a collision based on their energy and the cross-sections of all +the different collisional processes included. + +The MCC implementation assumes that the background neutral particles are **thermal**, +and are moving at non-relativistic velocities in the lab frame. For each +simulation particle considered for a collision, a velocity vector for a neutral +particle is randomly chosen given the user specified neutral temperature. The +particle velocity is then boosted to the stationary frame of the neutral through +a Galilean transformation. The energy of the collision is calculated using the +particle utility function, ``ParticleUtils::getCollisionEnergy()``, as .. math:: @@ -23,19 +41,55 @@ The MCC implementation assumes that the background neutral particles are **therm &= \frac{2Mmu^2}{M + m + \sqrt{M^2+m^2+2\gamma mM}}\frac{1}{\gamma + 1} \end{aligned} -where :math:`u` is the speed of the particle as tracked in WarpX (i.e. :math:`u = \gamma v` with :math:`v` the particle speed), while :math:`m` and :math:`M` are the rest masses of the simulation and background species, respectively. The Lorentz factor is defined in the usual way, :math:`\gamma = \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above expression clearly reduces to the classical equation :math:`E_{coll} = \frac{1}{2}\frac{Mm}{M+m} u^2`. The collision cross-sections for all scattering processes are evaluated at the energy as calculated above. +where :math:`u` is the speed of the particle as tracked in WarpX (i.e. +:math:`u = \gamma v` with :math:`v` the particle speed), while :math:`m` and +:math:`M` are the rest masses of the simulation and background species, +respectively. The Lorentz factor is defined in the usual way, +:math:`\gamma \def \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above +expression reduces to the classical equation +:math:`E_{coll} = \frac{1}{2}\frac{Mm}{M+m} u^2`. The collision cross-sections +for all scattering processes are evaluated at the energy as calculated above. Once a particle is selected for a specific collision process, that process determines how the particle is scattered as outlined below. +.. _theory-collisions-dsmc: + +Direct Simulation Monte Carlo (DSMC) +------------------------------------ + +The algorithm by which binary collisions are treated is outlined below. The +description assumes collisions between different species. + +1. Particles from both species are sorted by grid-cells. +2. The order of the particles in each cell is shuffled. +3. Within each cell, particles are paired to form collision partners. Particles + of the species with fewer members in a given cell is split in half so that + each particle has exactly one partner of the other species. +4. Each collision pair is considered for a collision using the same logic as in + the MCC description above. +5. Particles that are chosen for collision are scattered according to the + selected collision process. + +Scattering processes +-------------------- + Charge exchange ^^^^^^^^^^^^^^^ -This is the simplest scattering process. Under charge exchange the simulation particle's velocity is simply replaced with the sampled velocity of the neutral particle. Charge exchange usually has a cooling effect on the ions in the simulation by exchanging energetic ions for lower energy neutrals. +This process can occur when an ion and neutral (of the same species) collide +and results in the exchange of an electron. The ion velocity is simply replaced +with the neutral velocity and vice-versa. Elastic scattering ^^^^^^^^^^^^^^^^^^ -This scattering process as well as the ones below that relate to it, are all performed in the center-of-momentum (COM) frame. Designating the COM velocity of the particle as :math:`\vec{u}_c` and its labframe velocity as :math:`\vec{u}_l`, the transformation from lab frame to COM frame is done with a general Lorentz boost (see function ``ParticleUtils::doLorentzTransform()``): +The ``elastic`` option uses isotropic scattering, i.e., with a differential +cross section that is independent of angle. +This scattering process as well as the ones below that relate to it, are all +performed in the center-of-momentum (COM) frame. Designating the COM velocity of +the particle as :math:`\vec{u}_c` and its labframe velocity as :math:`\vec{u}_l`, +the transformation from lab frame to COM frame is done with a general Lorentz +boost (see function ``ParticleUtils::doLorentzTransform()``): .. math:: \begin{bmatrix} @@ -74,6 +128,12 @@ Excitation The process is also the same as for elastic scattering except the excitation energy cost is subtracted from the particle energy. This is done by reducing the velocity before a scattering angle is chosen. +Benchmarks +---------- + +See the :ref:`MCC example ` for a benchmark of the MCC +implementation against literature results. + Particle cooling due to elastic collisions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Docs/source/theory/input_output.rst b/Docs/source/theory/input_output.rst index 75427ad9c6d..21a5f5c8d2c 100644 --- a/Docs/source/theory/input_output.rst +++ b/Docs/source/theory/input_output.rst @@ -3,7 +3,7 @@ Inputs and Outputs ================== -Initialization of the plasma columns and drivers (laser or particle beam) is performed via the specification of multidimensional functions that describe the initial state with, if needed, a time dependence, or from reconstruction of distributions based on experimental data. Care is needed when initializing quantities in parallel to avoid double counting and ensure smoothness of the distributions at the interface of computational domains. When the sum of the initial distributions of charged particles is not charge neutral, initial fields are computed using generally a static approximation with Poisson solves accompanied by proper relativistic scalings (Vay 2008; Cowan et al. 2013). +Initialization of the plasma columns and drivers (laser or particle beam) is performed via the specification of multidimensional functions that describe the initial state with, if needed, a time dependence, or from reconstruction of distributions based on experimental data. Care is needed when initializing quantities in parallel to avoid double counting and ensure smoothness of the distributions at the interface of computational domains. When the sum of the initial distributions of charged particles is not charge neutral, initial fields are computed using generally a static approximation with Poisson solves accompanied by proper relativistic scalings :cite:p:`io-Vaypop2008, io-CowanPRSTAB13`. Outputs include dumps of particle and field quantities at regular intervals, histories of particle distributions moments, spectra, etc, and plots of the various quantities. In parallel simulations, the diagnostic subroutines need to handle additional complexity from the domain decomposition, as well as large amount of data that may necessitate data reduction in some form before saving to disk. @@ -12,16 +12,17 @@ Simulations in a Lorentz boosted frame require additional considerations, as des Inputs and outputs in a boosted frame simulation ------------------------------------------------ -.. _Fig_inputoutput: +.. _fig_inputoutput: + .. figure:: Input_output.png :alt: (top) Snapshot of a particle beam showing “frozen" (grey spheres) and “active" (colored spheres) macroparticles traversing the injection plane (red rectangle). (bottom) Snapshot of the beam macroparticles (colored spheres) passing through the background of electrons (dark brown streamlines) and the diagnostic stations (red rectangles). The electrons, the injection plane and the diagnostic stations are fixed in the laboratory plane, and are thus counter-propagating to the beam in a boosted frame. - :width: 120mm + :width: 100% (top) Snapshot of a particle beam showing “frozen" (grey spheres) and “active" (colored spheres) macroparticles traversing the injection plane (red rectangle). (bottom) Snapshot of the beam macroparticles (colored spheres) passing through the background of electrons (dark brown streamlines) and the diagnostic stations (red rectangles). The electrons, the injection plane and the diagnostic stations are fixed in the laboratory plane, and are thus counter-propagating to the beam in a boosted frame. The input and output data are often known from, or compared to, experimental data. Thus, calculating in a frame other than the laboratory entails transformations of the data between the calculation frame and the laboratory -frame. This section describes the procedures that have been implemented in the Particle-In-Cell framework Warp (Grote et al. 2005) to handle the input and output of data between the frame of calculation and the laboratory frame (Vay et al. 2011). Simultaneity of events between two frames is valid only for a plane that is perpendicular to the relative motion of the frame. As a result, the input/output processes involve the input of data (particles or fields) through a plane, as well as output through a series of planes, all of which are perpendicular to the direction of the relative velocity between the frame of calculation and the other frame of choice. +frame. This section describes the procedures that have been implemented in the Particle-In-Cell framework Warp :cite:p:`io-Warp` to handle the input and output of data between the frame of calculation and the laboratory frame :cite:p:`io-Vaypop2011`. Simultaneity of events between two frames is valid only for a plane that is perpendicular to the relative motion of the frame. As a result, the input/output processes involve the input of data (particles or fields) through a plane, as well as output through a series of planes, all of which are perpendicular to the direction of the relative velocity between the frame of calculation and the other frame of choice. Input in a boosted frame simulation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -29,17 +30,17 @@ Input in a boosted frame simulation Particles - ^^^^^^^^^^^^ -Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse “electron cloud” instability (Vay 2007). In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity :math:`\{x,y,z,v_x,v_y,v_z\}` for each beam macroparticle at time :math:`t=0` for a beam moving at the average velocity :math:`v_b=\beta_b c` (where :math:`c` is the speed of light) in the laboratory, and using the standard synchronization (:math:`z=z'=0` at :math:`t=t'=0`) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity :math:`\beta c` in the laboratory is as follows (the superscript :math:`'` relates to quantities known in the boosted frame while the superscript :math:`^*` relates to quantities that are know at a given longitudinal position :math:`z^*` but different times of arrival): +Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse “electron cloud” instability :cite:p:`io-Vayprl07`. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity :math:`\{x,y,z,v_x,v_y,v_z\}` for each beam macroparticle at time :math:`t=0` for a beam moving at the average velocity :math:`v_b=\beta_b c` (where :math:`c` is the speed of light) in the laboratory, and using the standard synchronization (:math:`z=z'=0` at :math:`t=t'=0`) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity :math:`\beta c` in the laboratory is as follows (the superscript :math:`'` relates to quantities known in the boosted frame while the superscript :math:`^*` relates to quantities that are know at a given longitudinal position :math:`z^*` but different times of arrival): #. project positions at :math:`z^*=0` assuming ballistic propagation .. math:: \begin{aligned} - t^* &=& \left(z-\bar{z}\right)/v_z \label{Eq:t*}\\ - x^* &=& x-v_x t^* \label{Eq:x*}\\ - y^* &=& y-v_y t^* \label{Eq:y*}\\ - z^* &=& 0 \label{Eq:z*}\end{aligned} + t^* &= \left(z-\bar{z}\right)/v_z \label{Eq:t*}\\ + x^* &= x-v_x t^* \label{Eq:x*}\\ + y^* &= y-v_y t^* \label{Eq:y*}\\ + z^* &= 0 \label{Eq:z*}\end{aligned} the velocity components being left unchanged, @@ -48,13 +49,13 @@ Particles are launched through a plane using a technique that is generic and app .. math:: \begin{aligned} - t'^* &=& -\gamma t^* \label{Eq:tp*}\\ - x'^* &=& x^* \label{Eq:xp*}\\ - y'^* &=& y^* \label{Eq:yp*}\\ - z'^* &=& \gamma\beta c t^* \label{Eq:zp*}\\ - v'^*_x&=&\frac{v_x^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vxp*}\\ - v'^*_y&=&\frac{v_y^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vyp*}\\ - v'^*_z&=&\frac{v_z^*-\beta c}{1-\beta \beta_b} \label{Eq:vzp*}\end{aligned} + t'^* &= -\gamma t^* \label{Eq:tp*}\\ + x'^* &= x^* \label{Eq:xp*}\\ + y'^* &= y^* \label{Eq:yp*}\\ + z'^* &= \gamma\beta c t^* \label{Eq:zp*}\\ + v'^*_x&=\frac{v_x^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vxp*}\\ + v'^*_y&=\frac{v_y^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vyp*}\\ + v'^*_z&=\frac{v_z^*-\beta c}{1-\beta \beta_b} \label{Eq:vzp*}\end{aligned} where :math:`\gamma=1/\sqrt{1-\beta^2}`. With the knowledge of the time at which each beam macroparticle crosses the plane into consideration, one can inject each beam macroparticle in the simulation at the appropriate location and time. @@ -63,11 +64,11 @@ Particles are launched through a plane using a technique that is generic and app .. math:: \begin{aligned} - z' &=& z'^*-\bar{v}'^*_z t'^* \label{Eq:zp}\end{aligned} + z' &= z'^*-\bar{v}'^*_z t'^* \label{Eq:zp}\end{aligned} - This additional step is needed for setting the electrostatic or electromagnetic fields at the plane of injection. In a Particle-In-Cell code, the three-dimensional fields are calculated by solving the Maxwell equations (or static approximation like Poisson, Darwin or other (Vay 2008)) on a grid on which the source term is obtained from the macroparticles distribution. This requires generation of a three-dimensional representation of the beam distribution of macroparticles at a given time before they cross the injection plane at :math:`z'^*`. This is accomplished by expanding the beam distribution longitudinally such that all macroparticles (so far known at different times of arrival at the injection plane) are synchronized to the same time in the boosted frame. To keep the beam shape constant, the particles are “frozen” until they cross that plane: the three velocity components and the two position components perpendicular to the boosted frame velocity are kept constant, while the remaining position component is advanced at the average beam velocity. As particles cross the plane of injection, they become regular “active” particles with full 6-D dynamics. + This additional step is needed for setting the electrostatic or electromagnetic fields at the plane of injection. In a Particle-In-Cell code, the three-dimensional fields are calculated by solving the Maxwell equations (or static approximation like Poisson, Darwin or other :cite:p:`io-Vaypop2008`) on a grid on which the source term is obtained from the macroparticles distribution. This requires generation of a three-dimensional representation of the beam distribution of macroparticles at a given time before they cross the injection plane at :math:`z'^*`. This is accomplished by expanding the beam distribution longitudinally such that all macroparticles (so far known at different times of arrival at the injection plane) are synchronized to the same time in the boosted frame. To keep the beam shape constant, the particles are “frozen” until they cross that plane: the three velocity components and the two position components perpendicular to the boosted frame velocity are kept constant, while the remaining position component is advanced at the average beam velocity. As particles cross the plane of injection, they become regular “active” particles with full 6-D dynamics. -Figure :numref:`Fig_inputoutput` (top) shows a snapshot of a beam that has passed partly through the injection plane. As the frozen beam macroparticles pass through the injection plane (which moves opposite to the beam in the boosted frame), they are converted to “active" macroparticles. The charge or current density is accumulated from the active and the frozen particles, thus ensuring that the fields at the plane of injection are consistent. +A snapshot of a beam that has passed partly through the injection plane in shown in :numref:`fig_inputoutput` (top). As the frozen beam macroparticles pass through the injection plane (which moves opposite to the beam in the boosted frame), they are converted to “active" macroparticles. The charge or current density is accumulated from the active and the frozen particles, thus ensuring that the fields at the plane of injection are consistent. Laser - ^^^^^^^^ @@ -89,93 +90,28 @@ If, for convenience, the injection plane is moving at constant velocity :math:`\ .. math:: \begin{aligned} - E_\perp\left(x,y,t\right)&=&\left(1-\beta_s\right)E_0 f\left(x,y,t\right)\nonumber \\ - &\times& \sin\left[\left(1-\beta_s\right)\omega t+\phi\left(x,y,\omega\right)\right].\end{aligned} + E_\perp\left(x,y,t\right)&=\left(1-\beta_s\right)E_0 f\left(x,y,t\right)\sin\left[\left(1-\beta_s\right)\omega t+\phi\left(x,y,\omega\right)\right] + \end{aligned} The injection of a laser of frequency :math:`\omega` is considered for a simulation using a boosted frame moving at :math:`\beta c` with respect to the laboratory. Assuming that the laser is injected at a plane that is fixed in the laboratory, and thus moving at :math:`\beta_s=-\beta` in the boosted frame, the injection in the boosted frame is given by .. math:: \begin{aligned} - E_\perp\left(x',y',t'\right)&=&\left(1-\beta_s\right)E'_0 f\left(x',y',t'\right)\nonumber \\ - &\times&\sin\left[\left(1-\beta_s\right)\omega' t'+\phi\left(x',y',\omega'\right)\right]\\ - &=&\left(E_0/\gamma\right) f\left(x',y',t'\right) \nonumber\\ - &\times&\sin\left[\omega t'/\gamma+\phi\left(x',y',\omega'\right)\right]\end{aligned} + E_\perp\left(x',y',t'\right)&=\left(1-\beta_s\right)E'_0 f\left(x',y',t'\right)\sin\left[\left(1-\beta_s\right)\omega' t'+\phi\left(x',y',\omega'\right)\right] + \\ + &=\left(E_0/\gamma\right) f\left(x',y',t'\right)\sin\left[\omega t'/\gamma+\phi\left(x',y',\omega'\right)\right] + \end{aligned} since :math:`E'_0/E_0=\omega'/\omega=1/\left(1+\beta\right)\gamma`. -The electric field is then converted into currents that get injected via a 2D array of macro-particles, with one positive and one dual negative macro-particle for each array cell in the plane of injection, whose weights and motion are governed by :math:`E_\perp\left(x',y',t'\right)`. Injecting using this dual array of macroparticles offers the advantage of automatically including the longitudinal component that arises from emitting into a boosted frame, and to automatically verify the discrete Gauss’ law thanks to using charge conserving (e.g. Esirkepov) current deposition scheme (Esirkepov 2001). +The electric field is then converted into currents that get injected via a 2D array of macro-particles, with one positive and one dual negative macro-particle for each array cell in the plane of injection, whose weights and motion are governed by :math:`E_\perp\left(x',y',t'\right)`. Injecting using this dual array of macroparticles offers the advantage of automatically including the longitudinal component that arises from emitting into a boosted frame, and to automatically verify the discrete Gauss’ law thanks to using charge conserving (e.g. Esirkepov) current deposition scheme :cite:p:`io-Esirkepovcpc01`. Output in a boosted frame simulation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some quantities, e.g. charge or dimensions perpendicular to the boost velocity, are Lorentz invariant. -Those quantities are thus readily available from standard diagnostics in the boosted frame calculations. Quantities that do not fall in this category are recorded at a number of regularly spaced “stations", immobile in the laboratory frame, at a succession of time intervals to record data history, or averaged over time. A visual example is given on Fig. :numref:`Fig_inputoutput` (bottom). Since the space-time locations of the diagnostic grids in the laboratory frame generally do not coincide with the space-time positions of the macroparticles and grid nodes used for the calculation in a boosted frame, some interpolation is performed at runtime during the data collection process. As a complement or an alternative, selected particle or field quantities can be dumped at regular intervals and quantities are reconstructed in the laboratory frame during a post-processing phase. The choice of the methods depends on the requirements of the diagnostics and particular implementations. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, Benjamin M, David L Bruhwiler, John R Cary, Estelle Cormier-Michel, and Cameron G R Geddes. 2013. “Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations.” *Physical Review Special Topics-Accelerators and Beams* 16 (4). https://doi.org/10.1103/PhysRevSTAB.16.041303. - -.. raw:: html - -
- -.. raw:: html - -
- -Esirkepov, Tz. 2001. “Exact Charge Conservation Scheme for Particle-in-Cell Simulation with an Arbitrary Form-Factor.” *Computer Physics Communications* 135 (2): 144–53. - -.. raw:: html - -
- -.. raw:: html - -
- -Grote, D P, A Friedman, J.-L. Vay, and I Haber. 2005. “The Warp Code: Modeling High Intensity Ion Beams.” In *Aip Conference Proceedings*, 55–58. 749. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L. 2008. “Simulation of Beams or Plasmas Crossing at Relativistic Velocity.” *Physics of Plasmas* 15 (5): 56701. https://doi.org/10.1063/1.2837054. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2007. “Noninvariance of Space- and Time-Scale Ranges Under A Lorentz Transformation and the Implications for the Study of Relativistic Interactions.” *Physical Review Letters* 98 (13): 130405/1–4. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -L., C G R Geddes, E Esarey, C B Schroeder, W P Leemans, E Cormier-Michel, and D P Grote. 2011. “Modeling of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations.” *Physics of Plasmas* 18 (12). https://doi.org/10.1063/1.3663841. - -.. raw:: html - -
- -.. raw:: html +Those quantities are thus readily available from standard diagnostics in the boosted frame calculations. Quantities that do not fall in this category are recorded at a number of regularly spaced “stations", immobile in the laboratory frame, at a succession of time intervals to record data history, or averaged over time. A visual example is given on :numref:`fig_inputoutput` (bottom). Since the space-time locations of the diagnostic grids in the laboratory frame generally do not coincide with the space-time positions of the macroparticles and grid nodes used for the calculation in a boosted frame, some interpolation is performed at runtime during the data collection process. As a complement or an alternative, selected particle or field quantities can be dumped at regular intervals and quantities are reconstructed in the laboratory frame during a post-processing phase. The choice of the methods depends on the requirements of the diagnostics and particular implementations. -
+.. bibliography:: + :keyprefix: io- diff --git a/Docs/source/theory/intro.rst b/Docs/source/theory/intro.rst index cbf56c919df..10fbba679c2 100644 --- a/Docs/source/theory/intro.rst +++ b/Docs/source/theory/intro.rst @@ -8,16 +8,17 @@ Introduction Plasma laser-driven (top) and charged-particles-driven (bottom) acceleration (rendering from 3-D Particle-In-Cell simulations). A laser beam (red and blue disks in top picture) or a charged particle beam (red dots in bottom picture) propagating (from left to right) through an under-dense plasma (not represented) displaces electrons, creating a plasma wakefield that supports very high electric fields (pale blue and yellow). These electric fields, which can be orders of magnitude larger than with conventional techniques, can be used to accelerate a short charged particle beam (white) to high-energy over a very short distance. -Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`Tsung2006,Geddes2008,Geddes2009,Geddes2010,Huang2009`, with -accurate modeling of wake formation, electron self-trapping and acceleration requiring fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`Leemans2014,Blumenfeld2007,Bulanov2014,Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. +Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`i-Tsungpop06,i-Geddesjp08,i-Geddesscidac09,i-Geddespac09,i-Huangscidac09`, with +accurate modeling of wake formation, electron self-trapping and acceleration requiring fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`i-LeemansPRL2014,i-Blumenfeld2007,i-BulanovSV2014,i-Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. For such simulations, the most popular algorithm is the Particle-In-Cell (or PIC) technique, which represents electromagnetic fields on a grid and particles by a sample of macroparticles. However, these simulations are extremely computationally intensive, due to the need to resolve the evolution of a driver (laser or particle beam) and an accelerated beam into a structure that is orders of magnitude longer and wider than the accelerated beam. -Various techniques or reduced models have been developed to allow multidimensional simulations at manageable computational costs: quasistatic approximation :cite:p:`Sprangle1990,Antonsen1992,Krall1993,Mora1997,Huang2006`, -ponderomotive guiding center (PGC) models :cite:p:`Antonsen1992,Krall1993,Huang2006,Benedetti2010,Cowan2011`, simulation in an optimal Lorentz boosted frame :cite:p:`Vay2007,Bruhwiler2009,Vay2009a,Vay2009b,Vay2010,Martins2010a,Martins2010b,Martins2010c,Vay2011a,Vay2011b,Vay2011c,Yu2016`, -expanding the fields into a truncated series of azimuthal modes :cite:p:`Godfrey1985,Lifschitz2009,Davidson2015,Lehe2016,Andriyash2016`, fluid approximation :cite:p:`Krall1993,Shadwick2009,Benedetti2010` and scaled parameters :cite:p:`CormierMichel2009`. +Various techniques or reduced models have been developed to allow multidimensional simulations at manageable computational costs: quasistatic approximation :cite:p:`i-Sprangleprl90,i-Antonsenprl1992,i-Krallpre1993,i-Morapop1997,i-Quickpic`, +ponderomotive guiding center (PGC) models :cite:p:`i-Antonsenprl1992,i-Krallpre1993,i-Quickpic,i-Benedettiaac2010,i-Cowanjcp11`, simulation in an optimal Lorentz boosted frame :cite:p:`i-Vayprl07,i-Bruhwileraac08,i-Vayscidac09,i-Vaypac09,i-Martinspac09,i-VayAAC2010,i-Martinsnaturephysics10,i-Martinspop10,i-Martinscpc10,i-Vayjcp2011,i-VayPOPL2011,i-Vaypop2011,i-Yu2016`, +expanding the fields into a truncated series of azimuthal modes :cite:p:`i-godfrey1985iprop,i-LifschitzJCP2009,i-DavidsonJCP2015,i-Lehe2016,i-AndriyashPoP2016`, fluid approximation :cite:p:`i-Krallpre1993,i-Shadwickpop09,i-Benedettiaac2010` and scaled parameters :cite:p:`i-Cormieraac08,i-Geddespac09`. .. bibliography:: + :keyprefix: i- diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 37e4955d665..854c4d5ada1 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -18,8 +18,8 @@ has to resolve the electron Debye length and CFL-condition based on the speed of light. Many authors have described variations of the kinetic ion & fluid electron model, -generally referred to as particle-fluid hybrid or just hybrid-PIC models. The implementation -in WarpX follows the outline from :cite:t:`c-winske2022hybrid`. +generally referred to as particle-fluid hybrid or just hybrid-PIC models. The +implementation in WarpX is described in detail in :cite:t:`kfhm-Groenewald2023`. This description follows mostly from that reference. Model @@ -29,31 +29,57 @@ The basic justification for the hybrid model is that the system to which it is applied is dominated by ion kinetics, with ions moving much slower than electrons and photons. In this scenario two critical approximations can be made, namely, neutrality (:math:`n_e=n_i`) and the Maxwell-Ampere equation can be simplified by -neglecting the displacement current term :cite:p:`c-NIELSON1976`, giving, +neglecting the displacement current term :cite:p:`kfhm-Nielson1976`, giving, .. math:: \mu_0\vec{J} = \vec{\nabla}\times\vec{B}, -where :math:`\vec{J} = \vec{J}_i - \vec{J}_e` is the total electrical current, i.e. -the sum of electron and ion currents. Since ions are treated in the regular -PIC manner, the ion current, :math:`\vec{J}_i`, is known during a simulation. Therefore, +where :math:`\vec{J} = \sum_{s\neq e}\vec{J}_s + \vec{J}_e + \vec{J}_{ext}` is the total electrical current, +i.e. the sum of electron and ion currents as well as any external current (not captured through plasma +particles). Since ions are treated in the regular +PIC manner, the ion current, :math:`\sum_{s\neq e}\vec{J}_s`, is known during a simulation. Therefore, given the magnetic field, the electron current can be calculated. -If we now further assume electrons are inertialess, the electron momentum -equation yields, +The electron momentum transport equation (obtained from multiplying the Vlasov equation by mass and +integrating over velocity), also called the generalized Ohm's law, is given by: .. math:: - \frac{d(n_em_e\vec{V}_e)}{dt} = 0 = -en_e\vec{E}-\vec{J}_e\times\vec{B}-\nabla\cdot\vec{P}_e+en_e\vec{\eta}\cdot\vec{J}, + en_e\vec{E} = \frac{m}{e}\frac{\partial \vec{J}_e}{\partial t} + \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e -where :math:`\vec{V_e}=\vec{J}_e/(en_e)`, :math:`\vec{P}_e` is the electron pressure -tensor and :math:`\vec{\eta}` is the resistivity tensor. An expression for the electric field -(generalized Ohm's law) can be obtained from the above as: +where :math:`\vec{U}_e = \vec{J}_e/(en_e)` is the electron fluid velocity, +:math:`{\overleftrightarrow P}_e` is the electron pressure tensor and +:math:`\vec{R}_e` is the drag force due to collisions between electrons and ions. +Applying the above momentum equation to the Maxwell-Faraday equation (:math:`\frac{\partial\vec{B}}{\partial t} = -\nabla\times\vec{E}`) +and substituting in :math:`\vec{J}` calculated from the Maxwell-Ampere equation, gives, .. math:: - \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla\cdot\vec{P}_e \right)+\vec{\eta}\cdot\vec{J}. + \frac{\partial\vec{J}_e}{\partial t} = -\frac{1}{\mu_0}\nabla\times\left(\nabla\times\vec{E}\right) - \frac{\partial\vec{J}_{ext}}{\partial t} - \sum_{s\neq e}\frac{\partial\vec{J}_s}{\partial t}. + +Plugging this back into the generalized Ohm' law gives: + + .. math:: + + \left(en_e +\frac{m}{e\mu_0}\nabla\times\nabla\times\right)\vec{E} =& + - \frac{m}{e}\left( \frac{\partial\vec{J}_{ext}}{\partial t} + \sum_{s\neq e}\frac{\partial\vec{J}_s}{\partial t} \right) \\ + &+ \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e. + +If we now further assume electrons are inertialess (i.e. :math:`m=0`), the above equation simplifies to, + + .. math:: + + en_e\vec{E} = -\vec{J}_e\times\vec{B}-\nabla\cdot{\overleftrightarrow P}_e+\vec{R}_e. + +Making the further simplifying assumptions that the electron pressure is isotropic and that +the electron drag term can be written as a simple resistance +i.e. :math:`\vec{R}_e = en_e\vec{\eta}\cdot\vec{J}`, brings us to the implemented form of +Ohm's law: + + .. math:: + + \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla P_e \right)+\vec{\eta}\cdot\vec{J}. Lastly, if an electron temperature is given from which the electron pressure can be calculated, the model is fully constrained and can be evolved given initial @@ -108,7 +134,7 @@ Extrapolation step Obtaining the E-field at timestep :math:`t=t_{n+1}` is a well documented issue for the hybrid model. Currently the approach in WarpX is to simply extrapolate -:math:`\vec{J}_i` foward in time, using +:math:`\vec{J}_i` forward in time, using .. math:: @@ -142,4 +168,4 @@ The isothermal limit is given by :math:`\gamma = 1` while :math:`\gamma = 5/3` (default) produces the adiabatic limit. .. bibliography:: - :keyprefix: c- + :keyprefix: kfhm- diff --git a/Docs/source/theory/pic.rst b/Docs/source/theory/pic.rst new file mode 100644 index 00000000000..820cdba50e6 --- /dev/null +++ b/Docs/source/theory/pic.rst @@ -0,0 +1,695 @@ +.. _theory-pic: + +Particle-in-Cell Method +======================= + +.. _fig-pic: + +.. figure:: PIC.png + :alt: [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). + + The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). + +In the *electromagnetic particle-in-cell method* :cite:p:`pt-Birdsalllangdon,pt-HockneyEastwoodBook`, +the electromagnetic fields are solved on a grid, usually using Maxwell’s +equations + +.. math:: + \frac{\mathbf{\partial B}}{\partial t} = -\nabla\times\mathbf{E} + :label: Faraday-1 + +.. math:: + \frac{\mathbf{\partial E}}{\partial t} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere-1 + +.. math:: + \nabla\cdot\mathbf{E} = \rho + :label: Gauss-1 + +.. math:: + \nabla\cdot\mathbf{B} = 0 + :label: divb-1 + +given here in natural units (:math:`\epsilon_0=\mu_0=c=1`), where :math:`t` is time, :math:`\mathbf{E}` and +:math:`\mathbf{B}` are the electric and magnetic field components, and +:math:`\rho` and :math:`\mathbf{J}` are the charge and current densities. The +charged particles are advanced in time using the Newton-Lorentz equations +of motion + +.. math:: + \frac{d\mathbf{x}}{dt} = \mathbf{v}, + :label: Lorentz_x-1 + +.. math:: + \frac{d\left(\gamma\mathbf{v}\right)}{dt} = \frac{q}{m}\left(\mathbf{E}+\mathbf{v}\times\mathbf{B}\right), + :label: Lorentz_v-1 + +where :math:`m`, :math:`q`, :math:`\mathbf{x}`, :math:`\mathbf{v}` and :math:`\gamma=1/\sqrt{1-v^{2}}` +are respectively the mass, charge, position, velocity and relativistic +factor of the particle given in natural units (:math:`c=1`). The charge and current densities are interpolated +on the grid from the particles’ positions and velocities, while the +electric and magnetic field components are interpolated from the grid +to the particles’ positions for the velocity update. + +.. _theory-pic-push: + +Particle push +------------- + +A centered finite-difference discretization of the Newton-Lorentz +equations of motion is given by + +.. math:: + \frac{\mathbf{x}^{i+1}-\mathbf{x}^{i}}{\Delta t} = \mathbf{v}^{i+1/2}, + :label: leapfrog_x + +.. math:: + \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}-\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{\Delta t} = \frac{q}{m}\left(\mathbf{E}^{i}+\mathbf{\bar{v}}^{i}\times\mathbf{B}^{i}\right). + :label: leapfrog_v + +In order to close the system, :math:`\bar{\mathbf{v}}^{i}` must be +expressed as a function of the other quantities. The two implementations that have become the most popular are presented below. + +.. _theory-pic-push-boris: + +Boris relativistic velocity rotation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The solution proposed by Boris :cite:p:`pt-BorisICNSP70` is given by + +.. math:: + \mathbf{\bar{v}}^{i} = \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}+\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{2\bar{\gamma}^{i}} + :label: boris_v + +where :math:`\bar{\gamma}^{i}` is defined by :math:`\bar{\gamma}^{i} \equiv (\gamma^{i+1/2}+\gamma^{i-1/2} )/2`. + +The system (:eq:`leapfrog_v`, :eq:`boris_v`) is solved very +efficiently following Boris’ method, where the electric field push +is decoupled from the magnetic push. Setting :math:`\mathbf{u}=\gamma\mathbf{v}`, the +velocity is updated using the following sequence: + +.. math:: + + \begin{aligned} + \mathbf{u^{-}} & = \mathbf{u}^{i-1/2}+\left(q\Delta t/2m\right)\mathbf{E}^{i} + \\ + \mathbf{u'} & = \mathbf{u}^{-}+\mathbf{u}^{-}\times\mathbf{t} + \\ + \mathbf{u}^{+} & = \mathbf{u}^{-}+\mathbf{u'}\times2\mathbf{t}/(1+\mathbf{t}^{2}) + \\ + \mathbf{u}^{i+1/2} & = \mathbf{u}^{+}+\left(q\Delta t/2m\right)\mathbf{E}^{i} + \end{aligned} + +where :math:`\mathbf{t}=\left(q\Delta t/2m\right)\mathbf{B}^{i}/\bar{\gamma}^{i}` and where +:math:`\bar{\gamma}^{i}` can be calculated as :math:`\bar{\gamma}^{i}=\sqrt{1+(\mathbf{u}^-/c)^2}`. + +The Boris implementation is second-order accurate, time-reversible and fast. Its implementation is very widespread and used in the vast majority of PIC codes. + +.. _theory-pic-push-vay: + +Vay Lorentz-invariant formulation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It was shown in :cite:t:`pt-Vaypop2008` that the Boris formulation is +not Lorentz invariant and can lead to significant errors in the treatment +of relativistic dynamics. A Lorentz invariant formulation is obtained +by considering the following velocity average + +.. math:: + \mathbf{\bar{v}}^{i} = \frac{\mathbf{v}^{i+1/2}+\mathbf{v}^{i-1/2}}{2}. + :label: new_v + +This gives a system that is solvable analytically (see :cite:t:`pt-Vaypop2008` +for a detailed derivation), giving the following velocity update: + +.. math:: + \mathbf{u^{*}} = \mathbf{u}^{i-1/2}+\frac{q\Delta t}{m}\left(\mathbf{E}^{i}+\frac{\mathbf{v}^{i-1/2}}{2}\times\mathbf{B}^{i}\right), + :label: pusher_gamma + +.. math:: + \mathbf{u}^{i+1/2} = \frac{\mathbf{u^{*}}+\left(\mathbf{u^{*}}\cdot\mathbf{t}\right)\mathbf{t}+\mathbf{u^{*}}\times\mathbf{t}}{1+\mathbf{t}^{2}}, + :label: pusher_upr + +where + +.. math:: + + \begin{align} + \mathbf{t} & = \boldsymbol{\tau}/\gamma^{i+1/2}, + \\ + \boldsymbol{\tau} & = \left(q\Delta t/2m\right)\mathbf{B}^{i}, + \\ + \gamma^{i+1/2} & = \sqrt{\sigma+\sqrt{\sigma^{2}+\left(\boldsymbol{\tau}^{2}+w^{2}\right)}}, + \\ + w & = \mathbf{u^{*}}\cdot\boldsymbol{\tau}, + \\ + \sigma & = \left(\gamma'^{2}-\boldsymbol{\tau}^{2}\right)/2, + \\ + \gamma' & = \sqrt{1+(\mathbf{u}^{*}/c)^{2}}. + \end{align} + +This Lorentz invariant formulation +is particularly well suited for the modeling of ultra-relativistic +charged particle beams, where the accurate account of the cancellation +of the self-generated electric and magnetic fields is essential, as +shown in :cite:t:`pt-Vaypop2008`. + +.. _theory-pic-mwsolve: + +Field solve +----------- + +Various methods are available for solving Maxwell’s equations on a +grid, based on finite-differences, finite-volume, finite-element, +spectral, or other discretization techniques that apply most commonly +on single structured or unstructured meshes and less commonly on multiblock +multiresolution grid structures. In this chapter, we summarize the widespread +second order finite-difference time-domain (FDTD) algorithm, its extension +to non-standard finite-differences as well as the pseudo-spectral +analytical time-domain (PSATD) and pseudo-spectral time-domain (PSTD) +algorithms. Extension to multiresolution (or mesh refinement) PIC +is described in, e.g., :cite:t:`pt-VayCSD12,pt-Vaycpc04`. + +.. _fig_yee_grid: + +.. figure:: Yee_grid.png + :alt: [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. + + (left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. + +.. _theory-pic-mwsolve-fdtd: + +Finite-Difference Time-Domain (FDTD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The most popular algorithm for electromagnetic PIC codes is the Finite-Difference +Time-Domain (or FDTD) solver + +.. math:: + D_{t}\mathbf{B} = -\nabla\times\mathbf{E} + :label: Faraday-2 + +.. math:: + D_{t}\mathbf{E} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere-2 + +.. math:: + \left[\nabla\cdot\mathbf{E} = \rho\right] + :label: Gauss-2 + +.. math:: + \left[\nabla\cdot\mathbf{B} = 0\right]. + :label: divb-2 + +The differential operator is defined as :math:`\nabla=D_{x}\mathbf{\hat{x}}+D_{y}\mathbf{\hat{y}}+D_{z}\mathbf{\hat{z}}` +and the finite-difference operators in time and space are defined +respectively as + +.. math:: + + \begin{align} + D_{t}G|_{i,j,k}^{n} & = \frac{(G|_{i,j,k}^{n+1/2}-G|_{i,j,k}^{n-1/2})}{\Delta t}, + \\ + D_{x}G|_{i,j,k}^{n} & = \frac{G|_{i+1/2,j,k}^{n}-G|_{i-1/2,j,k}^{n}}{\Delta x}, + \end{align} + +where :math:`\Delta t` and :math:`\Delta x` are respectively the time step and +the grid cell size along :math:`x`, :math:`n` is the time index and :math:`i`, :math:`j` +and :math:`k` are the spatial indices along :math:`x`, :math:`y` and :math:`z` respectively. +The difference operators along :math:`y` and :math:`z` are obtained by circular +permutation. The equations in brackets are given for completeness, +as they are often not actually solved, thanks to the usage of a so-called +charge conserving algorithm, as explained below. As shown in :numref:`fig_yee_grid`, +the quantities are given on a staggered (or “Yee”) +grid :cite:p:`pt-Yee`, where the electric field components are located +between nodes and the magnetic field components are located in the +center of the cell faces. Knowing the current densities at half-integer steps, +the electric field components are updated alternately with the magnetic +field components at integer and half-integer steps respectively. + +.. _theory-pic-mwsolve-nsfdtd: + +Non-Standard Finite-Difference Time-Domain (NSFDTD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An implementation of the source-free Maxwell’s wave equations for narrow-band +applications based on non-standard finite-differences (NSFD) +was introduced in :cite:t:`pt-Coleieee1997,pt-Coleieee2002`, and +was adapted for wideband applications in :cite:t:`pt-Karkicap06`. At +the Courant limit for the time step and for a given set of parameters, +the stencil proposed in :cite:t:`pt-Karkicap06` has no numerical dispersion +along the principal axes, provided that the cell size is the same +along each dimension (i.e. cubic cells in 3D). The “Cole-Karkkainen” +(or CK) solver uses the non-standard finite difference formulation +(based on extended stencils) of the Maxwell-Ampere equation and can be +implemented as follows :cite:p:`pt-Vayjcp2011`: + +.. math:: + D_{t}\mathbf{B} = -\nabla^{*}\times\mathbf{E} + :label: Faraday + +.. math:: + D_{t}\mathbf{E} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere + +.. math:: + \left[\nabla\cdot\mathbf{E} = \rho\right] + :label: Gauss + +.. math:: + \left[\nabla^{*}\cdot\mathbf{B}= 0\right] + :label: divb + +Eqs. (:eq:`Gauss`) and (:eq:`divb`) are not being solved explicitly +but verified via appropriate initial conditions and current deposition +procedure. The NSFD differential operator is given by + +.. math:: + + \nabla^{*}=D_{x}^{*}\mathbf{\hat{x}}+D_{y}^{*}\mathbf{\hat{y}}+D_{z}^{*}\mathbf{\hat{z}} + +where + +.. math:: + + D_{x}^{*}=\left(\alpha+\beta S_{x}^{1}+\xi S_{x}^{2}\right)D_{x} + +with + +.. math:: + + \begin{align} + S_{x}^{1}G|_{i,j,k}^{n} & = G|_{i,j+1,k}^{n}+G|_{i,j-1,k}^{n}+G|_{i,j,k+1}^{n}+G|_{i,j,k-1}^{n}, + \\ + S_{x}^{2}G|_{i,j,k}^{n} & = G|_{i,j+1,k+1}^{n}+G|_{i,j-1,k+1}^{n}+G|_{i,j+1,k-1}^{n}+G|_{i,j-1,k-1}^{n}. + \end{align} + +Here :math:`G` is a sample vector component, while :math:`\alpha`, :math:`\beta` and :math:`\xi` +are constant scalars satisfying :math:`\alpha+4\beta+4\xi=1`. As with +the FDTD algorithm, the quantities with half-integer are located between +the nodes (electric field components) or in the center of the cell +faces (magnetic field components). The operators along :math:`y` and :math:`z`, +i.e. :math:`D_{y}`, :math:`D_{z}`, :math:`D_{y}^{*}`, :math:`D_{z}^{*}`, :math:`S_{y}^{1}`, +:math:`S_{z}^{1}`, :math:`S_{y}^{2}`, and :math:`S_{z}^{2}`, are obtained by circular +permutation of the indices. + +Assuming cubic cells (:math:`\Delta x=\Delta y=\Delta z`), the coefficients +given in :cite:t:`pt-Karkicap06` (:math:`\alpha=7/12`, :math:`\beta=1/12` and :math:`\xi=1/48`) +allow for the Courant condition to be at :math:`\Delta t=\Delta x`, which +equates to having no numerical dispersion along the principal axes. +The algorithm reduces to the FDTD algorithm with :math:`\alpha=1` and :math:`\beta=\xi=0`. +An extension to non-cubic cells is provided in 3-D by :cite:t:`pt-CowanPRSTAB13` and in 2-D by +:cite:t:`pt-PukhovJPP99`. An alternative NSFDTD implementation that enables superluminous waves is also +given in :cite:t:`pt-LehePRSTAB13`. + +As mentioned above, a key feature of the algorithms based on NSFDTD +is that some implementations :cite:p:`pt-Karkicap06,pt-CowanPRSTAB13` enable the time step :math:`\Delta t=\Delta x` along one or +more axes and no numerical dispersion along those axes. However, as +shown in :cite:t:`pt-Vayjcp2011`, an instability develops at the Nyquist +wavelength at (or very near) such a timestep. It is also shown in +the same paper that removing the Nyquist component in all the source +terms using a bilinear filter (see description of the filter below) +suppresses this instability. + +.. _theory-pic-mwsolve-psatd: + +Pseudo Spectral Analytical Time Domain (PSATD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Maxwell’s equations in Fourier space are given by + +.. math:: \frac{\partial\mathbf{\tilde{E}}}{\partial t} = i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}} +.. math:: \frac{\partial\mathbf{\tilde{B}}}{\partial t} = -i\mathbf{k}\times\mathbf{\tilde{E}} +.. math:: {}[i\mathbf{k}\cdot\mathbf{\tilde{E}} = \tilde{\rho}] +.. math:: {}[i\mathbf{k}\cdot\mathbf{\tilde{B}} = 0] + +where :math:`\tilde{a}` is the Fourier Transform of the quantity :math:`a`. +As with the real space formulation, provided that the continuity equation +:math:`\partial\tilde{\rho}/\partial t+i\mathbf{k}\cdot\mathbf{\tilde{J}}=0` is satisfied, then +the last two equations will automatically be satisfied at any time +if satisfied initially and do not need to be explicitly integrated. + +Decomposing the electric field and current between longitudinal and +transverse components + +.. math:: + + \begin{aligned} + \mathbf{\tilde{E}} & = \mathbf{\tilde{E}}_{L}+\mathbf{\tilde{E}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{E}}) + \\ + \mathbf{\tilde{J}} & = \mathbf{\tilde{J}}_{L}+\mathbf{\tilde{J}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{J}}) + \end{aligned} + +gives + +.. math:: + + \begin{aligned} + \frac{\partial\mathbf{\tilde{E}}_{T}}{\partial t} & = i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}_{T}} + \\ + \frac{\partial\mathbf{\tilde{E}}_{L}}{\partial t} & = -\mathbf{\tilde{J}_{L}} + \\ + \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = -i\mathbf{k}\times\mathbf{\tilde{E}} + \end{aligned} + +with :math:`\mathbf{\hat{k}}=\mathbf{k}/k`. + +If the sources are assumed to be constant over a time interval :math:`\Delta t`, +the system of equations is solvable analytically and is given by (see :cite:t:`pt-Habericnsp73` for the original formulation and :cite:t:`pt-VayJCP2013` +for a more detailed derivation): + +.. math:: + \mathbf{\tilde{E}}_{T}^{n+1} = C\mathbf{\tilde{E}}_{T}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}_{T}^{n+1/2} + :label: PSATD_transverse_1 + +.. math:: + \mathbf{\tilde{E}}_{L}^{n+1} = \mathbf{\tilde{E}}_{L}^{n}-\Delta t\mathbf{\tilde{J}}_{L}^{n+1/2} + :label: PSATD_longitudinal + +.. math:: + \mathbf{\tilde{B}}^{n+1} = C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n} + i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2} + :label: PSATD_transverse_2 + +with :math:`C=\cos\left(k\Delta t\right)` and :math:`S=\sin\left(k\Delta t\right)`. + +Combining the transverse and longitudinal components, gives + +.. math:: + \begin{aligned} + \mathbf{\tilde{E}}^{n+1} & = C\mathbf{\tilde{E}}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}^{n+1/2} + \\ + & + (1-C)\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}}^{n})\nonumber + \\ + & + \mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}}^{n+1/2})\left(\frac{S}{k}-\Delta t\right), + \end{aligned} + :label: Eq_PSATD_1 + +.. math:: + \begin{aligned} + \mathbf{\tilde{B}}^{n+1} & = C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n} + \\ + & + i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}. + \end{aligned} + :label: Eq_PSATD_2 + +For fields generated by the source terms without the self-consistent +dynamics of the charged particles, this algorithm is free of numerical +dispersion and is not subject to a Courant condition. Furthermore, +this solution is exact for any time step size subject to the assumption +that the current source is constant over that time step. + +As shown in :cite:t:`pt-VayJCP2013`, by expanding the coefficients :math:`S_{h}` +and :math:`C_{h}` in Taylor series and keeping the leading terms, the PSATD +formulation reduces to the perhaps better known pseudo-spectral time-domain +(PSTD) formulation :cite:p:`pt-DawsonRMP83,pt-Liumotl1997`: + +.. math:: + + \begin{aligned} + \mathbf{\tilde{E}}^{n+1} & = \mathbf{\tilde{E}}^{n}+i\Delta t\mathbf{k}\times\mathbf{\tilde{B}}^{n+1/2}-\Delta t\mathbf{\tilde{J}}^{n+1/2}, + \\ + \mathbf{\tilde{B}}^{n+3/2} & = \mathbf{\tilde{B}}^{n+1/2}-i\Delta t\mathbf{k}\times\mathbf{\tilde{E}}^{n+1}. + \end{aligned} + +The dispersion relation of the PSTD solver is given by :math:`\sin(\frac{\omega\Delta t}{2})=\frac{k\Delta t}{2}.` +In contrast to the PSATD solver, the PSTD solver is subject to numerical +dispersion for a finite time step and to a Courant condition that +is given by :math:`\Delta t\leq \frac{2}{\pi}\left(\frac{1}{\Delta x^{2}}+\frac{1}{\Delta y^{2}}+\frac{1}{\Delta z^{2}}\right)^{-1/2}`. + +The PSATD and PSTD formulations that were just given apply to the +field components located at the nodes of the grid. As noted in :cite:t:`pt-Ohmurapiers2010`, +they can also be easily recast on a staggered Yee grid by multiplication +of the field components by the appropriate phase factors to shift +them from the collocated to the staggered locations. The choice between +a collocated and a staggered formulation is application-dependent. + +Spectral solvers used to be very popular in the years 1970s to early 1990s, before being replaced by finite-difference methods with the advent of parallel supercomputers that favored local methods. However, it was shown recently that standard domain decomposition with Fast Fourier Transforms that are local to each subdomain could be used effectively with PIC spectral methods :cite:p:`pt-VayJCP2013`, at the cost of truncation errors in the guard cells that could be neglected. A detailed analysis of the effectiveness of the method with exact evaluation of the magnitude of the effect of the truncation error is given in :cite:t:`pt-VincentiCPC2017a` for stencils of arbitrary order (up-to the infinite “spectral” order). + +WarpX also includes a kinetic-fluid hybrid model in which the electric field is +calculated using Ohm's law instead of directly evolving Maxwell's equations. This +approach allows reduced physics simulations to be done with significantly lower +spatial and temporal resolution than in the standard, fully kinetic, PIC. Details +of this model can be found in the section +:ref:`Kinetic-fluid hybrid model `. + +.. _current_deposition: + +Current deposition +------------------ + +The current densities are deposited on the computational grid from +the particle position and velocities, employing splines of various +orders :cite:p:`pt-Abejcp86`. + +.. math:: + + \begin{aligned} + \rho & = \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_nS_n + \\ + \mathbf{J} & = \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_n\mathbf{v_n}S_n + \end{aligned} + +In most applications, it is essential to prevent the accumulation +of errors resulting from the violation of the discretized Gauss’ Law. +This is accomplished by providing a method for depositing the current +from the particles to the grid that preserves the discretized Gauss’ +Law, or by providing a mechanism for “divergence cleaning” :cite:p:`pt-Birdsalllangdon,pt-Langdoncpc92,pt-Marderjcp87,pt-Vaypop98,pt-Munzjcp2000`. +For the former, schemes that allow a deposition of the current that +is exact when combined with the Yee solver is given in :cite:t:`pt-Villasenorcpc92` +for linear splines and in :cite:t:`pt-Esirkepovcpc01` for splines of arbitrary order. + +The NSFDTD formulations given above and in :cite:t:`pt-PukhovJPP99,pt-Vayjcp2011,pt-CowanPRSTAB13,pt-LehePRSTAB13` +apply to the Maxwell-Faraday +equation, while the discretized Maxwell-Ampere equation uses the FDTD +formulation. Consequently, the charge conserving algorithms developed +for current deposition :cite:p:`pt-Villasenorcpc92,pt-Esirkepovcpc01` apply +readily to those NSFDTD-based formulations. More details concerning +those implementations, including the expressions for the numerical +dispersion and Courant condition are given +in :cite:t:`pt-PukhovJPP99,pt-Vayjcp2011,pt-CowanPRSTAB13,pt-LehePRSTAB13`. + +Current correction +~~~~~~~~~~~~~~~~~~ + +In the case of the pseudospectral solvers, the current deposition +algorithm generally does not satisfy the discretized continuity equation +in Fourier space: + +.. math:: + \tilde{\rho}^{n+1}=\tilde{\rho}^{n}-i\Delta t\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}. + +In this case, a Boris correction :cite:p:`pt-Birdsalllangdon` can be applied +in :math:`k` space in the form + +.. math:: + \mathbf{\tilde{E}}_{c}^{n+1}=\mathbf{\tilde{E}}^{n+1}-\frac{\mathbf{k}\cdot\mathbf{\tilde{E}}^{n+1}+i\tilde{\rho}^{n+1}}{k}\mathbf{\hat{k}}, + +where :math:`\mathbf{\tilde{E}}_{c}` is the corrected field. Alternatively, a correction +to the current can be applied (with some similarity to the current +deposition presented by Morse and Nielson in their potential-based +model in :cite:t:`pt-Morsenielson1971`) using + +.. math:: \mathbf{\tilde{J}}_{c}^{n+1/2}=\mathbf{\tilde{J}}^{n+1/2}-\left[\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}-i\left(\tilde{\rho}^{n+1}-\tilde{\rho}^{n}\right)/\Delta t\right]\mathbf{\hat{k}}/k, + +where :math:`\mathbf{\tilde{J}}_{c}` is the corrected current. In this case, the transverse +component of the current is left untouched while the longitudinal +component is effectively replaced by the one obtained from integration +of the continuity equation, ensuring that the corrected current satisfies +the continuity equation. The advantage of correcting the current rather than +the electric field is that it is more local and thus more compatible with +domain decomposition of the fields for parallel computation :cite:p:`pt-VayJCP2013`. + +Vay deposition +~~~~~~~~~~~~~~ + +Alternatively, an exact current deposition can be written for the pseudo-spectral solvers, following the geometrical interpretation of existing methods in real space :cite:p:`pt-Morsenielson1971,pt-Villasenorcpc92,pt-Esirkepovcpc01`. + +The Vay deposition scheme is the generalization of the Esirkepov deposition scheme for the spectral case with arbitrary-order stencils :cite:p:`pt-VayJCP2013`. +The current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` in Fourier space is computed as :math:`\widehat{\boldsymbol{J}}^{\,n+1/2} = i \, \widehat{\boldsymbol{D}} / \boldsymbol{k}` when :math:`\boldsymbol{k} \neq 0` and set to zero otherwise. +The quantity :math:`\boldsymbol{D}` is deposited in real space by averaging the currents over all possible grid paths between the initial position :math:`\boldsymbol{x}^{\,n}` and the final position :math:`\boldsymbol{x}^{\,n+1}` and is defined as + +- 2D Cartesian geometry: + +.. math:: + \begin{align} + D_x & = \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n+1}) + + \Gamma(x_i^{n+1},z_i^{n}) - \Gamma(x_i^{n},z_i^{n}) + \bigg] + \\[8pt] + D_y & = \sum_i \frac{v_i^y}{\Delta x \Delta z} \frac{q_i w_i}{4} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) + \Gamma(x_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n},z_i^{n+1}) + \Gamma(x_i^{n},z_i^{n}) + \bigg] + \\[8pt] + D_z & = \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n}) + \bigg] + \end{align} + +- 3D Cartesian geometry: + +.. math:: + \begin{align} + \begin{split} + D_x & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \\[8pt] + \begin{split} + D_y & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \\[8pt] + \begin{split} + D_z & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + 2 \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \end{align} + +Here, :math:`w_i` represents the weight of the :math:`i`-th macro-particle and :math:`\Gamma` represents its shape factor. +Note that in 2D Cartesian geometry, :math:`D_y` is effectively :math:`J_y` and does not require additional operations in Fourier space. + +Field gather +------------ + +In general, the field is gathered from the mesh onto the macroparticles +using splines of the same order as for the current deposition :math:`\mathbf{S}=\left(S_{x},S_{y},S_{z}\right)`. +Three variations are considered: + +- “momentum conserving”: fields are interpolated from the grid nodes + to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` + for all field components (if the fields are known at staggered positions, + they are first interpolated to the nodes on an auxiliary grid), + +- “energy conserving (or Galerkin)”: fields are interpolated from + the staggered Yee grid to the macroparticles using :math:`\left(S_{nx-1},S_{ny},S_{nz}\right)` + for :math:`E_{x}`, :math:`\left(S_{nx},S_{ny-1},S_{nz}\right)` for :math:`E_{y}`, + :math:`\left(S_{nx},S_{ny},S_{nz-1}\right)` for :math:`E_{z}`, :math:`\left(S_{nx},S_{ny-1},S_{nz-1}\right)` + for :math:`B_{x}`, :math:`\left(S_{nx-1},S_{ny},S_{nz-1}\right)` for :math:`B{}_{y}` + and\ :math:`\left(S_{nx-1},S_{ny-1},S_{nz}\right)` for :math:`B_{z}` (if the fields + are known at the nodes, they are first interpolated to the staggered + positions on an auxiliary grid), + +- “uniform”: fields are interpolated directly form the Yee grid + to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` + for all field components (if the fields are known at the nodes, they + are first interpolated to the staggered positions on an auxiliary + grid). + +As shown in :cite:t:`pt-Birdsalllangdon,pt-HockneyEastwoodBook,pt-LewisJCP1972`, +the momentum and energy conserving schemes conserve momentum and energy +respectively at the limit of infinitesimal time steps and generally +offer better conservation of the respective quantities for a finite +time step. The uniform scheme does not conserve momentum nor energy +in the sense defined for the others but is given for completeness, +as it has been shown to offer some interesting properties in the modeling +of relativistically drifting plasmas :cite:p:`pt-GodfreyJCP2013`. + +.. _theory-pic-filter: + +Filtering +--------- + +It is common practice to apply digital filtering to the charge or +current density in Particle-In-Cell simulations as a complement or +an alternative to using higher order splines :cite:p:`pt-Birdsalllangdon`. +A commonly used filter in PIC simulations is the three points filter + +.. math:: + \phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-1}+\phi_{j+1}\right)/2 + +where :math:`\phi^{f}` is the filtered quantity. This filter is called +a bilinear filter when :math:`\alpha=0.5`. Assuming :math:`\phi=e^{jkx}` and +:math:`\phi^{f}=g\left(\alpha,k\right)e^{jkx}`, the filter gain :math:`g` is +given as a function of the filtering coefficient :math:`\alpha` and +the wavenumber :math:`k` by + +.. math:: + g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(k\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. + +The total attenuation :math:`G` for :math:`n` successive applications of filters +of coefficients :math:`\alpha_{1}`...\ :math:`\alpha_{n}` is given by + +.. math:: + G=\prod_{i=1}^{n}g\left(\alpha_{i},k\right)\approx1-\left(n-\sum_{i=1}^{n}\alpha_{i}\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. + +A sharper cutoff in :math:`k` space is provided by using :math:`\alpha_{n}=n-\sum_{i=1}^{n-1}\alpha_{i}`, +so that :math:`G\approx1+O\left(k^{4}\right)`. Such step is called a “compensation” +step :cite:p:`pt-Birdsalllangdon`. For the bilinear filter (:math:`\alpha=1/2`), +the compensation factor is :math:`\alpha_{c}=2-1/2=3/2`. For a succession +of :math:`n` applications of the bilinear factor, it is :math:`\alpha_{c}=n/2+1`. + +It is sometimes necessary to filter on a relatively wide band of wavelength, +necessitating the application of a large number of passes of the bilinear +filter or on the use of filters acting on many points. The former +can become very intensive computationally while the latter is problematic +for parallel computations using domain decomposition, as the footprint +of the filter may eventually surpass the size of subdomains. A workaround +is to use a combination of filters of limited footprint. A solution +based on the combination of three point filters with various strides +was proposed in :cite:t:`pt-Vayjcp2011` and operates as follows. + +The bilinear filter provides complete suppression of the signal at +the grid Nyquist wavelength (twice the grid cell size). Suppression +of the signal at integer multiples of the Nyquist wavelength can be +obtained by using a stride :math:`s` in the filter + +.. math:: + \phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-s}+\phi_{j+s}\right)/2 + +for which the gain is given by + +.. math:: + g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(sk\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(sk\Delta x\right)^{2}}{2}+O\left(k^{4}\right). + +For a given stride, the gain is given by the gain of the bilinear +filter shifted in k space, with the pole :math:`g=0` shifted from the wavelength +:math:`\lambda=2/\Delta x` to :math:`\lambda=2s/\Delta x`, with additional poles, +as given by :math:`sk\Delta x=\arccos\left(\frac{\alpha}{\alpha-1}\right)\pmod{2\pi}`. +The resulting filter is pass band between the poles, but since the +poles are spread at different integer values in k space, a wide band +low pass filter can be constructed by combining filters using different +strides. As shown in :cite:t:`pt-Vayjcp2011`, the successive application +of 4-passes + compensation of filters with strides 1, 2 and 4 has +a nearly equivalent fall-off in gain as 80 passes + compensation of +a bilinear filter. Yet, the strided filter solution needs only 15 +passes of a three-point filter, compared to 81 passes for an equivalent +n-pass bilinear filter, yielding a gain of 5.4 in number of operations +in favor of the combination of filters with stride. The width of the +filter with stride 4 extends only on 9 points, compared to 81 points +for a single pass equivalent filter, hence giving a gain of 9 in compactness +for the stride filters combination in comparison to the single-pass +filter with large stencil, resulting in more favorable scaling with the number +of computational cores for parallel calculations. + +.. bibliography:: + :keyprefix: pt- diff --git a/Docs/source/theory/picsar_theory.rst b/Docs/source/theory/picsar_theory.rst deleted file mode 100644 index 0aadbd558f0..00000000000 --- a/Docs/source/theory/picsar_theory.rst +++ /dev/null @@ -1,867 +0,0 @@ -.. raw:: latex - - \markboth{J.-L. Vay, R. Lehe}{Simulations for plasma and laser acceleration.} - -.. raw:: latex - - \maketitle - -.. raw:: latex - - \linenumbers - -.. _theory-pic: - -Particle-in-Cell Method -======================= - -.. figure:: PIC.png - :alt: [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). - - [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). - -In the *electromagnetic particle-in-cell method* (Birdsall and Langdon 1991), -the electromagnetic fields are solved on a grid, usually using Maxwell’s -equations - -.. math:: - - \begin{aligned} - \frac{\mathbf{\partial B}}{\partial t} & = & -\nabla\times\mathbf{E}\label{Eq:Faraday-1}\\ - \frac{\mathbf{\partial E}}{\partial t} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere-1}\\ - \nabla\cdot\mathbf{E} & = & \rho\label{Eq:Gauss-1}\\ - \nabla\cdot\mathbf{B} & = & 0\label{Eq:divb-1}\end{aligned} - -given here in natural units (:math:`\epsilon_0=\mu_0=c=1`), where :math:`t` is time, :math:`\mathbf{E}` and -:math:`\mathbf{B}` are the electric and magnetic field components, and -:math:`\rho` and :math:`\mathbf{J}` are the charge and current densities. The -charged particles are advanced in time using the Newton-Lorentz equations -of motion - -.. math:: - - \begin{aligned} - \frac{d\mathbf{x}}{dt}= & \mathbf{v},\label{Eq:Lorentz_x-1}\\ - \frac{d\left(\gamma\mathbf{v}\right)}{dt}= & \frac{q}{m}\left(\mathbf{E}+\mathbf{v}\times\mathbf{B}\right),\label{Eq:Lorentz_v-1}\end{aligned} - -where :math:`m`, :math:`q`, :math:`\mathbf{x}`, :math:`\mathbf{v}` and :math:`\gamma=1/\sqrt{1-v^{2}}` -are respectively the mass, charge, position, velocity and relativistic -factor of the particle given in natural units (:math:`c=1`). The charge and current densities are interpolated -on the grid from the particles’ positions and velocities, while the -electric and magnetic field components are interpolated from the grid -to the particles’ positions for the velocity update. - -.. _theory-pic-push: - -Particle push -------------- - -A centered finite-difference discretization of the Newton-Lorentz -equations of motion is given by - -.. math:: - - \begin{aligned} - \frac{\mathbf{x}^{i+1}-\mathbf{x}^{i}}{\Delta t}= & \mathbf{v}^{i+1/2},\label{Eq:leapfrog_x}\\ - \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}-\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{\Delta t}= & \frac{q}{m}\left(\mathbf{E}^{i}+\mathbf{\bar{v}}^{i}\times\mathbf{B}^{i}\right).\label{Eq:leapfrog_v}\end{aligned} - -In order to close the system, :math:`\bar{\mathbf{v}}^{i}` must be -expressed as a function of the other quantities. The two implementations that have become the most popular are presented below. - -.. _theory-pic-push-boris: - -Boris relativistic velocity rotation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The solution proposed by Boris (Boris 1970) is given by - -.. math:: - - \begin{aligned} - \mathbf{\bar{v}}^{i}= & \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}+\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{2\bar{\gamma}^{i}}.\label{Eq:boris_v}\end{aligned} - -where :math:`\bar{\gamma}^{i}` is defined by :math:`\bar{\gamma}^{i} \equiv (\gamma^{i+1/2}+\gamma^{i-1/2} )/2`. - -The system (`[Eq:leapfrog_v] <#Eq:leapfrog_v>`__,\ `[Eq:boris_v] <#Eq:boris_v>`__) is solved very -efficiently following Boris’ method, where the electric field push -is decoupled from the magnetic push. Setting :math:`\mathbf{u}=\gamma\mathbf{v}`, the -velocity is updated using the following sequence: - -.. math:: - - \begin{aligned} - \mathbf{u^{-}}= & \mathbf{u}^{i-1/2}+\left(q\Delta t/2m\right)\mathbf{E}^{i}\\ - \mathbf{u'}= & \mathbf{u}^{-}+\mathbf{u}^{-}\times\mathbf{t}\\ - \mathbf{u}^{+}= & \mathbf{u}^{-}+\mathbf{u'}\times2\mathbf{t}/(1+t^{2})\\ - \mathbf{u}^{i+1/2}= & \mathbf{u}^{+}+\left(q\Delta t/2m\right)\mathbf{E}^{i}\end{aligned} - -where :math:`\mathbf{t}=\left(q\Delta t/2m\right)\mathbf{B}^{i}/\bar{\gamma}^{i}` and where -:math:`\bar{\gamma}^{i}` can be calculated as :math:`\bar{\gamma}^{i}=\sqrt{1+(\mathbf{u}^-/c)^2}`. - -The Boris implementation is second-order accurate, time-reversible and fast. Its implementation is very widespread and used in the vast majority of PIC codes. - -.. _theory-pic-push-vay: - -Vay Lorentz-invariant formulation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It was shown in (Vay 2008) that the Boris formulation is -not Lorentz invariant and can lead to significant errors in the treatment -of relativistic dynamics. A Lorentz invariant formulation is obtained -by considering the following velocity average - -.. math:: - - \begin{aligned} - \mathbf{\bar{v}}^{i}= & \frac{\mathbf{v}^{i+1/2}+\mathbf{v}^{i-1/2}}{2},\label{Eq:new_v}\end{aligned} - -This gives a system that is solvable analytically (see (Vay 2008) -for a detailed derivation), giving the following velocity update: - -.. math:: - - \begin{aligned} - \mathbf{u^{*}}= & \mathbf{u}^{i-1/2}+\frac{q\Delta t}{m}\left(\mathbf{E}^{i}+\frac{\mathbf{v}^{i-1/2}}{2}\times\mathbf{B}^{i}\right),\label{pusher_gamma}\\ - \mathbf{u}^{i+1/2}= & \left[\mathbf{u^{*}}+\left(\mathbf{u^{*}}\cdot\mathbf{t}\right)\mathbf{t}+\mathbf{u^{*}}\times\mathbf{t}\right]/\left(1+t^{2}\right),\label{pusher_upr}\end{aligned} - -where :math:`\mathbf{t}=\boldsymbol{\tau}/\gamma^{i+1/2}`, :math:`\boldsymbol{\tau}=\left(q\Delta t/2m\right)\mathbf{B}^{i}`, -:math:`\gamma^{i+1/2}=\sqrt{\sigma+\sqrt{\sigma^{2}+\left(\tau^{2}+w^{2}\right)}}`, -:math:`w=\mathbf{u^{*}}\cdot\boldsymbol{\tau}`, :math:`\sigma=\left(\gamma'^{2}-\tau^{2}\right)/2` -and :math:`\gamma'=\sqrt{1+(\mathbf{u}^{*}/c)^{2}}`. This Lorentz invariant formulation -is particularly well suited for the modeling of ultra-relativistic -charged particle beams, where the accurate account of the cancellation -of the self-generated electric and magnetic fields is essential, as -shown in (Vay 2008). - -.. _theory-pic-mwsolve: - -Field solve ------------ - -Various methods are available for solving Maxwell’s equations on a -grid, based on finite-differences, finite-volume, finite-element, -spectral, or other discretization techniques that apply most commonly -on single structured or unstructured meshes and less commonly on multiblock -multiresolution grid structures. In this chapter, we summarize the widespread -second order finite-difference time-domain (FDTD) algorithm, its extension -to non-standard finite-differences as well as the pseudo-spectral -analytical time-domain (PSATD) and pseudo-spectral time-domain (PSTD) -algorithms. Extension to multiresolution (or mesh refinement) PIC -is described in, e.g. (Vay et al. 2012; Vay, Adam, and Heron 2004). - -.. _theory-pic-mwsolve-fdtd: - -Finite-Difference Time-Domain (FDTD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The most popular algorithm for electromagnetic PIC codes is the Finite-Difference -Time-Domain (or FDTD) solver - -.. math:: - - \begin{aligned} - D_{t}\mathbf{B} & = & -\nabla\times\mathbf{E}\label{Eq:Faraday-2}\\ - D_{t}\mathbf{E} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere-2}\\ - \left[\nabla\cdot\mathbf{E}\right. & = & \left.\rho\right]\label{Eq:Gauss-2}\\ - \left[\nabla\cdot\mathbf{B}\right. & = & \left.0\right].\label{Eq:divb-2}\end{aligned} - -.. figure:: Yee_grid.png - :alt: [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. - - [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. - -The differential operator is defined as :math:`\nabla=D_{x}\mathbf{\hat{x}}+D_{y}\mathbf{\hat{y}}+D_{z}\mathbf{\hat{z}}` -and the finite-difference operators in time and space are defined -respectively as - -.. math:: D_{t}G|_{i,j,k}^{n}=\left(G|_{i,j,k}^{n+1/2}-G|_{i,j,k}^{n-1/2}\right)/\Delta t - -and :math:`D_{x}G|_{i,j,k}^{n}=\left(G|_{i+1/2,j,k}^{n}-G|_{i-1/2,j,k}^{n}\right)/\Delta x`, -where :math:`\Delta t` and :math:`\Delta x` are respectively the time step and -the grid cell size along :math:`x`, :math:`n` is the time index and :math:`i`, :math:`j` -and :math:`k` are the spatial indices along :math:`x`, :math:`y` and :math:`z` respectively. -The difference operators along :math:`y` and :math:`z` are obtained by circular -permutation. The equations in brackets are given for completeness, -as they are often not actually solved, thanks to the usage of a so-called -charge conserving algorithm, as explained below. As shown in Figure -`[fig:yee_grid] <#fig:yee_grid>`__, the quantities are given on a staggered (or “Yee”) -grid (Yee 1966), where the electric field components are located -between nodes and the magnetic field components are located in the -center of the cell faces. Knowing the current densities at half-integer steps, -the electric field components are updated alternately with the magnetic -field components at integer and half-integer steps respectively. - -.. _theory-pic-mwsolve-nsfdtd: - -Non-Standard Finite-Difference Time-Domain (NSFDTD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In (Cole 1997, 2002), Cole introduced an implementation -of the source-free Maxwell’s wave equations for narrow-band applications -based on non-standard finite-differences (NSFD). In (Karkkainen et al. 2006), -Karkkainen *et al.* adapted it for wideband applications. At -the Courant limit for the time step and for a given set of parameters, -the stencil proposed in (Karkkainen et al. 2006) has no numerical dispersion -along the principal axes, provided that the cell size is the same -along each dimension (i.e. cubic cells in 3D). The “Cole-Karkkainnen” -(or CK) solver uses the non-standard finite difference formulation -(based on extended stencils) of the Maxwell-Ampere equation and can be -implemented as follows (Vay et al. 2011): - -.. math:: - - \begin{aligned} - D_{t}\mathbf{B} & = & -\nabla^{*}\times\mathbf{E}\label{Eq:Faraday}\\ - D_{t}\mathbf{E} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere}\\ - \left[\nabla\cdot\mathbf{E}\right. & = & \left.\rho\right]\label{Eq:Gauss}\\ - \left[\nabla^{*}\cdot\mathbf{B}\right. & = & \left.0\right]\label{Eq:divb}\end{aligned} - -Eq. `[Eq:Gauss] <#Eq:Gauss>`__ and `[Eq:divb] <#Eq:divb>`__ are not being solved explicitly -but verified via appropriate initial conditions and current deposition -procedure. The NSFD differential operators is given by :math:`\nabla^{*}=D_{x}^{*}\mathbf{\hat{x}}+D_{y}^{*}\mathbf{\hat{y}}+D_{z}^{*}\mathbf{\hat{z}}` -where :math:`D_{x}^{*}=\left(\alpha+\beta S_{x}^{1}+\xi S_{x}^{2}\right)D_{x}` -with :math:`S_{x}^{1}G|_{i,j,k}^{n}=G|_{i,j+1,k}^{n}+G|_{i,j-1,k}^{n}+G|_{i,j,k+1}^{n}+G|_{i,j,k-1}^{n}`, -:math:`S_{x}^{2}G|_{i,j,k}^{n}=G|_{i,j+1,k+1}^{n}+G|_{i,j-1,k+1}^{n}+G|_{i,j+1,k-1}^{n}+G|_{i,j-1,k-1}^{n}`. -:math:`G` is a sample vector component, while :math:`\alpha`, :math:`\beta` and :math:`\xi` -are constant scalars satisfying :math:`\alpha+4\beta+4\xi=1`. As with -the FDTD algorithm, the quantities with half-integer are located between -the nodes (electric field components) or in the center of the cell -faces (magnetic field components). The operators along :math:`y` and :math:`z`, -i.e. :math:`D_{y}`, :math:`D_{z}`, :math:`D_{y}^{*}`, :math:`D_{z}^{*}`, :math:`S_{y}^{1}`, -:math:`S_{z}^{1}`, :math:`S_{y}^{2}`, and :math:`S_{z}^{2}`, are obtained by circular -permutation of the indices. - -Assuming cubic cells (:math:`\Delta x=\Delta y=\Delta z`), the coefficients -given in (Karkkainen et al. 2006) (:math:`\alpha=7/12`, :math:`\beta=1/12` and :math:`\xi=1/48`) -allow for the Courant condition to be at :math:`\Delta t=\Delta x`, which -equates to having no numerical dispersion along the principal axes. -The algorithm reduces to the FDTD algorithm with :math:`\alpha=1` and :math:`\beta=\xi=0`. -An extension to non-cubic cells is provided by Cowan, *et al.* -in 3-D in (Cowan et al. 2013) and was given by Pukhov in 2-D in -(Pukhov 1999). An alternative NSFDTD implementation that enables superluminous waves is also -given by Lehe et al. in (Lehe et al. 2013). - -As mentioned above, a key feature of the algorithms based on NSFDTD -is that some implementations (Karkkainen et al. 2006; Cowan et al. 2013) enable the time step :math:`\Delta t=\Delta x` along one or -more axes and no numerical dispersion along those axes. However, as -shown in (Vay et al. 2011), an instability develops at the Nyquist -wavelength at (or very near) such a timestep. It is also shown in -the same paper that removing the Nyquist component in all the source -terms using a bilinear filter (see description of the filter below) -suppresses this instability. - -.. _theory-pic-mwsolve-psatd: - -Pseudo Spectral Analytical Time Domain (PSATD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Maxwell’s equations in Fourier space are given by - -.. math:: - - \begin{aligned} - \frac{\partial\mathbf{\tilde{E}}}{\partial t} & = & i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}}\\ - \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = & -i\mathbf{k}\times\mathbf{\tilde{E}}\\ - {}[i\mathbf{k}\cdot\mathbf{\tilde{E}}& = & \tilde{\rho}]\\ - {}[i\mathbf{k}\cdot\mathbf{\tilde{B}}& = & 0]\end{aligned} - -where :math:`\tilde{a}` is the Fourier Transform of the quantity :math:`a`. -As with the real space formulation, provided that the continuity equation -:math:`\partial\tilde{\rho}/\partial t+i\mathbf{k}\cdot\mathbf{\tilde{J}}=0` is satisfied, then -the last two equations will automatically be satisfied at any time -if satisfied initially and do not need to be explicitly integrated. - -Decomposing the electric field and current between longitudinal and -transverse components :math:`\mathbf{\tilde{E}}=\mathbf{\tilde{E}}_{L}+\mathbf{\tilde{E}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{E}})` -and :math:`\mathbf{\tilde{J}}=\mathbf{\tilde{J}}_{L}+\mathbf{\tilde{J}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{J}})` -gives - -.. math:: - - \begin{aligned} - \frac{\partial\mathbf{\tilde{E}}_{T}}{\partial t} & = & i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}_{T}}\\ - \frac{\partial\mathbf{\tilde{E}}_{L}}{\partial t} & = & -\mathbf{\tilde{J}_{L}}\\ - \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = & -i\mathbf{k}\times\mathbf{\tilde{E}}\end{aligned} - -with :math:`\mathbf{\hat{k}}=\mathbf{k}/k`. - -If the sources are assumed to be constant over a time interval :math:`\Delta t`, -the system of equations is solvable analytically and is given by (see -(Haber et al. 1973) for the original formulation and (Jean-Luc Vay, Haber, and Godfrey 2013) -for a more detailed derivation): - -[Eq:PSATD] - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}_{T}^{n+1} & = & C\mathbf{\tilde{E}}_{T}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}_{T}^{n+1/2}\label{Eq:PSATD_transverse_1}\\ - \mathbf{\tilde{E}}_{L}^{n+1} & = & \mathbf{\tilde{E}}_{L}^{n}-\Delta t\mathbf{\tilde{J}}_{L}^{n+1/2}\\ - \mathbf{\tilde{B}}^{n+1} & = & C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n}\\ - &+&i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}\label{Eq:PSATD_transverse_2}\end{aligned} - -with :math:`C=\cos\left(k\Delta t\right)` and :math:`S=\sin\left(k\Delta t\right)`. - -Combining the transverse and longitudinal components, gives - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}^{n+1} & = & C\mathbf{\tilde{E}}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}^{n+1/2}\\ - & + &(1-C)\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}}^{n})\nonumber \\ - & + & \mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}}^{n+1/2})\left(\frac{S}{k}-\Delta t\right),\label{Eq_PSATD_1}\\ - \mathbf{\tilde{B}}^{n+1} & = & C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n}\\ - &+&i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}.\label{Eq_PSATD_2}\end{aligned} - -For fields generated by the source terms without the self-consistent -dynamics of the charged particles, this algorithm is free of numerical -dispersion and is not subject to a Courant condition. Furthermore, -this solution is exact for any time step size subject to the assumption -that the current source is constant over that time step. - -As shown in (Jean-Luc Vay, Haber, and Godfrey 2013), by expanding the coefficients :math:`S_{h}` -and :math:`C_{h}` in Taylor series and keeping the leading terms, the PSATD -formulation reduces to the perhaps better known pseudo-spectral time-domain -(PSTD) formulation (Dawson 1983; Liu 1997): - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}^{n+1} & = & \mathbf{\tilde{E}}^{n}+i\Delta t\mathbf{k}\times\mathbf{\tilde{B}}^{n+1/2}-\Delta t\mathbf{\tilde{J}}^{n+1/2},\\ - \mathbf{\tilde{B}}^{n+3/2} & = & \mathbf{\tilde{B}}^{n+1/2}-i\Delta t\mathbf{k}\times\mathbf{\tilde{E}}^{n+1}.\end{aligned} - -The dispersion relation of the PSTD solver is given by :math:`\sin(\frac{\omega\Delta t}{2})=\frac{k\Delta t}{2}.` -In contrast to the PSATD solver, the PSTD solver is subject to numerical -dispersion for a finite time step and to a Courant condition that -is given by :math:`\Delta t\leq \frac{2}{\pi}\left(\frac{1}{\Delta x^{2}}+\frac{1}{\Delta y^{2}}+\frac{1}{\Delta x^{2}}\right)^{-1/2}.` - -The PSATD and PSTD formulations that were just given apply to the -field components located at the nodes of the grid. As noted in (Ohmura and Okamura 2010), -they can also be easily recast on a staggered Yee grid by multiplication -of the field components by the appropriate phase factors to shift -them from the collocated to the staggered locations. The choice between -a collocated and a staggered formulation is application-dependent. - -Spectral solvers used to be very popular in the years 1970s to early 1990s, before being replaced by finite-difference methods with the advent of parallel supercomputers that favored local methods. However, it was shown recently that standard domain decomposition with Fast Fourier Transforms that are local to each subdomain could be used effectively with PIC spectral methods (Jean-Luc Vay, Haber, and Godfrey 2013), at the cost of truncation errors in the guard cells that could be neglected. A detailed analysis of the effectiveness of the method with exact evaluation of the magnitude of the effect of the truncation error is given in (Vincenti and Vay 2016) for stencils of arbitrary order (up-to the infinite “spectral” order). - -WarpX also includes a kinetic-fluid hybrid model in which the electric field is -calculated using Ohm's law instead of directly evolving Maxwell's equations. This -approach allows reduced physics simulations to be done with significantly lower -spatial and temporal resolution than in the standard, fully kinetic, PIC. Details -of this model can be found in the section -:ref:`Kinetic-fluid hybrid model `. - -.. _current_deposition: - -Current deposition ------------------- - -The current densities are deposited on the computational grid from -the particle position and velocities, employing splines of various -orders (Abe et al. 1986). - -.. math:: - - \begin{aligned} - \rho & = & \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_nS_n\\ - \mathbf{J} & = & \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_n\mathbf{v_n}S_n\end{aligned} - -In most applications, it is essential to prevent the accumulation -of errors resulting from the violation of the discretized Gauss’ Law. -This is accomplished by providing a method for depositing the current -from the particles to the grid that preserves the discretized Gauss’ -Law, or by providing a mechanism for “divergence cleaning” (Birdsall and Langdon 1991; Langdon 1992; Marder 1987; Vay and Deutsch 1998; Munz et al. 2000). -For the former, schemes that allow a deposition of the current that -is exact when combined with the Yee solver is given in (Villasenor and Buneman 1992) -for linear splines and in (Esirkepov 2001) for splines of arbitrary order. - -The NSFDTD formulations given above and in (Pukhov 1999; Vay et al. 2011; Cowan et al. 2013; Lehe et al. 2013) -apply to the Maxwell-Faraday -equation, while the discretized Maxwell-Ampere equation uses the FDTD -formulation. Consequently, the charge conserving algorithms developed -for current deposition (Villasenor and Buneman 1992; Esirkepov 2001) apply -readily to those NSFDTD-based formulations. More details concerning -those implementations, including the expressions for the numerical -dispersion and Courant condition are given -in (Pukhov 1999; Vay et al. 2011; Cowan et al. 2013; Lehe et al. 2013). - -Current correction -~~~~~~~~~~~~~~~~~~ - -In the case of the pseudospectral solvers, the current deposition -algorithm generally does not satisfy the discretized continuity equation -in Fourier space :math:`\tilde{\rho}^{n+1}=\tilde{\rho}^{n}-i\Delta t\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}`. -In this case, a Boris correction (Birdsall and Langdon 1991) can be applied -in :math:`k` space in the form :math:`\mathbf{\tilde{E}}_{c}^{n+1}=\mathbf{\tilde{E}}^{n+1}-\left(\mathbf{k}\cdot\mathbf{\tilde{E}}^{n+1}+i\tilde{\rho}^{n+1}\right)\mathbf{\hat{k}}/k`, -where :math:`\mathbf{\tilde{E}}_{c}` is the corrected field. Alternatively, a correction -to the current can be applied (with some similarity to the current -deposition presented by Morse and Nielson in their potential-based -model in (Morse and Nielson 1971)) using :math:`\mathbf{\tilde{J}}_{c}^{n+1/2}=\mathbf{\tilde{J}}^{n+1/2}-\left[\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}-i\left(\tilde{\rho}^{n+1}-\tilde{\rho}^{n}\right)/\Delta t\right]\mathbf{\hat{k}}/k`, -where :math:`\mathbf{\tilde{J}}_{c}` is the corrected current. In this case, the transverse -component of the current is left untouched while the longitudinal -component is effectively replaced by the one obtained from integration -of the continuity equation, ensuring that the corrected current satisfies -the continuity equation. The advantage of correcting the current rather than -the electric field is that it is more local and thus more compatible with -domain decomposition of the fields for parallel computation (Jean Luc Vay, Haber, and Godfrey 2013). - -Vay deposition -~~~~~~~~~~~~~~ - -Alternatively, an exact current deposition can be written for the pseudo-spectral solvers, following the geometrical interpretation of existing methods in real space (`Morse and Nielson, 1971 `_; `Villasenor and Buneman, 1992 `_; `Esirkepov, 2001 `_). - -The Vay deposition scheme is the generalization of the Esirkepov deposition scheme for the spectral case with arbitrary-order stencils `(Vay et al, 2013) `_. -The current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` in Fourier space is computed as :math:`\widehat{\boldsymbol{J}}^{\,n+1/2} = i \, \widehat{\boldsymbol{D}} / \boldsymbol{k}` when :math:`\boldsymbol{k} \neq 0` and set to zero otherwise. -The quantity :math:`\boldsymbol{D}` is deposited in real space by averaging the currents over all possible grid paths between the initial position :math:`\boldsymbol{x}^{\,n}` and the final position :math:`\boldsymbol{x}^{\,n+1}` and is defined as - -- 2D Cartesian geometry: - -.. math:: - \begin{align} - D_x = & \: \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n+1}) - + \Gamma(x_i^{n+1},z_i^{n}) - \Gamma(x_i^{n},z_i^{n}) - \bigg] - \\[8pt] - D_y = & \: \sum_i \frac{v_i^y}{\Delta x \Delta z} \frac{q_i w_i}{4} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) + \Gamma(x_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n},z_i^{n+1}) + \Gamma(x_i^{n},z_i^{n}) - \bigg] - \\[8pt] - D_z = & \: \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n}) - \bigg] - \end{align} - -- 3D Cartesian geometry: - -.. math:: - \begin{align} - \begin{split} - D_x = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) \\[4pt] - & + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) - + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) \\[4pt] - & - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} \\[8pt] - \begin{split} - D_y = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) \\[4pt] - & + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) - + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) \\[4pt] - & - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} \\[8pt] - \begin{split} - D_z = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) \\[4pt] - & + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) \\[4pt] - & - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + 2 \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} - \end{align} - -Here, :math:`w_i` represents the weight of the :math:`i`-th macro-particle and :math:`\Gamma` represents its shape factor. -Note that in 2D Cartesian geometry, :math:`D_y` is effectively :math:`J_y` and does not require additional operations in Fourier space. - -Field gather ------------- - -In general, the field is gathered from the mesh onto the macroparticles -using splines of the same order as for the current deposition :math:`\mathbf{S}=\left(S_{x},S_{y},S_{z}\right)`. -Three variations are considered: - -- “momentum conserving”: fields are interpolated from the grid nodes - to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` - for all field components (if the fields are known at staggered positions, - they are first interpolated to the nodes on an auxiliary grid), - -- “energy conserving (or Galerkin)”: fields are interpolated from - the staggered Yee grid to the macroparticles using :math:`\left(S_{nx-1},S_{ny},S_{nz}\right)` - for :math:`E_{x}`, :math:`\left(S_{nx},S_{ny-1},S_{nz}\right)` for :math:`E_{y}`, - :math:`\left(S_{nx},S_{ny},S_{nz-1}\right)` for :math:`E_{z}`, :math:`\left(S_{nx},S_{ny-1},S_{nz-1}\right)` - for :math:`B_{x}`, :math:`\left(S_{nx-1},S_{ny},S_{nz-1}\right)` for :math:`B{}_{y}` - and\ :math:`\left(S_{nx-1},S_{ny-1},S_{nz}\right)` for :math:`B_{z}` (if the fields - are known at the nodes, they are first interpolated to the staggered - positions on an auxiliary grid), - -- “uniform”: fields are interpolated directly form the Yee grid - to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` - for all field components (if the fields are known at the nodes, they - are first interpolated to the staggered positions on an auxiliary - grid). - -As shown in :raw-latex:`\cite{BirdsallLangdon,HockneyEastwoodBook,LewisJCP1972}`, -the momentum and energy conserving schemes conserve momentum and energy -respectively at the limit of infinitesimal time steps and generally -offer better conservation of the respective quantities for a finite -time step. The uniform scheme does not conserve momentum nor energy -in the sense defined for the others but is given for completeness, -as it has been shown to offer some interesting properties in the modeling -of relativistically drifting plasmas :raw-latex:`\cite{GodfreyJCP2013}`. - -.. _theory-pic-filter: - -Filtering -========= - -It is common practice to apply digital filtering to the charge or -current density in Particle-In-Cell simulations as a complement or -an alternative to using higher order splines (Birdsall and Langdon 1991). -A commonly used filter in PIC simulations is the three points filter -:math:`\phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-1}+\phi_{j+1}\right)/2` -where :math:`\phi^{f}` is the filtered quantity. This filter is called -a bilinear filter when :math:`\alpha=0.5`. Assuming :math:`\phi=e^{jkx}` and -:math:`\phi^{f}=g\left(\alpha,k\right)e^{jkx}`, the filter gain :math:`g` is -given as a function of the filtering coefficient :math:`\alpha` and -the wavenumber :math:`k` by :math:`g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(k\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -The total attenuation :math:`G` for :math:`n` successive applications of filters -of coefficients :math:`\alpha_{1}`...\ :math:`\alpha_{n}` is given by :math:`G=\prod_{i=1}^{n}g\left(\alpha_{i},k\right)\approx1-\left(n-\sum_{i=1}^{n}\alpha_{i}\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -A sharper cutoff in :math:`k` space is provided by using :math:`\alpha_{n}=n-\sum_{i=1}^{n-1}\alpha_{i}`, -so that :math:`G\approx1+O\left(k^{4}\right)`. Such step is called a “compensation” -step (Birdsall and Langdon 1991). For the bilinear filter (:math:`\alpha=1/2`), -the compensation factor is :math:`\alpha_{c}=2-1/2=3/2`. For a succession -of :math:`n` applications of the bilinear factor, it is :math:`\alpha_{c}=n/2+1`. - -It is sometimes necessary to filter on a relatively wide band of wavelength, -necessitating the application of a large number of passes of the bilinear -filter or on the use of filters acting on many points. The former -can become very intensive computationally while the latter is problematic -for parallel computations using domain decomposition, as the footprint -of the filter may eventually surpass the size of subdomains. A workaround -is to use a combination of filters of limited footprint. A solution -based on the combination of three point filters with various strides -was proposed in (Vay et al. 2011) and operates as follows. - -The bilinear filter provides complete suppression of the signal at -the grid Nyquist wavelength (twice the grid cell size). Suppression -of the signal at integer multiples of the Nyquist wavelength can be -obtained by using a stride :math:`s` in the filter :math:`\phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-s}+\phi_{j+s}\right)/2` -for which the gain is given by :math:`g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(sk\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(sk\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -For a given stride, the gain is given by the gain of the bilinear -filter shifted in k space, with the pole :math:`g=0` shifted from the wavelength -:math:`\lambda=2/\Delta x` to :math:`\lambda=2s/\Delta x`, with additional poles, -as given by :math:`sk\Delta x=\arccos\left(\frac{\alpha}{\alpha-1}\right)\pmod{2\pi}`. -The resulting filter is pass band between the poles, but since the -poles are spread at different integer values in k space, a wide band -low pass filter can be constructed by combining filters using different -strides. As shown in (Vay et al. 2011), the successive application -of 4-passes + compensation of filters with strides 1, 2 and 4 has -a nearly equivalent fall-off in gain as 80 passes + compensation of -a bilinear filter. Yet, the strided filter solution needs only 15 -passes of a three-point filter, compared to 81 passes for an equivalent -n-pass bilinear filter, yielding a gain of 5.4 in number of operations -in favor of the combination of filters with stride. The width of the -filter with stride 4 extends only on 9 points, compared to 81 points -for a single pass equivalent filter, hence giving a gain of 9 in compactness -for the stride filters combination in comparison to the single-pass -filter with large stencil, resulting in more favorable scaling with the number -of computational cores for parallel calculations. - -.. raw:: latex - - \IfFileExists{\jobname.bbl}{} {\typeout{} \typeout{{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}} - \typeout{{*}{*} Please run \textquotedbl{}bibtex \jobname\textquotedbl{} - to optain} \typeout{{*}{*} the bibliography and then re-run LaTeX} - \typeout{{*}{*} twice to fix the references!} \typeout{{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}} - \typeout{} } - -.. raw:: html - -
- -.. raw:: html - -
- -Abe, H, N Sakairi, R Itatani, and H Okuda. 1986. “High-Order Spline Interpolations in the Particle Simulation.” *Journal of Computational Physics* 63 (2): 247–67. - -.. raw:: html - -
- -.. raw:: html - -
- -Birdsall, C K, and A B Langdon. 1991. *Plasma Physics via Computer Simulation*. Adam-Hilger. - -.. raw:: html - -
- -.. raw:: html - -
- -Boris, Jp. 1970. “Relativistic Plasma Simulation-Optimization of a Hybrid Code.” In *Proc. Fourth Conf. Num. Sim. Plasmas*, 3–67. Naval Res. Lab., Wash., D. C. - -.. raw:: html - -
- -.. raw:: html - -
- -Cole, J. B. 1997. “A High-Accuracy Realization of the Yee Algorithm Using Non-Standard Finite Differences.” *Ieee Transactions on Microwave Theory and Techniques* 45 (6): 991–96. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2002. “High-Accuracy Yee Algorithm Based on Nonstandard Finite Differences: New Developments and Verifications.” *Ieee Transactions on Antennas and Propagation* 50 (9): 1185–91. https://doi.org/10.1109/Tap.2002.801268. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, Benjamin M, David L Bruhwiler, John R Cary, Estelle Cormier-Michel, and Cameron G R Geddes. 2013. “Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations.” *Physical Review Special Topics-Accelerators and Beams* 16 (4). https://doi.org/10.1103/PhysRevSTAB.16.041303. - -.. raw:: html - -
- -.. raw:: html - -
- -Dawson, J M. 1983. “Particle Simulation of Plasmas.” *Reviews of Modern Physics* 55 (2): 403–47. https://doi.org/10.1103/RevModPhys.55.403. - -.. raw:: html - -
- -.. raw:: html - -
- -Esirkepov, Tz. 2001. “Exact Charge Conservation Scheme for Particle-in-Cell Simulation with an Arbitrary Form-Factor.” *Computer Physics Communications* 135 (2): 144–53. - -.. raw:: html - -
- -.. raw:: html - -
- -Haber, I, R Lee, Hh Klein, and Jp Boris. 1973. “Advances in Electromagnetic Simulation Techniques.” In *Proc. Sixth Conf. Num. Sim. Plasmas*, 46–48. Berkeley, Ca. - -.. raw:: html - -
- -.. raw:: html - -
- -Karkkainen, M, E Gjonaj, T Lau, and T Weiland. 2006. “Low-Dispersionwake Field Calculation Tools.” In *Proc. Of International Computational Accelerator Physics Conference*, 35–40. Chamonix, France. - -.. raw:: html - -
- -.. raw:: html - -
- -Langdon, A B. 1992. “On Enforcing Gauss Law in Electromagnetic Particle-in-Cell Codes.” *Computer Physics Communications* 70 (3): 447–50. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, R, A Lifschitz, C Thaury, V Malka, and X Davoine. 2013. “Numerical growth of emittance in simulations of laser-wakefield acceleration.” *Physical Review Special Topics-Accelerators and Beams* 16 (2). https://doi.org/10.1103/PhysRevSTAB.16.021301. - -.. raw:: html - -
- -.. raw:: html - -
- -Liu, Qh. 1997. “The PSTD Algorithm: A Time-Domain Method Requiring Only Two Cells Per Wavelength.” *Microwave and Optical Technology Letters* 15 (3): 158–65. `https://doi.org/10.1002/(SICI)1098-2760(19970620)15\:3\<158\:\:AID-MOP11\>3.0.CO\;2-3 3.0.CO\;2-3>`_. - -.. raw:: html - -
- -.. raw:: html - -
- -Marder, B. 1987. “A Method for Incorporating Gauss Law into Electromagnetic Pic Codes.” *Journal of Computational Physics* 68 (1): 48–55. - -.. raw:: html - -
- -.. raw:: html - -
- -Morse, Rl, and Cw Nielson. 1971. “Numerical Simulation of Weibel Instability in One and 2 Dimensions.” *Phys. Fluids* 14 (4): 830 –&. https://doi.org/10.1063/1.1693518. - -.. raw:: html - -
- -.. raw:: html - -
- -Munz, Cd, P Omnes, R Schneider, E Sonnendrucker, and U Voss. 2000. “Divergence Correction Techniques for Maxwell Solvers Based on A Hyperbolic Model.” *Journal of Computational Physics* 161 (2): 484–511. https://doi.org/10.1006/Jcph.2000.6507. - -.. raw:: html - -
- -.. raw:: html - -
- -Ohmura, Y, and Y Okamura. 2010. “Staggered Grid Pseudo-Spectral Time-Domain Method for Light Scattering Analysis.” *Piers Online* 6 (7): 632–35. - -.. raw:: html - -
- -.. raw:: html - -
- -Pukhov, A. 1999. “Three-dimensional electromagnetic relativistic particle-in-cell code VLPL (Virtual Laser Plasma Lab).” *Journal of Plasma Physics* 61 (3): 425–33. https://doi.org/10.1017/S0022377899007515. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean-Luc, Irving Haber, and Brendan B Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243 (June): 260–68. https://doi.org/10.1016/j.jcp.2013.03.010. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean Luc, Irving Haber, and Brendan B. Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243: 260–68. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L. 2008. “Simulation of Beams or Plasmas Crossing at Relativistic Velocity.” *Physics of Plasmas* 15 (5): 56701. https://doi.org/10.1063/1.2837054. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., J.-C. Adam, and A Heron. 2004. “Asymmetric Pml for the Absorption of Waves. Application to Mesh Refinement in Electromagnetic Particle-in-Cell Plasma Simulations.” *Computer Physics Communications* 164 (1-3): 171–77. https://doi.org/10.1016/J.Cpc.2004.06.026. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., and C Deutsch. 1998. “Charge Compensated Ion Beam Propagation in A Reactor Sized Chamber.” *Physics of Plasmas* 5 (4): 1190–7. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Numerical Methods for Instability Mitigation in the Modeling of Laser Wakefield Accelerators in A Lorentz-Boosted Frame.” *Journal of Computational Physics* 230 (15): 5908–29. https://doi.org/10.1016/J.Jcp.2011.04.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., D P Grote, R H Cohen, and A Friedman. 2012. “Novel methods in the particle-in-cell accelerator code-framework warp.” Journal Paper. *Computational Science and Discovery* 5 (1): 014019 (20 pp.). - -.. raw:: html - -
- -.. raw:: html - -
- -Villasenor, J, and O Buneman. 1992. “Rigorous Charge Conservation for Local Electromagnetic-Field Solvers.” *Computer Physics Communications* 69 (2-3): 306–16. - -.. raw:: html - -
- -.. raw:: html - -
- -Vincenti, H., and J.-L. Vay. 2016. “Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver.” *Computer Physics Communications* 200 (March). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 147–67. https://doi.org/10.1016/j.cpc.2015.11.009. - -.. raw:: html - -
- -.. raw:: html - -
- -Yee, Ks. 1966. “Numerical Solution of Initial Boundary Value Problems Involving Maxwells Equations in Isotropic Media.” *Ieee Transactions on Antennas and Propagation* Ap14 (3): 302–7. - -.. raw:: html - -
- -.. raw:: html - -
diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index e733938a93c..855faab392a 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -7,247 +7,169 @@ This section allows you to **download input files** that correspond to different We provide two kinds of inputs: -* AMReX ``inputs`` files, :ref:`with parameters described here `, * PICMI python input files, `with parameters described here `__. +* AMReX ``inputs`` files, :ref:`with parameters described here `, -For a complete list of all example input files, have a look at our ``Examples/`` directory. -It contains folders and subfolders with self-describing names that you can try. All these input files are automatically tested, so they should always be up-to-date. +For a complete list of all example input files, also have a look at our `Examples/ `__ directory. +It contains folders and subfolders with self-describing names that you can try. +All these input files are automatically tested, so they should always be up-to-date. -Beam-driven electron acceleration ---------------------------------- -AMReX ``inputs``: +Plasma-Based Acceleration +------------------------- -* :download:`2D case <../../../Examples/Physics_applications/plasma_acceleration/inputs_2d>` -* :download:`2D case in boosted frame <../../../Examples/Physics_applications/plasma_acceleration/inputs_2d_boost>` -* :download:`3D case in boosted frame <../../../Examples/Physics_applications/plasma_acceleration/inputs_3d_boost>` +.. toctree:: + :maxdepth: 1 -PICMI: + examples/lwfa/README.rst + examples/pwfa/README.rst + pwfa.rst -* :download:`Without mesh refinement <../../../Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py>` -* :download:`With mesh refinement <../../../Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py>` +Coming soon: -Laser-driven electron acceleration ----------------------------------- +* LWFA: External injection in the boosted frame +* LWFA: Ionization injection in the lab frame using a LASY data file +* PWFA: External injection in the boosted frame +* PWFA: Self-injection in the lab frame +* MR case? -AMReX ``inputs``: -* :download:`1D case <../../../Examples/Physics_applications/laser_acceleration/inputs_1d>` -* :download:`2D case <../../../Examples/Physics_applications/laser_acceleration/inputs_2d>` -* :download:`2D case in boosted frame <../../../Examples/Physics_applications/laser_acceleration/inputs_2d_boost>` -* :download:`3D case <../../../Examples/Physics_applications/laser_acceleration/inputs_3d>` -* :download:`RZ case <../../../Examples/Physics_applications/laser_acceleration/inputs_rz>` +Laser-Plasma Interaction +------------------------ -PICMI (Python) scripts: +.. toctree:: + :maxdepth: 1 -* :download:`1D case <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py>` -* :download:`2D case with mesh refinement <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py>` -* :download:`3D case <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py>` -* :download:`RZ case <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py>` + examples/laser_ion/README.rst + examples/plasma_mirror/README.rst -Plasma mirror -------------- +Coming soon: -:download:`2D case <../../../Examples/Physics_applications/plasma_mirror/inputs_2d>` +* MVA (3D & RZ) +* MR for the planar example? -Laser-ion acceleration ----------------------- -:download:`2D case <../../../Examples/Physics_applications/laser_ion/inputs>` +Particle Accelerator & Beam Physics +----------------------------------- -.. note:: +.. toctree:: + :maxdepth: 1 - The resolution of this 2D case is extremely low by default. - You will need a computing cluster for adequate resolution of the target density, see comments in the input file. + examples/gaussian_beam/README.rst + examples/beam-beam_collision/README.rst -.. warning:: +Coming soon: - It is strongly advised to set the parameters ``.zmin / zmax / xmin / ...`` when working with highly dense targets that are limited in one or multiple dimensions. - The particle creation routine will first create particles everywhere between these limits (`defaulting to box size if unset`), setting particles to invalid only afterwards based on the density profile. - Not setting these parameters can quickly lead to memory overflows. +* Beam-Beam Collision +* Beam Transport or Injector +* Cathode/source -Uniform plasma --------------- -:download:`2D case <../../../Examples/Physics_applications/uniform_plasma/inputs_2d>` -:download:`3D case <../../../Examples/Physics_applications/uniform_plasma/inputs_3d>` +High Energy Astrophysical Plasma Physics +---------------------------------------- -Capacitive discharge --------------------- +.. toctree:: + :maxdepth: 1 -The Monte-Carlo collision (MCC) model can be used to simulate electron and ion collisions with a neutral background gas. In particular this can be used to study capacitive discharges between parallel plates. The implementation has been tested against the benchmark results from :cite:t:`ex-Turner2013`. The figure below shows a comparison of the ion density as calculated in WarpX (in June 2022 with `PR #3118 `_) compared to the literature results (which can be found `here `__). + examples/ohm_solver_magnetic_reconnection/README.rst -.. figure:: https://user-images.githubusercontent.com/40245517/171573007-f7d733c7-c0de-490c-9ed6-ff4c02154358.png - :alt: MCC benchmark against Turner et. al. (2013). - :width: 80% -An input file to reproduce the benchmark calculations is linked below. -To run a given case ``-n``, from 1 to 4, execute: +Microelectronics +---------------- - .. code-block:: bash +`ARTEMIS (Adaptive mesh Refinement Time-domain ElectrodynaMIcs Solver) `__ is based on WarpX and couples the Maxwell's equations implementation in WarpX with classical equations that describe quantum material behavior (such as, LLG equation for micromagnetics and London equation for superconducting materials) for quantifying the performance of `next-generation microelectronics `__. - python3 PICMI_inputs_1d.py -n 1 +* `ARTEMIS examples `__ +* `ARTEMIS manual `__ -Once the simulation completes an output file ``avg_ion_density.npy`` will be created which can be compared to the literature results as in the plot above. Running case 1 on 4 processors takes roughly 20 minutes to complete. -* :download:`input file <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py>` +Nuclear Fusion +-------------- .. note:: - This example needs `additional calibration data for cross sections `__. - Download this data alongside your inputs file and update the paths in the inputs file: - - .. code-block:: bash + TODO - git clone https://github.com/ECP-WarpX/warpx-data.git +Coming soon: -Test cases ----------- +* Microchannel +* Magnetically Confined Plasma with a Single Coil - Magnetic bottle: simple geometry with an external field -PICMI (Python) test cases included that can be used as a reference: -* :download:`Gaussian beam <../../../Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py>` -* :download:`Langmuir plasma wave test in 3d <../../../Examples/Tests/langmuir/PICMI_inputs_langmuir_rt.py>` -* :download:`Langmuir plasma wave test in RZ <../../../Examples/Tests/langmuir/PICMI_inputs_langmuir_rz_multimode_analyze.py>` -* :download:`Langmuir plasma wave test in 2D <../../../Examples/Tests/langmuir/PICMI_inputs_langmuir2d.py>` - -Manipulating fields via Python ------------------------------- - -An example of using Python to access the simulation charge density, solve the Poisson equation (using ``superLU``) and write the resulting electrostatic potential back to the simulation is given in the input file below. This example uses the ``fields.py`` module included in the ``pywarpx`` library. - -* :download:`Direct Poisson solver example <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py>` - -An example of initializing the fields by accessing their data through Python, advancing the simulation for a chosen number of time steps, and plotting the fields again through Python. The simulation runs with 128 regular cells, 8 guard cells, and 10 PML cells, in each direction. Moreover, it uses div(E) and div(B) cleaning both in the regular grid and in the PML and initializes all available electromagnetic fields (E,B,F,G) identically. - -* :download:`Unit pulse with PML <../../../Examples/Tests/python_wrappers/PICMI_inputs_2d.py>` - -.. _examples-hybrid-model: - -Kinetic-fluid hybrid model +Fundamental Plasma Physics -------------------------- -Several examples and benchmarks of the kinetic-fluid hybrid model are shown below. The first few examples are replications -of the verification tests described in :cite:t:`ex-MUNOZ2018`. -The hybrid-PIC model was added to WarpX in `PR #3665 `_ - the figures below -were generated at that time. - -Electromagnetic modes -^^^^^^^^^^^^^^^^^^^^^ +.. toctree:: + :maxdepth: 1 -In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the -:math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are -analyzed for mode excitations. + examples/langmuir/README.rst + examples/capacitive_discharge/README.rst -Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except -in a region of thermal resonances as indicated on the plot below. +Coming soon: -.. figure:: https://user-images.githubusercontent.com/40245517/216207688-9c39374a-9e69-45b8-a588-35b087b83d27.png - :alt: Parallel EM modes in thermal ion plasma - :width: 70% +* Expanding Sphere example -Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. - -.. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png - :alt: Perpendicular EM modes in thermal ion plasma - :width: 50% - -The input file for these examples and the corresponding analysis can be found at: - -* :download:`EM modes input <../../../Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_EM_modes/analysis.py>` - -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the parallel propagating or -ion-Bernstein modes as indicated below. - - .. code-block:: bash - - python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y/z} - -Ion beam R instability -^^^^^^^^^^^^^^^^^^^^^^ - -In this example a low density ion beam interacts with a "core" plasma population which induces an instability. -Based on the relative density between the beam and the core plasma a resonant or non-resonant condition can -be accessed. The figures below show the evolution of the y-component of the magnetic field as the beam and -core plasma interact. - -.. figure:: https://user-images.githubusercontent.com/40245517/217923933-6bdb65cb-7d26-40d8-8687-7dd75274bd48.png - :alt: Resonant ion beam R instability - :width: 70% - -.. figure:: https://user-images.githubusercontent.com/40245517/217925983-b91d6482-69bc-43c1-8c7d-23ebe7c69d49.png - :alt: Non-resonant ion beam R instability - :width: 70% - -The growth rates of the strongest growing modes for the resonant case are compared -to theory (dashed lines) in the figure below. - -.. figure:: https://github.com/ECP-WarpX/WarpX/assets/40245517/a94bb6e5-30e9-4d8f-9e6b-844dc8f51d17 - :alt: Resonant ion beam R instability growth rates - :width: 50% +.. _examples-hybrid-model: -The input file for these examples and the corresponding analysis can be found at: +Kinetic-fluid Hybrid Models +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* :download:`Ion beam R instability input <../../../Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_ion_beam_instability/analysis.py>` +WarpX includes a reduced plasma model in which electrons are treated as a massless +fluid while ions are kinetically evolved, and Ohm's law is used to calculate +the electric field. This model is appropriate for problems in which ion kinetics +dominate (ion cyclotron waves, for instance). See the +:ref:`theory section ` for more details. Several +examples and benchmarks of this kinetic-fluid hybrid model are provided below. +A few of the examples are replications of the verification tests described in +:cite:t:`ex-MUNOZ2018`. The hybrid-PIC model was added to WarpX in +`PR #3665 `_ - the figures in the +examples below were generated at that time. -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the resonant or non-resonant -condition as indicated below. +.. toctree:: + :maxdepth: 1 - .. code-block:: bash + examples/ohm_solver_EM_modes/README.rst + examples/ohm_solver_ion_beam_instability/README.rst + examples/ohm_solver_ion_Landau_damping/README.rst - python3 PICMI_inputs.py -dim {1/2/3} --resonant -Ion Landau damping -^^^^^^^^^^^^^^^^^^ +High-Performance Computing and Numerics +--------------------------------------- -Landau damping is a well known process in which electrostatic (acoustic) waves -are damped by transferring energy to particles satisfying a resonance condition. -The process can be simulated by seeding a plasma with a specific acoustic mode -(density perturbation) and tracking the strength of the mode as a function of -time. The figure below shows a set of such simulations with parameters matching -those described in section 4.5 of :cite:t:`ex-MUNOZ2018`. The straight lines show -the theoretical damping rate for the given temperature ratios. +The following examples are commonly used to study the performance of WarpX, e.g., for computing efficiency, scalability, and I/O patterns. +While all prior examples are used for such studies as well, the examples here need less explanation on the physics, less-detail tuning on load balancing, and often simply scale (weak or strong) by changing the number of cells, AMReX block size and number of compute units. -.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png - :alt: Ion Landau damping - :width: 70% +.. toctree:: + :maxdepth: 1 -The input file for these examples and the corresponding analysis can be found at: + examples/uniform_plasma/README.rst -* :download:`Ion Landau damping input <../../../Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py>` -The same input script can be used for 1d, 2d or 3d simulations and to sweep different -temperature ratios. +Manipulating fields via Python +------------------------------ - .. code-block:: bash +.. note:: - python3 PICMI_inputs.py -dim {1/2/3} --temp_ratio {value} + TODO: The section needs to be sorted into either science cases (above) or later sections (workflows and Python API details). -Magnetic reconnection -^^^^^^^^^^^^^^^^^^^^^ +An example of using Python to access the simulation charge density, solve the Poisson equation (using ``superLU``) and write the resulting electrostatic potential back to the simulation is given in the input file below. This example uses the ``fields.py`` module included in the ``pywarpx`` library. -Hybrid-PIC codes are often used to simulate magnetic reconnection in space -plasmas. An example of magnetic reconnection from a force-free sheet is -provided, based on the simulation described in :cite:t:`ex-Le2016`. +* :download:`Direct Poisson solver example <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py>` -.. figure:: https://user-images.githubusercontent.com/40245517/229639784-b5d3b596-3550-4570-8761-8d9a67aa4b3b.gif - :alt: Magnetic reconnection - :width: 70% +An example of initializing the fields by accessing their data through Python, advancing the simulation for a chosen number of time steps, and plotting the fields again through Python. The simulation runs with 128 regular cells, 8 guard cells, and 10 PML cells, in each direction. Moreover, it uses div(E) and div(B) cleaning both in the regular grid and in the PML and initializes all available electromagnetic fields (E,B,F,G) identically. -The input file for this example and corresponding analysis can be found at: +* :download:`Unit pulse with PML <../../../Examples/Tests/python_wrappers/PICMI_inputs_2d.py>` -* :download:`Magnetic reconnection input <../../../Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py>` Many Further Examples, Demos and Tests -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------------------- WarpX runs over 200 integration tests on a variety of modeling cases, which validate and demonstrate its functionality. Please see the `Examples/Tests/ `__ directory for many more examples. +Example References +------------------ + .. bibliography:: :keyprefix: ex- diff --git a/Docs/source/usage/examples/beam-beam_collision b/Docs/source/usage/examples/beam-beam_collision new file mode 120000 index 00000000000..8c6ac6b30b1 --- /dev/null +++ b/Docs/source/usage/examples/beam-beam_collision @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/beam-beam_collision \ No newline at end of file diff --git a/Docs/source/usage/examples/capacitive_discharge b/Docs/source/usage/examples/capacitive_discharge new file mode 120000 index 00000000000..dd1493f8f70 --- /dev/null +++ b/Docs/source/usage/examples/capacitive_discharge @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/capacitive_discharge \ No newline at end of file diff --git a/Docs/source/usage/examples/gaussian_beam b/Docs/source/usage/examples/gaussian_beam new file mode 120000 index 00000000000..a03c49d4dda --- /dev/null +++ b/Docs/source/usage/examples/gaussian_beam @@ -0,0 +1 @@ +../../../../Examples/Tests/gaussian_beam \ No newline at end of file diff --git a/Docs/source/usage/examples/langmuir b/Docs/source/usage/examples/langmuir new file mode 120000 index 00000000000..f0f0c5d7bc3 --- /dev/null +++ b/Docs/source/usage/examples/langmuir @@ -0,0 +1 @@ +../../../../Examples/Tests/langmuir \ No newline at end of file diff --git a/Docs/source/usage/examples/laser_ion b/Docs/source/usage/examples/laser_ion new file mode 120000 index 00000000000..8d9aa37f076 --- /dev/null +++ b/Docs/source/usage/examples/laser_ion @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/laser_ion \ No newline at end of file diff --git a/Docs/source/usage/examples/lwfa b/Docs/source/usage/examples/lwfa new file mode 120000 index 00000000000..4ec620fb20e --- /dev/null +++ b/Docs/source/usage/examples/lwfa @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/laser_acceleration \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_EM_modes b/Docs/source/usage/examples/ohm_solver_EM_modes new file mode 120000 index 00000000000..485be7241ae --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_EM_modes @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_EM_modes \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_ion_Landau_damping b/Docs/source/usage/examples/ohm_solver_ion_Landau_damping new file mode 120000 index 00000000000..c70b062da77 --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_ion_Landau_damping @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_ion_Landau_damping \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_ion_beam_instability b/Docs/source/usage/examples/ohm_solver_ion_beam_instability new file mode 120000 index 00000000000..3864ca292b5 --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_ion_beam_instability @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_ion_beam_instability \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_magnetic_reconnection b/Docs/source/usage/examples/ohm_solver_magnetic_reconnection new file mode 120000 index 00000000000..1dd74c99f84 --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_magnetic_reconnection @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_magnetic_reconnection \ No newline at end of file diff --git a/Docs/source/usage/examples/plasma_mirror b/Docs/source/usage/examples/plasma_mirror new file mode 120000 index 00000000000..46f96c3eb8f --- /dev/null +++ b/Docs/source/usage/examples/plasma_mirror @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/plasma_mirror \ No newline at end of file diff --git a/Docs/source/usage/examples/pwfa b/Docs/source/usage/examples/pwfa new file mode 120000 index 00000000000..044974be6ce --- /dev/null +++ b/Docs/source/usage/examples/pwfa @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/plasma_acceleration \ No newline at end of file diff --git a/Docs/source/usage/examples/uniform_plasma b/Docs/source/usage/examples/uniform_plasma new file mode 120000 index 00000000000..fe1c540dfa9 --- /dev/null +++ b/Docs/source/usage/examples/uniform_plasma @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/uniform_plasma \ No newline at end of file diff --git a/Docs/source/usage/how_to_run.rst b/Docs/source/usage/how_to_run.rst index 26c56cb6007..ab5cc6e79bd 100644 --- a/Docs/source/usage/how_to_run.rst +++ b/Docs/source/usage/how_to_run.rst @@ -25,7 +25,7 @@ Where ```` by the actual path to the run directory. ------------- If you installed warpX with a :ref:`package manager `, a ``warpx``-prefixed executable will be available as a regular system command to you. -Depending on the choosen build options, the name is suffixed with more details. +Depending on the chosen build options, the name is suffixed with more details. Try it like this: .. code-block:: bash diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 27d7682899d..81580f5c342 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1,7 +1,11 @@ .. _running-cpp-parameters: -Input Parameters -================ +Parameters: Inputs File +======================= + +This documents on how to use WarpX with an inputs file (e.g., ``warpx.3d input_3d``). + +Complete example input files can be found in :ref:`the examples section `. .. note:: @@ -77,6 +81,43 @@ Overall simulation parameters one should not expect to obtain the same random numbers, even if a fixed ``warpx.random_seed`` is provided. +* ``algo.evolve_scheme`` (`string`, default: `explicit`) + Specifies the evolve scheme used by WarpX. + + * ``explicit``: Use an explicit solver, such as the standard FDTD or PSATD + + * ``implicit_picard``: Use an implicit solver with exact energy conservation that uses a Picard iteration to solve the system. + Note that this method is for demonstration only. It is inefficient and does not work well when + :math:`\omega_{pe} \Delta t` is close to or greater than one. + The method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. + The version implemented is an updated version that is relativistically correct, including the relativistic gamma factor for the particles. + For exact energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, + and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in + which case charge will also be conserved). + + * ``semi_implicit_picard``: Use an energy conserving semi-implicit solver that uses a Picard iteration to solve the system. + Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. It is inefficient and does not work well or at all when :math:`\omega_{pe} \Delta t` is close to or greater than one. + The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. + For energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, + and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in + which case charge will also be conserved). + +* ``algo.max_picard_iterations`` (`integer`, default: 10) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the maximum number of Picard + itearations that are done each time step. + +* ``algo.picard_iteration_tolerance`` (`float`, default: 1.e-7) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the convergence tolerance of + the iterations, the maximum of the relative change of the L2 norm of the field from one iteration to the next. + If this is set to zero, the maximum number of iterations will always be done with the change only calculated on the last + iteration (for a slight optimization). + +* ``algo.require_picard_convergence`` (`bool`, default: 1) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets whether the iteration each step + is required to converge. + If it is required, an abort is raised if it does not converge and the code then exits. + If not, then a warning is issued and the calculation continues. + * ``warpx.do_electrostatic`` (`string`) optional (default `none`) Specifies the electrostatic mode. When turned on, instead of updating the fields at each iteration with the full Maxwell equations, the fields @@ -151,7 +192,7 @@ Overall simulation parameters This only applies when warpx.do_electrostatic = labframe. * ``warpx.self_fields_verbosity`` (`integer`, default: 2) - The vebosity used for MLMG solver for space-charge fields calculation. Currently + The verbosity used for MLMG solver for space-charge fields calculation. Currently MLMG solver looks for verbosity levels from 0-5. A higher number results in more verbose output. @@ -167,6 +208,12 @@ Overall simulation parameters For all regular WarpX operations, we therefore do explicit memory transfers without the need for managed memory and thus changed the AMReX default to false. `Please also see the documentation in AMReX `__. +* ``amrex.omp_threads`` (``system``, ``nosmt`` or positive integer; default is ``nosmt``) + An integer number can be set in lieu of the ``OMP_NUM_THREADS`` environment variable to control the number of OpenMP threads to use for the ``OMP`` compute backend on CPUs. + By default, we use the ``nosmt`` option, which overwrites the OpenMP default of spawning one thread per logical CPU core, and instead only spawns a number of threads equal to the number of physical CPU cores on the machine. + If set, the environment variable ``OMP_NUM_THREADS`` takes precedence over ``system`` and ``nosmt``, but not over integer numbers set in this option. + + Signal Handling ^^^^^^^^^^^^^^^ @@ -284,6 +331,13 @@ Setting up the field mesh This patch is rectangular, and thus its extent is given here by the coordinates of the lower corner (``warpx.fine_tag_lo``) and upper corner (``warpx.fine_tag_hi``). +* ``warpx.ref_patch_function(x,y,z)`` (`string`) optional + A function of `x`, `y`, `z` that defines the extent of the refined patch when + using static mesh refinement with ``amr.max_level``>0. Note that the function can be used + to define distinct regions for refinement, however, the refined regions should be such that + the pml layer surrounding the patches should not overlap. For this reason, when defining + distinct patches, please ensure that they are sufficiently separated. + * ``warpx.refine_plasma`` (`integer`) optional (default `0`) Increase the number of macro-particles that are injected "ahead" of a mesh refinement patch in a moving window simulation. @@ -352,7 +406,7 @@ Domain Boundary Conditions * ``Absorbing``: Particles leaving the boundary will be deleted. - * ``Periodic``: Particles leaving the boundary will re-enter from the opposite boundary. The field boundary condition must be consistenly set to periodic and both lower and upper boundaries must be periodic. + * ``Periodic``: Particles leaving the boundary will re-enter from the opposite boundary. The field boundary condition must be consistently set to periodic and both lower and upper boundaries must be periodic. * ``Reflecting``: Particles leaving the boundary are reflected from the boundary back into the domain. When ``boundary.reflect_all_velocities`` is false, the sign of only the normal velocity is changed, otherwise the sign of all velocities are changed. @@ -669,6 +723,19 @@ Particle initialization When ``.xmin`` and ``.xmax`` are set, they delimit the region within which particles are injected. If periodic boundary conditions are used in direction ``i``, then the default (i.e. if the range is not specified) range will be the simulation box, ``[geometry.prob_hi[i], geometry.prob_lo[i]]``. +* ``.injection_sources`` (``list of strings``) optional + Names of additional injection sources. By default, WarpX assumes one injection source per species, hence all of the input + parameters below describing the injection are parameters directly of the species. However, this option allows + additional sources, the names of which are specified here. For each source, the name of the source is added to the + input parameters below. For instance, with ``.injection_sources = source1 source2`` there can be the two input + parameters ``.source1.injection_style`` and ``.source2.injection_style``. + For the parameters of each source, the parameter with the name of the source will be used. + If it is not given, the value of the parameter without the source name will be used. This allows parameters used for all + sources to be specified once. For example, if the ``source1`` and ``source2`` have the same value of ``uz_m``, then it can be + set using ``.uz_m`` instead of setting it for each source. + Note that since by default ``.injection_style = none``, all injection sources can be input this way. + Note that if a moving window is used, the bulk velocity of all of the sources must be the same since it is used when updating the window. + * ``.injection_style`` (`string`; default: ``none``) Determines how the (macro-)particles will be injected in the simulation. The number of particles per cell is always given with respect to the coarsest level (level 0/mother grid), even if particles are immediately assigned to a refined patch. @@ -683,29 +750,48 @@ Particle initialization * ``SingleParticle``: Inject a single macroparticle. This requires the additional parameters: - ``.single_particle_pos`` (`3 doubles`, particle 3D position [meter]) - ``.single_particle_u`` (`3 doubles`, particle 3D normalized momentum, i.e. :math:`\gamma \beta`) - ``.single_particle_weight`` ( `double`, macroparticle weight, i.e. number of physical particles it represents) + + * ``.single_particle_pos`` (`3 doubles`, particle 3D position [meter]) + + * ``.single_particle_u`` (`3 doubles`, particle 3D normalized momentum, i.e. :math:`\gamma \beta`) + + * ``.single_particle_weight`` ( `double`, macroparticle weight, i.e. number of physical particles it represents) * ``MultipleParticles``: Inject multiple macroparticles. This requires the additional parameters: - ``.multiple_particles_pos_x`` (list of `doubles`, X positions of the particles [meter]) - ``.multiple_particles_pos_y`` (list of `doubles`, Y positions of the particles [meter]) - ``.multiple_particles_pos_z`` (list of `doubles`, Z positions of the particles [meter]) - ``.multiple_particles_ux`` (list of `doubles`, X normalized momenta of the particles, i.e. :math:`\gamma \beta_x`) - ``.multiple_particles_uy`` (list of `doubles`, Y normalized momenta of the particles, i.e. :math:`\gamma \beta_y`) - ``.multiple_particles_uz`` (list of `doubles`, Z normalized momenta of the particles, i.e. :math:`\gamma \beta_z`) - ``.multiple_particles_weight`` (list of `doubles`, macroparticle weights, i.e. number of physical particles each represents) + + * ``.multiple_particles_pos_x`` (list of `doubles`, X positions of the particles [meter]) + + * ``.multiple_particles_pos_y`` (list of `doubles`, Y positions of the particles [meter]) + + * ``.multiple_particles_pos_z`` (list of `doubles`, Z positions of the particles [meter]) + + * ``.multiple_particles_ux`` (list of `doubles`, X normalized momenta of the particles, i.e. :math:`\gamma \beta_x`) + + * ``.multiple_particles_uy`` (list of `doubles`, Y normalized momenta of the particles, i.e. :math:`\gamma \beta_y`) + + * ``.multiple_particles_uz`` (list of `doubles`, Z normalized momenta of the particles, i.e. :math:`\gamma \beta_z`) + + * ``.multiple_particles_weight`` (list of `doubles`, macroparticle weights, i.e. number of physical particles each represents) * ``gaussian_beam``: Inject particle beam with gaussian distribution in space in all directions. This requires additional parameters: - ``.q_tot`` (beam charge), - ``.npart`` (number of particles in the beam), - ``.x/y/z_m`` (average position in `x/y/z`), - ``.x/y/z_rms`` (standard deviation in `x/y/z`), - ``.x/y/z_cut`` (optional, particles with ``abs(x-x_m) > x_cut*x_rms`` are not injected, same for y and z. ``.q_tot`` is the charge of the un-cut beam, so that cutting the distribution is likely to result in a lower total charge), - and optional arguments ``.do_symmetrize`` (whether to - symmetrize the beam) and ``.symmetrization_order`` (order of symmetrization, default is 4, can be 4 or 8). + + * ``.q_tot`` (beam charge), + + * ``.npart`` (number of particles in the beam), + + * ``.x/y/z_m`` (average position in `x/y/z`), + + * ``.x/y/z_rms`` (standard deviation in `x/y/z`), + + There are additional optional parameters: + + * ``.x/y/z_cut`` (optional, particles with ``abs(x-x_m) > x_cut*x_rms`` are not injected, same for y and z. ``.q_tot`` is the charge of the un-cut beam, so that cutting the distribution is likely to result in a lower total charge), + * ``.do_symmetrize`` (optional, whether to symmetrize the beam) + + * ``.symmetrization_order`` (order of symmetrization, default is 4, can be 4 or 8). + If ``.do_symmetrize`` is 0, no symmetrization occurs. If ``.do_symmetrize`` is 1, then the beam is symmetrized according to the value of ``.symmetrization_order``. If set to 4, symmetrization is in the x and y direction, (x,y) (-x,y) (x,-y) (-x,-y). @@ -713,11 +799,17 @@ Particle initialization * ``external_file``: Inject macroparticles with properties (mass, charge, position, and momentum - :math:`\gamma \beta m c`) read from an external openPMD file. With it users can specify the additional arguments: - ``.injection_file`` (`string`) openPMD file name and - ``.charge`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. - ``.mass`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. - ``.z_shift`` (`double`) optional (default is no shift) when set this value will be added to the longitudinal, ``z``, position of the particles. - ``.impose_t_lab_from_file`` (`bool`) optional (default is false) only read if warpx.gamma_boost > 1., it allows to set t_lab for the Lorentz Transform as being the time stored in the openPMD file. + + * ``.injection_file`` (`string`) openPMD file name and + + * ``.charge`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. + + * ``.mass`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. + + * ``.z_shift`` (`double`) optional (default is no shift) when set this value will be added to the longitudinal, ``z``, position of the particles. + + * ``.impose_t_lab_from_file`` (`bool`) optional (default is false) only read if warpx.gamma_boost > 1., it allows to set t_lab for the Lorentz Transform as being the time stored in the openPMD file. + Warning: ``q_tot!=0`` is not supported with the ``external_file`` injection style. If a value is provided, it is ignored and no re-scaling is done. The external file must include the species ``openPMD::Record`` labeled ``position`` and ``momentum`` (`double` arrays), with dimensionality and units set via ``openPMD::setUnitDimension`` and ``setUnitSI``. If the external file also contains ``openPMD::Records`` for ``mass`` and ``charge`` (constant `double` scalars) then the species will use these, unless overwritten in the input file (see ``.mass``, ``.charge`` or ``.species_type``). @@ -726,13 +818,20 @@ Particle initialization * ``NFluxPerCell``: Continuously inject a flux of macroparticles from a planar surface. This requires the additional parameters: - ``.flux_profile`` (see the description of this parameter further below) - ``.surface_flux_pos`` (`double`, location of the injection plane [meter]) - ``.flux_normal_axis`` (`x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) - ``.flux_direction`` (`-1` or `+1`, direction of flux relative to the plane) - ``.num_particles_per_cell`` (`double`) - ``.flux_tmin`` (`double`, Optional time at which the flux will be turned on. Ignored when negative.) - ``.flux_tmax`` (`double`, Optional time at which the flux will be turned off. Ignored when negative.) + + * ``.flux_profile`` (see the description of this parameter further below) + + * ``.surface_flux_pos`` (`double`, location of the injection plane [meter]) + + * ``.flux_normal_axis`` (`x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) + + * ``.flux_direction`` (`-1` or `+1`, direction of flux relative to the plane) + + * ``.num_particles_per_cell`` (`double`) + + * ``.flux_tmin`` (`double`, Optional time at which the flux will be turned on. Ignored when negative.) + + * ``.flux_tmax`` (`double`, Optional time at which the flux will be turned off. Ignored when negative.) * ``none``: Do not inject macro-particles (for example, in a simulation that starts with neutral, ionizable atoms, one may want to create the electrons species -- where ionized electrons can be stored later on -- without injecting electron macro-particles). @@ -831,7 +930,7 @@ Particle initialization ``.ux``, ``.uy`` and ``.uz``, the normalized momenta in the x, y and z direction respectively, which are all ``0.`` by default. - * ``uniform``: uniform probability distribution between a minumum and a maximum value. + * ``uniform``: uniform probability distribution between a minimum and a maximum value. The x, y and z directions are sampled independently and the final momentum space is a cuboid. The parameters that control the minimum and maximum domain of the distribution are ``.u_min`` and ``.u_max`` in each @@ -908,6 +1007,20 @@ Particle initialization ``.momentum_function_uy(x,y,z)`` and ``.momentum_function_uz(x,y,z)``, which gives the distribution of each component of the momentum as a function of space. + * ``gaussian_parse_momentum_function``: Gaussian momentum distribution where the mean and the standard deviation are given by functions of position in the input file. + Both are assumed to be non-relativistic. + The mean is the normalized momentum, :math:`u_m = \gamma v_m/c`. + The standard deviation is normalized, :math:`u_th = v_th/c`. + For example, this might be `u_th = sqrt(T*q_e/mass)/clight` given the temperature (in eV) and mass. + It requires the following arguments: + + * ``.momentum_function_ux_m(x,y,z)``: mean :math:`u_{x}` + * ``.momentum_function_uy_m(x,y,z)``: mean :math:`u_{y}` + * ``.momentum_function_uz_m(x,y,z)``: mean :math:`u_{z}` + * ``.momentum_function_ux_th(x,y,z)``: standard deviation of :math:`u_{x}` + * ``.momentum_function_uy_th(x,y,z)``: standard deviation of :math:`u_{y}` + * ``.momentum_function_uz_th(x,y,z)``: standard deviation of :math:`u_{z}` + * ``.theta_distribution_type`` (`string`) optional (default ``constant``) Only read if ``.momentum_distribution_type`` is ``maxwell_boltzmann`` or ``maxwell_juttner``. See documentation for these distributions (above) for constraints on values of theta. Temperatures less than zero are not allowed. @@ -1044,11 +1157,12 @@ Particle initialization the above mentioned function. .. note:: - When accessing the data via Python, the scraped particle buffer relies on the user - to clear the buffer after processing the data. The - buffer will grow unbounded as particles are scraped and therefore could - lead to memory issues if not periodically cleared. To clear the buffer - call ``warpx_clearParticleBoundaryBuffer()``. + + When accessing the data via Python, the scraped particle buffer relies on the user + to clear the buffer after processing the data. The + buffer will grow unbounded as particles are scraped and therefore could + lead to memory issues if not periodically cleared. To clear the buffer + call ``clear_buffer()``. * ``.do_field_ionization`` (`0` or `1`) optional (default `0`) Do field ionization for this species (using the ADK theory). @@ -1136,6 +1250,18 @@ Particle initialization Resampling is performed everytime the number of macroparticles per cell of the species averaged over the whole simulation domain exceeds this parameter. + +.. _running-cpp-parameters-fluids: + +Cold Relativistic Fluid initialization +-------------------------------------- + +* ``fluids.species_names`` (`strings`, separated by spaces) + Defines the names of each fluid species. It is a required input to create and evolve fluid species using the cold relativistic fluid equations. + Most of the parameters described in the section "Particle initialization" can also be used to initialize fluid properties (e.g. initial density distribution). + For fluid-specific inputs we use `` as a placeholder. Also see external fields + for how to specify these for fluids as the function names differ. + .. _running-cpp-parameters-laser: Laser initialization @@ -1186,7 +1312,7 @@ Laser initialization be parallel to ``warpx.boost_direction``, for now. * ``.e_max`` (`float` ; in V/m) - Peak amplitude of the laser field. + Peak amplitude of the laser field, in the focal plane. For a laser with a wavelength :math:`\lambda = 0.8\,\mu m`, the peak amplitude is related to :math:`a_0` by: @@ -1200,9 +1326,9 @@ Laser initialization perform the conversion to the boosted frame. * ``.a0`` (`float` ; dimensionless) - Peak normalized amplitude of the laser field (given in the lab frame, just as ``e_max`` above). + Peak normalized amplitude of the laser field, in the focal plane (given in the lab frame, just as ``e_max`` above). See the description of ``.e_max`` for the conversion between ``a0`` and ``e_max``. - Exactly one of ``a0`` and ``e_max`` must be specified. + Either ``a0`` or ``e_max`` must be specified. * ``.wavelength`` (`float`; in meters) The wavelength of the laser in vacuum. @@ -1251,14 +1377,16 @@ Laser initialization ``.e_max`` (i.e. in most cases the maximum of abs(E(x,y,t)) should be 1, so that the maximum field intensity can be set straightforwardly with ``.e_max``). The binary file has to respect the following format: - * flag to indicate the grid is uniform(1 byte, 0 means non-uniform, !=0 means uniform) - only uniform is supported - * nt, number of timesteps (uint32_t, must be >=2) - * nx, number of points along x (uint32_t, must be >=2) - * ny, number of points along y (uint32_t, must be 1 for 2D simulations and >=2 for 3D simulations) - * timesteps (double[2]=[t_min,t_max]) - * x_coords (double[2]=[x_min,x_max]) - * y_coords (double[1] in 2D, double[2]=[y_min,y_max] in 3D) - * field_data (double[nt * nx * ny], with nt being the slowest coordinate). + + * ``flag`` to indicate the grid is uniform (1 byte, 0 means non-uniform, !=0 means uniform) - only uniform is supported + * ``nt``, number of timesteps (``uint32_t``, must be >=2) + * ``nx``, number of points along x (``uint32_t``, must be >=2) + * ``ny``, number of points along y (``uint32_t``, must be 1 for 2D simulations and >=2 for 3D simulations) + * ``timesteps`` (``double[2]=[t_min,t_max]``) + * ``x_coords`` (``double[2]=[x_min,x_max]``) + * ``y_coords`` (``double[1]`` in 2D, ``double[2]=[y_min,y_max]`` in 3D) + * ``field_data`` (``double[nt x nx * ny]``, with ``nt`` being the slowest coordinate). + A binary file can be generated from Python, see an example at ``Examples/Tests/laser_injection_from_file`` * ``.profile_t_peak`` (`float`; in seconds) @@ -1278,12 +1406,15 @@ Laser initialization Note that :math:`\tau` relates to the full width at half maximum (FWHM) of *intensity*, which is closer to pulse length measurements in experiments, as :math:`\tau = \mathrm{FWHM}_I / \sqrt{2\ln(2)}` :math:`\approx \mathrm{FWHM}_I / 1.1774`. + For a chirped laser pulse (i.e. with a non-zero ``.phi2``), ``profile_duration`` is the Fourier-limited duration of the pulse, not the actual duration of the pulse. See the documentation for ``.phi2`` for more detail. + When running a **boosted-frame simulation**, provide the value of ``.profile_duration`` in the laboratory frame, and use ``warpx.gamma_boost`` to automatically perform the conversion to the boosted frame. * ``.profile_waist`` (`float` ; in meters) - The waist of the transverse Gaussian laser profile, defined as :math:`w_0` : + The waist of the transverse Gaussian :math:`w_0`, i.e. defined such that the electric field of the + laser pulse in the focal plane is of the form: .. math:: @@ -1316,8 +1447,23 @@ Laser initialization See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). * ``.phi2`` (`float`; in seconds**2) optional (default `0.`) - Temporal chirp at focus. - See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). + The amount of temporal chirp :math:`\phi^{(2)}`, at focus (in the lab frame). Namely, a wave packet + centered on the frequency :math:`(\omega_0 + \delta \omega)` will reach its peak intensity + at :math:`z(\delta \omega) = z_0 - c \phi^{(2)} \, \delta \omega`. Thus, a positive + :math:`\phi^{(2)}` corresponds to positive chirp, i.e. red part of the spectrum in the + front of the pulse and blue part of the spectrum in the back. More specifically, the electric + field in the focal plane is of the form: + + .. math:: + + E(\boldsymbol{x},t) \propto Re\left[ \exp\left( -\frac{(t-t_{peak})^2}{\tau^2 + 2i\phi^{(2)}} + i\omega_0 (t-t_{peak}) + i\phi_0 \right) \right] + + where :math:`\tau` is given by ``.profile_duration`` and represents the + Fourier-limited duration of the laser pulse. Thus, the actual duration of the chirped laser pulse is: + + .. math:: + + \tau' = \sqrt{ \tau^2 + 4 (\phi^{(2)})^2/\tau^2 } * ``.do_continuous_injection`` (`0` or `1`) optional (default `0`). Whether or not to use continuous injection. @@ -1365,10 +1511,10 @@ External fields Grid initialization ^^^^^^^^^^^^^^^^^^^ -* ``warpx.B_ext_grid_init_style`` (string) optional (default is "default") +* ``warpx.B_ext_grid_init_style`` (string) optional This parameter determines the type of initialization for the external - magnetic field. The "default" style initializes the - external magnetic field (Bx,By,Bz) to (0.0, 0.0, 0.0). + magnetic field. By default, the + external magnetic field (Bx,By,Bz) is initialized to (0.0, 0.0, 0.0). The string can be set to "constant" if a constant magnetic field is required to be set at initialization. If set to "constant", then an additional parameter, namely, ``warpx.B_external_grid`` must be specified. @@ -1396,9 +1542,9 @@ Grid initialization Regarding how to prepare the openPMD data file, one can refer to the `openPMD-example-datasets `__. -* ``warpx.E_ext_grid_init_style`` (string) optional (default is "default") +* ``warpx.E_ext_grid_init_style`` (string) optional This parameter determines the type of initialization for the external - electric field. The "default" style initializes the + electric field. By default, the external electric field (Ex,Ey,Ez) to (0.0, 0.0, 0.0). The string can be set to "constant" if a constant electric field is required to be set at initialization. If set to "constant", then an @@ -1440,6 +1586,14 @@ Grid initialization the field solver. In particular, do not use any other boundary condition than periodic. +* ``warpx.maxlevel_extEMfield_init`` (default is maximum number of levels in the simulation) + With this parameter, the externally applied electric and magnetic fields + will not be applied for levels greater than ``warpx.maxlevel_extEMfield_init``. + For some mesh-refinement simulations, + the external fields are only applied to the parent grid and not the refined patches. In such cases, + ``warpx.maxlevel_extEMfield_init`` can be set to 0. + In that case, the other levels have external field values of 0. + Applied to Particles ^^^^^^^^^^^^^^^^^^^^ @@ -1499,6 +1653,37 @@ Applied to Particles and :math:`E_z = 0`, and :math:`B_x = \mathrm{strength} \cdot y`, :math:`B_y = -\mathrm{strength} \cdot x`, and :math:`B_z = 0`. + +Applied to Cold Relativistic Fluids +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* ``.E_ext_init_style`` & ``.B_ext_init_style`` (string) optional (default "none") + These parameters determine the type of the external electric and + magnetic fields respectively that are applied directly to the cold relativistic fluids at every timestep. + The field values are specified in the lab frame. + With the default ``none`` style, no field is applied. + Possible values are ``parse_E_ext_function`` or ``parse_B_ext_function``. + + * ``parse_E_ext_function`` or ``parse_B_ext_function``: the field is specified as an analytic + expression that is a function of space (x,y,z) and time (t), relative to the lab frame. + The E-field is specified by the input parameters: + + * ``.Ex_external_function(x,y,z,t)`` + + * ``.Ey_external_function(x,y,z,t)`` + + * ``.Ez_external_function(x,y,z,t)`` + + The B-field is specified by the input parameters: + + * ``.Bx_external_function(x,y,z,t)`` + + * ``.By_external_function(x,y,z,t)`` + + * ``.Bz_external_function(x,y,z,t)`` + + Note that the position is defined in Cartesian coordinates, as a function of (x,y,z), even for RZ. + Accelerator Lattice ^^^^^^^^^^^^^^^^^^^ @@ -1565,6 +1750,7 @@ Collision models ---------------- WarpX provides several particle collision models, using varying degrees of approximation. +Details about the collision models can be found in the :ref:`theory section `. * ``collisions.collision_names`` (`strings`, separated by spaces) The name of each collision type. @@ -1574,29 +1760,29 @@ WarpX provides several particle collision models, using varying degrees of appro * ``.type`` (`string`) optional The type of collision. The types implemented are: - - ``pairwisecoulomb`` for pairwise Coulomb collisions, the default if unspecified. + - ``pairwisecoulomb`` for pair-wise Coulomb collisions, the default if unspecified. This provides a pair-wise relativistic elastic Monte Carlo binary Coulomb collision model, - following the algorithm given by `Perez et al. (Phys. Plasmas 19, 083104, 2012) `__. + following the algorithm given by :cite:t:`param-Perez2012`. When the RZ mode is used, `warpx.n_rz_azimuthal_modes` must be set to 1 at the moment, since the current implementation of the collision module assumes axisymmetry. - ``nuclearfusion`` for fusion reactions. - This implements the pair-wise fusion model by `Higginson et al. (JCP 388, 439-453, 2019) `__. + This implements the pair-wise fusion model by :cite:t:`param-Higginson2019`. Currently, WarpX supports deuterium-deuterium, deuterium-tritium, deuterium-helium and proton-boron fusion. When initializing the reactant and product species, you need to use ``species_type`` (see the documentation for this parameter), so that WarpX can identify the type of reaction to use. (e.g. ``.species_type = 'deuterium'``) + - ``dsmc`` for pair-wise, non-Coulomb collisions between kinetic species. + This is a "direct simulation Monte Carlo" treatment of collisions between + kinetic species. See :ref:`DSMC section `. - ``background_mcc`` for collisions between particles and a neutral background. This is a relativistic Monte Carlo treatment for particles colliding - with a neutral background gas. The implementation follows the so-called - null collision strategy discussed for example in `Birdsall (IEEE Transactions on - Plasma Science, vol. 19, no. 2, pp. 65-85, 1991) `_. - See also :ref:`collisions section `. + with a neutral background gas. See :ref:`MCC section `. - ``background_stopping`` for slowing of ions due to collisions with electrons or ions. This implements the approximate formulae as derived in Introduction to Plasma Physics, from Goldston and Rutherford, section 14.2. * ``.species`` (`strings`) - If using ``pairwisecoulomb`` or ``nuclearfusion``, this should be the name(s) of the species, + If using ``dsmc``, ``pairwisecoulomb`` or ``nuclearfusion``, this should be the name(s) of the species, between which the collision will be considered. (Provide only one name for intra-species collisions.) If using ``background_mcc`` or ``background_stopping`` type this should be the name of the species for which collisions with a background will be included. @@ -1619,7 +1805,7 @@ WarpX provides several particle collision models, using varying degrees of appro :math:`A` is the mass number. If this is not provided, or if a non-positive value is provided, a Coulomb logarithm will be computed automatically according to the algorithm in - `Perez et al. (Phys. Plasmas 19, 083104, 2012) `__. + :cite:t:`param-Perez2012`. * ``.fusion_multiplier`` (`float`) optional. Only for ``nuclearfusion``. @@ -1628,12 +1814,12 @@ WarpX provides several particle collision models, using varying degrees of appro total number of physical particle remains the same). This can improve the statistics of the simulation, in the case where fusion reactions are very rare. More specifically, in a fusion reaction between two macroparticles with weight ``w_1`` and ``w_2``, - the weight of the product macroparticles will be ``min(w_1,w_2)/fusion_multipler``. + the weight of the product macroparticles will be ``min(w_1,w_2)/fusion_multiplier``. (And the weights of the reactant macroparticles are reduced correspondingly after the reaction.) - See `Higginson et al. (JCP 388, 439-453, 2019) `__ - for more details. The default value of ``fusion_multiplier`` is 1. + See :cite:t:`param-Higginson2019` for more details. + The default value of ``fusion_multiplier`` is 1. -* ``.fusion_probability_threshold``(`float`) optional. +* ``.fusion_probability_threshold`` (`float`) optional. Only for ``nuclearfusion``. If the fusion multiplier is too high and results in a fusion probability that approaches 1 (for a given collision between two macroparticles), then @@ -1701,25 +1887,21 @@ WarpX provides several particle collision models, using varying degrees of appro where :math:`\beta` is the term on the r.h.s except :math:`W_b`. * ``.scattering_processes`` (`strings` separated by spaces) - Only for ``background_mcc``. The MCC scattering processes that should be + Only for ``dsmc`` and ``background_mcc``. The scattering processes that should be included. Available options are ``elastic``, ``back`` & ``charge_exchange`` for ions and ``elastic``, ``excitationX`` & ``ionization`` for electrons. - The ``elastic`` option uses hard-sphere scattering, with a differential - cross section that is independent of angle. - With ``charge_exchange``, the ion velocity is replaced with the neutral - velocity, chosen from a Maxwellian based on the value of - ``.background_temperature``. Multiple excitation events can be included for electrons corresponding to excitation to different levels, the ``X`` above can be changed to a unique identifier for each excitation process. For each scattering process specified - a path to a cross-section data file must also be given. We use + a path to a cross-section data file must also be given. We use ```` as a placeholder going forward. * ``._cross_section`` (`string`) - Only for ``background_mcc``. Path to the file containing cross-section data + Only for ``dsmc`` and ``background_mcc``. Path to the file containing cross-section data for the given scattering processes. The cross-section file must have exactly 2 columns of data, the first containing equally spaced energies in eV and the - second the corresponding cross-section in :math:`m^2`. + second the corresponding cross-section in :math:`m^2`. The energy column should + represent the kinetic energy of the colliding particles in the center-of-mass frame. * ``._energy`` (`float`) Only for ``background_mcc``. If the scattering process is either @@ -1781,7 +1963,9 @@ Particle push, charge and current deposition, field gathering Available options are: ``direct``, ``esirkepov``, and ``vay``. The default choice is ``esirkepov`` for FDTD maxwell solvers but ``direct`` for standard or Galilean PSATD solver (i.e. with ``algo.maxwell_solver = psatd``) and - for the hybrid-PIC solver (i.e. with ``algo.maxwell_solver = hybrid``). + for the hybrid-PIC solver (i.e. with ``algo.maxwell_solver = hybrid``) and for + diagnostics output with the electrostatic solvers (i.e., with + ``warpx.do_electrostatic = ...``). Note that ``vay`` is only available for ``algo.maxwell_solver = psatd``. 1. ``direct`` @@ -2033,25 +2217,28 @@ Maxwell solver: kinetic-fluid hybrid ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * ``hybrid_pic_model.elec_temp`` (`float`) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the electron temperature, in eV, used to calculate - the electron pressure (see :ref:`here `). + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the electron temperature, in eV, used to calculate + the electron pressure (see :ref:`here `). * ``hybrid_pic_model.n0_ref`` (`float`) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the reference density, in :math:`m^{-3}`, used to calculate - the electron pressure (see :ref:`here `). + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the reference density, in :math:`m^{-3}`, used to calculate + the electron pressure (see :ref:`here `). * ``hybrid_pic_model.gamma`` (`float`) optional (default ``5/3``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the exponent used to calculate - the electron pressure (see :ref:`here `). + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the exponent used to calculate + the electron pressure (see :ref:`here `). * ``hybrid_pic_model.plasma_resistivity(rho)`` (`float` or `str`) optional (default ``0``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. + +* ``hybrid_pic_model.J[x/y/z]_external_grid_function(x, y, z, t)`` (`float` or `str`) optional (default ``0``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the external current (on the grid) in :math:`A/m^2`. * ``hybrid_pic_model.n_floor`` (`float`) optional (default ``1``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma density floor, in :math:`m^{-3}`, which is useful since the generalized Ohm's law used to calculate the E-field includes a :math:`1/n` term. + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma density floor, in :math:`m^{-3}`, which is useful since the generalized Ohm's law used to calculate the E-field includes a :math:`1/n` term. * ``hybrid_pic_model.substeps`` (`int`) optional (default ``100``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the number of sub-steps to take during the B-field update. + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the number of sub-steps to take during the B-field update. .. note:: @@ -2166,8 +2353,8 @@ Additional parameters ``idx_type = {0, 0, 0}``: Sort particles to a cell centered grid ``idx_type = {1, 1, 1}``: Sort particles to a node centered grid ``idx_type = {2, 2, 2}``: Compromise between a cell and node centered grid. - In 2D (XZ and RZ), only the first two elements are read. - In 1D, only the first element is read. + In 2D (XZ and RZ), only the first two elements are read. + In 1D, only the first element is read. * ``warpx.sort_bin_size`` (list of `int`) optional (default ``1 1 1``) If ``sort_intervals`` is activated and ``sort_particles_for_deposition`` is ``false``, particles are sorted in bins of ``sort_bin_size`` cells. @@ -2196,7 +2383,7 @@ Additional parameters * ``warpx.shared_tilesize`` (list of `int`) optional (default `6 6 8` in 3D; `14 14` in 2D; `1s` otherwise) Used to tune performance when ``do_shared_mem_current_deposition`` or - ``do_shared_mem_charge_depostion`` is enabled. ``shared_tilesize`` is the + ``do_shared_mem_charge_deposition`` is enabled. ``shared_tilesize`` is the size of the temporary buffer allocated in shared memory for a threadblock. A larger tilesize requires more shared memory, but gives more work to each threadblock, which can lead to higher occupancy, and allows for more @@ -2550,6 +2737,8 @@ This can be important if a large number of particles are lost, avoiding filling In addition to their usual attributes, the saved particles have an integer attribute ``timestamp``, which indicates the PIC iteration at which each particle was absorbed at the boundary. +``BoundaryScrapingDiagnostics`` can be used with ``..random_fraction``, ``..uniform_stride``, and ``..plot_filter_function``, which have the same behavior as for ``FullDiagnostics``. For ``BoundaryScrapingDiagnostics``, these filters are applied at the time the data is written to file. An implication of this is that more particles may initially be accumulated in memory than are ultimately written. ``t`` in ``plot_filter_function`` refers to the time the diagnostic is written rather than the time the particle crossed the boundary. + .. _running-cpp-parameters-diagnostics-reduced: Reduced Diagnostics @@ -2681,8 +2870,9 @@ Reduced Diagnostics defaulting to ``1``. In RZ geometry, this only saves the 0'th azimuthal mode component of the fields. - Integrated electric and magnetic field components can instead be obtained by specifying - ``.integrate == true``. + Time integrated electric and magnetic field components can instead be obtained by specifying + ``.integrate = true``. + The integration is done every time step even when the data is written out less often. In a *moving window* simulation, the FieldProbe can be set to follow the moving frame by specifying ``.do_moving_window_FP = 1`` (default 0). .. warning:: @@ -3188,7 +3378,7 @@ The checkpoint capability can be turned with regular diagnostics: ``. Name of the checkpoint file to restart from. Returns an error if the folder does not exist or if it is not properly formatted. -* ``warpx.write_diagonstics_on_restart`` (`bool`) optional (default `false`) +* ``warpx.write_diagnostics_on_restart`` (`bool`) optional (default `false`) When `true`, write the diagnostics after restart at the time of the restart. Intervals parser diff --git a/Docs/source/usage/pwfa.rst b/Docs/source/usage/pwfa.rst index 44d62384699..5119184089c 100644 --- a/Docs/source/usage/pwfa.rst +++ b/Docs/source/usage/pwfa.rst @@ -1,3 +1,5 @@ +.. _examples-pwfa-boost: + In-Depth: PWFA ============== @@ -42,7 +44,7 @@ Listed below are the key arguments and best-practices relevant for choosing the Finite Difference Time Domain ----------------------------- - For standard plasma wakefield configurations, it is possible to model the physics correctly using the Particle-In-Cell (PIC) Finite Difference Time Domain (FDTD) algorithms (:doc:`../theory/picsar_theory`). + For standard plasma wakefield configurations, it is possible to model the physics correctly using the :ref:`Particle-In-Cell (PIC) ` Finite Difference Time Domain (FDTD) algorithms. If the simulation contains localised extremely high intensity fields, however, numerical instabilities might arise, such as the numerical Cherenkov instability (:doc:`../theory/boosted_frame`). In that case, it is recommended to use the Pseudo Spectral Analytical Time Domain (PSATD) or the Pseudo-Spectral Time-Domain (PSTD) algorithms. In the example we are describing, it is sufficient to use FDTD. @@ -96,7 +98,7 @@ Time step The time step (:math:`dt`) is used to iterated over the main PIC loop and is computed by WarpX differently depending on the Maxwell field FDTD solvers used: - * **For Yee** is equal to the CFL parameter chosen in the input file (:doc:`parameters`) times the Courant–Friedrichs–Lewy condition (CFL) that follows the analytical expression in :doc:`../theory/picsar_theory` + * **For Yee** is equal to the CFL parameter chosen in the input file (:doc:`parameters`) times the Courant–Friedrichs–Lewy condition (CFL) that follows the analytical expression in :ref:`theory-pic` * **For CKC** is equal to CFL times the minimum between the boosted frame cell dimensions where CFL is chosen to be below unity and set an optimal trade-off between making the simulation faster and avoiding NCI and other spurious effects. diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 90015a85b9a..448800f0d4d 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -1,337 +1,144 @@ .. _usage-picmi: +.. _usage-picmi-run: + +Parameters: Python (PICMI) +========================== -Python (PICMI) -============== +This documents on how to use WarpX as a Python script (e.g., ``python3 PICMI_script.py``). WarpX uses the `PICMI standard `__ for its Python input files. -Python version 3.8 or newer is required. +Complete example input files can be found in :ref:`the examples section `. -Example input files can be found in :ref:`the examples section `. In the input file, instances of classes are created defining the various aspects of the simulation. -The `Simulation` object is the central object, where the instances are passed, -defining the simulation time, field solver, registered species, etc. +A variable of type :py:class:`pywarpx.picmi.Simulation` is the central object to which all other options are passed, defining the simulation time, field solver, registered species, etc. -.. _usage-picmi-parameters: +Once the simulation is fully configured, it can be used in one of two modes. +**Interactive** use is the most common and can be :ref:`extended with custom runtime functionality `: -Classes ----------- +.. tab-set:: -Simulation and grid setup -^^^^^^^^^^^^^^^^^^^^^^^^^ + .. tab-item:: Interactive -Simulation -"""""""""" -.. autoclass:: pywarpx.picmi.Simulation - :members: step, add_species, add_laser, write_input_file + :py:meth:`~pywarpx.picmi.Simulation.step`: run directly from Python -Constants -""""""""" -For convenience, the PICMI interface defines the following constants, -which can be used directly inside any PICMI script. The values are in SI units. + .. tab-item:: Preprocessor -- ``picmi.constants.c``: The speed of light in vacuum. -- ``picmi.constants.ep0``: The vacuum permittivity :math:`\epsilon_0` -- ``picmi.constants.mu0``: The vacuum permeability :math:`\mu_0` -- ``picmi.constants.q_e``: The elementary charge (absolute value of the charge of an electron). -- ``picmi.constants.m_e``: The electron mass -- ``picmi.constants.m_p``: The proton mass + :py:meth:`~pywarpx.picmi.Simulation.write_input_file`: create an :ref:`inputs file for a WarpX executable ` -Field solvers define the updates of electric and magnetic fields. +When run directly from Python, one can also extend WarpX with further custom user logic. +See the :ref:`detailed workflow page ` on how to extend WarpX from Python. -ElectromagneticSolver -""""""""""""""""""""" -.. autoclass:: pywarpx.picmi.ElectromagneticSolver -ElectrostaticSolver -""""""""""""""""""" -.. autoclass:: pywarpx.picmi.ElectrostaticSolver +.. _usage-picmi-parameters: + +Simulation and Grid Setup +------------------------- + +.. autoclass:: pywarpx.picmi.Simulation + :members: step, add_species, add_laser, write_input_file -Cartesian3DGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.Cartesian3DGrid -Cartesian2DGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.Cartesian2DGrid -Cartesian1DGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.Cartesian1DGrid -CylindricalGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.CylindricalGrid -EmbeddedBoundary -"""""""""""""""" .. autoclass:: pywarpx.picmi.EmbeddedBoundary +Field solvers define the updates of electric and magnetic fields. + +.. autoclass:: pywarpx.picmi.ElectromagneticSolver + +.. autoclass:: pywarpx.picmi.ElectrostaticSolver + +Constants +--------- + +For convenience, the PICMI interface defines the following constants, +which can be used directly inside any PICMI script. The values are in SI units. + +- ``picmi.constants.c``: The speed of light in vacuum. +- ``picmi.constants.ep0``: The vacuum permittivity :math:`\epsilon_0` +- ``picmi.constants.mu0``: The vacuum permeability :math:`\mu_0` +- ``picmi.constants.q_e``: The elementary charge (absolute value of the charge of an electron). +- ``picmi.constants.m_e``: The electron mass +- ``picmi.constants.m_p``: The proton mass + Applied fields -^^^^^^^^^^^^^^ +-------------- + +.. autoclass:: pywarpx.picmi.AnalyticInitialField -ConstantAppliedField -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.ConstantAppliedField -AnalyticAppliedField -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticAppliedField -PlasmaLens -"""""""""" .. autoclass:: pywarpx.picmi.PlasmaLens -Mirror -"""""" .. autoclass:: pywarpx.picmi.Mirror Diagnostics -^^^^^^^^^^^ +----------- -ParticleDiagnostic -"""""""""""""""""" .. autoclass:: pywarpx.picmi.ParticleDiagnostic -FieldDiagnostic -""""""""""""""" .. autoclass:: pywarpx.picmi.FieldDiagnostic -ElectrostaticFieldDiagnostic -"""""""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.ElectrostaticFieldDiagnostic +.. autoclass:: pywarpx.picmi.Checkpoint + Lab-frame diagnostics diagnostics are used when running boosted-frame simulations. -LabFrameFieldDiagnostic -""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.LabFrameFieldDiagnostic -Checkpoint -"""""""""" -.. autoclass:: pywarpx.picmi.Checkpoint - Particles -^^^^^^^^^ +--------- Species objects are a collection of particles with similar properties. For instance, background plasma electrons, background plasma ions and an externally injected beam could each be their own particle species. -Species -""""""" .. autoclass:: pywarpx.picmi.Species -MultiSpecies -"""""""""""" .. autoclass:: pywarpx.picmi.MultiSpecies Particle distributions can be used for to initialize particles in a particle species. -GaussianBunchDistribution -""""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.GaussianBunchDistribution -UniformDistribution -""""""""""""""""""" .. autoclass:: pywarpx.picmi.UniformDistribution -AnalyticDistribution -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticDistribution -ParticleListDistribution -"""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.ParticleListDistribution Particle layouts determine how to microscopically place macro particles in a grid cell. -GriddedLayout -""""""""""""" .. autoclass:: pywarpx.picmi.GriddedLayout -PseudoRandomLayout -"""""""""""""""""" .. autoclass:: pywarpx.picmi.PseudoRandomLayout -Other operations related to particles +Other operations related to particles: -CoulombCollisions -""""""""""""""""" .. autoclass:: pywarpx.picmi.CoulombCollisions -MCCCollisions -""""""""""""" +.. autoclass:: pywarpx.picmi.DSMCCollisions + .. autoclass:: pywarpx.picmi.MCCCollisions -FieldIonization -""""""""""""""" .. autoclass:: pywarpx.picmi.FieldIonization -Lasers -^^^^^^ +Laser Pulses +------------ Laser profiles can be used to initialize laser pulses in the simulation. -GaussianLaser -""""""""""""" .. autoclass:: pywarpx.picmi.GaussianLaser -AnalyticLaser -""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticLaser Laser injectors control where to initialize laser pulses on the simulation grid. -LaserAntenna -"""""""""""" .. autoclass:: pywarpx.picmi.LaserAntenna - - -.. _usage-picmi-run: - -Running -------- - -WarpX can be run in one of two modes. It can run as a preprocessor, using the -Python input file to generate an input file to be used by the C++ version, or -it can be run directly from Python. -The examples support running in both modes by commenting and uncommenting the appropriate lines. - -In either mode, if using a `virtual environment `__, be sure to activate it before compiling and running WarpX. - - -Running WarpX directly from Python -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -For this, a full Python installation of WarpX is required, as described in :ref:`the install documentation ` (:ref:`developers `). - -In order to run a new simulation: - -* Create a **new directory**, where the simulation will be run. - -* Add a **Python script** in the directory. - -The input file should have the line ``sim.step()`` which runs the simulation. - -* **Run** the script with Python: - -.. code-block:: bash - - mpirun -np python - -where ```` is the number of MPI ranks used, and ```` -is the name of the script. - - -Extending a Simulation from Python -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When running WarpX directly from Python it is possible to interact with the simulation -by installing ``CallbackFunctions``, which will execute a given Python function at a -specific location in the WarpX simulation loop. - -.. autoclass:: pywarpx.callbacks.CallbackFunctions - -Places in the WarpX loop where callbacks are available include: -``afterinit``, ``beforecollisions``, ``aftercollisions``, ``beforeEsolve``, ``afterEsolve``, -``beforeInitEsolve``, ``afterInitEsolve``, ``beforedeposition``, ``afterdeposition``, -``beforestep``, ``afterstep``, ``afterdiagnostics``,``afterrestart`` and ``oncheckpointsignal``. -See the examples in *Examples/Tests/ParticleDataPython* for references on how to use -``callbacks``. - -There are several "hooks" available via the ``libwarpx`` shared library to access and manipulate -simulation objects (particles, fields and memory buffers) as well as general properties -(such as processor number). These "hooks" are accessible through the `Simulation.extension` object. - -.. autofunction:: pywarpx.picmi.Simulation.extension.getNProcs - -.. autofunction:: pywarpx.picmi.Simulation.extension.getMyProc - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_nattr - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_nattr_species - -.. autofunction:: pywarpx.picmi.Simulation.extension.getistep - -.. autofunction:: pywarpx.picmi.Simulation.extension.gett_new - -.. autofunction:: pywarpx.picmi.Simulation.extension.evolve - -.. autofunction:: pywarpx.picmi.Simulation.extension.finalize - -.. autofunction:: pywarpx.picmi.Simulation.extension.getProbLo - -.. autofunction:: pywarpx.picmi.Simulation.extension.getProbHi - -.. autofunction:: pywarpx.picmi.Simulation.extension.getCellSize - -Particles can be added to the simulation at specific positions and with specific -attribute values: - -.. autofunction:: pywarpx.picmi.Simulation.extension.add_particles - -Properties of the particles already in the simulation can be obtained with various -functions. - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_particle_count - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_particle_structs - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_particle_arrays - -The ``get_particle_structs()`` and ``get_particle_arrays()`` functions are called -by several utility functions of the form ``get_particle_{comp_name}`` where -``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, -``weight``, ``ux``, ``uy`` or ``uz``. - -The index of some specific component of the particle data can be obtained. - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_particle_comp_index - -New components can be added via Python. - -.. autofunction:: pywarpx.picmi.Simulation.extension.add_real_comp - -Various diagnostics are also accessible from Python. -This includes getting the deposited or total charge density from a given species -as well as accessing the scraped particle buffer. See the example in -*Examples/Tests/ParticleBoudaryScrape* for a reference on how to interact -with scraped particle data. - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_species_charge_sum - -.. autofunction:: pywarpx.picmi.Simulation.extension.depositChargeDensity - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_particle_boundary_buffer_size - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_particle_boundary_buffer_structs - -.. autofunction:: pywarpx.picmi.Simulation.extension.get_particle_boundary_buffer - -.. autofunction:: pywarpx.picmi.Simulation.extension.clearParticleBoundaryBuffer - -The embedded boundary conditions can be modified when using the electrostatic solver. - -.. autofunction:: pywarpx.picmi.Simulation.extension.set_potential_EB - -Using Python input as a preprocessor -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In this case, only the pure Python version needs to be installed, as described :ref:`here `. - -In order to run a new simulation: - -* Create a **new directory**, where the simulation will be run. - -* Add a **Python script** in the directory. - -The input file should have the line like ``sim.write_input_file(file_name = 'inputs_from_PICMI')`` -which runs the preprocessor, generating the AMReX inputs file. - -* **Run** the script with Python: - -.. code-block:: bash - - python - -where ```` is the name of the script. -This creates the WarpX input file that you can run as normal with the WarpX executable. diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index 6fefcd9a243..e68ec9391be 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -8,9 +8,12 @@ This section collects typical user workflows and best practices for WarpX. .. toctree:: :maxdepth: 2 + workflows/python_extend + workflows/domain_decomposition + workflows/plot_distribution_mapping workflows/debugging workflows/libensemble workflows/plot_timestep_duration - workflows/plot_distribution_mapping workflows/psatd_stencil workflows/archiving + workflows/ml_dataset_training diff --git a/Docs/source/usage/workflows/debugging.rst b/Docs/source/usage/workflows/debugging.rst index 9a7d7efacd0..519649e4b46 100644 --- a/Docs/source/usage/workflows/debugging.rst +++ b/Docs/source/usage/workflows/debugging.rst @@ -23,6 +23,9 @@ Try the following steps to debug a simulation: Do you spot numerical artifacts or instabilities that could point to missing resolution or unexpected/incompatible numerical parameters? #. Did the job output files indicate a crash? Check the ``Backtrace.`` files for the location of the code that triggered the crash. Backtraces are read from bottom (high-level) to top (most specific line that crashed). + + #. Was this a segmentation fault in C++, but the run was controlled from Python (PICMI)? + To get the last called Python line for the backtrace, run again and add the Python ``faulthandler``, e.g., with ``python3 -X faulthandler PICMI_your_script_here.py``. #. Try to make the reproducible scenario as small as possible by modifying the inputs file. Reduce number of cells, particles and MPI processes to something as small and as quick to execute as possible. The next steps in debugging will increase runtime, so you will benefit from a fast reproducer. diff --git a/Docs/source/usage/domain_decomposition.rst b/Docs/source/usage/workflows/domain_decomposition.rst similarity index 98% rename from Docs/source/usage/domain_decomposition.rst rename to Docs/source/usage/workflows/domain_decomposition.rst index d3df5b78f5c..1340ecc10d9 100644 --- a/Docs/source/usage/domain_decomposition.rst +++ b/Docs/source/usage/workflows/domain_decomposition.rst @@ -66,7 +66,7 @@ and the details of the on-node parallelization and computer architecture used fo Because these parameters put additional constraints on the domain size for a simulation, it can be cumbersome to calculate the number of cells and the physical size of the computational domain for a given resolution. This -:download:`Python script <../../../Tools/DevUtils/compute_domain.py>` does it +:download:`Python script <../../../../Tools/DevUtils/compute_domain.py>` does it automatically. When using the RZ spectral solver, the values of ``amr.max_grid_size`` and ``amr.blocking_factor`` are constrained since the solver diff --git a/Docs/source/usage/workflows/ml_dataset_training.rst b/Docs/source/usage/workflows/ml_dataset_training.rst new file mode 100644 index 00000000000..6e60a318bee --- /dev/null +++ b/Docs/source/usage/workflows/ml_dataset_training.rst @@ -0,0 +1,267 @@ +.. _ml_dataset_training: + +Training a Surrogate Model from WarpX Data +========================================== + +Suppose we have a WarpX simulation that we wish to replace with a neural network surrogate model. +For example, a simulation determined by the following input script + +.. dropdown:: Python Input for Training Simulation + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/run_warpx_training.py + :language: python + +In this section we walk through a workflow for data processing and model training. +This workflow was developed and first presented in :cite:t:`ml-SandbergIPAC23,ml-SandbergPASC24`. + +This assumes you have an up-to-date environment with PyTorch and openPMD. + +Data Cleaning +------------- + +It is important to inspect the data for artifacts to +check that input/output data make sense. +If we plot the final phase space for beams 1-8, +the particle data is distributed in a single blob, +as shown by :numref:`fig_phase_space_beam_1` for beam 1. +This is as we expect and what is optimal for training neural networks. + +.. _fig_phase_space_beam_1: + +.. figure:: https://user-images.githubusercontent.com/10621396/290010209-c55baf1c-dd98-4d56-a675-ad3729481eee.png + :alt: Plot showing the final phase space projections for beam 1 of the training data, for a surrogate to stage 1. + + The final phase space projections for beam 1 of the training data, for a surrogate to stage 1. + +.. _fig_phase_space_beam_0: + +.. figure:: https://user-images.githubusercontent.com/10621396/290010282-40560ac4-8509-4599-82ca-167bb1739cff.png + :alt: Plot showing the final phase space projections for beam 0 of the training data, for a surrogate to stage 0. + + The final phase space projections for beam 0 of the training data, for a surrogate to stage 0 + +On the other hand, the final phase space for beam 0, shown in :numref:`fig_phase_space_beam_1`, +has a halo of outlying particles. +Looking closer at the z-pz space, we see that some particles got caught in a decelerating +region of the wake, have slipped back and are much slower than the rest of the beam. +To assist our neural network in learning dynamics of interest, we filter out these particles. +It is sufficient for our purposes to select particles that are not too far back, setting +``particle_selection={'z':[0.28002, None]}``. Then a particle tracker is set up to make sure +we consistently filter out these particles from both the initial and final data. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Particle tracking START + :end-before: # Manual: Particle tracking END + +Create Normalized Dataset +------------------------- + +Having chosen training data we are content with, we now need to format the data, +normalize it, and store the normalized data as well as the normalizations. +The script below will take the openPMD data we have selected and +format, normalize, and store it. + +.. dropdown:: Python dataset creation + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/create_dataset.py + :language: python + +Load openPMD Data +^^^^^^^^^^^^^^^^^ + +First the openPMD data is loaded, using the particle selector as chosen above. +The neural network will make predictions from the initial phase space coordinates, +using the final phase space coordinates to measure how well it is making predictions. +Hence we load two sets of particle data, the source and target particle arrays. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Load openPMD START + :end-before: # Manual: Load openPMD END + +Normalize Data +^^^^^^^^^^^^^^ + +Neural networks learn better on appropriately normalized data. +Here we subtract out the mean in each coordinate direction and +divide by the standard deviation in each coordinate direction, +for normalized data that is centered on the origin with unit variance. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Normalization START + :end-before: # Manual: Normalization END + +openPMD to PyTorch Data +^^^^^^^^^^^^^^^^^^^^^^^ + +With the data normalized, it must be stored in a form PyTorch recognizes. +The openPMD data are 6 lists of arrays, for each of the 6 phase space coordinates +:math:`x, y, z, p_x, p_y,` and :math:`p_z`. +This data are converted to an :math:`N\times 6` numpy array and then to a PyTorch :math:`N\times 6` tensor. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Format data START + :end-before: # Manual: Format data END + +Save Normalizations and Normalized Data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +With the data properly normalized, it and the normalizations are saved to file for +use in training and inference. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Save dataset START + :end-before: # Manual: Save dataset END + +Neural Network Structure +------------------------ + +It was found in :cite:t:`ml-SandbergPASC24` that reasonable surrogate models are obtained with +shallow feedforward neural networks consisting of fewer than 10 hidden layers and +just under 1000 nodes per layer. +The example shown here uses 3 hidden layers and 20 nodes per layer +and is trained for 10 epochs. + + + +Train and Save Neural Network +----------------------------- + +The script below trains the neural network on the dataset just created. +In subsequent sections we discuss the various parts of the training process. + +.. dropdown:: Python neural network training + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/train.py + :language: python3 + +Training Function +^^^^^^^^^^^^^^^^^ + +In the training function, the model weights are updated. +Iterating through batches, the loss function is evaluated on each batch. +PyTorch provides automatic differentiation, so the direction of steepest descent +is determined when the loss function is evaluated and the ``loss.backward()`` function +is invoked. +The optimizer uses this information to update the weights in the ``optimizer.step()`` call. +The training loop then resets the optimizer and updates the summed error for the whole dataset +with the error on the batch and continues iterating through batches. +Note that this function returns the sum of all errors across the entire dataset, +which is later divided by the size of the dataset in the training loop. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Train function START + :end-before: # Manual: Train function END + +Testing Function +^^^^^^^^^^^^^^^^ + +The testing function just evaluates the neural network on the testing data that has not been used +to update the model parameters. +This testing function requires that the testing dataset is small enough to be loaded all at once. +The PyTorch dataloader can load data in batches if this size assumption is not satisfied. +The error, measured by the loss function, is returned by the testing function to be aggregated and stored. +Note that this function returns the sum of all errors across the entire dataset, +which is later divided by the size of the dataset in the training loop. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Test function START + :end-before: # Manual: Test function END + +Train Loop +^^^^^^^^^^ + +The full training loop performs ``n_epochs`` number of iterations. +At each iteration the training and testing functions are called, +the respective errors are divided by the size of the dataset and recorded, +and a status update is printed to the console. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Training loop START + :end-before: # Manual: Training loop END + +Save Neural Network Parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The model weights are saved after training to record the updates to the model parameters. +Additionally, we save some model metainformation with the model for convenience, +including the model hyperparameters, the training and testing losses, and how long the training took. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Save model START + :end-before: # Manual: Save model END + +Evaluate +-------- + +In this section we show two ways to diagnose how well the neural network is learning the data. +First we consider the train-test loss curves, shown in :numref:`fig_train_test_loss`. +This figure shows the model error on the training data (in blue) and testing data (in green) as a function of the number of epochs seen. +The training data is used to update the model parameters, so training error should be lower than testing error. +A key feature to look for in the train-test loss curve is the inflection point in the test loss trend. +The testing data is set aside as a sample of data the neural network hasn't seen before. +The testing error serves as a metric of model generalizability, indicating how well the model performs +on data it hasn't seen yet. +When the test-loss starts to trend flat or even upward, the neural network is no longer improving its ability to generalize to new data. + +.. _fig_train_test_loss: + +.. figure:: https://user-images.githubusercontent.com/10621396/290010428-f83725ab-a08f-494c-b075-314b0d26cb9a.png + :alt: Plot of training and testing loss curves versus number of training epochs. + + Training (in blue) and testing (in green) loss curves versus number of training epochs. + +.. _fig_train_evaluation: + +.. figure:: https://user-images.githubusercontent.com/10621396/290010486-4a3541e7-e0be-4cf1-b33b-57d5e5985196.png + :alt: Plot comparing model prediction with simulation output. + + A comparison of model prediction (yellow-red dots, colored by mean-squared error) with simulation output (black dots). + +A visual inspection of the model prediction can be seen in :numref:`fig_train_evaluation`. +This plot compares the model prediction, with dots colored by mean-square error, on the testing data with the actual simulation output in black. +The model obtained with the hyperparameters chosen here trains quickly but is not very accurate. +A more accurate model is obtained with 5 hidden layers and 800 nodes per layer, +as discussed in :cite:t:`ml-SandbergPASC24`. + +These figures can be generated with the following Python script. + +.. dropdown:: Python visualization of progress training neural network + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/visualize.py + :language: python3 + + +Surrogate Usage in Accelerator Physics +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A neural network such as the one we trained here can be incorporated in other BLAST codes. +`Consider the example using neural networks in ImpactX `__. + +.. bibliography:: + :keyprefix: ml- diff --git a/Docs/source/usage/workflows/ml_materials/create_dataset.py b/Docs/source/usage/workflows/ml_materials/create_dataset.py new file mode 100644 index 00000000000..08000105479 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/create_dataset.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +import os +import tarfile +from urllib import request + +import numpy as np + +c = 2.998e8 + +from openpmd_viewer import OpenPMDTimeSeries, ParticleTracker +import torch + +############### + +def sanitize_dir_strings(*dir_strings): + """append '/' to a string for concatenation in building up file tree descriptions + """ + dir_strings = list(dir_strings) + for ii, dir_string in enumerate(dir_strings): + if dir_string[-1] != '/': + dir_strings[ii] = dir_string + '/' + + return dir_strings + +def download_and_unzip(url, data_dir): + request.urlretrieve(url, data_dir) + with tarfile.open(data_dir) as tar_dataset: + tar_dataset.extractall() + +def create_source_target_data(data_dir, + species, + source_index=0, + target_index=-1, + particle_selection=None + ): + """Create dataset from openPMD files + + Parameters + --- + data_dir : string, location of diagnostic data + source_index : int, which index to take source data from + target_index : int, which index to take target data from + particle_selection: dictionary, optional, selection criterion for dataset + + Returns + --- + source_data: Nx6 array of source particle data + source_means: 6 element array of source particle coordinate means + source_stds: 6 element array of source particle coordinate standard deviations + target_data: Nx6 array of target particle data + target_means: 6 element array of target particle coordinate means + target_stds: 6 element array of source particle coordinate standard deviations + relevant times: 2 element array of source and target times + """ + data_dir, = sanitize_dir_strings(data_dir) + data_path = data_dir + print('loading openPMD data from', data_path) + ts = OpenPMDTimeSeries(data_path) + relevant_times = [ts.t[source_index], ts.t[target_index]] + + # Manual: Particle tracking START + iteration = ts.iterations[target_index] + pt = ParticleTracker( ts, + species=species, + iteration=iteration, + select=particle_selection) + # Manual: Particle tracking END + + #### create normalized source, target data sets #### + print('creating data sets') + + # Manual: Load openPMD START + iteration = ts.iterations[source_index] + source_data = ts.get_particle(species=species, + iteration=iteration, + var_list=['x','y','z','ux','uy','uz'], + select=pt) + + iteration = ts.iterations[target_index] + target_data = ts.get_particle(species=species, + iteration=iteration, + var_list=['x','y','z','ux','uy','uz'], + select=pt) + # Manual: Load openPMD END + + # Manual: Normalization START + target_means = np.zeros(6) + target_stds = np.zeros(6) + source_means = np.zeros(6) + source_stds = np.zeros(6) + for jj in range(6): + source_means[jj] = source_data[jj].mean() + source_stds[jj] = source_data[jj].std() + source_data[jj] -= source_means[jj] + source_data[jj] /= source_stds[jj] + + for jj in range(6): + target_means[jj] = target_data[jj].mean() + target_stds[jj] = target_data[jj].std() + target_data[jj] -= target_means[jj] + target_data[jj] /= target_stds[jj] + # Manual: Normalization END + + # Manual: Format data START + source_data = torch.tensor(np.column_stack(source_data)) + target_data = torch.tensor(np.column_stack(target_data)) + # Manual: Format data END + + return source_data, source_means, source_stds, target_data, target_means, target_stds, relevant_times + + +def save_warpx_surrogate_data(dataset_fullpath_filename, + diag_dir, + species, + training_frac, + batch_size, + source_index, + target_index, + survivor_select_index, + particle_selection=None + ): + + source_target_data = create_source_target_data(data_dir=diag_dir, + species=species, + source_index=source_index, + target_index=target_index, + particle_selection=particle_selection + ) + source_data, source_means, source_stds, target_data, target_means, target_stds, times = source_target_data + + # Manual: Save dataset START + full_dataset = torch.utils.data.TensorDataset(source_data.float(), target_data.float()) + + n_samples = full_dataset.tensors[0].size(0) + n_train = int(training_frac*n_samples) + n_test = n_samples - n_train + + train_data, test_data = torch.utils.data.random_split(full_dataset, [n_train, n_test]) + + torch.save({'dataset':full_dataset, + 'train_indices':train_data.indices, + 'test_indices':test_data.indices, + 'source_means':source_means, + 'source_stds':source_stds, + 'target_means':target_means, + 'target_stds':target_stds, + 'times':times, + }, + dataset_fullpath_filename + ) + # Manual: Save dataset END + +######## end utility functions ############# +######## start dataset creation ############ + +data_url = "https://zenodo.org/records/10368972/files/ml_example_training.tar.gz?download=1" +download_and_unzip(data_url, "training_dataset.tar.gz") +data_dir = "lab_particle_diags/lab_particle_diags/" + + +# create data set + +source_index = 0 +target_index = 1 +survivor_select_index = 1 +batch_size=1200 +training_frac = 0.7 + +os.makedirs('datasets', exist_ok=True) + +# improve stage 0 dataset +stage_i = 0 +select = {'z':[0.28002, None]} +species = f'beam_stage_{stage_i}' +dataset_filename = f'dataset_{species}.pt' +dataset_file = 'datasets/' + dataset_filename +save_warpx_surrogate_data(dataset_fullpath_filename=dataset_file, + diag_dir=data_dir, + species=species, + training_frac=training_frac, + batch_size=batch_size, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index, + particle_selection=select + ) + +for stage_i in range(1,9): + species = f'beam_stage_{stage_i}' + dataset_filename = f'dataset_{species}.pt' + dataset_file = 'datasets/' + dataset_filename + save_warpx_surrogate_data(dataset_fullpath_filename=dataset_file, + diag_dir=data_dir, + species=species, + training_frac=training_frac, + batch_size=batch_size, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index + ) diff --git a/Docs/source/usage/workflows/ml_materials/neural_network_classes.py b/Docs/source/usage/workflows/ml_materials/neural_network_classes.py new file mode 100644 index 00000000000..58b51a1d364 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/neural_network_classes.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +from enum import Enum + +from torch import nn + + +class ActivationType(Enum): + """ + Activation class provides an enumeration type for the supported activation layers + """ + ReLU = 1 + Tanh = 2 + PReLU = 3 + Sigmoid = 4 + +def get_enum_type(type_to_test, EnumClass): + """ + Returns the enumeration type associated to type_to_test in EnumClass + + Parameters + ---------- + type_to_test: EnumClass, int or str + object whose Enum class is to be obtained + EnumClass: Enum class + Enum class to test + """ + if type(type_to_test) is EnumClass: + return type_to_test + if type(type_to_test) is int: + return EnumClass(type_to_test) + if type(type_to_test) is str: + return getattr(EnumClass, type_to_test) + else: + raise Exception("unsupported type entered") + + + +class ConnectedNN(nn.Module): + """ + ConnectedNN is a class of fully connected neural networks + """ + def __init__(self, layers): + super().__init__() + self.stack = nn.Sequential(*layers) + def forward(self, x): + return self.stack(x) + +class OneActNN(ConnectedNN): + """ + OneActNN is class of fully connected neural networks admitting only one activation function + """ + def __init__(self, + n_in, + n_out, + n_hidden_nodes, + n_hidden_layers, + act): + + self.n_in = n_in + self.n_out = n_out + self.n_hidden_layers = n_hidden_layers + self.n_hidden_nodes = n_hidden_nodes + + self.act = get_enum_type(act, ActivationType) + + layers = [nn.Linear(self.n_in, self.n_hidden_nodes)] + + for ii in range(self.n_hidden_layers): + if self.act is ActivationType.ReLU: + layers += [nn.ReLU()] + if self.act is ActivationType.Tanh: + layers += [nn.Tanh()] + if self.act is ActivationType.PReLU: + layers += [nn.PReLU()] + if self.act is ActivationType.Sigmoid: + layers += [nn.Sigmoid()] + + if ii < self.n_hidden_layers - 1: + layers += [nn.Linear(self.n_hidden_nodes,self.n_hidden_nodes)] + + layers += [nn.Linear(self.n_hidden_nodes, self.n_out)] + + super().__init__(layers) diff --git a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py new file mode 100644 index 00000000000..8ee098e3e29 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +# +# Copyright 2022-2023 WarpX contributors +# Authors: WarpX team +# License: BSD-3-Clause-LBNL +# +# -*- coding: utf-8 -*- + +import math + +import numpy as np + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e +m_e = picmi.constants.m_e +m_p = picmi.constants.m_p +ep0 = picmi.constants.ep0 + +# Number of cells +dim = '3' +nx = ny = 128 +nz = 8832 +if dim == 'rz': + nr = nx//2 + +# Computational domain +rmin = 0. +rmax = 128e-6 +zmin = -180e-6 +zmax = 0. + +# Number of processes for static load balancing +# Check with your submit script +num_procs = [1, 1, 16*4] +if dim == 'rz': + num_procs = [1, 16] + +# Number of time steps +gamma_boost = 60. +beta_boost = np.sqrt(1.-gamma_boost**-2) + +# Create grid +if dim == 'rz': + grid = picmi.CylindricalGrid( + number_of_cells=[nr, nz], + guard_cells=[32, 32], + n_azimuthal_modes=2, + lower_bound=[rmin, zmin], + upper_bound=[rmax, zmax], + lower_boundary_conditions=['none', 'damped'], + upper_boundary_conditions=['none', 'damped'], + lower_boundary_conditions_particles=['absorbing', 'absorbing'], + upper_boundary_conditions_particles=['absorbing', 'absorbing'], + moving_window_velocity=[0., c], + warpx_max_grid_size=256, + warpx_blocking_factor=64) +else: + grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + guard_cells=[11, 11, 12], + lower_bound=[-rmax, -rmax, zmin], + upper_bound=[rmax, rmax, zmax], + lower_boundary_conditions=['periodic', 'periodic', 'damped'], + upper_boundary_conditions=['periodic', 'periodic', 'damped'], + lower_boundary_conditions_particles=['periodic', 'periodic', 'absorbing'], + upper_boundary_conditions_particles=['periodic', 'periodic', 'absorbing'], + moving_window_velocity=[0., 0., c], + warpx_max_grid_size=256, + warpx_blocking_factor=32) + + +# plasma region +plasma_rlim = 100.e-6 +N_stage = 9 +L_plasma_bulk = 0.28 +L_ramp = 1.e-9 +L_ramp_up = L_ramp +L_ramp_down = L_ramp +L_stage = L_plasma_bulk + 2*L_ramp + +# focusing +# lens external fields +beam_gamma1 = 15095 +lens_focal_length = 0.015 +lens_width = 0.003 + +stage_spacing = L_plasma_bulk + 2*lens_focal_length + +def get_species_of_accelerator_stage(stage_idx, stage_zmin, stage_zmax, + stage_xmin=-plasma_rlim, stage_xmax=plasma_rlim, + stage_ymin=-plasma_rlim, stage_ymax=plasma_rlim, + Lplus = L_ramp_up, Lp = L_plasma_bulk, + Lminus = L_ramp_down): + # Parabolic density profile + n0 = 1.7e23 + Rc = 40.e-6 + Lstage = Lplus + Lp + Lminus + if not np.isclose(stage_zmax-stage_zmin, Lstage): + print('Warning: zmax disagrees with stage length') + parabolic_distribution = picmi.AnalyticDistribution( + density_expression= + f'n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.-cos(pi*(z-{stage_zmin})/Lplus)))*((z-{stage_zmin})=Lplus)*((z-{stage_zmin})<(Lplus+Lp))' \ + + f'+n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.+cos(pi*((z-{stage_zmin})-Lplus-Lp)/Lminus)))*((z-{stage_zmin})>=(Lplus+Lp))*((z-{stage_zmin})<(Lplus+Lp+Lminus))', + pi=3.141592653589793, + n0=n0, + kp=q_e/c*math.sqrt(n0/(m_e*ep0)), + Rc=Rc, + Lplus=Lplus, + Lp=Lp, + Lminus=Lminus, + lower_bound=[stage_xmin, stage_ymin, stage_zmin], + upper_bound=[stage_xmax, stage_ymax, stage_zmax], + fill_in=True) + + electrons = picmi.Species( + particle_type='electron', + name=f'electrons{stage_idx}', + initial_distribution=parabolic_distribution) + + ions = picmi.Species( + particle_type='proton', + name=f'ions{stage_idx}', + initial_distribution=parabolic_distribution) + + return electrons, ions + +species_list = [] +for i_stage in range(1): + # Add plasma + zmin_stage = i_stage * stage_spacing + zmax_stage = zmin_stage + L_stage + electrons, ions = get_species_of_accelerator_stage(i_stage+1, zmin_stage, zmax_stage) + species_list.append(electrons) + species_list.append(ions) + +# add beam to species_list +beam_charge = -10.e-15 # in Coulombs +N_beam_particles = int(1e6) +beam_centroid_z = -107.e-6 +beam_rms_z = 2.e-6 +#beam_gammas = [2000 + 13000 * i_stage for i_stage in range(N_stage)] +beam_gammas = [1957, 15188, 28432, 41678, 54926, 68174, 81423,94672, 107922,121171] # From 3D run +beams = [] +for i_stage in range(N_stage): + beam_gamma = beam_gammas[i_stage] + sigma_gamma = 0.10 * beam_gamma + gaussian_distribution = picmi.GaussianBunchDistribution( + n_physical_particles= abs(beam_charge) / q_e, + rms_bunch_size=[2.e-6, 2.e-6, beam_rms_z], + rms_velocity=[8*c, 8*c, sigma_gamma*c], + centroid_position=[0., 0., beam_centroid_z], + centroid_velocity=[0., 0., beam_gamma*c], + ) + beam = picmi.Species( + particle_type='electron', + name=f'beam_stage_{i_stage}', + initial_distribution= gaussian_distribution + ) + beams.append(beam) + +# Laser +antenna_z = -1e-9 +profile_t_peak = 1.46764864e-13 +def get_laser(antenna_z, profile_t_peak, fill_in=True): + profile_focal_distance = 0. + laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=36e-06, + duration=7.33841e-14, + focal_position=[0., 0., profile_focal_distance + antenna_z], + centroid_position=[0., 0., antenna_z - c*profile_t_peak], + propagation_direction=[0., 0., 1.], + polarization_direction=[0., 1., 0.], + a0=2.36, + fill_in=fill_in) + laser_antenna = picmi.LaserAntenna( + position=[0., 0., antenna_z], + normal_vector=[0., 0., 1.]) + return (laser, laser_antenna) +lasers = [] +for i_stage in range(1): + fill_in = True + if i_stage == 0: + fill_in = False + lasers.append( + get_laser(antenna_z + i_stage*stage_spacing, + profile_t_peak + i_stage*stage_spacing/c, + fill_in) + ) + +# Electromagnetic solver + +psatd_algo = 'multij' +if psatd_algo == 'galilean': + galilean_velocity = [0.,0.] if dim=='3' else [0.] + galilean_velocity += [-c*beta_boost] + n_pass_z = 1 + do_multiJ = None + do_multi_J_n_depositions=None + J_in_time = None + current_correction = True + divE_cleaning = False +elif psatd_algo == 'multij': + n_pass_z = 4 + galilean_velocity = None + do_multiJ = True + do_multi_J_n_depositions = 2 + J_in_time = "linear" + current_correction = False + divE_cleaning = True +else: + raise Exception(f'PSATD algorithm \'{psatd_algo}\' is not recognized!\n'\ + 'Valid options are \'multiJ\' or \'galilean\'.') +if dim == 'rz': + stencil_order = [8, 16] + smoother = picmi.BinomialSmoother(n_pass=[1,n_pass_z]) + grid_type = 'collocated' +else: + stencil_order = [8, 8, 16] + smoother = picmi.BinomialSmoother(n_pass=[1,1,n_pass_z]) + grid_type = 'hybrid' + + +solver = picmi.ElectromagneticSolver( + grid=grid, + method='PSATD', + cfl=0.9999, + source_smoother=smoother, + stencil_order=stencil_order, + galilean_velocity=galilean_velocity, + warpx_psatd_update_with_rho=True, + warpx_current_correction=current_correction, + divE_cleaning=divE_cleaning, + warpx_psatd_J_in_time=J_in_time + ) + +# Diagnostics +diag_field_list = ['B', 'E', 'J', 'rho'] +diag_particle_list = ['weighting','position','momentum'] +coarse_btd_end = int((L_plasma_bulk+0.001+stage_spacing*(N_stage-1))*100000) +stage_end_snapshots=[f'{int((L_plasma_bulk+stage_spacing*ii)*100000)}:{int((L_plasma_bulk+stage_spacing*ii)*100000+50)}:5' for ii in range(1)] +btd_particle_diag = picmi.LabFrameParticleDiagnostic( + name='lab_particle_diags', + species=beams, + grid=grid, + num_snapshots=25*N_stage, + #warpx_intervals=', '.join([f':{coarse_btd_end}:1000']+stage_end_snapshots), + warpx_intervals=', '.join(['0:0']+stage_end_snapshots), + dt_snapshots=0.00001/c, + data_list=diag_particle_list, + write_dir='lab_particle_diags', + warpx_format='openpmd', + warpx_openpmd_backend='bp') + +btd_field_diag = picmi.LabFrameFieldDiagnostic( + name='lab_field_diags', + grid=grid, + num_snapshots=25*N_stage, + dt_snapshots=stage_spacing/25/c, + data_list=diag_field_list, + warpx_lower_bound=[-128.e-6, 0.e-6, -180.e-6], + warpx_upper_bound=[128.e-6, 0.e-6, 0.], + write_dir='lab_field_diags', + warpx_format='openpmd', + warpx_openpmd_backend='bp') + +field_diag = picmi.FieldDiagnostic( + name='field_diags', + data_list=diag_field_list, + grid=grid, + period=100, + write_dir='field_diags', + lower_bound=[-128.e-6, 0.e-6, -180.e-6], + upper_bound=[128.e-6, 0.e-6, 0.], + warpx_format='openpmd', + warpx_openpmd_backend='h5') + +particle_diag = picmi.ParticleDiagnostic( + name='particle_diags', + species=beams, + period=100, + write_dir='particle_diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5') + +beamrel_red_diag = picmi.ReducedDiagnostic( + diag_type='BeamRelevant', + name='beamrel', + species=beam, + period=1) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + warpx_numprocs=num_procs, + warpx_compute_max_step_from_btd=True, + verbose=2, + particle_shape='cubic', + gamma_boost=gamma_boost, + warpx_charge_deposition_algo='standard', + warpx_current_deposition_algo='direct', + warpx_field_gathering_algo='momentum-conserving', + warpx_particle_pusher_algo='vay', + warpx_amrex_the_arena_is_managed=False, + warpx_amrex_use_gpu_aware_mpi=True, + warpx_do_multi_J=do_multiJ, + warpx_do_multi_J_n_depositions=do_multi_J_n_depositions, + warpx_grid_type=grid_type, + # default: 2 for staggered grids, 8 for hybrid grids + warpx_field_centering_order=[16,16,16], + # only for hybrid grids, default: 8 + warpx_current_centering_order=[16,16,16] + ) + +for species in species_list: + if dim=='rz': + n_macroparticle_per_cell=[2,4,2] + else: + n_macroparticle_per_cell=[2,2,2] + sim.add_species( + species, + layout=picmi.GriddedLayout(grid=grid, + n_macroparticle_per_cell=n_macroparticle_per_cell) + ) + +for i_stage in range(N_stage): + sim.add_species_through_plane( + species=beams[i_stage], + layout=picmi.PseudoRandomLayout(grid=grid, n_macroparticles=N_beam_particles), + injection_plane_position=0., + injection_plane_normal_vector=[0.,0.,1.]) + +for i_stage in range(1): + # Add laser + (laser, laser_antenna) = lasers[i_stage] + sim.add_laser(laser, injection_method=laser_antenna) + +# Add diagnostics +sim.add_diagnostic(btd_particle_diag) +#sim.add_diagnostic(btd_field_diag) +#sim.add_diagnostic(field_diag) +#sim.add_diagnostic(particle_diag) + +# Add reduced diagnostic +sim.add_diagnostic(beamrel_red_diag) + +sim.write_input_file(f'inputs_training_{N_stage}_stages') + +# Advance simulation until last time step +sim.step() diff --git a/Docs/source/usage/workflows/ml_materials/train.py b/Docs/source/usage/workflows/ml_materials/train.py new file mode 100644 index 00000000000..dc62819061c --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/train.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +import os +import time + +import neural_network_classes as mynn +import torch +import torch.nn.functional as F +import torch.optim as optim + +############# set model parameters ################# + +stage_i = 0 +species = f'beam_stage_{stage_i}' +source_index = 0 +target_index = 4 +survivor_select_index = 4 + +data_dim = 6 +n_in = data_dim +n_out = data_dim + +learning_rate = 0.0001 +n_epochs = 10 +batch_size = 1200 + +loss_fun = F.mse_loss + +n_hidden_nodes = 20 +n_hidden_layers = 3 +activation_type = 'ReLU' + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +print(f'device={device}') +#################### load dataset ################ +dataset_filename = f'dataset_{species}.pt' +dataset_file = 'datasets/' + dataset_filename + +print(f"trying to load dataset+test-train split in {dataset_file}") + +dataset_with_indices = torch.load(dataset_file) +train_data = torch.utils.data.dataset.Subset(dataset_with_indices['dataset'], dataset_with_indices['train_indices']) +test_data = torch.utils.data.dataset.Subset(dataset_with_indices['dataset'], dataset_with_indices['test_indices']) +source_data = dataset_with_indices['dataset'] +source_means = dataset_with_indices['source_means'] +source_stds = dataset_with_indices['source_stds'] +target_means = dataset_with_indices['target_means'] +target_stds = dataset_with_indices['target_stds'] +print("able to load data and test/train split") + +###### move data to device (GPU) if available ######## +source_device = train_data.dataset.tensors[0].to(device) # equivalently, test_data.tensors[0].to(device) +target_device = train_data.dataset.tensors[1].to(device) +full_dataset_device = torch.utils.data.TensorDataset(source_device.float(), target_device.float()) + +train_data_device = torch.utils.data.dataset.Subset(full_dataset_device, train_data.indices) +test_data_device = torch.utils.data.dataset.Subset(full_dataset_device, test_data.indices) + +train_loader_device = torch.utils.data.DataLoader(train_data_device, batch_size=batch_size, shuffle=True) +test_loader_device = torch.utils.data.DataLoader(test_data_device, batch_size=batch_size, shuffle=True) + +test_source_device = test_data_device.dataset.tensors[0] +test_target_device = test_data_device.dataset.tensors[1] + +training_set_size = len(train_data_device.indices) +testing_set_size = len(test_data_device.indices) + +###### create model ########### + +model = mynn.OneActNN(n_in = n_in, + n_out = n_out, + n_hidden_nodes=n_hidden_nodes, + n_hidden_layers = n_hidden_layers, + act=activation_type + ) + +training_time = 0 +train_loss_list = [] +test_loss_list = [] + +model.to(device=device); + +########## train and test functions #### +# Manual: Train function START +def train(model, optimizer, train_loader, loss_fun): + model.train() + total_loss = 0. + for batch_idx, (data, target) in enumerate(train_loader): + #evaluate network with data + output = model(data) + #compute loss + # sum the differences squared, take mean afterward + loss = loss_fun(output, target,reduction='sum') + #backpropagation: step optimizer and reset gradients + loss.backward() + optimizer.step() + optimizer.zero_grad() + total_loss += loss.item() + return total_loss +# Manual: Train function END + +def test(model, test_loader, loss_fun): + model.eval() + total_loss = 0. + with torch.no_grad(): + for batch_idx, (data, target) in enumerate(test_loader): + output = model(data) + total_loss += loss_fun(output, target, reduction='sum').item() + return total_loss + +# Manual: Test function START +def test_dataset(model, test_source, test_target, loss_fun): + model.eval() + with torch.no_grad(): + output = model(test_source) + return loss_fun(output, test_target, reduction='sum').item() +# Manual: Test function END + +######## training loop ######## + +optimizer = optim.Adam(model.parameters(), lr=learning_rate) + +do_print = True + +t3 = time.time() +# Manual: Training loop START +for epoch in range(n_epochs): + if do_print: + t1 = time.time() + ave_train_loss = train(model, optimizer, train_loader_device, loss_fun) / data_dim / training_set_size + ave_test_loss = test_dataset(model, test_source_device, test_target_device, loss_fun) / data_dim / training_set_size + train_loss_list.append(ave_train_loss) + test_loss_list.append(ave_test_loss) + + if do_print: + t2 = time.time() + print('Train Epoch: {:04d} \tTrain Loss: {:.6f} \tTest Loss: {:.6f}, this epoch: {:.3f} s'.format( + epoch + 1, ave_train_loss, ave_test_loss, t2-t1)) +# Manual: Training loop END +t4 = time.time() +print(f'total training time: {t4-t3:.3f}s') + +######### save model ######### + +os.makedirs('models', exist_ok=True) + +# Manual: Save model START +model.to(device='cpu') +torch.save({ + 'n_hidden_layers':n_hidden_layers, + 'n_hidden_nodes':n_hidden_nodes, + 'activation':activation_type, + 'model_state_dict': model.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'train_loss_list': train_loss_list, + 'test_loss_list': test_loss_list, + 'training_time': training_time, + }, f'models/{species}_model.pt') +# Manual: Save model END diff --git a/Docs/source/usage/workflows/ml_materials/visualize.py b/Docs/source/usage/workflows/ml_materials/visualize.py new file mode 100644 index 00000000000..e6da1029b67 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/visualize.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +from matplotlib import pyplot as plt +import neural_network_classes as mynn +import numpy as np +import torch +import torch.nn.functional as F + +c = 2.998e8 + + +# open model file +stage_i = 0 +species = f'beam_stage_{stage_i}' +model_data = torch.load(f'models/{species}_model.pt',map_location=torch.device('cpu')) +data_dim = 6 +n_in = data_dim +n_out = data_dim +n_hidden_layers = model_data['n_hidden_layers'] +n_hidden_nodes = model_data['n_hidden_nodes'] +activation_type = model_data['activation'] +train_loss_list = model_data['train_loss_list'] +test_loss_list = model_data['test_loss_list'] +training_time = model_data['training_time'] +loss_fun = F.mse_loss + + +n_epochs = len(train_loss_list) +train_counter = np.arange(n_epochs)+1 +test_counter = train_counter + +do_log_plot = False +fig, ax = plt.subplots() +if do_log_plot: + ax.semilogy(train_counter, train_loss_list, '.-',color='blue',label='training loss') + ax.semilogy(test_counter, test_loss_list, color='green',label='testing loss') +else: + ax.plot(train_counter, train_loss_list, '.-',color='blue',label='training loss') + ax.plot(test_counter, test_loss_list, color='green',label='testing loss') +ax.set_xlabel('number of epochs seen') +ax.set_ylabel(' loss') +ax.legend() +fig_dir = 'figures/' +ax.set_title(f'final test error = {test_loss_list[-1]:.3e} ') +ax.grid() +plt.tight_layout() +plt.savefig(f'{species}_training_testing_error.png') + + +######### plot phase space comparison ####### +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +print(f'device={device}') + +model = mynn.OneActNN(n_in = n_in, + n_out = n_out, + n_hidden_nodes=n_hidden_nodes, + n_hidden_layers = n_hidden_layers, + act = activation_type + ) +model.load_state_dict(model_data['model_state_dict']) +model.to(device=device); + +###### load model data ############### +dataset_filename = f'dataset_{species}.pt' +dataset_dir = 'datasets/' +model_input_data = torch.load(dataset_dir + dataset_filename) +dataset = model_input_data['dataset'] +train_indices = model_input_data['train_indices'] +test_indices = model_input_data['test_indices'] +source_means = model_input_data['source_means'] +source_stds = model_input_data['source_stds'] +target_means = model_input_data['target_means'] +target_stds = model_input_data['target_stds'] +source_time, target_time = model_input_data['times'] + + +source = dataset.tensors[0] +test_source = source[test_indices] +test_source_device = test_source.to(device) +with torch.no_grad(): + evaluation_device = model(test_source_device.float()) +eval_cpu = evaluation_device.to('cpu') + +target = dataset.tensors[1] +test_target = target[test_indices] + +target_si = test_target * target_stds + target_means +eval_cpu_si = eval_cpu * target_stds + target_means +target_mu = np.copy(target_si) +eval_cpu_mu = np.copy(eval_cpu_si) +target_mu[:,2] -= c*target_time +eval_cpu_mu[:,2] -= c*target_time +target_mu[:,:3] *= 1e6 +eval_cpu_mu[:,:3] *= 1e6 + + + +loss_tensor = torch.sum(loss_fun(eval_cpu, + test_target, + reduction='none'), + axis=1)/6 +loss_array = loss_tensor.detach().numpy() + +tinds = np.nonzero(loss_array > 0.0)[0] +skip = 10 + +plt.figure() +fig, axT = plt.subplots(3,3) +axes_label = {0:r'x [$\mu$m]', 1:r'y [$\mu$m]', 2:r'z - %.2f cm [$\mu$m]'%(c*target_time),3:r'$p_x$',4:r'$p_y$',5:r'$p_z$'} +xy_inds = [(0,1),(2,0),(2,1)] +def set_axes(ax, indx, indy): + ax.scatter(target_mu[::skip,indx],target_mu[::skip,indy],s=8,c='k', label='simulation') + ax.scatter(eval_cpu_mu[::skip,indx],eval_cpu_mu[::skip,indy],marker='*',c=loss_array[::skip],s=0.02, label='surrogate',cmap='YlOrRd') + ax.set_xlabel(axes_label[indx]) + ax.set_ylabel(axes_label[indy]) + # return + + +for ii in range(3): + ax = axT[0,ii] + indx,indy = xy_inds[ii] + set_axes(ax,indx,indy) + +for ii in range(2): + indx,indy = xy_inds[ii] + ax = axT[1,ii] + set_axes(ax,indx+3,indy+3) + +for ii in range(3): + ax = axT[2,ii] + indx = ii + indy = ii+3 + set_axes(ax, indx, indy) + + +ax = axT[1,2] +indx = 5 +indy = 4 +ax.scatter(target_mu[::skip,indx],target_mu[::skip,indy],s=8,c='k', label='simulation') +evalplt = ax.scatter(eval_cpu_mu[::skip,indx],eval_cpu_mu[::skip,indy],marker='*',c=loss_array[::skip],s=2, label='surrogate',cmap='YlOrRd') +ax.set_xlabel(axes_label[indx]) +ax.set_ylabel(axes_label[indy]) + +cb = plt.colorbar(evalplt, ax=ax) +cb.set_label('MSE loss') + +fig.suptitle(f'stage {stage_i} prediction') + +plt.tight_layout() + +plt.savefig(f'{species}_model_evaluation.png') diff --git a/Docs/source/usage/workflows/python_extend.rst b/Docs/source/usage/workflows/python_extend.rst new file mode 100644 index 00000000000..6c9286c02ce --- /dev/null +++ b/Docs/source/usage/workflows/python_extend.rst @@ -0,0 +1,279 @@ +.. _usage-python-extend: + +Extend a Simulation with Python +=============================== + +When running WarpX directly :ref:`from Python ` it is possible to interact with the simulation. + +For instance, with the :py:meth:`~pywarpx.picmi.Simulation.step` method of the simulation class, one could run ``sim.step(nsteps=1)`` in a loop: + +.. code-block:: python3 + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + steps = 1000 + for _ in range(steps): + sim.step(nsteps=1) + + # do something custom with the sim object + +As a more flexible alternative, one can install `callback functions `__, which will execute a given Python function at a +specific location in the WarpX simulation loop. + +.. automodule:: pywarpx.callbacks + :members: installcallback, uninstallcallback, isinstalled + + +pyAMReX +------- + +Many of the following classes are provided through `pyAMReX `__. +After the simulation is initialized, the pyAMReX module can be accessed via + +.. code-block:: python + + from pywarpx import picmi, libwarpx + + # ... simulation definition ... + + # equivalent to + # import amrex.space3d as amr + # for a 3D simulation + amr = libwarpx.amr # picks the right 1d, 2d or 3d variant + + +Full details for pyAMReX APIs are `documented here `__. +Important APIs include: + +* `amr.ParallelDescriptor `__: MPI-parallel rank information +* `amr.MultiFab `__: MPI-parallel field data +* `amr.ParticleContainer_* `__: MPI-parallel particle data for a particle species + + +Data Access +----------- + +While the simulation is running, callbacks can have read and write access the WarpX simulation data *in situ*. + +An important object in the ``pywarpx.picmi`` module for data access is ``Simulation.extension.warpx``, which is available only during the simulation run. +This object is the Python equivalent to the C++ ``WarpX`` simulation class. + +.. py:class:: WarpX + + .. py:method:: getistep(lev: int) + + Get the current step on mesh-refinement level ``lev``. + + .. py:method:: gett_new(lev: int) + + Get the current physical time on mesh-refinement level ``lev``. + + .. py:method:: getdt(lev: int) + + Get the current physical time step size on mesh-refinement level ``lev``. + + .. py:method:: multifab(multifab_name: str) + + Return MultiFabs by name, e.g., ``"Efield_aux[x][level=0]"``, ``"Efield_cp[x][level=0]"``, ... + + The physical fields in WarpX have the following naming: + + - ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level + - ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. + - ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher). + + .. py:method:: multi_particle_container + + .. py:method:: get_particle_boundary_buffer + + .. py:method:: set_potential_on_eb(potential: str) + + The embedded boundary (EB) conditions can be modified when using the electrostatic solver. + This set the EB potential string and updates the function parser. + + .. py:method:: evolve(numsteps=-1) + + Evolve the simulation the specified number of steps. + + .. autofunction:: pywarpx.picmi.Simulation.extension.finalize + +.. py::def:: pywarpx.picmi.Simulation.extension.get_instance + + Return a reference to the WarpX object. + + +The :py:class:`WarpX` also provides read and write access to field ``MultiFab`` and ``ParticleContainer`` data, shown in the following examples. + +Fields +^^^^^^ + +This example accesses the :math:`E_x(x,y,z)` field at level 0 after every time step and calculate the largest value in it. + +.. code-block:: python3 + + from pywarpx import picmi + from pywarpx.callbacks import callfromafterstep + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + + @callfromafterstep + def set_E_x(): + warpx = sim.extension.warpx + + # data access + E_x_mf = warpx.multifab(f"Efield_fp[x][level=0]") + + # compute + # iterate over mesh-refinement levels + for lev in range(warpx.finest_level + 1): + # grow (aka guard/ghost/halo) regions + ngv = E_x_mf.n_grow_vect + + # get every local block of the field + for mfi in E_x_mf: + # global index space box, including guards + bx = mfi.tilebox().grow(ngv) + print(bx) # note: global index space of this block + + # numpy representation: non-copying view, including the + # guard/ghost region; .to_cupy() for GPU! + E_x_np = E_x_mf.array(mfi).to_numpy() + + # notes on indexing in E_x_np: + # - numpy uses locally zero-based indexing + # - layout is F_CONTIGUOUS by default, just like AMReX + + # notes: + # Only the next lines are the "HOT LOOP" of the computation. + # For efficiency, use numpy array operation for speed on CPUs. + # For GPUs use .to_cupy() above and compute with cupy or numba. + E_x_np[()] = 42.0 + + + sim.step(nsteps=100) + +For further details on how to `access GPU data `__ or compute on ``E_x``, please see the `pyAMReX documentation `__. + + +High-Level Field Wrapper +"""""""""""""""""""""""" + +.. note:: + + TODO + +.. note:: + + TODO: What are the benefits of using the high-level wrapper? + TODO: What are the limitations (e.g., in memory usage or compute scalability) of using the high-level wrapper? + + +Particles +^^^^^^^^^ + +.. code-block:: python3 + + from pywarpx import picmi + from pywarpx.callbacks import callfromafterstep + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + @callfromafterstep + def my_after_step_callback(): + warpx = sim.extension.warpx + + # data access + multi_pc = warpx.multi_particle_container() + pc = multi_pc.get_particle_container_from_name("electrons") + + # compute + # iterate over mesh-refinement levels + for lvl in range(pc.finest_level + 1): + # get every local chunk of particles + for pti in pc.iterator(pc, level=lvl): + # default layout: AoS with positions and cpuid + aos = pti.aos().to_numpy() + + # additional compile-time and runtime attributes in SoA format + soa = pti.soa().to_numpy() + + # notes: + # Only the next lines are the "HOT LOOP" of the computation. + # For efficiency, use numpy array operation for speed on CPUs. + # For GPUs use .to_cupy() above and compute with cupy or numba. + + # write to all particles in the chunk + # note: careful, if you change particle positions, you need to + # redistribute particles before continuing the simulation step + # aos[()]["x"] = 0.30 + # aos[()]["y"] = 0.35 + # aos[()]["z"] = 0.40 + + for soa_real in soa.real: + soa_real[()] = 42.0 + + for soa_int in soa.int: + soa_int[()] = 12 + + + sim.step(nsteps=100) + +For further details on how to `access GPU data `__ or compute on ``electrons``, please see the `pyAMReX documentation `__. + + +High-Level Particle Wrapper +""""""""""""""""""""""""""" + +.. note:: + + TODO: What are the benefits of using the high-level wrapper? + TODO: What are the limitations (e.g., in memory usage or compute scalability) of using the high-level wrapper? + +Particles can be added to the simulation at specific positions and with specific attribute values: + +.. code-block:: python + + from pywarpx import particle_containers, picmi + + # ... + + electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") + + +.. autoclass:: pywarpx.particle_containers.ParticleContainerWrapper + :members: + +The ``get_particle_structs()`` and ``get_particle_arrays()`` functions are called +by several utility functions of the form ``get_particle_{comp_name}`` where +``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, +``weight``, ``ux``, ``uy`` or ``uz``. + + +Diagnostics +----------- + +Various diagnostics are also accessible from Python. +This includes getting the deposited or total charge density from a given species as well as accessing the scraped particle buffer. +See the example in ``Examples/Tests/ParticleBoundaryScrape`` for a reference on how to interact with scraped particle data. + + +.. autoclass:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper + :members: + + +Modify Solvers +-------------- + +From Python, one can also replace numerical solvers in the PIC loop or add new physical processes into the time step loop. +Examples: + +* :ref:`Capacitive Discharge `: replaces the Poisson solver of an electrostatic simulation (default: MLMG) with a python function that uses `superLU `__ to directly solve the Poisson equation. diff --git a/Docs/spack.yaml b/Docs/spack.yaml index 9f4cc71b184..f17c36e5d97 100644 --- a/Docs/spack.yaml +++ b/Docs/spack.yaml @@ -19,10 +19,13 @@ spack: - doxygen - graphviz - python - - py-sphinx + - py-openpmd-viewer - py-breathe - - py-recommonmark + - py-pybtex - py-pygments + - py-recommonmark + - py-sphinx - py-sphinx-copybutton - py-sphinx-design - py-sphinx-rtd-theme + - py-yt diff --git a/Examples/Physics_applications/beam-beam_collision/README.rst b/Examples/Physics_applications/beam-beam_collision/README.rst new file mode 100644 index 00000000000..559a81277db --- /dev/null +++ b/Examples/Physics_applications/beam-beam_collision/README.rst @@ -0,0 +1,70 @@ +.. _examples-beam-beam_collision: + +Beam-beam collision +==================== + +This example shows how to simulate the collision between two ultra-relativistic particle beams. +This is representative of what happens at the interaction point of a linear collider. +We consider a right-propagating electron bunch colliding against a left-propagating positron bunch. + +We turn on the Quantum Synchrotron QED module for photon emission (also known as beamstrahlung in the collider community) and +the Breit-Wheeler QED module for the generation of electron-positron pairs (also known as coherent pair generation in the collider community). + +To solve for the electromagnetic field we use the nodal version of the electrostatic relativistic solver. +This solver computes the average velocity of each species, and solves the corresponding relativistic Poisson equation (see the WarpX documentation for `warpx.do_electrostatic = relativistic` for more detail). This solver accurately reproduced the subtle cancellation that occur for some component of the ``E + v x B`` terms which are crucial in simulations of relativistic particles. + + +This example is based on the following paper :cite:t:`ex-Yakimenko2019`. + + +Run +--- + +The PICMI input file is not available for this example yet. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/inputs``. + + +Visualize +--------- + +The figure below shows the number of photons emitted per beam particle (left) and the number of secondary pairs generated per beam particle (right). + +We compare different results: +* (red) simplified WarpX simulation as the example stored in the directory ``/Examples/Physics_applications/beam-beam_collision``; +* (blue) large-scale WarpX simulation (high resolution and ad hoc generated tables ; +* (black) literature results from :cite:t:`ex-Yakimenko2019`. + +The small-scale simulation has been performed with a resolution of ``nx = 64, ny = 64, nz = 128`` grid cells, while the large-scale one has a much higher resolution of ``nx = 512, ny = 512, nz = 1024``. Moreover, the large-scale simulation uses dedicated QED lookup tables instead of the builtin tables. To generate the tables within WarpX, the code must be compiled with the flag ``-DWarpX_QED_TABLE_GEN=ON``. For the large-scale simulation we have used the following options: + +.. code-block:: ini + + qed_qs.lookup_table_mode = generate + qed_bw.lookup_table_mode = generate + qed_qs.tab_dndt_chi_min=1e-3 + qed_qs.tab_dndt_chi_max=2e3 + qed_qs.tab_dndt_how_many=512 + qed_qs.tab_em_chi_min=1e-3 + qed_qs.tab_em_chi_max=2e3 + qed_qs.tab_em_chi_how_many=512 + qed_qs.tab_em_frac_how_many=512 + qed_qs.tab_em_frac_min=1e-12 + qed_qs.save_table_in=my_qs_table.txt + qed_bw.tab_dndt_chi_min=1e-2 + qed_bw.tab_dndt_chi_max=2e3 + qed_bw.tab_dndt_how_many=512 + qed_bw.tab_pair_chi_min=1e-2 + qed_bw.tab_pair_chi_max=2e3 + qed_bw.tab_pair_chi_how_many=512 + qed_bw.tab_pair_frac_how_many=512 + qed_bw.save_table_in=my_bw_table.txt + +.. figure:: https://user-images.githubusercontent.com/17280419/291749626-aa61fff2-e6d2-45a3-80ee-84b2851ea0bf.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDMwMzQzNTEsIm5iZiI6MTcwMzAzNDA1MSwicGF0aCI6Ii8xNzI4MDQxOS8yOTE3NDk2MjYtYWE2MWZmZjItZTZkMi00NWEzLTgwZWUtODRiMjg1MWVhMGJmLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFJV05KWUFYNENTVkVINTNBJTJGMjAyMzEyMjAlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjMxMjIwVDAxMDA1MVomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPWFiYzY2MGQyYzIyZGIzYzUxOWI3MzNjZTk5ZDM1YzgyNmY4ZDYxOGRlZjAyZTIwNTAyMTc3NTgwN2Q0YjEwNGMmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0JmFjdG9yX2lkPTAma2V5X2lkPTAmcmVwb19pZD0wIn0.I96LQpjqmFXirPDVnBlFQIkCuenR6IuOSY0OIIQvtCo + :alt: Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. + :width: 100% + + Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. diff --git a/Examples/Physics_applications/beam-beam_collision/inputs b/Examples/Physics_applications/beam-beam_collision/inputs new file mode 100644 index 00000000000..e66f0e63050 --- /dev/null +++ b/Examples/Physics_applications/beam-beam_collision/inputs @@ -0,0 +1,241 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.mc2 = m_e*clight*clight +my_constants.nano = 1.0e-9 +my_constants.GeV = q_e*1.e9 + +# BEAMS +my_constants.beam_energy = 125.*GeV +my_constants.beam_uz = beam_energy/(mc2) +my_constants.beam_charge = 0.14*nano +my_constants.sigmax = 10*nano +my_constants.sigmay = 10*nano +my_constants.sigmaz = 10*nano +my_constants.beam_uth = 0.1/100.*beam_uz +my_constants.n0 = beam_charge / (q_e * sigmax * sigmay * sigmaz * (2.*pi)**(3./2.)) +my_constants.omegab = sqrt(n0 * q_e**2 / (epsilon0*m_e)) +my_constants.mux = 0.0 +my_constants.muy = 0.0 +my_constants.muz = -0.5*Lz+3.2*sigmaz + +# BOX +my_constants.Lx = 100.0*clight/omegab +my_constants.Ly = 100.0*clight/omegab +my_constants.Lz = 180.0*clight/omegab + +# for a full scale simulation use: nx, ny, nz = 512, 512, 1024 +my_constants.nx = 64 +my_constants.ny = 64 +my_constants.nz = 128 + + +# TIME +my_constants.T = 0.7*Lz/clight +my_constants.dt = sigmaz/clight/10. + +# DIAGS +my_constants.every_red = 1. +warpx.used_inputs_file = warpx_used_inputs.txt + +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = T +amr.n_cell = nx ny nz +amr.max_grid_size = 128 +amr.blocking_factor = 2 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz +geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = PEC PEC PEC +boundary.field_hi = PEC PEC PEC +boundary.particle_lo = Absorbing Absorbing Absorbing +boundary.particle_hi = Absorbing Absorbing Absorbing + +################################# +############ NUMERICS ########### +################################# +warpx.do_electrostatic = relativistic +warpx.const_dt = dt +warpx.grid_type = collocated +algo.particle_shape = 3 +algo.load_balance_intervals=100 +algo.particle_pusher = vay + +################################# +########### PARTICLES ########### +################################# +particles.species_names = beam1 beam2 pho1 pho2 ele_nlbw1 pos_nlbw1 ele_nlbw2 pos_nlbw2 pho ele pos +particles.photon_species = pho1 pho2 pho + +beam1.species_type = electron +beam1.injection_style = NUniformPerCell +beam1.num_particles_per_cell_each_dim = 1 1 1 +beam1.profile = parse_density_function +beam1.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z-muz)**2/(2*sigmaz**2))" +beam1.density_min = n0 / 1e3 +beam1.momentum_distribution_type = gaussian +beam1.uz_m = beam_uz +beam1.uy_m = 0.0 +beam1.ux_m = 0.0 +beam1.ux_th = beam_uth +beam1.uy_th = beam_uth +beam1.uz_th = beam_uth +beam1.initialize_self_fields = 1 +beam1.self_fields_required_precision = 5e-10 +beam1.self_fields_max_iters = 10000 +beam1.do_qed_quantum_sync = 1 +beam1.qed_quantum_sync_phot_product_species = pho1 +beam1.do_classical_radiation_reaction = 0 + +beam2.species_type = positron +beam2.injection_style = NUniformPerCell +beam2.num_particles_per_cell_each_dim = 1 1 1 +beam2.profile = parse_density_function +beam2.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z+muz)**2/(2*sigmaz**2))" +beam2.density_min = n0 / 1e3 +beam2.momentum_distribution_type = gaussian +beam2.uz_m = -beam_uz +beam2.uy_m = 0.0 +beam2.ux_m = 0.0 +beam2.ux_th = beam_uth +beam2.uy_th = beam_uth +beam2.uz_th = beam_uth +beam2.initialize_self_fields = 1 +beam2.self_fields_required_precision = 5e-10 +beam2.self_fields_max_iters = 10000 +beam2.do_qed_quantum_sync = 1 +beam2.qed_quantum_sync_phot_product_species = pho2 +beam2.do_classical_radiation_reaction = 0 + +pho1.species_type = photon +pho1.injection_style = none +pho1.do_qed_breit_wheeler = 1 +pho1.qed_breit_wheeler_ele_product_species = ele_nlbw1 +pho1.qed_breit_wheeler_pos_product_species = pos_nlbw1 + +pho2.species_type = photon +pho2.injection_style = none +pho2.do_qed_breit_wheeler = 1 +pho2.qed_breit_wheeler_ele_product_species = ele_nlbw2 +pho2.qed_breit_wheeler_pos_product_species = pos_nlbw2 + +ele_nlbw1.species_type = electron +ele_nlbw1.injection_style = none +ele_nlbw1.self_fields_required_precision = 1e-11 +ele_nlbw1.self_fields_max_iters = 10000 +ele_nlbw1.do_qed_quantum_sync = 1 +ele_nlbw1.qed_quantum_sync_phot_product_species = pho +ele_nlbw1.do_classical_radiation_reaction = 0 + +pos_nlbw1.species_type = positron +pos_nlbw1.injection_style = none +pos_nlbw1.self_fields_required_precision = 1e-11 +pos_nlbw1.self_fields_max_iters = 10000 +pos_nlbw1.do_qed_quantum_sync = 1 +pos_nlbw1.qed_quantum_sync_phot_product_species = pho +pos_nlbw1.do_classical_radiation_reaction = 0 + +ele_nlbw2.species_type = electron +ele_nlbw2.injection_style = none +ele_nlbw2.self_fields_required_precision = 1e-11 +ele_nlbw2.self_fields_max_iters = 10000 +ele_nlbw2.do_qed_quantum_sync = 1 +ele_nlbw2.qed_quantum_sync_phot_product_species = pho +ele_nlbw2.do_classical_radiation_reaction = 0 + +pos_nlbw2.species_type = positron +pos_nlbw2.injection_style = none +pos_nlbw2.self_fields_required_precision = 1e-11 +pos_nlbw2.self_fields_max_iters = 10000 +pos_nlbw2.do_qed_quantum_sync = 1 +pos_nlbw2.qed_quantum_sync_phot_product_species = pho +pos_nlbw2.do_classical_radiation_reaction = 0 + +pho.species_type = photon +pho.injection_style = none +pho.do_qed_breit_wheeler = 1 +pho.qed_breit_wheeler_ele_product_species = ele +pho.qed_breit_wheeler_pos_product_species = pos + +ele.species_type = electron +ele.injection_style = none +ele.self_fields_required_precision = 1e-11 +ele.self_fields_max_iters = 10000 +ele.do_qed_quantum_sync = 1 +ele.qed_quantum_sync_phot_product_species = pho +ele.do_classical_radiation_reaction = 0 + +pos.species_type = positron +pos.injection_style = none +pos.self_fields_required_precision = 1e-11 +pos.self_fields_max_iters = 10000 +pos.do_qed_quantum_sync = 1 +pos.qed_quantum_sync_phot_product_species = pho +pos.do_classical_radiation_reaction = 0 + +################################# +############# QED ############### +################################# +qed_qs.photon_creation_energy_threshold = 0. + +qed_qs.lookup_table_mode = builtin +qed_qs.chi_min = 1.e-3 + +qed_bw.lookup_table_mode = builtin +qed_bw.chi_min = 1.e-2 + +# for accurate results use the generated tables with +# the following parameters +# note: must compile with -DWarpX_QED_TABLE_GEN=ON +#qed_qs.lookup_table_mode = generate +#qed_bw.lookup_table_mode = generate +#qed_qs.tab_dndt_chi_min=1e-3 +#qed_qs.tab_dndt_chi_max=2e3 +#qed_qs.tab_dndt_how_many=512 +#qed_qs.tab_em_chi_min=1e-3 +#qed_qs.tab_em_chi_max=2e3 +#qed_qs.tab_em_chi_how_many=512 +#qed_qs.tab_em_frac_how_many=512 +#qed_qs.tab_em_frac_min=1e-12 +#qed_qs.save_table_in=my_qs_table.txt +#qed_bw.tab_dndt_chi_min=1e-2 +#qed_bw.tab_dndt_chi_max=2e3 +#qed_bw.tab_dndt_how_many=512 +#qed_bw.tab_pair_chi_min=1e-2 +#qed_bw.tab_pair_chi_max=2e3 +#qed_bw.tab_pair_chi_how_many=512 +#qed_bw.tab_pair_frac_how_many=512 +#qed_bw.save_table_in=my_bw_table.txt + +warpx.do_qed_schwinger = 0. + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 + +diag1.intervals = 0 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam1 rho_beam2 rho_ele_nlbw1 rho_pos_nlbw1 rho_ele_nlbw2 rho_pos_nlbw2 rho_ele rho_pos +diag1.format = openpmd +diag1.dump_last_timestep = 1 +diag1.species = pho1 pho2 pho ele_nlbw1 pos_nlbw1 ele_nlbw2 pos_nlbw2 ele pos beam1 beam2 + +# REDUCED +warpx.reduced_diags_names = ParticleNumber ColliderRelevant_beam1_beam2 + +ColliderRelevant_beam1_beam2.type = ColliderRelevant +ColliderRelevant_beam1_beam2.intervals = every_red +ColliderRelevant_beam1_beam2.species = beam1 beam2 + +ParticleNumber.type = ParticleNumber +ParticleNumber.intervals = every_red diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index 766e7693b6d..38b83e46948 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# --- Copyright 2021 Modern Electron +# --- Copyright 2021 Modern Electron (DSMC test added in 2023 by TAE Technologies) # --- Monte-Carlo Collision script to reproduce the benchmark tests from # --- Turner et al. (2013) - https://doi.org/10.1063/1.4775084 @@ -11,7 +11,7 @@ from scipy.sparse import csc_matrix from scipy.sparse import linalg as sla -from pywarpx import callbacks, fields, particle_containers, picmi +from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi constants = picmi.constants @@ -46,9 +46,7 @@ def initialize_inputs(self): WarpX parser. """ # grab the boundary potentials from the grid object - self.right_voltage = ( - self.grid.potential_zmax.replace('sin', 'np.sin').replace('pi', 'np.pi') - ) + self.right_voltage = self.grid.potential_zmax # set WarpX boundary potentials to None since we will handle it # ourselves in this solver @@ -108,8 +106,12 @@ def solve(self): calculating phi from rho.""" left_voltage = 0.0 - t = self.sim_ext.gett_new() - right_voltage = eval(self.right_voltage) + right_voltage = eval( + self.right_voltage, { + 't': self.sim.extension.warpx.gett_new(0), + 'sin': np.sin, 'pi': np.pi + } + ) # Construct b vector rho = -self.rho_data / constants.ep0 @@ -158,11 +160,12 @@ class CapacitiveDischargeExample(object): # Time (in seconds) between diagnostic evaluations diag_interval = 32 / freq - def __init__(self, n=0, test=False, pythonsolver=False): + def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): """Get input parameters for the specific case (n) desired.""" self.n = n self.test = test self.pythonsolver = pythonsolver + self.dsmc = dsmc # Case specific input parameters self.voltage = f"{self.voltage[n]}*sin(2*pi*{self.freq:.5e}*t)" @@ -184,6 +187,9 @@ def __init__(self, n=0, test=False, pythonsolver=False): else: self.mcc_subcycling_steps = None + if self.dsmc: + self.rng = np.random.default_rng(23094290) + self.ion_density_array = np.zeros(self.nz + 1) self.setup_run() @@ -236,13 +242,26 @@ def setup_run(self): rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, ) ) + if self.dsmc: + self.neutrals = picmi.Species( + particle_type='He', name='neutrals', + charge=0, mass=self.m_ion, + warpx_reflection_model_zlo=1.0, + warpx_reflection_model_zhi=1.0, + warpx_do_resampling=True, + warpx_resampling_trigger_max_avg_ppc=int(self.seed_nppc*1.5), + initial_distribution=picmi.UniformDistribution( + density=self.gas_density, + rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, + ) + ) ####################################################################### # Collision initialization # ####################################################################### cross_sec_direc = '../../../../warpx-data/MCC_cross_sections/He/' - mcc_electrons = picmi.MCCCollisions( + electron_colls = picmi.MCCCollisions( name='coll_elec', species=self.electrons, background_density=self.gas_density, @@ -269,24 +288,26 @@ def setup_run(self): } ) - mcc_ions = picmi.MCCCollisions( - name='coll_ion', - species=self.ions, - background_density=self.gas_density, - background_temperature=self.gas_temp, - ndt=self.mcc_subcycling_steps, - scattering_processes={ - 'elastic' : { - 'cross_section' : cross_sec_direc+'ion_scattering.dat' - }, - 'back' : { - 'cross_section' : cross_sec_direc+'ion_back_scatter.dat' - }, - # 'charge_exchange' : { - # 'cross_section' : cross_sec_direc+'charge_exchange.dat' - # } - } - ) + ion_scattering_processes={ + 'elastic': {'cross_section': cross_sec_direc+'ion_scattering.dat'}, + 'back': {'cross_section': cross_sec_direc+'ion_back_scatter.dat'}, + # 'charge_exchange': {'cross_section': cross_sec_direc+'charge_exchange.dat'} + } + if self.dsmc: + ion_colls = picmi.DSMCCollisions( + name='coll_ion', + species=[self.ions, self.neutrals], + ndt=5, scattering_processes=ion_scattering_processes + ) + else: + ion_colls = picmi.MCCCollisions( + name='coll_ion', + species=self.ions, + background_density=self.gas_density, + background_temperature=self.gas_temp, + ndt=self.mcc_subcycling_steps, + scattering_processes=ion_scattering_processes + ) ####################################################################### # Initialize simulation # @@ -296,10 +317,10 @@ def setup_run(self): solver=self.solver, time_step_size=self.dt, max_steps=self.max_steps, - warpx_collisions=[mcc_electrons, mcc_ions], - warpx_load_balance_intervals=self.max_steps//5000, + warpx_collisions=[electron_colls, ion_colls], verbose=self.test ) + self.solver.sim = self.sim self.sim.add_species( self.electrons, @@ -313,17 +334,41 @@ def setup_run(self): n_macroparticle_per_cell=[self.seed_nppc], grid=self.grid ) ) + if self.dsmc: + self.sim.add_species( + self.neutrals, + layout = picmi.GriddedLayout( + n_macroparticle_per_cell=[self.seed_nppc//2], grid=self.grid + ) + ) self.solver.sim_ext = self.sim.extension + if self.dsmc: + # Periodically reset neutral density to starting temperature + callbacks.installbeforecollisions(self.rethermalize_neutrals) + ####################################################################### # Add diagnostics for the CI test to be happy # ####################################################################### - if self.pythonsolver: - file_prefix = 'Python_background_mcc_1d_plt' + if self.dsmc: + file_prefix = 'Python_dsmc_1d_plt' else: - file_prefix = 'Python_background_mcc_1d_tridiag_plt' - + if self.pythonsolver: + file_prefix = 'Python_background_mcc_1d_plt' + else: + file_prefix = 'Python_background_mcc_1d_tridiag_plt' + + species = [self.electrons, self.ions] + if self.dsmc: + species.append(self.neutrals) + particle_diag = picmi.ParticleDiagnostic( + species=species, + name='diag1', + period=0, + write_dir='.', + warpx_file_prefix=file_prefix + ) field_diag = picmi.FieldDiagnostic( name='diag1', grid=self.grid, @@ -332,11 +377,37 @@ def setup_run(self): write_dir='.', warpx_file_prefix=file_prefix ) + self.sim.add_diagnostic(particle_diag) self.sim.add_diagnostic(field_diag) + def rethermalize_neutrals(self): + # When using DSMC the neutral temperature will change due to collisions + # with the ions. This is not captured in the original MCC test. + # Re-thermalize the neutrals every 1000 steps + step = self.sim.extension.warpx.getistep(lev=0) + if step % 1000 != 10: + return + + if not hasattr(self, 'neutral_cont'): + self.neutral_cont = particle_containers.ParticleContainerWrapper( + self.neutrals.name + ) + + ux_arrays = self.neutral_cont.uxp + uy_arrays = self.neutral_cont.uyp + uz_arrays = self.neutral_cont.uzp + + vel_std = np.sqrt(constants.kb * self.gas_temp / self.m_ion) + for ii in range(len(ux_arrays)): + nps = len(ux_arrays[ii]) + ux_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + uy_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + uz_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + def _get_rho_ions(self): # deposit the ion density in rho_fp - self.sim.extension.depositChargeDensity('he_ions', 0) + he_ions_wrapper = particle_containers.ParticleContainerWrapper('he_ions') + he_ions_wrapper.deposit_charge_density(level=0) rho_data = self.rho_wrapper[...] self.ion_density_array += rho_data / constants.q_e / self.diag_steps @@ -350,7 +421,7 @@ def run_sim(self): self.sim.step(self.diag_steps) - if self.sim.extension.getMyProc() == 0: + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: np.save(f'ion_density_case_{self.n+1}.npy', self.ion_density_array) # query the particle z-coordinates if this is run during CI testing @@ -380,11 +451,17 @@ def run_sim(self): '--pythonsolver', help='toggle whether to use the Python level solver', action='store_true' ) +parser.add_argument( + '--dsmc', help='toggle whether to use DSMC for ions in place of MCC', + action='store_true' +) args, left = parser.parse_known_args() sys.argv = sys.argv[:1]+left if args.n < 1 or args.n > 4: raise AttributeError('Test number must be an integer from 1 to 4.') -run = CapacitiveDischargeExample(n=args.n-1, test=args.test, pythonsolver=args.pythonsolver) +run = CapacitiveDischargeExample( + n=args.n-1, test=args.test, pythonsolver=args.pythonsolver, dsmc=args.dsmc +) run.run_sim() diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py index 2a365a3729c..71e1070ef2f 100755 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py @@ -177,7 +177,7 @@ def solve(self): calculating phi from rho.""" right_voltage = eval( self.right_voltage, - {'t':sim.extension.gett_new(0), 'sin':np.sin, 'pi':np.pi} + {'t': sim.extension.warpx.gett_new(0), 'sin': np.sin, 'pi': np.pi} ) left_voltage = 0.0 @@ -312,6 +312,12 @@ def solve(self): # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = diagnostic_intervals, + write_dir = '.', + warpx_file_prefix = 'Python_background_mcc_plt' +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -345,6 +351,7 @@ def solve(self): ) ) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) ########################## diff --git a/Examples/Physics_applications/capacitive_discharge/README.md b/Examples/Physics_applications/capacitive_discharge/README.md deleted file mode 100644 index f5653b2831a..00000000000 --- a/Examples/Physics_applications/capacitive_discharge/README.md +++ /dev/null @@ -1,7 +0,0 @@ -The examples in this directory are based on the benchmark cases from Turner et -al. (Phys. Plasmas 20, 013507, 2013). -See the 'Examples' section in the documentation for previously computed -comparisons between WarpX and the literature results. -The 1D PICMI input file can be used to reproduce the results from Turner et al. -for a given case, N, by executing: - `python3 PICMI_inputs_1d.py -n N` diff --git a/Examples/Physics_applications/capacitive_discharge/README.rst b/Examples/Physics_applications/capacitive_discharge/README.rst new file mode 100644 index 00000000000..708b4528cd5 --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/README.rst @@ -0,0 +1,54 @@ +.. _examples-capacitive-discharge: + +Capacitive Discharge +==================== + +The examples in this directory are based on the benchmark cases from Turner et al. (Phys. Plasmas 20, 013507, 2013) :cite:p:`ex-Turner2013`. + +The Monte-Carlo collision (MCC) model can be used to simulate electron and ion collisions with a neutral background gas. +In particular this can be used to study capacitive discharges between parallel plates. +The implementation has been tested against the benchmark results from :cite:t:`ex-Turner2013`. + +.. note:: + + This example needs `additional calibration data for cross sections `__. + Download this data alongside your inputs file and update the paths in the inputs file: + + .. code-block:: bash + + git clone https://github.com/ECP-WarpX/warpx-data.git + + +Run +--- + +The 1D PICMI input file can be used to reproduce the results from Turner et al. for a given case, ``N`` from 1 to 4, by executing ``python3 PICMI_inputs_1d.py -n N``, e.g., + +.. code-block:: bash + + python3 PICMI_inputs_1d.py -n 1 + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: PICMI_inputs_1d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py``. + + +Analyze +------- + +Once the simulation completes an output file ``avg_ion_density.npy`` will be created which can be compared to the literature results as in the plot below. +Running case ``1`` on four CPU processors takes roughly 20 minutes to complete. + + +Visualize +--------- + +The figure below shows a comparison of the ion density as calculated in WarpX (in June 2022 with `PR #3118 `_) compared to the literature results (which can be found `in the supplementary materials of Turner et al. `__). + +.. figure:: https://user-images.githubusercontent.com/40245517/171573007-f7d733c7-c0de-490c-9ed6-ff4c02154358.png + :alt: MCC benchmark against :cite:t:`ex-Turner2013`. + :width: 80% + + MCC benchmark against :cite:t:`ex-Turner2013`. diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py new file mode 100755 index 00000000000..df773dd9deb --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +# 2023 TAE Technologies + +import os +import sys + +import numpy as np + +sys.path.append('../../../../warpx/Regression/Checksum/') + +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] + +my_check = checksumAPI.evaluate_checksum( + test_name, fn, do_particles=True, rtol=0.01 +) + +ref_density = np.array([ + 1.27953969e+14, 2.23553999e+14, 2.55384510e+14, 2.55663110e+14, + 2.55805760e+14, 2.55812087e+14, 2.55813911e+14, 2.55754104e+14, + 2.55929601e+14, 2.56085472e+14, 2.55932867e+14, 2.55828121e+14, + 2.55901711e+14, 2.55985074e+14, 2.56182697e+14, 2.56446847e+14, + 2.56483696e+14, 2.56301187e+14, 2.56245301e+14, 2.56797584e+14, + 2.57257907e+14, 2.57023627e+14, 2.56500876e+14, 2.56106851e+14, + 2.56283546e+14, 2.56723967e+14, 2.56960855e+14, 2.56825486e+14, + 2.56674669e+14, 2.56567191e+14, 2.56310927e+14, 2.56361171e+14, + 2.56692197e+14, 2.56743606e+14, 2.56653108e+14, 2.56883854e+14, + 2.56763228e+14, 2.56343726e+14, 2.56385489e+14, 2.56570110e+14, + 2.56538112e+14, 2.56472179e+14, 2.56322922e+14, 2.56195384e+14, + 2.56474576e+14, 2.56764233e+14, 2.56533016e+14, 2.56257170e+14, + 2.56362463e+14, 2.56363962e+14, 2.56311292e+14, 2.56678788e+14, + 2.57061138e+14, 2.56785892e+14, 2.56406603e+14, 2.56334908e+14, + 2.56120051e+14, 2.56003269e+14, 2.56132187e+14, 2.56329572e+14, + 2.56535713e+14, 2.56708950e+14, 2.56661860e+14, 2.56448986e+14, + 2.56386823e+14, 2.56233660e+14, 2.56137632e+14, 2.56206263e+14, + 2.56364996e+14, 2.56483536e+14, 2.56308741e+14, 2.56447231e+14, + 2.56896301e+14, 2.56691405e+14, 2.56170780e+14, 2.56122216e+14, + 2.56427399e+14, 2.56897558e+14, 2.56928868e+14, 2.56659033e+14, + 2.56749993e+14, 2.56952497e+14, 2.56798907e+14, 2.56377081e+14, + 2.56453057e+14, 2.56796632e+14, 2.56944576e+14, 2.57248469e+14, + 2.57279426e+14, 2.56849516e+14, 2.56601834e+14, 2.56850545e+14, + 2.56953072e+14, 2.56442586e+14, 2.56329006e+14, 2.56790661e+14, + 2.57083582e+14, 2.57075550e+14, 2.56719615e+14, 2.56220486e+14, + 2.56222323e+14, 2.56547365e+14, 2.56499423e+14, 2.56434041e+14, + 2.56378587e+14, 2.56249892e+14, 2.56380492e+14, 2.56504513e+14, + 2.56337631e+14, 2.56204891e+14, 2.56325116e+14, 2.56297798e+14, + 2.56112782e+14, 2.56054218e+14, 2.56320120e+14, 2.56580938e+14, + 2.56446800e+14, 2.56267011e+14, 2.56372853e+14, 2.56617592e+14, + 2.56630745e+14, 2.56615242e+14, 2.56625259e+14, 2.56561320e+14, + 2.56640072e+14, 2.56693273e+14, 2.56613237e+14, 2.24169847e+14, + 1.27683197e+14 +]) + +density_data = np.load( 'ion_density_case_1.npy' ) +print(repr(density_data)) +assert np.allclose(density_data, ref_density, rtol=0.01) diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py index 008311e6bbe..d8bdddfaca6 100755 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py @@ -79,6 +79,11 @@ # Diagnostics diag_field_list = ['B', 'E', 'J', 'rho'] +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 100, + write_dir = '.', + warpx_file_prefix = 'Python_LaserAcceleration_1d_plt') field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -108,6 +113,7 @@ injection_method = laser_antenna) # Add diagnostics +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) # Write input file that can be used to run with the compiled version diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py index fbc04a3e42c..b50e16bfc0a 100755 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py @@ -112,6 +112,11 @@ # Diagnostics diag_field_list = ['B', 'E', 'J', 'rho'] +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 200, + write_dir = '.', + warpx_file_prefix = 'Python_LaserAccelerationMR_plt') field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -145,6 +150,7 @@ injection_method = laser_antenna) # Add diagnostics +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) # Write input file that can be used to run with the compiled version diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py index b41d5c6df4b..ac3398e43fc 100755 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py +++ b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py @@ -110,6 +110,11 @@ # Diagnostics diag_field_list = ['B', 'E', 'J', 'rho'] +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 100, + write_dir = '.', + warpx_file_prefix = 'Python_LaserAcceleration_plt') field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -144,6 +149,7 @@ injection_method = laser_antenna) # Add diagnostics +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) # Write input file that can be used to run with the compiled version diff --git a/Examples/Physics_applications/laser_acceleration/README.md b/Examples/Physics_applications/laser_acceleration/README.md deleted file mode 100644 index 01a87fdce8c..00000000000 --- a/Examples/Physics_applications/laser_acceleration/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Examples of laser acceleration simulations. - -Examples are provided using the executable or Python-driven version of WarpX. - -## Using the executable version: - - inputs.2d: 2d simulation of LWFA in the laboratory frame. - - inputs.3d: 3d simulation of LWFA in the laboratory frame. - - inputs.2d.boost: 2d simulation of LWFA in a boosted frame. This script also - uses rigid injection and a parser for the plasma density. - -## Using the python-driven version: - - laser_acceleration_PICMI.py diff --git a/Examples/Physics_applications/laser_acceleration/README.rst b/Examples/Physics_applications/laser_acceleration/README.rst new file mode 100644 index 00000000000..0ba3b5382f2 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/README.rst @@ -0,0 +1,87 @@ +.. _examples-lwfa: + +Laser-Wakefield Acceleration of Electrons +========================================= + +This example shows how to model a laser-wakefield accelerator (LWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. + +Laser-wakefield acceleration is best performed in 3D or quasi-cylindrical (RZ) geometry, in order to correctly capture some of the key physics (laser diffraction, beamloading, shape of the accelerating bubble in the blowout regime, etc.). +For physical situations that have close-to-cylindrical symmetry, simulations in RZ geometry capture the relevant physics at a fraction of the computational cost of a 3D simulation. +On the other hand, for physical situation with strong asymmetries (e.g., non-round laser driver, strong hosing of the accelerated beam, etc.), only 3D simulations are suitable. + +For LWFA scenarios with long propagation lengths, use the :ref:`boosted frame method `. +An example can be seen in the :ref:`PWFA example `. + + +Run +--- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: 3D + + This example can be run **either** as: + + * **Python** script: ``python3 PICMI_inputs_3d.py`` or + * WarpX **executable** using an input file: ``warpx.3d inputs_3d max_step=400`` + + .. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_3d``. + + .. tab-item:: RZ + + This example can be run **either** as: + + * **Python** script: ``python3 PICMI_inputs_rz.py`` or + * WarpX **executable** using an input file: ``warpx.rz inputs_3d max_step=400`` + + .. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_rz + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_rz``. + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +You can run the following script to visualize the beam evolution over time: + +.. dropdown:: Script ``plot_3d.py`` + + .. literalinclude:: plot_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/plot_3d.py diags/diag1000400/``. + +.. figure:: https://user-images.githubusercontent.com/1353258/287800852-f994a020-4ecc-4987-bffc-2cb7df6144a9.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDE3MTYyNjksIm5iZiI6MTcwMTcxNTk2OSwicGF0aCI6Ii8xMzUzMjU4LzI4NzgwMDg1Mi1mOTk0YTAyMC00ZWNjLTQ5ODctYmZmYy0yY2I3ZGY2MTQ0YTkucG5nP1gtQW16LUFsZ29yaXRobT1BV1M0LUhNQUMtU0hBMjU2JlgtQW16LUNyZWRlbnRpYWw9QUtJQUlXTkpZQVg0Q1NWRUg1M0ElMkYyMDIzMTIwNCUyRnVzLWVhc3QtMSUyRnMzJTJGYXdzNF9yZXF1ZXN0JlgtQW16LURhdGU9MjAyMzEyMDRUMTg1MjQ5WiZYLUFtei1FeHBpcmVzPTMwMCZYLUFtei1TaWduYXR1cmU9NDkyNWJkMTg2NWM3ZjcwZjVkMjlmNDE1NmRjNWEyZWM5MzgxMWJhZTVjMGMxNjdkZDg1Zjk0NmQ1NGEwMjNiMiZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmYWN0b3JfaWQ9MCZrZXlfaWQ9MCZyZXBvX2lkPTAifQ.C_NQceQcqiCDzBoSzIjm3c8QdTLNDdtjJmkQjkhW4c8 + :alt: (top) Electric field of the laser pulse and (bottom) absolute density. + + (top) Electric field of the laser pulse and (bottom) absolute density. diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py new file mode 100755 index 00000000000..a33b82ebc02 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2023 Grant Johnson, Remi Lehe +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D WFA with Pondermotive Envelope: +# REF: (Equations 20-23) https://journals.aps.org/rmp/pdf/10.1103/RevModPhys.81.1229 +import os +import sys + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +n0 = 20.e23 +# Plasma frequency +wp = np.sqrt((n0*e**2)/(m_e*epsilon_0)) +kp = wp/c +tau = 15.e-15 +a0 = 2.491668 +e = -e #Electrons +lambda_laser = 0.8e-6 + +zmin = -20e-6; zmax = 100.e-6; Nz = 10240 + +# Compute the theory + +# Call the ode solver +from scipy.integrate import odeint + + +# ODE Function +def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): + phi1, phi2 = phi + a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2))*np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 + dphi1_dxi = phi2 + dphi2_dxi = kp**2 * ((1 + a_sq) / (2 * (1 + phi1)**2) - 0.5) + return [dphi1_dxi, dphi2_dxi] + +# Call odeint to solve the ODE +xi_span = [-20e-6, 100e-6] +xi_0 = 0e-6 +phi0 = [0.0, 0.0] +dxi = (zmax-zmin)/Nz +xi = zmin + dxi*( 0.5 + np.arange(Nz) ) +phi = odeint(odefcn, phi0, xi, args=(kp, a0, c, tau, xi_0, lambda_laser)) + +# Change array direction to match the simulations +xi = -xi[::-1] +phi = phi[::-1] +xi_0 = -0e-6 +phi2 = phi[:, 0] +Ez = -phi[:, 1] + +# Compute the derived quantities +a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2)) *np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 +gamma_perp_sq = 1 + a_sq +n = n0 * (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)**2) +uz = (gamma_perp_sq - (1 + phi2)**2) / (2 * (1 + phi2)) +gamma = (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)) + +# Theory Components [convert to si] +uz *= c +J_th = np.multiply( np.divide(uz,gamma), n ) +J_th *= e +rho_th = e*n +E_th = Ez +E_th *= ((m_e*c*c)/e) +V_th = np.divide(uz,gamma) +V_th /= c +# Remove the ions +rho_th = rho_th - e*n0 + +# Dictate which region to compare solutions over +# (Currently this is the full domain) +min_i = 0 +max_i = 10240 + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, + dims=ds.domain_dimensions) +# Check the validity of the fields +error_rel = 0 +for field in ['Ez']: + E_sim = data[('mesh',field)].to_ndarray()[:,0,0] + #E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim[min_i:max_i]-E_th[min_i:max_i]).max()/abs(E_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the currents +for field in ['Jz']: + J_sim = data[('mesh',field)].to_ndarray()[:,0,0] + #J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim[min_i:max_i]-J_th[min_i:max_i]).max()/abs(J_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray()[:,0,0] + #rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim[min_i:max_i]-rho_th[min_i:max_i]).max()/abs(rho_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +V_sim = np.divide(J_sim,rho_sim) +V_sim /= c + +# Create a figure with 2 rows and 2 columns +fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(12, 8)) + +# Titles and labels +titles = ['Ez', 'rho', 'Jz', 'Vz/c'] +xlabel = r'Xi' +ylabel = ['Ez', 'rho', 'Jz', 'Vz/c'] + +# Plotting loop +for i in range(3): + ax = axes[i // 2, i % 2] # Get the current subplot + + # Plot theoretical data + ax.plot(xi, [E_th, rho_th, J_th, V_th][i], label='Theoretical') + + # Plot simulated data + ax.plot(xi, [E_sim, rho_sim, J_sim, V_sim][i], label='Simulated') + + # Set titles and labels + ax.set_title(f'{titles[i]} vs Xi') + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel[i]) + + # Add legend + ax.legend() + +# Adjust subplot layout +plt.tight_layout() + +# Save the figure +plt.savefig('wfa_fluid_nonlinear_1d_analysis.png') + +plt.show() + + +tolerance_rel = 0.20 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py new file mode 100755 index 00000000000..30301996921 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2023 Grant Johnson, Remi Lehe +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D WFA with Pondermotive Envelope: +# REF: (Equations 20-23) https://journals.aps.org/rmp/pdf/10.1103/RevModPhys.81.1229 +import os +import sys + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +n0 = 20.e23 +# Plasma frequency +wp = np.sqrt((n0*e**2)/(m_e*epsilon_0)) +kp = wp/c +tau = 15.e-15 +a0 = 2.491668 +e = -e #Electrons +lambda_laser = 0.8e-6 + +zmin = -20e-6; zmax = 100.e-6; Nz = 4864 + +# Compute the theory + +# Call the ode solver +from scipy.integrate import odeint + + +# ODE Function +def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): + phi1, phi2 = phi + a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2))*np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 + dphi1_dxi = phi2 + dphi2_dxi = kp**2 * ((1 + a_sq) / (2 * (1 + phi1)**2) - 0.5) + return [dphi1_dxi, dphi2_dxi] + +# Call odeint to solve the ODE +xi_span = [-20e-6, 100e-6] +xi_0 = 0e-6 +phi0 = [0.0, 0.0] +dxi = (zmax-zmin)/Nz +xi = zmin + dxi*( 0.5 + np.arange(Nz) ) +phi = odeint(odefcn, phi0, xi, args=(kp, a0, c, tau, xi_0, lambda_laser)) + +# Change array direction to match the simulations +xi = -xi[::-1] +phi = phi[::-1] +xi_0 = -0e-6 +phi2 = phi[:, 0] +Ez = -phi[:, 1] + +# Compute the derived quantities +a_sq = a0**2 * np.exp(-2 * (xi - xi_0)**2 / (c**2 * tau**2)) *np.sin(2*np.pi*(xi - xi_0)/lambda_laser)**2 +gamma_perp_sq = 1 + a_sq +n = n0 * (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)**2) +uz = (gamma_perp_sq - (1 + phi2)**2) / (2 * (1 + phi2)) +gamma = (gamma_perp_sq + (1 + phi2)**2) / (2 * (1 + phi2)) + +# Theory Components [convert to si] +uz *= c +J_th = np.multiply( np.divide(uz,gamma), n ) +J_th *= e +rho_th = e*n +E_th = Ez +E_th *= ((m_e*c*c)/e) +V_th = np.divide(uz,gamma) +V_th /= c +# Remove the ions +rho_th = rho_th - e*n0 + +# Dictate which region to compare solutions over (cuttoff 0's from BTD extra) +min_i = 200 +max_i = 4864 + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, + dims=ds.domain_dimensions) +# Check the validity of the fields +error_rel = 0 +for field in ['Ez']: + E_sim = data[('mesh',field)].to_ndarray()[:,0,0] + #E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim[min_i:max_i]-E_th[min_i:max_i]).max()/abs(E_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the currents +for field in ['Jz']: + J_sim = data[('mesh',field)].to_ndarray()[:,0,0] + #J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim[min_i:max_i]-J_th[min_i:max_i]).max()/abs(J_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray()[:,0,0] + #rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim[min_i:max_i]-rho_th[min_i:max_i]).max()/abs(rho_th[min_i:max_i]).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +V_sim = np.divide(J_sim,rho_sim) +V_sim /= c + +# Create a figure with 2 rows and 2 columns +fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(12, 8)) + +# Titles and labels +titles = ['Ez', 'rho', 'Jz', 'Vz/c'] +xlabel = r'Xi' +ylabel = ['Ez', 'rho', 'Jz', 'Vz/c'] + +# Plotting loop +for i in range(3): + ax = axes[i // 2, i % 2] # Get the current subplot + + # Plot theoretical data + ax.plot(xi, [E_th, rho_th, J_th, V_th][i], label='Theoretical') + + # Plot simulated data + ax.plot(xi, [E_sim, rho_sim, J_sim, V_sim][i], label='Simulated') + + # Set titles and labels + ax.set_title(f'{titles[i]} vs Xi') + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel[i]) + + # Add legend + ax.legend() + +# Adjust subplot layout +plt.tight_layout() + +# Save the figure +plt.savefig('wfa_fluid_nonlinear_1d_analysis.png') + +plt.show() + + +tolerance_rel = 0.30 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py b/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py index e7d26d2cb30..a66c838fe9d 100755 --- a/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py +++ b/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py @@ -43,7 +43,7 @@ n_move = 192 # ref ratio = 2 1 -# Refined only transversly. Longitudinal spacing between particles in each stream is the same in both coarse and fine regions +# Refined only transversely. Longitudinal spacing between particles in each stream is the same in both coarse and fine regions rr_longitudinal = 1 np_expected = (n_coarse + n_fine*rr_longitudinal)*(n_0 + n_move) diff --git a/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids b/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids new file mode 100644 index 00000000000..73fa6b7283f --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids @@ -0,0 +1,72 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 40000 +amr.n_cell = 10240 +amr.max_grid_size = 512 # maximum size of each AMReX box, used to decompose the domain +amr.blocking_factor = 512 # minimum size of each AMReX box, used to decompose the domain +geometry.dims = 1 +geometry.prob_lo = -120.e-6 # physical domain +geometry.prob_hi = 0.e-6 +amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = pec +boundary.field_hi = pec + +################################# +############ NUMERICS ########### +################################# +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.use_filter = 0 +warpx.cfl = 0.45 #Fluid CFL < 0.5 +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 # units of speed of light +warpx.do_dynamic_scheduling = 0 +warpx.serialize_initial_conditions = 1 + +################################# +############ PLASMA ############# +################################# +fluids.species_names = electrons ions + +electrons.species_type = electron +electrons.profile = parse_density_function +electrons.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" +electrons.momentum_distribution_type = "at_rest" + +ions.charge = q_e +ions.mass = m_p +ions.profile = parse_density_function +ions.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" +ions.momentum_distribution_type = "at_rest" + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############ LASER ############## +################################# +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. -11.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 0. 1. 0. # The main polarization vector +laser1.e_max = 10.e12 # Maximum amplitude of the laser field (in V/m) +laser1.profile_waist = 5.e-6 # The waist of the laser (in m) +laser1.profile_duration = 15.e-15 # The duration of the laser (in s) +laser1.profile_t_peak = 30.e-15 # Time at which the laser reaches its peak (in s) +laser1.profile_focal_distance = 100.e-6 # Focal distance from the antenna (in m) +laser1.wavelength = 0.8e-6 # The wavelength of the laser (in m) + +# Diagnostics +diagnostics.diags_names = diag1 + +# LAB +diag1.intervals = 20000 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted b/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted new file mode 100644 index 00000000000..f9437716f66 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted @@ -0,0 +1,79 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 10000 +amr.n_cell = 5120 +amr.max_grid_size = 1024 # maximum size of each AMReX box, used to decompose the domain +amr.blocking_factor = 1024 # minimum size of each AMReX box, used to decompose the domain +geometry.dims = 1 +geometry.prob_lo = -120.e-6 # physical domain +geometry.prob_hi = 0.e-6 +amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = pec +boundary.field_hi = pec + +################################# +############ NUMERICS ########### +################################# +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.use_filter = 0 +warpx.cfl = 0.45 #Fluid CFL < 0.5 +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 # units of speed of light +warpx.do_dynamic_scheduling = 0 +warpx.serialize_initial_conditions = 1 + +### BOOST ### +warpx.gamma_boost = 1.5 +warpx.boost_direction = z + +################################# +############ PLASMA ############# +################################# +fluids.species_names = electrons ions + +electrons.species_type = electron +electrons.profile = parse_density_function +electrons.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" +electrons.momentum_distribution_type = "at_rest" + +ions.charge = q_e +ions.mass = m_p +ions.profile = parse_density_function +ions.density_function(x,y,z) = "1.0e10 + 20.e23*((z*5.e4 + -0.5)*(z>10.e-6)*(z<30.e-6)) + 20.e23*((z>30.e-6))" +ions.momentum_distribution_type = "at_rest" + +# Order of particle shape factors +algo.particle_shape = 3 + +################################# +############ LASER ############## +################################# +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. -11.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 0. 1. 0. # The main polarization vector +laser1.e_max = 10.e12 # Maximum amplitude of the laser field (in V/m) +laser1.profile_waist = 5.e-6 # The waist of the laser (in m) +laser1.profile_duration = 15.e-15 # The duration of the laser (in s) +laser1.profile_t_peak = 30.e-15 # Time at which the laser reaches its peak (in s) +laser1.profile_focal_distance = 100.e-6 # Focal distance from the antenna (in m) +laser1.wavelength = 0.8e-6 # The wavelength of the laser (in m) + +# Diagnostics +diagnostics.diags_names = diag1 + + +# BOOST +diag1.diag_type = BackTransformed +diag1.do_back_transformed_fields = 1 +diag1.num_snapshots_lab = 2 +diag1.dt_snapshots_lab = 6.e-13 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diff --git a/Examples/Physics_applications/laser_acceleration/inputs_3d b/Examples/Physics_applications/laser_acceleration/inputs_3d index 27ecc00117b..bdcfd7676a4 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_3d +++ b/Examples/Physics_applications/laser_acceleration/inputs_3d @@ -77,6 +77,7 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho +diag1.format = openpmd # Reduced Diagnostics warpx.reduced_diags_names = FP diff --git a/Examples/Physics_applications/laser_acceleration/plot_3d.py b/Examples/Physics_applications/laser_acceleration/plot_3d.py new file mode 100755 index 00000000000..00222ff43c8 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/plot_3d.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl +# License: BSD-3-Clause-LBNL +# +# This is a script plots the wakefield of an LWFA simulation. + +import sys + +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + + +def plot_lwfa(): + # this will be the name of the plot file + fn = sys.argv[1] + + # Read the file + ds = yt.load(fn) + + # plot the laser field and absolute density + fields = ["Ey", "rho"] + normal = "y" + sl = yt.SlicePlot(ds, normal=normal, fields=fields) + for field in fields: + sl.set_log(field, False) + + sl.set_figure_size((4, 8)) + fig = sl.export_to_mpl_figure(nrows_ncols=(2, 1)) + fig.tight_layout() + plt.show() + +if __name__ == "__main__": + plot_lwfa() diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst new file mode 100644 index 00000000000..f382b36bc85 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -0,0 +1,65 @@ +.. _examples-laser-ion: + +Laser-Ion Acceleration with a Planar Target +=========================================== + +This example shows how to model laser-ion acceleration with planar targets of solid density :cite:p:`ex-Wilks2001,ex-Bulanov2008,ex-Macchi2013`. +The acceleration mechanism in this scenario depends on target parameters + +Although laser-ion acceleration requires full 3D modeling for adequate description of the acceleration dynamics, especially the acceleration field lengths and decay times, this example models a 2D example. +2D modeling can often hint at a qualitative overview of the dynamics, but mostly saves computational costs since the plasma frequency (and Debye length) of the plasma determines the resolution need in laser-solid interaction modeling. + +.. note:: + + TODO: The Python (PICMI) input file needs to be created. + +.. note:: + + The resolution of this 2D case is extremely low by default. + You will need a computing cluster for adequate resolution of the target density, see comments in the input file. + +.. warning:: + + It is strongly advised to set the parameters ``.zmin / zmax / xmin / ...`` when working with highly dense targets that are limited in one or multiple dimensions. + The particle creation routine will first create particles everywhere between these limits (`defaulting to box size if unset`), setting particles to invalid only afterwards based on the density profile. + Not setting these parameters can quickly lead to memory overflows. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: (*TODO*) or +* WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_2d`` file. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/inputs_2d``. + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Physics_applications/laser_ion/inputs b/Examples/Physics_applications/laser_ion/inputs_2d similarity index 100% rename from Examples/Physics_applications/laser_ion/inputs rename to Examples/Physics_applications/laser_ion/inputs_2d diff --git a/Examples/Physics_applications/plasma_acceleration/README.rst b/Examples/Physics_applications/plasma_acceleration/README.rst new file mode 100644 index 00000000000..d5775e93aa8 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/README.rst @@ -0,0 +1,63 @@ +.. _examples-pwfa: + +Beam-Driven Wakefield Acceleration of Electrons +=============================================== + +This example shows how to model a beam-driven plasma-wakefield accelerator (PWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. + +PWFA is best performed in 3D or quasi-cylindrical (RZ) geometry, in order to correctly capture some of the key physics (structure of the space-charge fields, beamloading, shape of the accelerating bubble in the blowout regime, etc.). +For physical situations that have close-to-cylindrical symmetry, simulations in RZ geometry capture the relevant physics at a fraction of the computational cost of a 3D simulation. +On the other hand, for physical situation with strong asymmetries (e.g., non-round driver, strong hosing of the accelerated beam, etc.), only 3D simulations are suitable. + +Additionally, to speed up computation, this example uses the :ref:`boosted frame method ` to effectively model long acceleration lengths. + +Alternatively, an other common approximation for PWFAs is quasi-static modeling, e.g., if effects such as self-injection can be ignored. +In the Beam, Plasma & Accelerator Simulation Toolkit (BLAST), `HiPACE++ `__ provides such methods. + +.. note:: + + TODO: The Python (PICMI) input file should use the boosted frame method, like the ``inputs_3d_boost`` file. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: ``python3 PICMI_inputs_plasma_acceleration.py`` or +* WarpX **executable** using an input file: ``warpx.3d inputs_3d_boost`` + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should use the boosted frame method, like the ``inputs_3d_boost`` file. + + .. literalinclude:: PICMI_inputs_plasma_acceleration.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_3d_boost + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/inputs_3d_boost``. + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Physics_applications/plasma_mirror/README.rst b/Examples/Physics_applications/plasma_mirror/README.rst new file mode 100644 index 00000000000..8741db09699 --- /dev/null +++ b/Examples/Physics_applications/plasma_mirror/README.rst @@ -0,0 +1,54 @@ +.. _examples-plasma-mirror: + +Plasma-Mirror +============= + +This example shows how to model a plasma mirror, using a planar target of solid density :cite:p:`ex-Dromey2004,ex-Roedel2010`. + +Although laser-solid interaction modeling requires full 3D modeling for adequate description of the dynamics at play, this example models a 2D example. +2D modeling provide a qualitative overview of the dynamics, but mostly saves computational costs since the plasma frequency (and Debye length) of the surface plasma determines the resolution need in laser-solid interaction modeling. + +.. note:: + + TODO: The Python (PICMI) input file needs to be created. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: (*TODO*) or +* WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_2d`` file. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/plasma_mirror/inputs_2d``. + + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Physics_applications/uniform_plasma/README.rst b/Examples/Physics_applications/uniform_plasma/README.rst new file mode 100644 index 00000000000..50d132712c6 --- /dev/null +++ b/Examples/Physics_applications/uniform_plasma/README.rst @@ -0,0 +1,65 @@ +.. _examples-uniform-plasma: + +Uniform Plasma +============== + +This example evolves a uniformly distributed, hot plasma over time. + + +Run +--- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: 3D + + .. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_3d`` file. + + .. tab-item:: Executable: Input File + + This example can be run **either** as WarpX **executable** using an input file: ``warpx.3d inputs_3d`` + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``usage/examples/lwfa/inputs_3d``. + + .. tab-item:: 2D + + .. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_2d`` file. + + .. tab-item:: Executable: Input File + + This example can be run **either** as WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``usage/examples/lwfa/inputs_2d``. + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Tests/Implicit/analysis_1d.py b/Examples/Tests/Implicit/analysis_1d.py new file mode 100755 index 00000000000..0f00010a505 --- /dev/null +++ b/Examples/Tests/Implicit/analysis_1d.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Copyright 2023 David Grote +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D periodic plasma using the implicit solver. +import os +import re +import sys + +import numpy as np + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +field_energy = np.loadtxt('diags/reducedfiles/field_energy.txt', skiprows=1) +particle_energy = np.loadtxt('diags/reducedfiles/particle_energy.txt', skiprows=1) + +total_energy = field_energy[:,2] + particle_energy[:,2] + +delta_E = (total_energy - total_energy[0])/total_energy[0] +max_delta_E = np.abs(delta_E).max() + +if re.match('SemiImplicitPicard_1d', fn): + tolerance_rel = 2.5e-5 +elif re.match('ImplicitPicard_1d', fn): + # This case should have near machine precision conservation of energy + tolerance_rel = 1.e-14 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel}") + +assert( max_delta_E < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d new file mode 100644 index 00000000000..50d28a2db75 --- /dev/null +++ b/Examples/Tests/Implicit/inputs_1d @@ -0,0 +1,81 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.const_dt = dt +algo.evolve_scheme = implicit_picard +algo.max_picard_iterations = 31 +algo.picard_iteration_tolerance = 0. +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit new file mode 100644 index 00000000000..2271a0bb1bc --- /dev/null +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -0,0 +1,81 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.const_dt = dt +algo.evolve_scheme = semi_implicit_picard +algo.max_picard_iterations = 5 +algo.picard_iteration_tolerance = 0. +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py b/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py index 857318baaa1..849d4d2be08 100644 --- a/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py +++ b/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py @@ -87,10 +87,19 @@ ######### DIAGNOSTICS ########### ################################# +particle_diag = picmi.ParticleDiagnostic( + name='diag1', + period=300, + species=[ions], + data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], + write_dir='.', + warpx_file_prefix='Python_LoadExternalField3D_plt' + ) field_diag = picmi.FieldDiagnostic( name='diag1', grid=grid, period=300, + data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], write_dir='.', warpx_file_prefix='Python_LoadExternalField3D_plt' ) @@ -121,6 +130,7 @@ ) sim.add_diagnostic(field_diag) +sim.add_diagnostic(particle_diag) ################################# ##### SIMULATION EXECUTION ###### diff --git a/Examples/Tests/boosted_diags/analysis.py b/Examples/Tests/boosted_diags/analysis.py index c6c089f9807..c0b03f4a20b 100755 --- a/Examples/Tests/boosted_diags/analysis.py +++ b/Examples/Tests/boosted_diags/analysis.py @@ -21,6 +21,7 @@ import numpy as np import openpmd_api as io +from openpmd_viewer import OpenPMDTimeSeries import yt yt.funcs.mylog.setLevel(0) @@ -48,9 +49,13 @@ Ez_openpmd = ds_openpmd.meshes['E']['z'].load_chunk() Ez_openpmd = Ez_openpmd.transpose() series.flush() - # Compare arrays to check consistency between new BTD formats (plotfile and openPMD) assert(np.allclose(Ez_plotfile, Ez_openpmd, rtol=rtol, atol=atol)) +# Check that particle random sub-selection has been applied +ts = OpenPMDTimeSeries('./diags/diag2/') +w, = ts.get_particle(['w'], species='beam', iteration=3) +assert (400 < len(w)) & (len(w) < 600) + test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/boosted_diags/inputs_3d b/Examples/Tests/boosted_diags/inputs_3d index ba98558be47..1b6b3448f26 100644 --- a/Examples/Tests/boosted_diags/inputs_3d +++ b/Examples/Tests/boosted_diags/inputs_3d @@ -122,3 +122,4 @@ diag2.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diag2.format = openpmd diag2.buffer_size = 32 diag2.openpmd_backend = h5 +diag2.beam.random_fraction = 0.5 diff --git a/Examples/Tests/collision/PICMI_inputs_2d.py b/Examples/Tests/collision/PICMI_inputs_2d.py index f1c41827093..99e217b0afc 100755 --- a/Examples/Tests/collision/PICMI_inputs_2d.py +++ b/Examples/Tests/collision/PICMI_inputs_2d.py @@ -106,6 +106,12 @@ ######### DIAGNOSTICS ########### ################################# +particle_diag = picmi.ParticleDiagnostic( + name='diag1', + period=10, + write_dir='.', + warpx_file_prefix='Python_collisionXZ_plt' +) field_diag = picmi.FieldDiagnostic( name='diag1', grid=grid, @@ -140,6 +146,7 @@ ) ) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) ################################# diff --git a/Examples/Tests/electrostatic_dirichlet_bc/PICMI_inputs_2d.py b/Examples/Tests/electrostatic_dirichlet_bc/PICMI_inputs_2d.py index 309251149ef..e4dd530c3bc 100755 --- a/Examples/Tests/electrostatic_dirichlet_bc/PICMI_inputs_2d.py +++ b/Examples/Tests/electrostatic_dirichlet_bc/PICMI_inputs_2d.py @@ -58,6 +58,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 4, + write_dir = '.', + warpx_file_prefix = 'Python_dirichletbc_plt' +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -79,6 +85,7 @@ verbose = 0 ) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) ########################## diff --git a/Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py b/Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py index 63b4ab8ea38..55fbc87bd9e 100755 --- a/Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py +++ b/Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py @@ -70,6 +70,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 1, + write_dir = '.', + warpx_file_prefix = 'Python_ElectrostaticSphereEB_plt' +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -102,6 +108,7 @@ warpx_field_gathering_algo='momentum-conserving' ) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) sim.add_diagnostic(reduced_diag) sim.add_diagnostic(reduced_diag_one_eighth) @@ -112,6 +119,6 @@ sim.step(1) -sim.extension.set_potential_EB("2.") +sim.extension.warpx.set_potential_on_eb("2.") sim.step(1) diff --git a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py b/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py index c047106b4ac..faec3ed4668 100755 --- a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py +++ b/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py @@ -58,6 +58,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 1, + write_dir = '.', + warpx_file_prefix = "embedded_boundary_python_API_plt" +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -78,6 +84,7 @@ verbose = 1 ) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) sim.initialize_inputs() diff --git a/Examples/Tests/field_probe/analysis_field_probe.py b/Examples/Tests/field_probe/analysis_field_probe.py index e038bfd0f48..e167942d77c 100755 --- a/Examples/Tests/field_probe/analysis_field_probe.py +++ b/Examples/Tests/field_probe/analysis_field_probe.py @@ -9,9 +9,9 @@ """ This script tests the accuracy of the FieldProbe diagnostic by observing a plane wave undergoing single slit diffraction. The input file inputs_2d is used. This -file defines the simulation box, laser pulse, embeded boundary with single slit, +file defines the simulation box, laser pulse, embedded boundary with single slit, and line of detector points. The plane wave initializes near the negative Z end -of the simulation box. The wave interacts with the embeded boundary at Z=0. The +of the simulation box. The wave interacts with the embedded boundary at Z=0. The wave undergoes diffraction at the slit. The electromagnetic flux is calculated at the line detector which is placed perpendicular to Z beyond the slit. This test will check if the detected EM flux matches expected values, @@ -39,7 +39,7 @@ def I_envelope (x, lam = 0.2e-6, a = 0.3e-6, D = 1.7e-6): arg = np.pi * a / lam * np.sin(np.arctan(x / D)) return np.sinc( arg / np.pi )**2 -# Count non-outlyer values away from simulation boundaries +# Count non-outlier values away from simulation boundaries counter = np.arange(60, 140, 2) # Count average error from expected values diff --git a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py index 048ef70f9cc..470cbfe6065 100755 --- a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py +++ b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py @@ -17,7 +17,7 @@ After the particles are emitted with flux injection, this script produces histograms of the velocity distribution and compares it with the expected -velocity distibution (Gaussian or Gaussian-flux depending on the direction +velocity distribution (Gaussian or Gaussian-flux depending on the direction of space) """ import os @@ -57,11 +57,8 @@ def gaussian_dist(u, u_th): return 1./((2*np.pi)**.5*u_th) * np.exp(-u**2/(2*u_th**2) ) def gaussian_flux_dist(u, u_th, u_m): - au_m = np.abs(u_m) - normalization_factor = u_th**2 * np.exp(-au_m**2/(2*u_th**2)) + (np.pi/2)**.5*au_m*u_th * (1 + erf(au_m/(2**.5*u_th))) - result = 1./normalization_factor * np.where( u>0, u * np.exp(-(u-au_m)**2/(2*u_th**2)), 0 ) - if u_m < 0.: - result = result[::-1] + normalization_factor = u_th**2 * np.exp(-u_m**2/(2*u_th**2)) + (np.pi/2)**.5*u_m*u_th * (1 + erf(u_m/(2**.5*u_th))) + result = 1./normalization_factor * np.where( u>0, u * np.exp(-(u-u_m)**2/(2*u_th**2)), 0 ) return result def compare_gaussian(u, w, u_th, label=''): @@ -84,10 +81,10 @@ def compare_gaussian_flux(u, w, u_th, u_m, label=''): # Load data and perform check -plt.figure(figsize=(5,7)) +plt.figure(figsize=(8,7)) -plt.subplot(211) -plt.title('Electrons') +plt.subplot(221) +plt.title('Electrons u_m=0.07') ux = ad['electron','particle_momentum_x'].to_ndarray()/(m_e*c) uy = ad['electron','particle_momentum_y'].to_ndarray()/(m_e*c) @@ -97,21 +94,46 @@ def compare_gaussian_flux(u, w, u_th, u_m, label=''): compare_gaussian(ux, w, u_th=0.1, label='u_x') compare_gaussian_flux(uy, w, u_th=0.1, u_m=0.07, label='u_y') compare_gaussian(uz, w, u_th=0.1, label='u_z') -plt.legend(loc=0) -plt.subplot(212) -plt.title('Protons') +plt.subplot(223) +plt.title('Protons u_m=0.05') ux = ad['proton','particle_momentum_x'].to_ndarray()/(m_p*c) uy = ad['proton','particle_momentum_y'].to_ndarray()/(m_p*c) uz = ad['proton','particle_momentum_z'].to_ndarray()/(m_p*c) w = ad['proton', 'particle_weight'].to_ndarray() -compare_gaussian_flux(ux, w, u_th=0.1, u_m=-0.05, label='u_x') +compare_gaussian_flux(-ux, w, u_th=0.1, u_m=0.05, label='u_x') compare_gaussian(uy, w, u_th=0.1, label='u_y') compare_gaussian(uz, w, u_th=0.1, label='u_z') -plt.legend(loc=0) +plt.subplot(222) +plt.title('Electrons u_m=-0.07') + +ux = ad['electron_negative','particle_momentum_x'].to_ndarray()/(m_e*c) +uy = ad['electron_negative','particle_momentum_y'].to_ndarray()/(m_e*c) +uz = ad['electron_negative','particle_momentum_z'].to_ndarray()/(m_e*c) +w = ad['electron_negative', 'particle_weight'].to_ndarray() + +compare_gaussian(ux, w, u_th=0.1, label='u_x') +compare_gaussian(uy, w, u_th=0.1, label='u_y') +compare_gaussian_flux(uz, w, u_th=0.1, u_m=-0.07, label='u_z') +plt.legend(loc=(1.02, 0.5)) + +plt.subplot(224) +plt.title('Protons u_m=-0.05') + +ux = ad['proton_negative','particle_momentum_x'].to_ndarray()/(m_p*c) +uy = ad['proton_negative','particle_momentum_y'].to_ndarray()/(m_p*c) +uz = ad['proton_negative','particle_momentum_z'].to_ndarray()/(m_p*c) +w = ad['proton_negative', 'particle_weight'].to_ndarray() + +compare_gaussian(ux, w, u_th=0.1, label='u_x') +compare_gaussian(uy, w, u_th=0.1, label='u_y') +compare_gaussian_flux(-uz, w, u_th=0.1, u_m=-0.05, label='u_z') +#plt.legend(loc=0) + +plt.tight_layout() plt.savefig('Distribution.png') # Verify checksum diff --git a/Examples/Tests/flux_injection/inputs_3d b/Examples/Tests/flux_injection/inputs_3d index 7d277e7c292..80ee23feff9 100644 --- a/Examples/Tests/flux_injection/inputs_3d +++ b/Examples/Tests/flux_injection/inputs_3d @@ -28,7 +28,7 @@ boundary.field_lo = periodic periodic periodic boundary.field_hi = periodic periodic periodic # particles -particles.species_names = electron proton +particles.species_names = electron proton electron_negative proton_negative algo.particle_shape = 3 electron.charge = -q_e @@ -61,6 +61,37 @@ proton.ux_m = 0.05 proton.uy_th = 0.1 proton.uz_th = 0.1 +# "negative" is negative u_m +electron_negative.charge = -q_e +electron_negative.mass = m_e +electron_negative.injection_style = NFluxPerCell +electron_negative.num_particles_per_cell = 100 +electron_negative.surface_flux_pos = -4. +electron_negative.flux_normal_axis = z +electron_negative.flux_direction = +1 +electron_negative.flux_profile = parse_flux_function +electron_negative.flux_function(x,y,z,t) = "1." +electron_negative.momentum_distribution_type = gaussianflux +electron_negative.ux_th = 0.1 +electron_negative.uy_th = 0.1 +electron_negative.uz_th = 0.1 +electron_negative.uz_m = -0.07 + +proton_negative.charge = +q_e +proton_negative.mass = m_p +proton_negative.injection_style = NFluxPerCell +proton_negative.num_particles_per_cell = 100 +proton_negative.surface_flux_pos = 4. +proton_negative.flux_normal_axis = z +proton_negative.flux_direction = -1 +proton_negative.flux_profile = constant +proton_negative.flux = 1. +proton_negative.momentum_distribution_type = gaussianflux +proton_negative.ux_th = 0.1 +proton_negative.uy_th = 0.1 +proton_negative.uz_th = 0.1 +proton_negative.uz_m = -0.05 + # Diagnostics diagnostics.diags_names = diag1 diag1.intervals = 1000 diff --git a/Examples/Tests/gaussian_beam/README.rst b/Examples/Tests/gaussian_beam/README.rst new file mode 100644 index 00000000000..bfca2bb2398 --- /dev/null +++ b/Examples/Tests/gaussian_beam/README.rst @@ -0,0 +1,47 @@ +.. _examples-gaussian-beam: + +Gaussian Beam +============= + +This example initializes a Gaussian beam distribution. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: ``python3 PICMI_inputs_gaussian_beam.py`` or +* WarpX **executable** using an input file: (*TODO*) + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_gaussian_beam.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py``. + + .. tab-item:: Executable: Input File + + .. note:: + + TODO: This input file should be created following the ``PICMI_inputs_gaussian_beam.py`` file. + + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Tests/initial_distribution/analysis_distribution.py b/Examples/Tests/initial_distribution/analysis_distribution.py index a15b688faad..d5301c077d2 100755 --- a/Examples/Tests/initial_distribution/analysis_distribution.py +++ b/Examples/Tests/initial_distribution/analysis_distribution.py @@ -14,6 +14,8 @@ # 5 denotes maxwell-juttner distribution w/ spatially varying temperature # 6 denotes maxwell-boltzmann distribution w/ constant velocity # 7 denotes maxwell-boltzmann distribution w/ spatially-varying velocity +# 8 denotes uniform distribution +# 9 denotes gaussian_parser distribution w/ spatially-varying mean and thermal velocity # The distribution is obtained through reduced diagnostic ParticleHistogram. import os @@ -340,6 +342,33 @@ def check_validity_uniform(bins, histogram, u_min, u_max, Ntrials=1000): check_validity_uniform(bin_value_y, h8y[timestep] / N0, uy_min, uy_max) check_validity_uniform(bin_value_z, h8z[timestep] / N0, uz_min, uz_max) +#================================================= +# Gaussian with parser mean and standard deviation +#================================================= + +# load data +bin_value_ux, bin_data_ux = read_reduced_diags_histogram("h9x.txt")[2:] +bin_value_uy, bin_data_uy = read_reduced_diags_histogram("h9y.txt")[2:] +bin_value_uz, bin_data_uz = read_reduced_diags_histogram("h9z.txt")[2:] + +def Gaussian(mean, sigma, u): + V = 8.0 # volume in m^3 + n = 1.0e21 # number density in 1/m^3 + return (n*V/(sigma*np.sqrt(2.*np.pi)))*np.exp(-(u - mean)**2/(2.*sigma**2)) + +du = 2./50 +f_ux = Gaussian(0.1 , 0.2 , bin_value_ux)*du +f_uy = Gaussian(0.12, 0.21, bin_value_uy)*du +f_uz = Gaussian(0.14, 0.22, bin_value_uz)*du + +f9_error = np.sum(np.abs(f_ux - bin_data_ux)/f_ux.max() + +np.abs(f_uy - bin_data_uy)/f_ux.max() + +np.abs(f_uz - bin_data_uz)/f_uz.max()) / bin_value_ux.size + +print('gaussian_parse_momentum_function velocity difference:', f9_error) + +assert(f9_error < tolerance) + test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/initial_distribution/inputs b/Examples/Tests/initial_distribution/inputs index 14683adeb91..5fa463604e2 100644 --- a/Examples/Tests/initial_distribution/inputs +++ b/Examples/Tests/initial_distribution/inputs @@ -29,7 +29,7 @@ algo.particle_shape = 1 ################################# ############ PLASMA ############# ################################# -particles.species_names = gaussian maxwell_boltzmann maxwell_juttner beam maxwell_juttner_parser velocity_constant velocity_parser uniform +particles.species_names = gaussian maxwell_boltzmann maxwell_juttner beam maxwell_juttner_parser velocity_constant velocity_parser uniform gaussian_parser particles.rigid_injected_species = beam gaussian.charge = -q_e @@ -133,6 +133,20 @@ uniform.uy_max = 0.1 uniform.uz_min = 10 uniform.uz_max = 11.2 +gaussian_parser.charge = -q_e +gaussian_parser.mass = m_e +gaussian_parser.injection_style = "NRandomPerCell" +gaussian_parser.num_particles_per_cell = 1000 +gaussian_parser.profile = constant +gaussian_parser.density = 1.0e21 +gaussian_parser.momentum_distribution_type = "gaussian_parse_momentum_function" +gaussian_parser.momentum_function_ux_m(x,y,z) = 0.1*z +gaussian_parser.momentum_function_uy_m(x,y,z) = 0.12*z +gaussian_parser.momentum_function_uz_m(x,y,z) = 0.14*z +gaussian_parser.momentum_function_ux_th(x,y,z) = 0.2*z +gaussian_parser.momentum_function_uy_th(x,y,z) = 0.21*z +gaussian_parser.momentum_function_uz_th(x,y,z) = 0.22*z + ################################# ########## DIAGNOSTIC ########### ################################# @@ -144,7 +158,7 @@ uniform.uz_max = 11.2 # 6 for maxwell-boltzmann with constant velocity # 7 for maxwell-boltzmann with parser velocity # 8 for cuboid in momentum space -warpx.reduced_diags_names = h1x h1y h1z h2x h2y h2z h3 h3_filtered h4x h4y h4z bmmntr h5_neg h5_pos h6 h6uy h7 h7uy_pos h7uy_neg h8x h8y h8z +warpx.reduced_diags_names = h1x h1y h1z h2x h2y h2z h3 h3_filtered h4x h4y h4z bmmntr h5_neg h5_pos h6 h6uy h7 h7uy_pos h7uy_neg h8x h8y h8z h9x h9y h9z h1x.type = ParticleHistogram h1x.intervals = 1 @@ -350,6 +364,33 @@ h8z.bin_min = 0 h8z.bin_max = 15 h8z.histogram_function(t,x,y,z,ux,uy,uz) = "uz" +h9x.type = ParticleHistogram +h9x.intervals = 1 +h9x.path = "./" +h9x.species = gaussian_parser +h9x.bin_number = 50 +h9x.bin_min = -1 +h9x.bin_max = 1 +h9x.histogram_function(t,x,y,z,ux,uy,uz) = "ux/z" + +h9y.type = ParticleHistogram +h9y.intervals = 1 +h9y.path = "./" +h9y.species = gaussian_parser +h9y.bin_number = 50 +h9y.bin_min = -1 +h9y.bin_max = 1 +h9y.histogram_function(t,x,y,z,ux,uy,uz) = "uy/z" + +h9z.type = ParticleHistogram +h9z.intervals = 1 +h9z.path = "./" +h9z.species = gaussian_parser +h9z.bin_number = 50 +h9z.bin_min = -1 +h9z.bin_max = 1 +h9z.histogram_function(t,x,y,z,ux,uy,uz) = "uz/z" + # our little beam monitor bmmntr.type = BeamRelevant bmmntr.intervals = 1 diff --git a/Examples/Tests/ion_stopping/analysis_ion_stopping.py b/Examples/Tests/ion_stopping/analysis_ion_stopping.py index dd9ee0f8744..b090014c0cf 100755 --- a/Examples/Tests/ion_stopping/analysis_ion_stopping.py +++ b/Examples/Tests/ion_stopping/analysis_ion_stopping.py @@ -73,7 +73,7 @@ def stopping_from_ions(dt, ni, Ti, mi, Zi, Zb, ion_mass, ion_energy): ion_energy = (f1)**(2./3.)/e return ion_energy -# Fetch background parameters and inital particle data +# Fetch background parameters and initial particle data ds0 = yt.load(f'{prefix}{len(last_it)*"0"}') ad0 = ds0.all_data() diff --git a/Examples/Tests/ionization/PICMI_inputs_2d.py b/Examples/Tests/ionization/PICMI_inputs_2d.py index 81b55d06600..86d24181a75 100644 --- a/Examples/Tests/ionization/PICMI_inputs_2d.py +++ b/Examples/Tests/ionization/PICMI_inputs_2d.py @@ -84,10 +84,18 @@ cfl = 0.999) # Diagnostics -diag = picmi.FieldDiagnostic( +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 10000, + species = [electrons, ions], + data_list = ['ux', 'uy', 'uz', 'x', 'y', 'weighting'], + write_dir = '.', + warpx_file_prefix = 'Python_ionization_plt') +field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, period = 10000, + data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], write_dir = '.', warpx_file_prefix = 'Python_ionization_plt') @@ -115,7 +123,8 @@ injection_method = laser_antenna) # Add diagnostics -sim.add_diagnostic(diag) +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) # Write input file that can be used to run with the compiled version sim.write_input_file(file_name = 'inputs_2d_picmi') diff --git a/Examples/Tests/ionization/analysis_ionization.py b/Examples/Tests/ionization/analysis_ionization.py index e5d61fc0c0a..95732b03e36 100755 --- a/Examples/Tests/ionization/analysis_ionization.py +++ b/Examples/Tests/ionization/analysis_ionization.py @@ -93,5 +93,13 @@ assert( error_rel < tolerance_rel ) +# Check that the user runtime component (if it exists) worked as expected +try: + orig_z = ad['electrons', 'particle_orig_z'].v + assert np.all( (orig_z > 0) & (orig_z < 1.5e-5) ) + print('particle_orig_z has reasonable values') +except yt.utilities.exceptions.YTFieldNotFound: + pass # Some of the tested script to not have the quantity orig_z + test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/ionization/inputs_2d_rt b/Examples/Tests/ionization/inputs_2d_rt index 130eb1cc46a..f7035c567ac 100644 --- a/Examples/Tests/ionization/inputs_2d_rt +++ b/Examples/Tests/ionization/inputs_2d_rt @@ -36,6 +36,8 @@ ions.physical_element = N electrons.mass = m_e electrons.charge = -q_e electrons.injection_style = none +electrons.addRealAttributes = orig_z +electrons.attribute.orig_z(x,y,z,ux,uy,uz,t) = z lasers.names = laser1 laser1.profile = Gaussian diff --git a/Examples/Tests/langmuir/PICMI_inputs_rz.py b/Examples/Tests/langmuir/PICMI_inputs_rz.py index 018303a9611..8328aba5185 100755 --- a/Examples/Tests/langmuir/PICMI_inputs_rz.py +++ b/Examples/Tests/langmuir/PICMI_inputs_rz.py @@ -178,7 +178,7 @@ def calcEz( z, r, k0, w0, wp, t, epsilons) : return( Ez_array ) # Current time of the simulation -t0 = sim.extension.gett_new(0) +t0 = sim.extension.warpx.gett_new(0) # Get the raw field data. Note that these are the real and imaginary # parts of the fields for each azimuthal mode. diff --git a/Examples/Tests/langmuir/README.md b/Examples/Tests/langmuir/README.md deleted file mode 100644 index febf280d490..00000000000 --- a/Examples/Tests/langmuir/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Examples of Langmuir oscillations in a uniform plasma in 1D, 2D, 3D, and RZ - -In each case, a uniform plasma is setup with a sinusoidal perturbation in the -electron momentum along each axis. The plasma is followed for a short period -of time, long enough so that E fields develop. The resulting fields can be -compared to the analytic solutions. - -# Input files (for C++ version) - - inputs_1d - inputs_2d - inputs_3d - inputs_rz - -# Input files (for Python version) - - PICMI_inputs_2d.py - PICMI_inputs_3d.py - PICMI_inputs_rz.py - -# Analysis scripts to check the results - - analysis_1d.py - analysis_2d.py - analysis_3d.py - analysis_rz.py diff --git a/Examples/Tests/langmuir/README.rst b/Examples/Tests/langmuir/README.rst new file mode 100644 index 00000000000..60c0018744c --- /dev/null +++ b/Examples/Tests/langmuir/README.rst @@ -0,0 +1,146 @@ +.. _examples-langmuir: + +Langmuir Waves +============== + +These are examples of Plasma oscillations (`Langmuir waves `__) in a uniform plasma in 1D, 2D, 3D, and RZ. + +In each case, a uniform plasma is setup with a sinusoidal perturbation in the electron momentum along each axis. +The plasma is followed for a short period of time, long enough so that E fields develop. +The resulting fields can be compared to the analytic solutions. + + +Run +--- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: 3D + + .. tab-set:: + + .. tab-item:: Python: Script + + This example can be run as a **Python** script: ``python3 PICMI_inputs_3d.py``. + + .. literalinclude:: PICMI_inputs_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_3d.py``. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.3d inputs_3d`` + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_3d``. + + .. tab-item:: 2D + + .. tab-set:: + + .. tab-item:: Python: Script + + This example can be run as a **Python** script: ``python3 PICMI_inputs_2d.py``. + + .. literalinclude:: PICMI_inputs_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_2d.py``. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_2d``. + + + .. tab-item:: RZ + + .. tab-set:: + + .. tab-item:: Python: Script + + This example can be run as a **Python** script: ``python3 PICMI_inputs_rz.py``. + + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_rz.py``. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.rz inputs_rz`` + + .. literalinclude:: inputs_rz + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_rz``. + + + .. tab-item:: 1D + + .. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created, like the ``inputs_1d`` file. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.1d inputs_1d`` + + .. literalinclude:: inputs_1d + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_1d``. + + +Analyze +------- + +We run the following script to analyze correctness: + +.. tab-set:: + + .. tab-item:: 3D + + .. dropdown:: Script ``analysis_3d.py`` + + .. literalinclude:: analysis_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_3d.py``. + + .. tab-item:: 2D + + .. dropdown:: Script ``analysis_2d.py`` + + .. literalinclude:: analysis_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_2d.py``. + + .. tab-item:: RZ + + .. dropdown:: Script ``analysis_rz.py`` + + .. literalinclude:: analysis_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_rz.py``. + + .. tab-item:: 1D + + .. dropdown:: Script ``analysis_1d.py`` + + .. literalinclude:: analysis_1d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_1d.py``. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Tests/langmuir_fluids/analysis_1d.py b/Examples/Tests/langmuir_fluids/analysis_1d.py new file mode 100755 index 00000000000..2d1a8f69d1d --- /dev/null +++ b/Examples/Tests/langmuir_fluids/analysis_1d.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2022 Jean-Luc Vay, Maxence Thevenet, Remi Lehe, Prabhat Kumar, Axel Huebl, Grant Johnson +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs.multi.rt`. This simulates a 1D periodic plasma wave. +# The electric field in the simulation is given (in theory) by: +# $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\sin(k_z z)\sin( \omega_p t)$$ +import os +import sys + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +epsilon = 0.01 +n = 4.e24 +n_osc_z = 2 +zmin = -20e-6; zmax = 20.e-6; Nz = 128 + +# Wave vector of the wave +kz = 2.*np.pi*n_osc_z/(zmax-zmin) +# Plasma frequency +wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) + +k = {'Ez':kz,'Jz':kz} +cos = {'Ez':(1,1,0), 'Jz':(1,1,0)} +cos_rho = {'rho': (1,1,1)} + +def get_contribution( is_cos, k ): + du = (zmax-zmin)/Nz + u = zmin + du*( 0.5 + np.arange(Nz) ) + if is_cos == 1: + return( np.cos(k*u) ) + else: + return( np.sin(k*u) ) + +def get_theoretical_field( field, t ): + amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) + cos_flag = cos[field] + z_contribution = get_contribution( cos_flag[2], kz ) + + E = amplitude * z_contribution + + return( E ) + +def get_theoretical_J_field( field, t ): + # wpdt/2 accounts for the Yee halfstep offset of the current + dt = t / 80 # SPECIFIC to config parameters! + amplitude = - epsilon_0 * wp * epsilon * (m_e*c**2*k[field])/e * np.cos(wp*t-wp*dt/2) + cos_flag = cos[field] + + z_contribution = get_contribution( cos_flag[2], kz ) + + J = amplitude * z_contribution + + return( J ) + +def get_theoretical_rho_field( field, t ): + amplitude = epsilon_0 * epsilon * (m_e*c**2*(kz*kz))/e * np.sin(wp*t) + cos_flag = cos_rho[field] + z_contribution = get_contribution( cos_flag[2], kz) + + rho = amplitude * z_contribution + + return( rho ) + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, + dims=ds.domain_dimensions) +# Check the validity of the fields +error_rel = 0 +for field in ['Ez']: + E_sim = data[('mesh',field)].to_ndarray()[:,0,0] + E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim-E_th).max()/abs(E_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the currents +for field in ['Jz']: + J_sim = data[('mesh',field)].to_ndarray()[:,0,0] + J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim-J_th).max()/abs(J_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray()[:,0,0] + rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Plot the last field from the loop (Ez at iteration 80) +plt.subplot2grid( (1,2), (0,0) ) +plt.plot( E_sim ) +#plt.colorbar() +plt.title('Ez, last iteration\n(simulation)') +plt.subplot2grid( (1,2), (0,1) ) +plt.plot( E_th ) +#plt.colorbar() +plt.title('Ez, last iteration\n(theory)') +plt.tight_layout() +plt.savefig('langmuir_fluid_multi_1d_analysis.png') + +tolerance_rel = 0.05 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/langmuir_fluids/analysis_2d.py b/Examples/Tests/langmuir_fluids/analysis_2d.py new file mode 100755 index 00000000000..cf5d2fb44de --- /dev/null +++ b/Examples/Tests/langmuir_fluids/analysis_2d.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Jean-Luc Vay, Maxence Thevenet, Remi Lehe, Grant Johnson +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs.multi.rt`. This simulates a 3D periodic plasma wave. +# The electric field in the simulation is given (in theory) by: +# $$ E_x = \epsilon \,\frac{m_e c^2 k_x}{q_e}\sin(k_x x)\cos(k_y y)\cos(k_z z)\sin( \omega_p t)$$ +# $$ E_y = \epsilon \,\frac{m_e c^2 k_y}{q_e}\cos(k_x x)\sin(k_y y)\cos(k_z z)\sin( \omega_p t)$$ +# $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\cos(k_x x)\cos(k_y y)\sin(k_z z)\sin( \omega_p t)$$ +import os +import sys + +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +epsilon = 0.01 +n = 4.e24 +n_osc_x = 2 +n_osc_z = 2 +xmin = -20e-6; xmax = 20.e-6; Nx = 128 +zmin = -20e-6; zmax = 20.e-6; Nz = 128 + +# Wave vector of the wave +kx = 2.*np.pi*n_osc_x/(xmax-xmin) +kz = 2.*np.pi*n_osc_z/(zmax-zmin) +# Plasma frequency +wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) + +k = {'Ex':kx, 'Ez':kz, 'Jx':kx, 'Jz':kz} +cos = {'Ex': (0,1,1), 'Ez':(1,1,0),'Jx': (0,1,1), 'Jz':(1,1,0)} +cos_rho = {'rho': (1,1,1)} + +def get_contribution( is_cos, k ): + du = (xmax-xmin)/Nx + u = xmin + du*( 0.5 + np.arange(Nx) ) + if is_cos == 1: + return( np.cos(k*u) ) + else: + return( np.sin(k*u) ) + +def get_theoretical_field( field, t ): + amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) + cos_flag = cos[field] + x_contribution = get_contribution( cos_flag[0], kx ) + z_contribution = get_contribution( cos_flag[2], kz ) + + E = amplitude * x_contribution[:, np.newaxis ] \ + * z_contribution[np.newaxis, :] + + return( E ) + +def get_theoretical_J_field( field, t ): + # wpdt/2 accounts for the Yee halfstep offset of the current + dt = t / 40 # SPECIFIC to config parameters! + amplitude = - epsilon_0 * wp * epsilon * (m_e*c**2*k[field])/e * np.cos(wp*t-wp*dt/2) + cos_flag = cos[field] + x_contribution = get_contribution( cos_flag[0], kx ) + z_contribution = get_contribution( cos_flag[2], kz ) + + J = amplitude * x_contribution[:, np.newaxis] \ + * z_contribution[np.newaxis, :] + + return( J ) + +def get_theoretical_rho_field( field, t ): + amplitude = epsilon_0 * epsilon * (m_e*c**2*(kx*kx+kz*kz))/e * np.sin(wp*t) + cos_flag = cos_rho[field] + x_contribution = get_contribution( cos_flag[0], kx ) + z_contribution = get_contribution( cos_flag[2], kz ) + + rho = amplitude * x_contribution[:, np.newaxis] \ + * z_contribution[ np.newaxis, :] + + return( rho ) + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) +edge = np.array([(ds.domain_left_edge[1]).item(), (ds.domain_right_edge[1]).item(), \ + (ds.domain_left_edge[0]).item(), (ds.domain_right_edge[0]).item()]) + +# Check the validity of the fields +error_rel = 0 +for field in ['Ex', 'Ez']: + E_sim = data[('mesh',field)].to_ndarray()[:,:,0] + E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim-E_th).max()/abs(E_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the currents +for field in ['Jx', 'Jz']: + J_sim = data[('mesh',field)].to_ndarray()[:,:,0] + J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim-J_th).max()/abs(J_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray()[:,:,0] + rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Plot the last field from the loop (Ez at iteration 40) +fig, (ax1, ax2) = plt.subplots(1, 2, dpi = 100) +# First plot +vmin = E_sim.min() +vmax = E_sim.max() +cax1 = make_axes_locatable(ax1).append_axes('right', size = '5%', pad = '5%') +im1 = ax1.imshow(E_sim, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) +cb1 = fig.colorbar(im1, cax = cax1) +ax1.set_xlabel(r'$z$') +ax1.set_ylabel(r'$x$') +ax1.set_title(r'$E_z$ (sim)') +# Second plot +vmin = E_th.min() +vmax = E_th.max() +cax2 = make_axes_locatable(ax2).append_axes('right', size = '5%', pad = '5%') +im2 = ax2.imshow(E_th, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) +cb2 = fig.colorbar(im2, cax = cax2) +ax2.set_xlabel(r'$z$') +ax2.set_ylabel(r'$x$') +ax2.set_title(r'$E_z$ (theory)') +# Save figure +fig.tight_layout() +fig.savefig('Langmuir_fluid_multi_2d_analysis.png', dpi = 200) + +tolerance_rel = 0.05 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/langmuir_fluids/analysis_3d.py b/Examples/Tests/langmuir_fluids/analysis_3d.py new file mode 100755 index 00000000000..0211956859e --- /dev/null +++ b/Examples/Tests/langmuir_fluids/analysis_3d.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2022 Jean-Luc Vay, Maxence Thevenet, Remi Lehe, Axel Huebl, Grant Johnson +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs.multi.rt`. This simulates a 3D periodic plasma wave. +# The electric field in the simulation is given (in theory) by: +# $$ E_x = \epsilon \,\frac{m_e c^2 k_x}{q_e}\sin(k_x x)\cos(k_y y)\cos(k_z z)\sin( \omega_p t)$$ +# $$ E_y = \epsilon \,\frac{m_e c^2 k_y}{q_e}\cos(k_x x)\sin(k_y y)\cos(k_z z)\sin( \omega_p t)$$ +# $$ E_z = \epsilon \,\frac{m_e c^2 k_z}{q_e}\cos(k_x x)\cos(k_y y)\sin(k_z z)\sin( \omega_p t)$$ +import os +import re +import sys + +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Parameters (these parameters must match the parameters in `inputs.multi.rt`) +epsilon = 0.01 +n = 4.e24 +n_osc_x = 2 +n_osc_y = 2 +n_osc_z = 2 +lo = [-20.e-6, -20.e-6, -20.e-6] +hi = [ 20.e-6, 20.e-6, 20.e-6] +Ncell = [64, 64, 64] + +# Wave vector of the wave +kx = 2.*np.pi*n_osc_x/(hi[0]-lo[0]) +ky = 2.*np.pi*n_osc_y/(hi[1]-lo[1]) +kz = 2.*np.pi*n_osc_z/(hi[2]-lo[2]) +# Plasma frequency +wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) + +k = {'Ex':kx, 'Ey':ky, 'Ez':kz, 'Jx':kx, 'Jy':ky, 'Jz':kz} +cos = {'Ex': (0,1,1), 'Ey':(1,0,1), 'Ez':(1,1,0),'Jx': (0,1,1), 'Jy':(1,0,1), 'Jz':(1,1,0)} +cos_rho = {'rho': (1,1,1)} + +def get_contribution( is_cos, k, idim ): + du = (hi[idim]-lo[idim])/Ncell[idim] + u = lo[idim] + du*( 0.5 + np.arange(Ncell[idim]) ) + if is_cos[idim] == 1: + return( np.cos(k*u) ) + else: + return( np.sin(k*u) ) + +def get_theoretical_field( field, t ): + amplitude = epsilon * (m_e*c**2*k[field])/e * np.sin(wp*t) + cos_flag = cos[field] + x_contribution = get_contribution( cos_flag, kx, 0 ) + y_contribution = get_contribution( cos_flag, ky, 1 ) + z_contribution = get_contribution( cos_flag, kz, 2 ) + + E = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ + * y_contribution[np.newaxis, :, np.newaxis] \ + * z_contribution[np.newaxis, np.newaxis, :] + + return( E ) + +def get_theoretical_J_field( field, t ): + # wpdt/2 accounts for the Yee halfstep offset of the current + dt = t / 40 # SPECIFIC to config parameters! + amplitude = - epsilon_0 * wp * epsilon * (m_e*c**2*k[field])/e * np.cos(wp*t-wp*dt/2) + cos_flag = cos[field] + x_contribution = get_contribution( cos_flag, kx, 0 ) + y_contribution = get_contribution( cos_flag, ky, 1 ) + z_contribution = get_contribution( cos_flag, kz, 2 ) + + J = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ + * y_contribution[np.newaxis, :, np.newaxis] \ + * z_contribution[np.newaxis, np.newaxis, :] + + return( J ) + +def get_theoretical_rho_field( field, t ): + amplitude = epsilon_0 * epsilon * (m_e*c**2*(kx*kx+ky*ky+kz*kz))/e * np.sin(wp*t) + cos_flag = cos_rho[field] + x_contribution = get_contribution( cos_flag, kx, 0 ) + y_contribution = get_contribution( cos_flag, ky, 1 ) + z_contribution = get_contribution( cos_flag, kz, 2 ) + + rho = amplitude * x_contribution[:, np.newaxis, np.newaxis] \ + * y_contribution[np.newaxis, :, np.newaxis] \ + * z_contribution[np.newaxis, np.newaxis, :] + + return( rho ) + +# Read the file +ds = yt.load(fn) + + +t0 = ds.current_time.to_value() +data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) +edge = np.array([(ds.domain_left_edge[2]).item(), (ds.domain_right_edge[2]).item(), \ + (ds.domain_left_edge[0]).item(), (ds.domain_right_edge[0]).item()]) + +# Check the validity of the fields +error_rel = 0 +for field in ['Ex', 'Ey', 'Ez']: + E_sim = data[('mesh',field)].to_ndarray() + E_th = get_theoretical_field(field, t0) + max_error = abs(E_sim-E_th).max()/abs(E_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + + +# Check the validity of the currents +for field in ['Jx', 'Jy', 'Jz']: + J_sim = data[('mesh',field)].to_ndarray() + J_th = get_theoretical_J_field(field, t0) + max_error = abs(J_sim-J_th).max()/abs(J_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + +# Check the validity of the charge +for field in ['rho']: + rho_sim = data[('boxlib',field)].to_ndarray() + rho_th = get_theoretical_rho_field(field, t0) + max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() + print('%s: Max error: %.2e' %(field,max_error)) + error_rel = max( error_rel, max_error ) + + +# Plot the last field from the loop (Ez at iteration 40) +fig, (ax1, ax2) = plt.subplots(1, 2, dpi = 100) +# First plot (slice at y=0) +E_plot = E_sim[:,Ncell[1]//2+1,:] +vmin = E_plot.min() +vmax = E_plot.max() +cax1 = make_axes_locatable(ax1).append_axes('right', size = '5%', pad = '5%') +im1 = ax1.imshow(E_plot, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) +cb1 = fig.colorbar(im1, cax = cax1) +ax1.set_xlabel(r'$z$') +ax1.set_ylabel(r'$x$') +ax1.set_title(r'$E_z$ (sim)') +# Second plot (slice at y=0) +E_plot = E_th[:,Ncell[1]//2+1,:] +vmin = E_plot.min() +vmax = E_plot.max() +cax2 = make_axes_locatable(ax2).append_axes('right', size = '5%', pad = '5%') +im2 = ax2.imshow(E_plot, origin = 'lower', extent = edge, vmin = vmin, vmax = vmax) +cb2 = fig.colorbar(im2, cax = cax2) +ax2.set_xlabel(r'$z$') +ax2.set_ylabel(r'$x$') +ax2.set_title(r'$E_z$ (theory)') +# Save figure +fig.tight_layout() +fig.savefig('Langmuir_fluid_multi_analysis.png', dpi = 200) + +tolerance_rel = 5e-2 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] + +if re.search( 'single_precision', fn ): + checksumAPI.evaluate_checksum(test_name, fn, rtol=1.e-3) +else: + checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/langmuir_fluids/analysis_rz.py b/Examples/Tests/langmuir_fluids/analysis_rz.py new file mode 100755 index 00000000000..108d054e75a --- /dev/null +++ b/Examples/Tests/langmuir_fluids/analysis_rz.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +# Copyright 2019 David Grote, Maxence Thevenet, Grant Johnson, Remi Lehe +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs.multi.rz.rt`. This simulates a RZ periodic plasma wave. +# The electric field in the simulation is given (in theory) by: +# $$ E_r = -\partial_r \phi = \epsilon \,\frac{mc^2}{e}\frac{2\,r}{w_0^2} \exp\left(-\frac{r^2}{w_0^2}\right) \sin(k_0 z) \sin(\omega_p t) +# $$ E_z = -\partial_z \phi = - \epsilon \,\frac{mc^2}{e} k_0 \exp\left(-\frac{r^2}{w_0^2}\right) \cos(k_0 z) \sin(\omega_p t) +# Unrelated to the Langmuir waves, we also test the plotfile particle filter function in this +# analysis script. +import os +import re +import sys + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + +import numpy as np +from scipy.constants import c, e, epsilon_0, m_e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +test_name = os.path.split(os.getcwd())[1] + +# Parse test name and check if current correction (psatd.current_correction) is applied +current_correction = True if re.search('current_correction', fn) else False + +# Parameters (these parameters must match the parameters in `inputs.multi.rz.rt`) +epsilon = 0.01 +n = 2.e24 +w0 = 5.e-6 +n_osc_z = 2 +rmin = 0e-6; rmax = 20.e-6; Nr = 64 +zmin = -20e-6; zmax = 20.e-6; Nz = 128 + +# Wave vector of the wave +k0 = 2.*np.pi*n_osc_z/(zmax-zmin) +# Plasma frequency +wp = np.sqrt((n*e**2)/(m_e*epsilon_0)) +kp = wp/c + +def Er( z, r, epsilon, k0, w0, wp, t) : + """ + Return the radial electric field as an array + of the same length as z and r, in the half-plane theta=0 + """ + Er_array = \ + epsilon * m_e*c**2/e * 2*r/w0**2 * \ + np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) + return( Er_array ) + +def Ez( z, r, epsilon, k0, w0, wp, t) : + """ + Return the longitudinal electric field as an array + of the same length as z and r, in the half-plane theta=0 + """ + Ez_array = \ + - epsilon * m_e*c**2/e * k0 * \ + np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t ) + return( Ez_array ) + +def Jr( z, r, epsilon, k0, w0, wp, t) : + """ + Return the radial current density as an array + of the same length as z and r, in the half-plane theta=0 + """ + dt = t / 80 # SPECIFIC to config parameters! + Jr_array = \ + - epsilon_0 * epsilon * m_e*c**2/e * 2*r/w0**2 * \ + np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.cos( wp*t -wp*dt/2) * wp #phase_error = wp*dt/2 + return( Jr_array ) + +def Jz( z, r, epsilon, k0, w0, wp, t) : + """ + Return the longitudinal current density as an array + of the same length as z and r, in the half-plane theta=0 + """ + dt = t / 80 # SPECIFIC to config parameters! + Jz_array = \ + epsilon_0 * epsilon * m_e*c**2/e * k0 * \ + np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.cos( wp*t -wp*dt/2) * wp #phase_error = wp*dt/2 + return( Jz_array ) + +def rho( z, r, epsilon, k0, w0, wp, t) : + """ + Return the charge density as an array + of the same length as z and r, in the half-plane theta=0 + """ + rho_array = \ + epsilon_0 * epsilon * m_e*c**2/e * np.sin( wp*t ) * np.sin( k0*z ) * np.exp( -r**2/w0**2 ) * \ + ((4.0/(w0**2))*(1 - (r**2)/(w0**2)) + k0**2) + return( rho_array ) + +# Read the file +ds = yt.load(fn) +t0 = ds.current_time.to_value() +data = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, + dims=ds.domain_dimensions) + +# Get cell centered coordinates +dr = (rmax - rmin)/Nr +dz = (zmax - zmin)/Nz +coords = np.indices([Nr, Nz],'d') +rr = rmin + (coords[0] + 0.5)*dr +zz = zmin + (coords[1] + 0.5)*dz + +# Check the validity of the fields +overall_max_error = 0 +Er_sim = data[('boxlib','Er')].to_ndarray()[:,:,0] +Er_th = Er(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(Er_sim-Er_th).max()/abs(Er_th).max() +print('Er: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +Ez_sim = data[('boxlib','Ez')].to_ndarray()[:,:,0] +Ez_th = Ez(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(Ez_sim-Ez_th).max()/abs(Ez_th).max() +print('Ez: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +Jr_sim = data[('boxlib','jr')].to_ndarray()[:,:,0] +Jr_th = Jr(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(Jr_sim-Jr_th).max()/abs(Jr_th).max() +print('Jr: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +Jz_sim = data[('boxlib','jz')].to_ndarray()[:,:,0] +Jz_th = Jz(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(Jz_sim-Jz_th).max()/abs(Jz_th).max() +print('Jz: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +rho_sim = data[('boxlib','rho')].to_ndarray()[:,:,0] +rho_th = rho(zz, rr, epsilon, k0, w0, wp, t0) +max_error = abs(rho_sim-rho_th).max()/abs(rho_th).max() +print('rho: Max error: %.2e' %(max_error)) +overall_max_error = max( overall_max_error, max_error ) + +# Plot the last field from the loop (Ez at iteration 40) +plt.subplot2grid( (1,2), (0,0) ) +plt.imshow( Ez_sim ) +plt.colorbar() +plt.title('Ez, last iteration\n(simulation)') +plt.subplot2grid( (1,2), (0,1) ) +plt.imshow( Ez_th ) +plt.colorbar() +plt.title('Ez, last iteration\n(theory)') +plt.tight_layout() +plt.savefig(test_name+'_analysis.png') + +error_rel = overall_max_error + +tolerance_rel = 0.08 + +print("error_rel : " + str(error_rel)) +print("tolerance_rel: " + str(tolerance_rel)) + +assert( error_rel < tolerance_rel ) + +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/langmuir_fluids/inputs_1d b/Examples/Tests/langmuir_fluids/inputs_1d new file mode 100644 index 00000000000..48fda36c61f --- /dev/null +++ b/Examples/Tests/langmuir_fluids/inputs_1d @@ -0,0 +1,72 @@ +# Maximum number of time steps +max_step = 80 + +# number of grid points +amr.n_cell = 128 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 64 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 1 +geometry.prob_lo = -20.e-6 # physical domain +geometry.prob_hi = 20.e-6 + +# Boundary condition +boundary.field_lo = periodic +boundary.field_hi = periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +warpx.use_filter = 0 + +# CFL +warpx.cfl = 0.8 + +# Parameters for the plasma wave +my_constants.epsilon = 0.01 +my_constants.n0 = 2.e24 # electron and positron densities, #/m^3 +my_constants.wp = sqrt(2.*n0*q_e**2/(epsilon0*m_e)) # plasma frequency +my_constants.kp = wp/clight # plasma wavenumber +my_constants.k = 2.*pi/20.e-6 # perturbation wavenumber +# Note: kp is calculated in SI for a density of 4e24 (i.e. 2e24 electrons + 2e24 positrons) +# k is calculated so as to have 2 periods within the 40e-6 wide box. + +# Particles +fluids.species_names = electrons positrons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.profile = constant +electrons.density = n0 # number of electrons per m^3 +electrons.momentum_distribution_type = parse_momentum_function +electrons.momentum_function_ux(x,y,z) = "epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +electrons.momentum_function_uy(x,y,z) = "epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +electrons.momentum_function_uz(x,y,z) = "epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +positrons.charge = q_e +positrons.mass = m_e +positrons.profile = constant +positrons.density = n0 # number of positrons per m^3 +positrons.momentum_distribution_type = parse_momentum_function +positrons.momentum_function_ux(x,y,z) = "-epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +positrons.momentum_function_uy(x,y,z) = "-epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +positrons.momentum_function_uz(x,y,z) = "-epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +# Diagnostics +diagnostics.diags_names = diag1 openpmd +diag1.intervals = 40 +diag1.diag_type = Full +diag1.fields_to_plot = Ez jz rho + +openpmd.intervals = 40 +openpmd.diag_type = Full +openpmd.format = openpmd diff --git a/Examples/Tests/langmuir_fluids/inputs_2d b/Examples/Tests/langmuir_fluids/inputs_2d new file mode 100644 index 00000000000..d75fa0f3393 --- /dev/null +++ b/Examples/Tests/langmuir_fluids/inputs_2d @@ -0,0 +1,69 @@ +# Maximum number of time steps +max_step = 80 + +# number of grid points +amr.n_cell = 128 128 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 64 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = -20.e-6 -20.e-6 # physical domain +geometry.prob_hi = 20.e-6 20.e-6 + +# Boundary condition +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.field_gathering = energy-conserving +warpx.use_filter = 0 + +# CFL +warpx.cfl = 1.0 + +# Parameters for the plasma wave +my_constants.epsilon = 0.01 +my_constants.n0 = 2.e24 # electron and positron densities, #/m^3 +my_constants.wp = sqrt(2.*n0*q_e**2/(epsilon0*m_e)) # plasma frequency +my_constants.kp = wp/clight # plasma wavenumber +my_constants.k = 2.*pi/20.e-6 # perturbation wavenumber +# Note: kp is calculated in SI for a density of 4e24 (i.e. 2e24 electrons + 2e24 positrons) +# k is calculated so as to have 2 periods within the 40e-6 wide box. + +# Fluids +fluids.species_names = electrons positrons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.profile = constant +electrons.density = n0 # number of electrons per m^3 +electrons.momentum_distribution_type = parse_momentum_function +electrons.momentum_function_ux(x,y,z) = "epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +electrons.momentum_function_uy(x,y,z) = "epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +electrons.momentum_function_uz(x,y,z) = "epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +positrons.charge = q_e +positrons.mass = m_e +positrons.profile = constant +positrons.density = n0 # number of positrons per m^3 +positrons.momentum_distribution_type = parse_momentum_function +positrons.momentum_function_ux(x,y,z) = "-epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +positrons.momentum_function_uy(x,y,z) = "-epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +positrons.momentum_function_uz(x,y,z) = "-epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 40 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ez jx jz rho diff --git a/Examples/Tests/langmuir_fluids/inputs_3d b/Examples/Tests/langmuir_fluids/inputs_3d new file mode 100644 index 00000000000..f92ccf0ed60 --- /dev/null +++ b/Examples/Tests/langmuir_fluids/inputs_3d @@ -0,0 +1,74 @@ +# Parameters for the plasma wave +my_constants.max_step = 40 +my_constants.lx = 40.e-6 # length of sides +my_constants.dx = 6.25e-07 # grid cell size +my_constants.nx = lx/dx # number of cells in each dimension +my_constants.epsilon = 0.01 +my_constants.n0 = 2.e24 # electron and positron densities, #/m^3 +my_constants.wp = sqrt(2.*n0*q_e**2/(epsilon0*m_e)) # plasma frequency +my_constants.kp = wp/clight # plasma wavenumber +my_constants.k = 2.*2.*pi/lx # perturbation wavenumber +# Note: kp is calculated in SI for a density of 4e24 (i.e. 2e24 electrons + 2e24 positrons) +# k is calculated so as to have 2 periods within the 40e-6 wide box. + +# Maximum number of time steps +max_step = max_step + +# number of grid points +amr.n_cell = nx nx nx + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = nx nx nx + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 3 +geometry.prob_lo = -lx/2. -lx/2. -lx/2. # physical domain +geometry.prob_hi = lx/2. lx/2. lx/2. + +# Boundary condition +boundary.field_lo = periodic periodic periodic +boundary.field_hi = periodic periodic periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +warpx.use_filter = 0 + +# CFL +warpx.cfl = 1.0 + +# Fluids +fluids.species_names = electrons positrons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.profile = constant +electrons.density = n0 # number of electrons per m^3 +electrons.momentum_distribution_type = parse_momentum_function +electrons.momentum_function_ux(x,y,z) = "epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +electrons.momentum_function_uy(x,y,z) = "epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +electrons.momentum_function_uz(x,y,z) = "epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +positrons.charge = q_e +positrons.mass = m_e +positrons.profile = constant +positrons.density = n0 # number of positrons per m^3 +positrons.momentum_distribution_type = parse_momentum_function +positrons.momentum_function_ux(x,y,z) = "-epsilon * k/kp * sin(k*x) * cos(k*y) * cos(k*z)" +positrons.momentum_function_uy(x,y,z) = "-epsilon * k/kp * cos(k*x) * sin(k*y) * cos(k*z)" +positrons.momentum_function_uz(x,y,z) = "-epsilon * k/kp * cos(k*x) * cos(k*y) * sin(k*z)" + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = max_step +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho diff --git a/Examples/Tests/langmuir_fluids/inputs_rz b/Examples/Tests/langmuir_fluids/inputs_rz new file mode 100644 index 00000000000..2be427cbce5 --- /dev/null +++ b/Examples/Tests/langmuir_fluids/inputs_rz @@ -0,0 +1,72 @@ +# Parameters for the plasma wave +my_constants.max_step = 80 +my_constants.epsilon = 0.01 +my_constants.n0 = 2.e24 # electron density, #/m^3 +my_constants.wp = sqrt(n0*q_e**2/(epsilon0*m_e)) # plasma frequency +my_constants.kp = wp/clight # plasma wavenumber +my_constants.k0 = 2.*pi/20.e-6 # longitudianl perturbation wavenumber +my_constants.w0 = 5.e-6 # transverse perturbation length +# Note: kp is calculated in SI for a density of 2e24 +# k0 is calculated so as to have 2 periods within the 40e-6 wide box. + +# Maximum number of time steps +max_step = max_step + +# number of grid points +amr.n_cell = 64 128 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 64 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = RZ +geometry.prob_lo = 0.e-6 -20.e-6 # physical domain +geometry.prob_hi = 20.e-6 20.e-6 +boundary.field_lo = none periodic +boundary.field_hi = none periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.field_gathering = energy-conserving +algo.current_deposition = esirkepov +warpx.use_filter = 0 + +# CFL +warpx.cfl = 1.0 + +# Having this turned on makes for a more sensitive test +warpx.do_dive_cleaning = 1 + +# Fluids +fluids.species_names = electrons ions + +electrons.charge = -q_e +electrons.mass = m_e +electrons.profile = constant +electrons.density = n0 # number of electrons per m^3 +electrons.momentum_distribution_type = parse_momentum_function +electrons.momentum_function_ux(x,y,z) = "epsilon/kp*2*x/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z)" +electrons.momentum_function_uy(x,y,z) = "epsilon/kp*2*y/w0**2*exp(-(x**2+y**2)/w0**2)*sin(k0*z)" +electrons.momentum_function_uz(x,y,z) = "-epsilon/kp*k0*exp(-(x**2+y**2)/w0**2)*cos(k0*z)" + + +ions.charge = q_e +ions.mass = m_p +ions.profile = constant +ions.density = n0 # number of ions per m^3 +ions.momentum_distribution_type = at_rest + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 80 +diag1.diag_type = Full + +diag1.fields_to_plot = jr jz Er Ez Bt rho diff --git a/Examples/Tests/laser_injection_from_file/analysis_2d_binary.py b/Examples/Tests/laser_injection_from_file/analysis_2d_binary.py index fd59fcf33dd..c5bdd84d023 100755 --- a/Examples/Tests/laser_injection_from_file/analysis_2d_binary.py +++ b/Examples/Tests/laser_injection_from_file/analysis_2d_binary.py @@ -121,9 +121,9 @@ def write_file(fname, x, y, t, E): file.write(E.tobytes()) def create_gaussian_2d(): - T, X, Y = np.meshgrid(tcoords, xcoords, np.array([0.0]), indexing='ij') - E_t = gauss(T,X,Y,'2d') - write_file("gauss_2d", xcoords, np.array([0.0]), tcoords, E_t) + T, X, Y = np.meshgrid(tcoords, xcoords, np.array([0.0]), indexing='ij') + E_t = gauss(T,X,Y,'2d') + write_file("gauss_2d", xcoords, np.array([0.0]), tcoords, E_t) def do_analysis(fname, compname, steps): ds = yt.load(fname) diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py index 18c62ee523f..8f205724563 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py @@ -113,6 +113,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 1, + write_dir = '.', + warpx_file_prefix = 'Python_magnetostatic_eb_3d_plt' +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -140,6 +146,7 @@ sim.add_species(beam, layout=beam_layout, initialize_self_field=True) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) ########################## diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py index 693bc1b888d..1268f7a02b0 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py @@ -110,6 +110,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 1, + write_dir = '.', + warpx_file_prefix = 'Python_magnetostatic_eb_rz_plt' +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -137,6 +143,7 @@ sim.add_species(beam, layout=beam_layout, initialize_self_field=True) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) ########################## diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index 8550b8767c6..eec2ba4fffb 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -65,11 +65,11 @@ keV_to_Joule = scc.e*1e3 MeV_to_Joule = scc.e*1e6 barn_to_square_meter = 1.e-28 -m_p = scc.m_p # Proton mass +m_p = 1.00782503223*scc.m_u # Proton mass m_b = 11.00930536*scc.m_u # Boron 11 mass m_reduced = m_p*m_b/(m_p+m_b) -m_a = 4.002602*scc.m_u # Alpha mass -m_be = 7.94748*m_p # Beryllium 8 mass +m_a = 4.00260325413*scc.m_u # Alpha mass +m_be = 7.94748*scc.m_p # Beryllium 8 mass Z_boron = 5. Z_proton = 1. E_Gamow = (Z_boron*Z_proton*np.pi*scc.fine_structure)**2*2.*m_reduced*scc.c**2 @@ -395,7 +395,7 @@ def p_sq_boron_frame_to_E_COM_frame(p_proton_sq): # Use invariant E**2 - p**2c**2 of 4-momentum norm to compute energy in center of mass frame E_com = np.sqrt(E_lab**2 - p_proton_sq*scc.c**2) # Corresponding kinetic energy - E_com_kin = E_com - (m_b+scc.m_p)*scc.c**2 + E_com_kin = E_com - (m_b+m_p)*scc.c**2 return E_com_kin*(p_proton_sq>0.) def p_sq_to_kinetic_energy(p_sq, m): @@ -521,11 +521,11 @@ def check_initial_energy2(data): # Loop over all slices (i.e. cells in the z direction) for slice_number in range(1, size_z): - ## For simplicity, all the calculations in this functino are done nonrelativistically + ## For simplicity, all the calculations in this function are done nonrelativistically ## Proton kinetic energy in the lab frame before fusion E_proton_nonrelativistic = Energy_step*slice_number**2 ## Corresponding square norm of proton momentum - p_proton_sq = 2.*scc.m_p*E_proton_nonrelativistic + p_proton_sq = 2.*m_p*E_proton_nonrelativistic ## Kinetic energy in the lab frame after ## proton + boron 11 -> alpha + beryllium 8 E_after_fusion = E_proton_nonrelativistic + E_fusion diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d index 3a8d2002a40..0943eee1d22 100644 --- a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d +++ b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d @@ -74,18 +74,22 @@ neutron_1.do_not_deposit = 1 my_constants.background_dens = 1.e26 my_constants.beam_dens = 1.e20 -deuterium_2.species_type = deuterium -deuterium_2.injection_style = "NRandomPerCell" -deuterium_2.num_particles_per_cell = 1000 -deuterium_2.profile = "parse_density_function" ## A tenth of the macroparticles in each cell is made of immobile high-density background deuteriums. ## The other nine tenths are made of fast low-density beam deuteriums. -deuterium_2.density_function(x,y,z) = if(y - floor(y) < 0.1, 10.*background_dens, 10./9.*beam_dens) -deuterium_2.momentum_distribution_type = "parse_momentum_function" -deuterium_2.momentum_function_ux(x,y,z) = 0. -deuterium_2.momentum_function_uy(x,y,z) = 0. -deuterium_2.momentum_function_uz(x,y,z) = "if(y - floor(y) < 0.1, - 0., sqrt(2*m_deuterium*Energy_step*(floor(z)**2))/(m_deuterium*clight))" +deuterium_2.species_type = deuterium +deuterium_2.injection_sources = high_density low_density +deuterium_2.profile = constant +deuterium_2.high_density.injection_style = "NRandomPerCell" +deuterium_2.high_density.num_particles_per_cell = 100 +deuterium_2.high_density.density = 10.*background_dens +deuterium_2.high_density.momentum_distribution_type = at_rest +deuterium_2.low_density.injection_style = "NRandomPerCell" +deuterium_2.low_density.num_particles_per_cell = 900 +deuterium_2.low_density.density = 10./9.*beam_dens +deuterium_2.low_density.momentum_distribution_type = "parse_momentum_function" +deuterium_2.low_density.momentum_function_ux(x,y,z) = 0. +deuterium_2.low_density.momentum_function_uy(x,y,z) = 0. +deuterium_2.low_density.momentum_function_uz(x,y,z) = sqrt(2*m_deuterium*Energy_step*(floor(z)**2))/(m_deuterium*clight) deuterium_2.do_not_push = 1 deuterium_2.do_not_deposit = 1 diff --git a/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d b/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d index 36414552c11..30988b1ca2f 100644 --- a/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d +++ b/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d @@ -33,11 +33,12 @@ particles.species_names = proton1 boron1 alpha1 proton2 boron2 alpha2 proton3 bo proton4 boron4 alpha4 proton5 boron5 alpha5 my_constants.m_b11 = 11.00930536*m_u # Boron 11 mass -my_constants.m_reduced = m_p*m_b11/(m_p+m_b11) +my_constants.m_h = 1.00782503223*m_u # Hydrogen 1 mass +my_constants.m_reduced = m_h*m_b11/(m_h+m_b11) my_constants.keV_to_J = 1.e3*q_e my_constants.Energy_step = 22. * keV_to_J -proton1.species_type = proton +proton1.species_type = hydrogen1 proton1.injection_style = "NRandomPerCell" proton1.num_particles_per_cell = 80000 proton1.profile = constant @@ -46,7 +47,7 @@ proton1.momentum_distribution_type = "parse_momentum_function" proton1.momentum_function_ux(x,y,z) = 0. proton1.momentum_function_uy(x,y,z) = 0. ## Thanks to the floor, all particles in the same cell have the exact same momentum -proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_p*clight) +proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_h*clight) proton1.do_not_push = 1 proton1.do_not_deposit = 1 @@ -63,14 +64,14 @@ boron1.momentum_function_uz(x,y,z) = -sqrt(2*m_reduced*Energy_step*(floor(z)**2) boron1.do_not_push = 1 boron1.do_not_deposit = 1 -alpha1.species_type = helium +alpha1.species_type = helium4 alpha1.do_not_push = 1 alpha1.do_not_deposit = 1 my_constants.background_dens = 1.e26 my_constants.beam_dens = 1.e20 -proton2.species_type = proton +proton2.species_type = hydrogen1 proton2.injection_style = "NRandomPerCell" proton2.num_particles_per_cell = 8000 proton2.profile = "parse_density_function" @@ -81,7 +82,7 @@ proton2.momentum_distribution_type = "parse_momentum_function" proton2.momentum_function_ux(x,y,z) = 0. proton2.momentum_function_uy(x,y,z) = 0. proton2.momentum_function_uz(x,y,z) = "if(x - floor(x) < 0.1, - 0., sqrt(2*m_p*Energy_step*(floor(z)**2))/(m_p*clight))" + 0., sqrt(2*m_h*Energy_step*(floor(z)**2))/(m_h*clight))" proton2.do_not_push = 1 proton2.do_not_deposit = 1 @@ -94,19 +95,19 @@ boron2.momentum_distribution_type = "constant" boron2.do_not_push = 1 boron2.do_not_deposit = 1 -alpha2.species_type = helium +alpha2.species_type = helium4 alpha2.do_not_push = 1 alpha2.do_not_deposit = 1 my_constants.temperature = 44. * keV_to_J -proton3.species_type = proton +proton3.species_type = hydrogen1 proton3.injection_style = "NRandomPerCell" proton3.num_particles_per_cell = 4800 proton3.profile = constant proton3.density = 1.e28 proton3.momentum_distribution_type = "maxwell_boltzmann" -proton3.theta = temperature/(m_p*clight**2) +proton3.theta = temperature/(m_h*clight**2) proton3.do_not_push = 1 proton3.do_not_deposit = 1 @@ -120,19 +121,19 @@ boron3.theta = temperature/(m_b11*clight**2) boron3.do_not_push = 1 boron3.do_not_deposit = 1 -alpha3.species_type = helium +alpha3.species_type = helium4 alpha3.do_not_push = 1 alpha3.do_not_deposit = 1 my_constants.proton4_energy = 550*keV_to_J -proton4.species_type = proton +proton4.species_type = hydrogen1 proton4.injection_style = "NRandomPerCell" proton4.num_particles_per_cell = 800 proton4.profile = "constant" proton4.density = 1.e35 proton4.momentum_distribution_type = "constant" -proton4.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton4.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton4.do_not_push = 1 proton4.do_not_deposit = 1 @@ -145,17 +146,17 @@ boron4.momentum_distribution_type = "constant" boron4.do_not_push = 1 boron4.do_not_deposit = 1 -alpha4.species_type = helium +alpha4.species_type = helium4 alpha4.do_not_push = 1 alpha4.do_not_deposit = 1 -proton5.species_type = proton +proton5.species_type = hydrogen1 proton5.injection_style = "NRandomPerCell" proton5.num_particles_per_cell = 800 proton5.profile = "constant" proton5.density = 1.e35 proton5.momentum_distribution_type = "constant" -proton5.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton5.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton5.do_not_push = 1 proton5.do_not_deposit = 1 @@ -168,7 +169,7 @@ boron5.momentum_distribution_type = "constant" boron5.do_not_push = 1 boron5.do_not_deposit = 1 -alpha5.species_type = helium +alpha5.species_type = helium4 alpha5.do_not_push = 1 alpha5.do_not_deposit = 1 diff --git a/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d b/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d index 46744013986..421e39d2765 100644 --- a/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d +++ b/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d @@ -33,11 +33,12 @@ particles.species_names = proton1 boron1 alpha1 proton2 boron2 alpha2 proton3 bo proton4 boron4 alpha4 proton5 boron5 alpha5 my_constants.m_b11 = 11.00930536*m_u # Boron 11 mass -my_constants.m_reduced = m_p*m_b11/(m_p+m_b11) +my_constants.m_h = 1.00782503223*m_u # Hydrogen 1 mass +my_constants.m_reduced = m_h*m_b11/(m_h+m_b11) my_constants.keV_to_J = 1.e3*q_e my_constants.Energy_step = 22. * keV_to_J -proton1.species_type = proton +proton1.species_type = hydrogen1 proton1.injection_style = "NRandomPerCell" proton1.num_particles_per_cell = 10000 proton1.profile = constant @@ -46,7 +47,7 @@ proton1.momentum_distribution_type = "parse_momentum_function" proton1.momentum_function_ux(x,y,z) = 0. proton1.momentum_function_uy(x,y,z) = 0. ## Thanks to the floor, all particles in the same cell have the exact same momentum -proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_p*clight) +proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_h*clight) proton1.do_not_push = 1 proton1.do_not_deposit = 1 @@ -63,14 +64,14 @@ boron1.momentum_function_uz(x,y,z) = -sqrt(2*m_reduced*Energy_step*(floor(z)**2) boron1.do_not_push = 1 boron1.do_not_deposit = 1 -alpha1.species_type = helium +alpha1.species_type = helium4 alpha1.do_not_push = 1 alpha1.do_not_deposit = 1 my_constants.background_dens = 1.e26 my_constants.beam_dens = 1.e20 -proton2.species_type = proton +proton2.species_type = hydrogen1 proton2.injection_style = "NRandomPerCell" proton2.num_particles_per_cell = 1000 proton2.profile = "parse_density_function" @@ -81,7 +82,7 @@ proton2.momentum_distribution_type = "parse_momentum_function" proton2.momentum_function_ux(x,y,z) = 0. proton2.momentum_function_uy(x,y,z) = 0. proton2.momentum_function_uz(x,y,z) = "if(y - floor(y) < 0.1, - 0., sqrt(2*m_p*Energy_step*(floor(z)**2))/(m_p*clight))" + 0., sqrt(2*m_h*Energy_step*(floor(z)**2))/(m_h*clight))" proton2.do_not_push = 1 proton2.do_not_deposit = 1 @@ -94,19 +95,19 @@ boron2.momentum_distribution_type = "constant" boron2.do_not_push = 1 boron2.do_not_deposit = 1 -alpha2.species_type = helium +alpha2.species_type = helium4 alpha2.do_not_push = 1 alpha2.do_not_deposit = 1 my_constants.temperature = 44. * keV_to_J -proton3.species_type = proton +proton3.species_type = hydrogen1 proton3.injection_style = "NRandomPerCell" proton3.num_particles_per_cell = 600 proton3.profile = constant proton3.density = 1.e28 proton3.momentum_distribution_type = "maxwell_boltzmann" -proton3.theta = temperature/(m_p*clight**2) +proton3.theta = temperature/(m_h*clight**2) proton3.do_not_push = 1 proton3.do_not_deposit = 1 @@ -120,19 +121,19 @@ boron3.theta = temperature/(m_b11*clight**2) boron3.do_not_push = 1 boron3.do_not_deposit = 1 -alpha3.species_type = helium +alpha3.species_type = helium4 alpha3.do_not_push = 1 alpha3.do_not_deposit = 1 my_constants.proton4_energy = 550*keV_to_J -proton4.species_type = proton +proton4.species_type = hydrogen1 proton4.injection_style = "NRandomPerCell" proton4.num_particles_per_cell = 100 proton4.profile = "constant" proton4.density = 1.e35 proton4.momentum_distribution_type = "constant" -proton4.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton4.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton4.do_not_push = 1 proton4.do_not_deposit = 1 @@ -145,17 +146,17 @@ boron4.momentum_distribution_type = "constant" boron4.do_not_push = 1 boron4.do_not_deposit = 1 -alpha4.species_type = helium +alpha4.species_type = helium4 alpha4.do_not_push = 1 alpha4.do_not_deposit = 1 -proton5.species_type = proton +proton5.species_type = hydrogen1 proton5.injection_style = "NRandomPerCell" proton5.num_particles_per_cell = 100 proton5.profile = "constant" proton5.density = 1.e35 proton5.momentum_distribution_type = "constant" -proton5.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton5.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton5.do_not_push = 1 proton5.do_not_deposit = 1 @@ -168,7 +169,7 @@ boron5.momentum_distribution_type = "constant" boron5.do_not_push = 1 boron5.do_not_deposit = 1 -alpha5.species_type = helium +alpha5.species_type = helium4 alpha5.do_not_push = 1 alpha5.do_not_deposit = 1 diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py index 78d7c847699..9363bcebd18 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py @@ -4,8 +4,8 @@ # --- treated as kinetic particles and electrons as an isothermal, inertialess # --- background fluid. The script is set up to produce either parallel or # --- perpendicular (Bernstein) EM modes and can be run in 1d, 2d or 3d -# --- Cartesian geometries. See Section 4.2 and 4.3 of Munoz et al. (2018) As a -# --- CI test only a small number of steps are taken using the 1d version. +# --- Cartesian geometries. See Section 4.2 and 4.3 of Munoz et al. (2018). +# --- As a CI test only a small number of steps are taken using the 1d version. import argparse import os @@ -15,15 +15,16 @@ from mpi4py import MPI as mpi import numpy as np -from pywarpx import callbacks, fields, picmi +from pywarpx import callbacks, fields, libwarpx, picmi constants = picmi.constants comm = mpi.COMM_WORLD -simulation = picmi.Simulation(verbose=0) -# make a shorthand for simulation.extension since we use it a lot -sim_ext = simulation.extension +simulation = picmi.Simulation( + warpx_serialize_initial_conditions=True, + verbose=0 +) class EMModes(object): @@ -256,6 +257,15 @@ def setup_run(self): self.output_file_name = 'perp_field_data.txt' if self.test: + particle_diag = picmi.ParticleDiagnostic( + name='field_diag', + period=self.total_steps, + write_dir='.', + warpx_file_prefix='Python_ohms_law_solver_EM_modes_1d_plt', + # warpx_format = 'openpmd', + # warpx_openpmd_backend = 'h5' + ) + simulation.add_diagnostic(particle_diag) field_diag = picmi.FieldDiagnostic( name='field_diag', grid=self.grid, @@ -306,16 +316,16 @@ def _record_average_fields(self): similar format as the reduced diagnostic so that the same analysis script can be used regardless of the simulation dimension. """ - step = sim_ext.getistep() - 1 + step = simulation.extension.warpx.getistep(lev=0) - 1 if step % self.diag_steps != 0: return Bx_warpx = fields.BxWrapper()[...] - By_warpx = fields.BxWrapper()[...] + By_warpx = fields.ByWrapper()[...] Ez_warpx = fields.EzWrapper()[...] - if sim_ext.getMyProc() != 0: + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: return t = step * self.dt diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py new file mode 100644 index 00000000000..21d8cafe750 --- /dev/null +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# +# --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are +# --- treated as kinetic particles and electrons as an isothermal, inertialess +# --- background fluid. The script is set up to produce parallel normal EM modes +# --- in a metallic cylinder and is run in RZ geometry. +# --- As a CI test only a small number of steps are taken. + +import argparse +import sys + +import dill +from mpi4py import MPI as mpi +import numpy as np + +from pywarpx import picmi + +constants = picmi.constants + +comm = mpi.COMM_WORLD + +simulation = picmi.Simulation(verbose=0) + + +class CylindricalNormalModes(object): + '''The following runs a simulation of an uniform plasma at a set ion + temperature (and Te = 0) with an external magnetic field applied in the + z-direction (parallel to domain). + The analysis script (in this same directory) analyzes the output field + data for EM modes. + ''' + # Applied field parameters + B0 = 0.5 # Initial magnetic field strength (T) + beta = 0.01 # Plasma beta, used to calculate temperature + + # Plasma species parameters + m_ion = 400.0 # Ion mass (electron masses) + vA_over_c = 5e-3 # ratio of Alfven speed and the speed of light + + # Spatial domain + Nz = 512 # number of cells in z direction + Nr = 128 # number of cells in r direction + + # Temporal domain (if not run as a CI test) + LT = 800.0 # Simulation temporal length (ion cyclotron periods) + + # Numerical parameters + NPPC = 8000 # Seed number of particles per cell + DZ = 0.4 # Cell size (ion skin depths) + DR = 0.4 # Cell size (ion skin depths) + DT = 0.02 # Time step (ion cyclotron periods) + + # Plasma resistivity - used to dampen the mode excitation + eta = 5e-4 + # Number of substeps used to update B + substeps = 250 + + def __init__(self, test, verbose): + """Get input parameters for the specific case desired.""" + self.test = test + self.verbose = verbose or self.test + + # calculate various plasma parameters based on the simulation input + self.get_plasma_quantities() + + if not self.test: + self.total_steps = int(self.LT / self.DT) + else: + # if this is a test case run for only a small number of steps + self.total_steps = 100 + # and make the grid and particle count smaller + self.Nz = 128 + self.Nr = 64 + self.NPPC = 200 + # output diagnostics 5 times per cyclotron period + self.diag_steps = max(10, int(1.0 / 5 / self.DT)) + + self.Lz = self.Nz * self.DZ * self.l_i + self.Lr = self.Nr * self.DR * self.l_i + + self.dt = self.DT * self.t_ci + + # dump all the current attributes to a dill pickle file + if comm.rank == 0: + with open(f'sim_parameters.dpkl', 'wb') as f: + dill.dump(self, f) + + # print out plasma parameters + if comm.rank == 0: + print( + f"Initializing simulation with input parameters:\n" + f"\tT = {self.T_plasma:.3f} eV\n" + f"\tn = {self.n_plasma:.1e} m^-3\n" + f"\tB0 = {self.B0:.2f} T\n" + f"\tM/m = {self.m_ion:.0f}\n" + ) + print( + f"Plasma parameters:\n" + f"\tl_i = {self.l_i:.1e} m\n" + f"\tt_ci = {self.t_ci:.1e} s\n" + f"\tv_ti = {self.v_ti:.1e} m/s\n" + f"\tvA = {self.vA:.1e} m/s\n" + ) + print( + f"Numerical parameters:\n" + f"\tdt = {self.dt:.1e} s\n" + f"\tdiag steps = {self.diag_steps:d}\n" + f"\ttotal steps = {self.total_steps:d}\n", + flush=True + ) + self.setup_run() + + def get_plasma_quantities(self): + """Calculate various plasma parameters based on the simulation input.""" + # Ion mass (kg) + self.M = self.m_ion * constants.m_e + + # Cyclotron angular frequency (rad/s) and period (s) + self.w_ci = constants.q_e * abs(self.B0) / self.M + self.t_ci = 2.0 * np.pi / self.w_ci + + # Alfven speed (m/s): vA = B / sqrt(mu0 * n * (M + m)) = c * omega_ci / w_pi + self.vA = self.vA_over_c * constants.c + self.n_plasma = ( + (self.B0 / self.vA)**2 / (constants.mu0 * (self.M + constants.m_e)) + ) + + # Ion plasma frequency (Hz) + self.w_pi = np.sqrt( + constants.q_e**2 * self.n_plasma / (self.M * constants.ep0) + ) + + # Skin depth (m) + self.l_i = constants.c / self.w_pi + + # Ion thermal velocity (m/s) from beta = 2 * (v_ti / vA)**2 + self.v_ti = np.sqrt(self.beta / 2.0) * self.vA + + # Temperature (eV) from thermal speed: v_ti = sqrt(kT / M) + self.T_plasma = self.v_ti**2 * self.M / constants.q_e # eV + + # Larmor radius (m) + self.rho_i = self.v_ti / self.w_ci + + def setup_run(self): + """Setup simulation components.""" + + ####################################################################### + # Set geometry and boundary conditions # + ####################################################################### + + self.grid = picmi.CylindricalGrid( + number_of_cells=[self.Nr, self.Nz], + warpx_max_grid_size=self.Nz, + lower_bound=[0, -self.Lz/2.0], + upper_bound=[self.Lr, self.Lz/2.0], + lower_boundary_conditions = ['none', 'periodic'], + upper_boundary_conditions = ['dirichlet', 'periodic'], + lower_boundary_conditions_particles = ['absorbing', 'periodic'], + upper_boundary_conditions_particles = ['reflecting', 'periodic'] + ) + simulation.time_step_size = self.dt + simulation.max_steps = self.total_steps + simulation.current_deposition_algo = 'direct' + simulation.particle_shape = 1 + simulation.verbose = self.verbose + + ####################################################################### + # Field solver and external field # + ####################################################################### + + self.solver = picmi.HybridPICSolver( + grid=self.grid, + Te=0.0, n0=self.n_plasma, plasma_resistivity=self.eta, + substeps=self.substeps, + n_floor=self.n_plasma*0.05 + ) + simulation.solver = self.solver + + B_ext = picmi.AnalyticInitialField( + Bz_expression=self.B0 + ) + simulation.add_applied_field(B_ext) + + ####################################################################### + # Particle types setup # + ####################################################################### + + self.ions = picmi.Species( + name='ions', charge='q_e', mass=self.M, + initial_distribution=picmi.UniformDistribution( + density=self.n_plasma, + rms_velocity=[self.v_ti]*3, + ) + ) + simulation.add_species( + self.ions, + layout=picmi.PseudoRandomLayout( + grid=self.grid, n_macroparticles_per_cell=self.NPPC + ) + ) + + ####################################################################### + # Add diagnostics # + ####################################################################### + + field_diag = picmi.FieldDiagnostic( + name='field_diag', + grid=self.grid, + period=self.diag_steps, + data_list=['B', 'E'], + write_dir='diags', + warpx_file_prefix='field_diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5', + ) + simulation.add_diagnostic(field_diag) + + # add particle diagnostic for checksum + if self.test: + part_diag = picmi.ParticleDiagnostic( + name='diag1', + period=self.total_steps, + species=[self.ions], + data_list=['ux', 'uy', 'uz', 'weighting'], + write_dir='.', + warpx_file_prefix='Python_ohms_law_solver_EM_modes_rz_plt' + ) + simulation.add_diagnostic(part_diag) + + +########################## +# parse input parameters +########################## + +parser = argparse.ArgumentParser() +parser.add_argument( + '-t', '--test', help='toggle whether this script is run as a short CI test', + action='store_true', +) +parser.add_argument( + '-v', '--verbose', help='Verbose output', action='store_true', +) +args, left = parser.parse_known_args() +sys.argv = sys.argv[:1]+left + +run = CylindricalNormalModes(test=args.test, verbose=args.verbose) +simulation.step() diff --git a/Examples/Tests/ohm_solver_EM_modes/README.rst b/Examples/Tests/ohm_solver_EM_modes/README.rst new file mode 100644 index 00000000000..034ee5815f0 --- /dev/null +++ b/Examples/Tests/ohm_solver_EM_modes/README.rst @@ -0,0 +1,119 @@ +.. _examples-ohm-solver-em-modes: + +Ohm solver: Electromagnetic modes +================================= + +In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the +:math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are +Fourier analyzed for Alfvén mode excitations. + +Run +--- + +The same input script can be used for 1d, 2d or 3d Cartesian simulations as well +as replicating either the parallel propagating or ion-Bernstein modes as indicated below. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Parallel propagating waves + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --bdir z + + .. tab-item:: Perpendicular propagating waves + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y} + +Analyze +------- + +The following script reads the simulation output from the above example, performs +Fourier transforms of the field data and compares the calculated spectrum +to the theoretical dispersions. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis.py``. + +Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except +in a region of thermal resonances as indicated on the plot below. + +.. figure:: https://user-images.githubusercontent.com/40245517/216207688-9c39374a-9e69-45b8-a588-35b087b83d27.png + :alt: Parallel EM modes in thermal ion plasma + :width: 70% + + Calculated Alvén waves spectrum with the theoretical dispersions overlaid. + +Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. + +.. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png + :alt: Perpendicular modes in thermal ion plasma + :width: 50% + + Calculated ion Bernstein waves spectrum with the theoretical dispersion overlaid. + +Ohm solver: Cylindrical normal modes +==================================== + +A RZ-geometry example case for normal modes propagating along an applied magnetic +field in a cylinder is also available. The analytical solution for these modes +are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. + +Run +--- + +The following script initializes a thermal plasma in a metallic cylinder with +periodic boundaries at the cylinder ends. + +.. dropdown:: Script ``PICMI_inputs_rz.py`` + + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py``. + +The example can be executed using: + +.. code-block:: bash + + python3 PICMI_inputs_rz.py + +Analyze +------- + +After the simulation completes the following script can be used to analyze the +field evolution and extract the normal mode dispersion relation. It performs a +standard Fourier transform along the cylinder axis and a Hankel transform in the +radial direction. + +.. dropdown:: Script ``analysis_rz.py`` + + .. literalinclude:: analysis_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis_rz.py``. + +The following figure was produced with the above analysis script, showing excellent +agreement between the calculated and theoretical dispersion relations. + +.. figure:: https://user-images.githubusercontent.com/40245517/259251824-33e78375-81d8-410d-a147-3fa0498c66be.png + :alt: Normal EM modes in a metallic cylinder + :width: 90% + + Cylindrical normal mode dispersion comparing the calculated spectrum with the + theoretical one. diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py new file mode 100755 index 00000000000..49746a73a77 --- /dev/null +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# +# --- Analysis script for the hybrid-PIC example producing EM modes. + +import dill +from matplotlib import colors +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +import scipy.fft as fft +from scipy.interpolate import RegularGridInterpolator +from scipy.special import j1, jn, jn_zeros + +from pywarpx import picmi + +constants = picmi.constants + +# load simulation parameters +with open(f'sim_parameters.dpkl', 'rb') as f: + sim = dill.load(f) + +diag_dir = "diags/field_diags" + +ts = OpenPMDTimeSeries(diag_dir, check_all_files=True) + +def transform_spatially(data_for_transform): + # interpolate from regular r-grid to special r-grid + interp = RegularGridInterpolator( + (info.z, info.r), data_for_transform, + method='linear' + ) + data_interp = interp((zg, rg)) + + # Applying manual hankel in r + # Fmz = np.sum(proj*data_for_transform, axis=(2,3)) + Fmz = np.einsum('ijkl,kl->ij', proj, data_interp) + # Standard fourier in z + Fmn = fft.fftshift(fft.fft(Fmz, axis=1), axes=1) + return Fmn + +def process(it): + print(f"Processing iteration {it}", flush=True) + field, info = ts.get_field('E', 'y', iteration=it) + F_k = transform_spatially(field) + return F_k + +# grab the first iteration to get the grids +Bz, info = ts.get_field('B', 'z', iteration=0) + +nr = len(info.r) +nz = len(info.z) + +nkr = 12 # number of radial modes to solve for + +r_max = np.max(info.r) + +# create r-grid with points spaced out according to zeros of the Bessel function +r_grid = jn_zeros(1, nr) / jn_zeros(1, nr)[-1] * r_max + +zg, rg = np.meshgrid(info.z, r_grid) + +# Setup Hankel Transform +j_1M = jn_zeros(1, nr)[-1] +r_modes = np.arange(nkr) + +A = ( + 4.0 * np.pi * r_max**2 / j_1M**2 + * j1(np.outer(jn_zeros(1, max(r_modes)+1)[r_modes], jn_zeros(1, nr)) / j_1M) + / jn(2 ,jn_zeros(1, nr))**2 +) + +# No transformation for z +B = np.identity(nz) + +# combine projection arrays +proj = np.einsum('ab,cd->acbd', A, B) + +results = np.zeros((len(ts.t), nkr, nz), dtype=complex) +for ii, it in enumerate(ts.iterations): + results[ii] = process(it) + +# now Fourier transform in time +F_kw = fft.fftshift(fft.fft(results, axis=0), axes=0) + +dz = info.z[1] - info.z[0] +kz = 2*np.pi*fft.fftshift(fft.fftfreq(F_kw[0].shape[1], dz)) +dt = ts.iterations[1] - ts.iterations[0] +omega = 2*np.pi*fft.fftshift(fft.fftfreq(F_kw.shape[0], sim.dt*dt)) + +# Save data for future plotting purposes +np.savez( + "diags/spectrograms.npz", + F_kw=F_kw, dz=dz, kz=kz, dt=dt, omega=omega +) + +# plot the resulting dispersions +k = np.linspace(0, 250, 500) +kappa = k * sim.l_i + +fig, axes = plt.subplots(2, 2, sharex=True, sharey=True, figsize=(6.75, 5)) + +vmin = [2e-3, 1.5e-3, 7.5e-4, 5e-4] +vmax = 1.0 + +# plot m = 1 +for ii, m in enumerate([1, 3, 6, 8]): + ax = axes.flatten()[ii] + ax.set_title(f"m = {m}", fontsize=11) + m -= 1 + pm1 = ax.pcolormesh( + kz*sim.l_i, omega/sim.w_ci, + abs(F_kw[:, m, :])/np.max(abs(F_kw[:, m, :])), + norm=colors.LogNorm(vmin=vmin[ii], vmax=vmax), + cmap='inferno' + ) + cb = fig.colorbar(pm1, ax=ax) + cb.set_label(r'Normalized $E_\theta(k_z, m, \omega)$') + + # Get dispersion relation - see for example + # T. Stix, Waves in Plasmas (American Inst. of Physics, 1992), Chap 6, Sec 2 + nu_m = jn_zeros(1, m+1)[-1] / sim.Lr + R2 = 0.5 * (nu_m**2 * (1.0 + kappa**2) + k**2 * (kappa**2 + 2.0)) + P4 = k**2 * (nu_m**2 + k**2) + omega_fast = sim.vA * np.sqrt(R2 + np.sqrt(R2**2 - P4)) + omega_slow = sim.vA * np.sqrt(R2 - np.sqrt(R2**2 - P4)) + # Upper right corner + ax.plot(k*sim.l_i, omega_fast/sim.w_ci, 'w--', label = f"$\omega_{{fast}}$") + ax.plot(k*sim.l_i, omega_slow/sim.w_ci, color='white', linestyle='--', label = f"$\omega_{{slow}}$") + # Thermal resonance + thermal_res = sim.w_ci + 3*sim.v_ti*k + ax.plot(k*sim.l_i, thermal_res/sim.w_ci, color='magenta', linestyle='--', label = "$\omega = \Omega_i + 3v_{th,i}k$") + ax.plot(-k*sim.l_i, thermal_res/sim.w_ci, color='magenta', linestyle='--', label = "") + thermal_res = sim.w_ci - 3*sim.v_ti*k + ax.plot(k*sim.l_i, thermal_res/sim.w_ci, color='magenta', linestyle='--', label = "$\omega = \Omega_i + 3v_{th,i}k$") + ax.plot(-k*sim.l_i, thermal_res/sim.w_ci, color='magenta', linestyle='--', label = "") + + +for ax in axes.flatten(): + ax.set_xlim(-1.75, 1.75) + ax.set_ylim(0, 1.6) + +axes[0, 0].set_ylabel('$\omega/\Omega_{ci}$') +axes[1, 0].set_ylabel('$\omega/\Omega_{ci}$') +axes[1, 0].set_xlabel('$k_zl_i$') +axes[1, 1].set_xlabel('$k_zl_i$') + +plt.savefig('normal_modes_disp.png', dpi=600) +if not sim.test: + plt.show() +else: + plt.close() + + # check if power spectrum sampling match earlier results + amps = np.abs(F_kw[2, 1, len(kz)//2-2:len(kz)//2+2]) + print("Amplitude sample: ", amps) + assert np.allclose( + amps, np.array([61.4170941, 19.39380715, 101.08640009, 11.09261815]) + ) + +if sim.test: + import os + import sys + sys.path.insert(1, '../../../../warpx/Regression/Checksum/') + import checksumAPI + + # this will be the name of the plot file + fn = sys.argv[1] + test_name = os.path.split(os.getcwd())[1] + checksumAPI.evaluate_checksum(test_name, fn, rtol=1e-6) diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py index 8706ceb518c..d18c2025e45 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py @@ -2,7 +2,7 @@ # # --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are # --- treated as kinetic particles and electrons as an isothermal, inertialess -# --- background fluid. The script simulates ion Landau damping as descibed +# --- background fluid. The script simulates ion Landau damping as described # --- in section 4.5 of Munoz et al. (2018). import argparse @@ -14,15 +14,16 @@ from mpi4py import MPI as mpi import numpy as np -from pywarpx import callbacks, fields, particle_containers, picmi +from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi constants = picmi.constants comm = mpi.COMM_WORLD -simulation = picmi.Simulation(verbose=0) -# make a shorthand for simulation.extension since we use it a lot -sim_ext = simulation.extension +simulation = picmi.Simulation( + warpx_serialize_initial_conditions=True, + verbose=0 +) class IonLandauDamping(object): @@ -216,11 +217,21 @@ def setup_run(self): callbacks.installafterstep(self.text_diag) if self.test: + particle_diag = picmi.ParticleDiagnostic( + name='diag1', + period=100, + write_dir='.', + species=[self.ions], + data_list = ['ux', 'uy', 'uz', 'x', 'y', 'weighting'], + warpx_file_prefix=f'Python_ohms_law_solver_landau_damping_{self.dim}d_plt', + ) + simulation.add_diagnostic(particle_diag) field_diag = picmi.FieldDiagnostic( - name='field_diag', + name='diag1', grid=self.grid, period=100, write_dir='.', + data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], warpx_file_prefix=f'Python_ohms_law_solver_landau_damping_{self.dim}d_plt', ) simulation.add_diagnostic(field_diag) @@ -254,7 +265,8 @@ def setup_run(self): def text_diag(self): """Diagnostic function to print out timing data and particle numbers.""" - step = sim_ext.getistep(0) + step = simulation.extension.warpx.getistep(lev=0) - 1 + if step % (self.total_steps // 10) != 0: return @@ -278,7 +290,7 @@ def text_diag(self): "{step_rate:4.2f} steps/s" ) - if sim_ext.getMyProc() == 0: + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: print(diag_string.format(**status_dict)) self.prev_time = time.time() @@ -289,14 +301,14 @@ def _record_average_fields(self): similar format as the reduced diagnostic so that the same analysis script can be used regardless of the simulation dimension. """ - step = sim_ext.getistep() - 1 + step = simulation.extension.warpx.getistep(lev=0) - 1 if step % self.diag_steps != 0: return Ez_warpx = fields.EzWrapper()[...] - if sim_ext.getMyProc() != 0: + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: return t = step * self.dt diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst new file mode 100644 index 00000000000..dd4f94b4edf --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst @@ -0,0 +1,50 @@ +.. _examples-ohm-solver-ion-landau-damping: + +Ohm solver: Ion Landau Damping +============================== + +Landau damping is a well known process in which electrostatic (acoustic) waves are +damped by transferring energy to particles satisfying a resonance condition. +The process can be simulated by seeding a plasma with a specific acoustic mode +(density perturbation) and tracking the strength of the mode as a function of time. + +Run +--- + +The same input script can be used for 1d, 2d or 3d simulations and to sweep different +temperature ratios. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --temp_ratio {value} + +Analyze +------- + +The following script extracts the amplitude of the seeded mode as a function +of time and compares it to the theoretical damping rate. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py``. + +The figure below shows a set of such simulations with parameters matching those +described in section 4.5 of :cite:t:`ex-MUNOZ2018`. +The straight lines show the theoretical damping rate for the given temperature ratios. + +.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png + :alt: Ion Landau damping + :width: 70% + + Decay of seeded modes as a function of time for different electron-ion temperature ratios. + The theoretical damping of the given modes are shown in dashed lines. diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py index 08f0bb22bb6..7a3b976b092 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -15,15 +15,16 @@ from mpi4py import MPI as mpi import numpy as np -from pywarpx import callbacks, fields, particle_containers, picmi +from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi constants = picmi.constants comm = mpi.COMM_WORLD -simulation = picmi.Simulation(verbose=0) -# make a shorthand for simulation.extension since we use it a lot -sim_ext = simulation.extension +simulation = picmi.Simulation( + warpx_serialize_initial_conditions=True, + verbose=0 +) class HybridPICBeamInstability(object): @@ -256,10 +257,20 @@ def setup_run(self): callbacks.installafterstep(self.text_diag) if self.test: + part_diag = picmi.ParticleDiagnostic( + name='diag1', + period=1250, + species=[self.ions, self.beam_ions], + data_list = ['ux', 'uy', 'uz', 'x', 'weighting'], + write_dir='.', + warpx_file_prefix='Python_ohms_law_solver_ion_beam_1d_plt', + ) + simulation.add_diagnostic(part_diag) field_diag = picmi.FieldDiagnostic( - name='field_diag', + name='diag1', grid=self.grid, period=1250, + data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], write_dir='.', warpx_file_prefix='Python_ohms_law_solver_ion_beam_1d_plt', ) @@ -324,13 +335,17 @@ def _create_data_arrays(self): self.start_time = self.prev_time self.prev_step = 0 - if sim_ext.getMyProc() == 0: + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: # allocate arrays for storing energy values self.energy_vals = np.zeros((self.total_steps//self.diag_steps, 4)) def text_diag(self): """Diagnostic function to print out timing data and particle numbers.""" - step = sim_ext.getistep(0) + step = simulation.extension.warpx.getistep(lev=0) - 1 + + if not hasattr(self, "prev_time"): + self._create_data_arrays() + if step % (self.total_steps // 10) != 0: return @@ -356,17 +371,17 @@ def text_diag(self): "{step_rate:4.2f} steps/s" ) - if sim_ext.getMyProc() == 0: + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: print(diag_string.format(**status_dict)) self.prev_time = time.time() self.prev_step = step def energy_diagnostic(self): - """Diangostic to get the total, magnetic and kinetic energies in the + """Diagnostic to get the total, magnetic and kinetic energies in the simulation.""" + step = simulation.extension.warpx.getistep(lev=0) - 1 - step = sim_ext.getistep(0) if step % self.diag_steps != 1: return @@ -379,7 +394,7 @@ def energy_diagnostic(self): Ec_par, Ec_perp = self._get_kinetic_energy(self.ion_container_wrapper) Eb_par, Eb_perp = self._get_kinetic_energy(self.beam_ion_container_wrapper) - if sim_ext.getMyProc() != 0: + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: return self.energy_vals[idx, 0] = Ec_par @@ -414,14 +429,14 @@ def _record_average_fields(self): similar format as the reduced diagnostic so that the same analysis script can be used regardless of the simulation dimension. """ - step = sim_ext.getistep() - 1 + step = simulation.extension.warpx.getistep(lev=0) - 1 if step % self.diag_steps != 0: return By_warpx = fields.BxWrapper()[...] - if sim_ext.getMyProc() != 0: + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: return t = step * self.dt diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/README.rst b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst new file mode 100644 index 00000000000..59469cf4aa9 --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst @@ -0,0 +1,75 @@ +.. _examples-ohm-solver-ion-beam-instability: + +Ohm solver: Ion Beam R Instability +================================== + +In this example a low density ion beam interacts with a "core" plasma population which induces an instability. +Based on the relative density between the beam and the core plasma a resonant or non-resonant condition can +be accessed. + +Run +--- + +The same input script can be used for 1d, 2d or 3d simulations as well as +replicating either the resonant or non-resonant condition as indicated below. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Resonant case + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --resonant + + .. tab-item:: Non-resonant case + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} + +Analyze +------- + +The following script reads the simulation output from the above example, performs +Fourier transforms of the field data and outputs the figures shown below. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/analysis.py``. + +The figures below show the evolution of the y-component of the magnetic field as the beam and +core plasma interact. + +.. figure:: https://user-images.githubusercontent.com/40245517/217923933-6bdb65cb-7d26-40d8-8687-7dd75274bd48.png + :alt: Resonant ion beam R instability + :width: 70% + +.. figure:: https://user-images.githubusercontent.com/40245517/217925983-b91d6482-69bc-43c1-8c7d-23ebe7c69d49.png + :alt: Non-resonant ion beam R instability + :width: 70% + + Evolution of :math:`B_y` for resonant (top) and non-resonant (bottom) conditions. + +The growth rates of the strongest growing modes for the resonant case are compared +to theory (dashed lines) in the figure below. + +.. figure:: https://github.com/ECP-WarpX/WarpX/assets/40245517/a94bb6e5-30e9-4d8f-9e6b-844dc8f51d17 + :alt: Resonant ion beam R instability growth rates + :width: 50% + + Time series of the mode amplitudes for m = 4, 5, 6 from simulation. The + theoretical growth for these modes are also shown as dashed lines. diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py index b541a83283a..8c5a819a9e0 100755 --- a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py @@ -85,7 +85,7 @@ plt.plot(t_grid, np.abs(field_kt[:, 5] / sim.B0), 'b', label=f'm = 5, $kl_i={k[5]:.2f}$') plt.plot(t_grid, np.abs(field_kt[:, 6] / sim.B0), 'k', label=f'm = 6, $kl_i={k[6]:.2f}$') - # The theoretical growth rates for the 4th, 5th and 6th Foruier modes of + # The theoretical growth rates for the 4th, 5th and 6th Fourier modes of # the By-field was obtained from Fig. 12a of Munoz et al. # Note the rates here are gamma / w_ci gamma4 = 0.1915611861780133 diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py index 7b42e57d557..143f1f3e82e 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py @@ -16,15 +16,16 @@ from mpi4py import MPI as mpi import numpy as np -from pywarpx import callbacks, fields, picmi +from pywarpx import callbacks, fields, libwarpx, picmi constants = picmi.constants comm = mpi.COMM_WORLD -simulation = picmi.Simulation(verbose=0) -# make a shorthand for simulation.extension since we use it a lot -sim_ext = simulation.extension +simulation = picmi.Simulation( + warpx_serialize_initial_conditions=True, + verbose=0 +) class ForceFreeSheetReconnection(object): @@ -249,19 +250,30 @@ def setup_run(self): callbacks.installafterEsolve(self.check_fields) if self.test: + particle_diag = picmi.ParticleDiagnostic( + name='diag1', + period=self.total_steps, + write_dir='.', + species=[self.ions], + data_list=['ux', 'uy', 'uz', 'x', 'y', 'weighting'], + warpx_file_prefix='Python_ohms_law_solver_magnetic_reconnection_2d_plt', + # warpx_format='openpmd', + # warpx_openpmd_backend='h5', + ) + simulation.add_diagnostic(particle_diag) field_diag = picmi.FieldDiagnostic( - name='field_diag', + name='diag1', grid=self.grid, period=self.total_steps, - data_list=['B', 'E', 'j'], + data_list=['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez'], write_dir='.', warpx_file_prefix='Python_ohms_law_solver_magnetic_reconnection_2d_plt', # warpx_format='openpmd', # warpx_openpmd_backend='h5', - warpx_write_species=True ) simulation.add_diagnostic(field_diag) + # reduced diagnostics for reconnection rate calculation # create a 2 l_i box around the X-point on which to measure # magnetic flux changes @@ -293,7 +305,7 @@ def setup_run(self): def check_fields(self): - step = sim_ext.getistep() + step = simulation.extension.warpx.getistep(lev=0) - 1 if not (step == 1 or step%self.diag_steps == 0): return @@ -305,7 +317,7 @@ def check_fields(self): By = fields.ByFPWrapper(include_ghosts=False)[...] / self.B0 Bz = fields.BzFPWrapper(include_ghosts=False)[...] / self.B0 - if sim_ext.getMyProc() != 0: + if libwarpx.amr.ParallelDescriptor.MyProc() != 0: return # save the fields to file diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst new file mode 100644 index 00000000000..943b5bd0248 --- /dev/null +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst @@ -0,0 +1,45 @@ +.. _examples-ohm-solver-magnetic-reconnection: + +Ohm Solver: Magnetic Reconnection +================================= + +Hybrid-PIC codes are often used to simulate magnetic reconnection in space plasmas. +An example of magnetic reconnection from a force-free sheet is provided, based on +the simulation described in :cite:t:`ex-Le2016`. + +Run +--- + +The following **Python** script configures and launches the simulation. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py``. + +Running the full simulation should take about 4 hours if executed on 1 V100 GPU. +For `MPI-parallel `__ runs, prefix these lines with +``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + + .. code-block:: bash + + python3 PICMI_inputs.py + +Analyze +------- + +The following script extracts the reconnection rate as a function of time and +animates the evolution of the magnetic field (as shown below). + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py``. + +.. figure:: https://user-images.githubusercontent.com/40245517/229639784-b5d3b596-3550-4570-8761-8d9a67aa4b3b.gif + :alt: Magnetic reconnection. + :width: 70% + + Magnetic reconnection from a force-free sheet. diff --git a/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py b/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py index 18d91ad45fa..9ac40818e12 100755 --- a/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py +++ b/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py @@ -74,14 +74,19 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 10, + write_dir = '.', + warpx_file_prefix = 'Python_particle_reflection_plt' +) field_diag = picmi.FieldDiagnostic( grid=grid, name = 'diag1', data_list=['E'], period = 10, write_dir = '.', - warpx_file_prefix = 'Python_particle_reflection_plt', - warpx_write_species=False + warpx_file_prefix = 'Python_particle_reflection_plt' ) ########################## @@ -102,6 +107,7 @@ n_macroparticle_per_cell=[5, 2], grid=grid ) ) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) ########################## diff --git a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py index 66a1c0c39e1..9871bdac655 100755 --- a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py +++ b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py @@ -5,7 +5,7 @@ import numpy as np -from pywarpx import particle_containers, picmi +from pywarpx import libwarpx, particle_containers, picmi ########################## # numerics parameters @@ -73,6 +73,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = diagnostic_intervals, + write_dir = '.', + warpx_file_prefix = 'Python_particle_scrape_plt' +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -102,6 +108,7 @@ ) ) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) ########################## @@ -118,7 +125,7 @@ from mpi4py import MPI as mpi -my_id = sim.extension.getMyProc() +my_id = libwarpx.amr.ParallelDescriptor.MyProc() particle_buffer = particle_containers.ParticleBoundaryBufferWrapper() diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py index a7387832a7e..a4b7d9e134e 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py @@ -4,7 +4,7 @@ import numpy as np -from pywarpx import callbacks, particle_containers, picmi +from pywarpx import callbacks, libwarpx, particle_containers, picmi # Create the parser and add the argument parser = argparse.ArgumentParser() @@ -70,6 +70,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 10, + write_dir = '.', + warpx_file_prefix = f"Python_particle_attr_access_{'unique_' if args.unique else ''}plt" +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -96,6 +102,7 @@ n_macroparticle_per_cell=[0, 0], grid=grid ) ) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) sim.initialize_inputs() @@ -112,7 +119,7 @@ elec_wrapper = particle_containers.ParticleContainerWrapper('electrons') elec_wrapper.add_real_comp('newPid') -my_id = sim.extension.getMyProc() +my_id = libwarpx.amr.ParallelDescriptor.MyProc() def add_particles(): diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py index e9f53801752..1becd4464e7 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py @@ -71,15 +71,21 @@ # diagnostics ########################## -field_diag = picmi.ParticleDiagnostic( - species=electrons, +part_diag = picmi.ParticleDiagnostic( name = 'diag1', - data_list=['previous_positions'], period = 10, + species=[electrons], + write_dir = '.', + warpx_file_prefix = 'Python_prev_positions_plt' +) +field_diag = picmi.FieldDiagnostic( + name = 'diag1', + data_list=['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], + period = 10, + grid=grid, write_dir = '.', warpx_file_prefix = 'Python_prev_positions_plt' ) - ########################## # simulation setup ########################## @@ -97,8 +103,8 @@ n_macroparticle_per_cell=[1, 1], grid=grid ) ) +sim.add_diagnostic(part_diag) sim.add_diagnostic(field_diag) - ########################## # simulation run ########################## diff --git a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py b/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py index b420b71c3aa..66f259da2ef 100755 --- a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py +++ b/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py @@ -143,4 +143,4 @@ # verify that amrex proc ranks are offset by -1 from # world comm proc ranks -# assert sim.extension.getMyProc() == rank - 1 +# assert libwarpx.amr.ParallelDescriptor.MyProc() == rank - 1 diff --git a/Examples/Tests/pec/analysis_pec.py b/Examples/Tests/pec/analysis_pec.py index 0414420ef91..841cc06ae22 100755 --- a/Examples/Tests/pec/analysis_pec.py +++ b/Examples/Tests/pec/analysis_pec.py @@ -7,7 +7,7 @@ # License: BSD-3-Clause-LBNL # # This is a script that analyses the simulation results from -# the script `inputs_field_PEC_3d`. This simulates a sinusoindal wave. +# the script `inputs_field_PEC_3d`. This simulates a sinusoidal wave. # The electric field (Ey) is a standing wave due to the PEC boundary condition, # and as a result, the minimum and maximum value after reflection would be two times the value at initialization due to constructive interference. # Additionally, the value of Ey at the boundary must be equal to zero. diff --git a/Examples/Tests/pec/analysis_pec_mr.py b/Examples/Tests/pec/analysis_pec_mr.py index 1f9e5276c51..e8aab4dcd6f 100755 --- a/Examples/Tests/pec/analysis_pec_mr.py +++ b/Examples/Tests/pec/analysis_pec_mr.py @@ -7,7 +7,7 @@ # License: BSD-3-Clause-LBNL # # This is a script that analyses the simulation results from -# the script `inputs_field_PEC_mr_3d`. This simulates a sinusoindal wave. +# the script `inputs_field_PEC_mr_3d`. This simulates a sinusoidal wave. # The electric field (Ey) is a standing wave due to the PEC boundary condition, # and as a result, the minimum and maximum value after reflection would be two times the value at initialization due to constructive interference. # Additionally, the value of Ey at the boundary must be equal to zero. diff --git a/Examples/Tests/plasma_lens/PICMI_inputs_3d.py b/Examples/Tests/plasma_lens/PICMI_inputs_3d.py index ed64ae3e53f..50d222bbf36 100644 --- a/Examples/Tests/plasma_lens/PICMI_inputs_3d.py +++ b/Examples/Tests/plasma_lens/PICMI_inputs_3d.py @@ -33,17 +33,25 @@ # Particles vel_z = 0.5*c -multiparticles_distribution = picmi.ParticleListDistribution(x = [0.05, 0.], - y = [0., 0.04], - z = [0.05, 0.05], - ux = [0., 0.], - uy = [0., 0.], - uz = [vel_z, vel_z], - weight = [1., 1.]) +offset_x_particle = picmi.ParticleListDistribution(x = [0.05], + y = [0.], + z = [0.05], + ux = [0.], + uy = [0.], + uz = [vel_z], + weight = [1.]) + +offset_y_particle = picmi.ParticleListDistribution(x = [0.], + y = [0.04], + z = [0.05], + ux = [0.], + uy = [0.], + uz = [vel_z], + weight = [1.]) electrons = picmi.Species(particle_type = 'electron', name = 'electrons', - initial_distribution = multiparticles_distribution) + initial_distribution = [offset_x_particle, offset_y_particle]) # Plasma lenses plasma_lenses = picmi.PlasmaLens(period = 0.5, @@ -61,10 +69,16 @@ part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', period = max_steps, species = [electrons], - data_list = ['ux', 'uy', 'uz'], + data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z'], write_dir = '.', warpx_file_prefix = 'Python_plasma_lens_plt') +field_diag1 = picmi.FieldDiagnostic(name = 'diag1', + grid = grid, + period = max_steps, + data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], + write_dir = '.', + warpx_file_prefix = 'Python_plasma_lens_plt') # Set up simulation sim = picmi.Simulation(solver = solver, max_steps = max_steps, @@ -81,6 +95,7 @@ # Add diagnostics sim.add_diagnostic(part_diag1) +sim.add_diagnostic(field_diag1) # Write input file that can be used to run with the compiled version #sim.write_input_file(file_name = 'inputs_3d_picmi') diff --git a/Examples/Tests/plasma_lens/analysis.py b/Examples/Tests/plasma_lens/analysis.py index b410c6eb128..9ce82d903f8 100755 --- a/Examples/Tests/plasma_lens/analysis.py +++ b/Examples/Tests/plasma_lens/analysis.py @@ -92,12 +92,22 @@ def applylens(x0, vx0, vz0, gamma, lens_length, lens_strength): plasma_lens_strengths_B = np.zeros(len(plasma_lens_zstarts)) -x0 = float(ds.parameters.get('electrons.multiple_particles_pos_x').split()[0]) -y0 = float(ds.parameters.get('electrons.multiple_particles_pos_y').split()[1]) -z0 = float(ds.parameters.get('electrons.multiple_particles_pos_z').split()[0]) -ux0 = float(ds.parameters.get('electrons.multiple_particles_ux').split()[0])*c -uy0 = float(ds.parameters.get('electrons.multiple_particles_uy').split()[1])*c -uz0 = eval(ds.parameters.get('electrons.multiple_particles_uz').split()[0])*c +try: + # The picmi version + x0 = float(ds.parameters.get('electrons.dist0.multiple_particles_pos_x')) + y0 = float(ds.parameters.get('electrons.dist1.multiple_particles_pos_y')) + z0 = float(ds.parameters.get('electrons.dist0.multiple_particles_pos_z')) + ux0 = float(ds.parameters.get('electrons.dist0.multiple_particles_ux'))*c + uy0 = float(ds.parameters.get('electrons.dist1.multiple_particles_uy'))*c + uz0 = eval(ds.parameters.get('electrons.dist0.multiple_particles_uz'))*c +except TypeError: + # The inputs version + x0 = float(ds.parameters.get('electrons.multiple_particles_pos_x').split()[0]) + y0 = float(ds.parameters.get('electrons.multiple_particles_pos_y').split()[1]) + z0 = float(ds.parameters.get('electrons.multiple_particles_pos_z').split()[0]) + ux0 = float(ds.parameters.get('electrons.multiple_particles_ux').split()[0])*c + uy0 = float(ds.parameters.get('electrons.multiple_particles_uy').split()[1])*c + uz0 = eval(ds.parameters.get('electrons.multiple_particles_uz').split()[0])*c tt = 0. xx = x0 diff --git a/Examples/Tests/pml/analysis_pml_psatd.py b/Examples/Tests/pml/analysis_pml_psatd.py index ca3d7cc3012..50d0b2ac1c1 100755 --- a/Examples/Tests/pml/analysis_pml_psatd.py +++ b/Examples/Tests/pml/analysis_pml_psatd.py @@ -8,6 +8,7 @@ # License: BSD-3-Clause-LBNL import os +import re import sys import numpy as np @@ -19,13 +20,18 @@ filename = sys.argv[1] -############################ -### INITIAL LASER ENERGY ### -############################ -energy_start = 7.282940107273505e-08 # electromagnetic energy at iteration 50 +galilean = True if re.search("galilean", filename) else False + +# Initial laser energy (at iteration 50) +if galilean: + filename_init = 'pml_x_galilean_plt000050' + energy_start = 4.439376199524034e-08 +else: + filename_init = 'pml_x_psatd_plt000050' + energy_start = 7.282940107273505e-08 # Check consistency of field energy diagnostics with initial energy above -ds = yt.load('pml_x_psatd_plt000050') +ds = yt.load(filename_init) all_data_level_0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) Bx = all_data_level_0['boxlib', 'Bx'].v.squeeze() By = all_data_level_0['boxlib', 'By'].v.squeeze() @@ -43,10 +49,8 @@ print("relative error = " + str(error)) assert (error < tolerance) -########################## -### FINAL LASER ENERGY ### -########################## -ds = yt.load( filename ) +# Final laser energy +ds = yt.load(filename) all_data_level_0 = ds.covering_grid(level=0,left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) Bx = all_data_level_0['boxlib', 'Bx'].v.squeeze() By = all_data_level_0['boxlib', 'By'].v.squeeze() @@ -58,8 +62,8 @@ energyB = np.sum(0.5 / scc.mu_0 * (Bx**2 + By**2 + Bz**2)) energy_end = energyE + energyB -reflectivity = energy_end / energy_start -reflectivity_max = 1e-06 +reflectivity = energy_end / energy_start_diags +reflectivity_max = 1e-6 print("reflectivity = " + str(reflectivity)) print("reflectivity_max = " + str(reflectivity_max)) @@ -70,7 +74,8 @@ sys.path.insert(0, '../../../../warpx/Examples/') from analysis_default_restart import check_restart -check_restart(filename) +if not galilean: + check_restart(filename) test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py index 2301a558b90..7104963c752 100755 --- a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py +++ b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py @@ -62,6 +62,11 @@ # Initialize diagnostics diag_field_list = ["E", "B"] +particle_diag = picmi.ParticleDiagnostic(name = 'diag1', + period = 10, + write_dir = '.', + warpx_file_prefix = 'Python_wrappers_plt', + data_list = diag_field_list) field_diag = picmi.FieldDiagnostic(name = 'diag1', grid = grid, period = 10, @@ -80,6 +85,7 @@ warpx_use_filter = 1) # Add diagnostics to simulation +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) # Write input file to run with compiled version diff --git a/Examples/Tests/reduced_diags/PICMI_inputs_loadbalancecosts.py b/Examples/Tests/reduced_diags/PICMI_inputs_loadbalancecosts.py index a8c74bf6227..0583a6fe1d0 100644 --- a/Examples/Tests/reduced_diags/PICMI_inputs_loadbalancecosts.py +++ b/Examples/Tests/reduced_diags/PICMI_inputs_loadbalancecosts.py @@ -78,9 +78,19 @@ )) # Diagnostic -diag = picmi.FieldDiagnostic( +particle_diag = picmi.ParticleDiagnostic( + name='diag1', + period=3, + species=[electrons], + data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], + write_dir='.', + warpx_file_prefix='Python_reduced_diags_loadbalancecosts_timers_plt' +) +field_diag = picmi.FieldDiagnostic( + name='diag1', grid=grid, period=3, + data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], write_dir='.', warpx_file_prefix='Python_reduced_diags_loadbalancecosts_timers_plt' ) @@ -104,7 +114,8 @@ sim.add_diagnostic(reduced_diag) # Add diagnostics -sim.add_diagnostic(diag) +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) # Advance simulation until last time step # sim.write_input_file("test_input") diff --git a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py b/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py index 86699304091..d400924a378 100755 --- a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py +++ b/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py @@ -62,6 +62,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 10, + write_dir = '.', + warpx_file_prefix = f'Python_restart_runtime_components_plt' +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -103,6 +109,7 @@ sim.amr_restart = restart_file_name sys.argv.remove(arg) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) sim.add_diagnostic(checkpoint) sim.initialize_inputs() @@ -143,7 +150,7 @@ def add_particles(): # simulation run ########################## -step_number = sim.extension.getistep(0) +step_number = sim.extension.warpx.getistep(lev=0) sim.step(max_steps) ############################################### diff --git a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py index b7c97c6c1f9..706dedb6959 100755 --- a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py +++ b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py @@ -63,6 +63,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = 10, + write_dir = '.', + warpx_file_prefix = f'Python_restart_runtime_components_plt' +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -104,6 +110,7 @@ sim.amr_restart = restart_file_name sys.argv.remove(arg) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) sim.add_diagnostic(checkpoint) sim.initialize_inputs() @@ -143,7 +150,7 @@ def add_particles(): # simulation run ########################## -step_number = sim.extension.getistep(0) +step_number = sim.extension.warpx.getistep(lev=0) sim.step(max_steps - 1 - step_number) ########################## diff --git a/Examples/Tests/restart_eb/PICMI_inputs_restart_eb.py b/Examples/Tests/restart_eb/PICMI_inputs_restart_eb.py index 240283e2a2e..a4727053334 100755 --- a/Examples/Tests/restart_eb/PICMI_inputs_restart_eb.py +++ b/Examples/Tests/restart_eb/PICMI_inputs_restart_eb.py @@ -73,6 +73,12 @@ # diagnostics ########################## +particle_diag = picmi.ParticleDiagnostic( + name = 'diag1', + period = diagnostic_intervals, + write_dir = '.', + warpx_file_prefix = 'Python_restart_eb_plt' +) field_diag = picmi.FieldDiagnostic( name = 'diag1', grid = grid, @@ -116,6 +122,7 @@ sim.amr_restart = restart_file_name sys.argv.remove(arg) +sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) sim.add_diagnostic(checkpoint) sim.initialize_inputs() @@ -125,5 +132,5 @@ # simulation run ########################## -step_number = sim.extension.getistep(0) +step_number = sim.extension.warpx.getistep(lev=0) sim.step(max_steps - step_number) diff --git a/Examples/Tests/scraping/analysis_rz_filter.py b/Examples/Tests/scraping/analysis_rz_filter.py new file mode 100755 index 00000000000..b97b6e0eb5a --- /dev/null +++ b/Examples/Tests/scraping/analysis_rz_filter.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Copyright 2023 Kale Weichman +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the particle scraping for the embedded boundary in RZ. +# Particles are initialized between r=0.15 and r=0.2 +# having a negative radial velocity. +# A cylindrical embedded surface is placed at r=0.1. +# Upon reaching the surface, particles should be removed. +# At the end of the simulation, i.e., at time step 37, +# there should be 512 particles left. +# This test checks that plot_filter_fucntion works with the boundary scraping diagnostic +# by making sure that the particles removed from only half the domain (z>0) have been recorded. + +# Possible errors: 0 +# tolerance: 0 +# Possible running time: < 1 s + +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +import yt + +tolerance = 0 + +fn = sys.argv[1] +ds = yt.load( fn ) +ad = ds.all_data() +x = ad['electron', 'particle_position_x'].v + +error = len(x)-512 +print('error = ', error) +print('tolerance = ', tolerance) +assert(error==tolerance) + +# Check that all the removed particles are properly recorded +# by making sure that, at each iteration, the sum of the number of +# remaining particles and scraped particles is equal to half the +# original number of particles +# also check that no particles with z <= 0 have been scraped +ts_full = OpenPMDTimeSeries('./diags/diag2/') +ts_scraping = OpenPMDTimeSeries('./diags/diag3/particles_at_eb') + +def n_remaining_particles( iteration ): + w, = ts_full.get_particle(['w'], iteration=iteration) + return len(w) +def n_scraped_particles( iteration ): + timestamp = ts_scraping.get_particle( ['timestamp'], iteration=ts_scraping.iterations[0] ) + return (timestamp <= iteration).sum() +def n_scraped_z_leq_zero( iteration ): + z_pos, = ts_scraping.get_particle( ['z'], iteration=ts_scraping.iterations[0] ) + return (z_pos <= 0).sum() +n_remaining = np.array([ n_remaining_particles(iteration) for iteration in ts_full.iterations ]) +n_scraped = np.array([ n_scraped_particles(iteration) for iteration in ts_full.iterations ]) +n_z_leq_zero = np.array([ n_scraped_z_leq_zero(iteration) for iteration in ts_full.iterations ]) +n_total = n_remaining[0] +assert np.all( 2*n_scraped+n_remaining == n_total) +assert np.all( n_z_leq_zero == 0) diff --git a/Examples/Tests/scraping/inputs_rz_filter b/Examples/Tests/scraping/inputs_rz_filter new file mode 100644 index 00000000000..0d67fb96f6c --- /dev/null +++ b/Examples/Tests/scraping/inputs_rz_filter @@ -0,0 +1,58 @@ +amr.max_level = 1 +warpx.fine_tag_lo = 0.0 -0.5 +warpx.fine_tag_hi = 0.25 0.0 + +max_step = 37 + +amr.n_cell = 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.5 +geometry.prob_hi = 0.5 0.5 + +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.const_dt = 1.216119097e-11 +warpx.eb_implicit_function = "-(x**2-0.1**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 4 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "(x*x+y*y>0.15*0.15)*(x*x+y*y<0.2*0.2)*1.0e21" +electron.momentum_distribution_type = parse_momentum_function +electron.momentum_function_ux(x,y,z) = "if(x*x+y*y>0.0, -1.0*x/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uy(x,y,z) = "if(x*x+y*y>0.0, -1.0*y/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uz(x,y,z) = "0" +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 diag3 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.fields_to_plot = Er +diag2.format = openpmd + +diag3.diag_type = BoundaryScraping +diag3.format = openpmd +diag3.electron.plot_filter_function(t,x,y,z,ux,uy,uz) = "z > 0" diff --git a/Examples/Tests/single_particle/analysis_bilinear_filter.py b/Examples/Tests/single_particle/analysis_bilinear_filter.py index 95ee0761943..db7250dc3bb 100755 --- a/Examples/Tests/single_particle/analysis_bilinear_filter.py +++ b/Examples/Tests/single_particle/analysis_bilinear_filter.py @@ -42,7 +42,7 @@ my_order[1] -= 1 my_filter = my_filterx[:,None]*my_filtery -# Apply filter. my_F_filetered is the theoretical value for filtered field +# Apply filter. my_F_filtered is the theoretical value for filtered field my_F_filtered = signal.convolve2d(my_F_nofilter, my_filter, boundary='symm', mode='same') # Get simulation result for F_filtered diff --git a/Examples/analysis_default_openpmd_regression.py b/Examples/analysis_default_openpmd_regression.py new file mode 100755 index 00000000000..3aadc49ac51 --- /dev/null +++ b/Examples/analysis_default_openpmd_regression.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import os +import re +import sys + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Get name of the test +test_name = os.path.split(os.getcwd())[1] + +# Run checksum regression test +if re.search( 'single_precision', fn ): + checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd', rtol=2.e-6) +else: + checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd') diff --git a/Python/pywarpx/Bucket.py b/Python/pywarpx/Bucket.py index 6beb97c8291..91a34b9d3d6 100644 --- a/Python/pywarpx/Bucket.py +++ b/Python/pywarpx/Bucket.py @@ -23,7 +23,7 @@ def _localsetattr(self, name, value): object.__setattr__(self, name, value) def add_new_attr(self, name, value): - """Names starting with "_" are make instance attributes. + """Names starting with "_" are made instance attributes. Otherwise the attribute is added to the args list. """ if name.startswith('_'): @@ -31,6 +31,15 @@ def add_new_attr(self, name, value): else: self.argvattrs[name] = value + def add_new_group_attr(self, group, name, value): + """The attribute is added to the args list in the form "group.name" if + group is not an empty string, otherwise as only "name". + """ + if group: + self.argvattrs[f'{group}.{name}'] = value + else: + self.argvattrs[name] = value + def __setattr__(self, name, value): self.add_new_attr(name, value) diff --git a/Python/pywarpx/Diagnostics.py b/Python/pywarpx/Diagnostics.py index 06562c28d77..19860e9b7ee 100644 --- a/Python/pywarpx/Diagnostics.py +++ b/Python/pywarpx/Diagnostics.py @@ -26,3 +26,12 @@ def add_new_attr_with_check(self, name, value): def __setattr__(self, name, value): self.add_new_attr_with_check(name, value) + + def set_or_replace_attr(self, name, value): + """ + Explicitly set or replace an existing attribute + (since __setattr__ cannot be used for replacing + as it would raise an Exception) + """ + assert not name.startswith('_') + self.argvattrs[name] = value diff --git a/Python/pywarpx/LoadThirdParty.py b/Python/pywarpx/LoadThirdParty.py new file mode 100644 index 00000000000..ea62d558eeb --- /dev/null +++ b/Python/pywarpx/LoadThirdParty.py @@ -0,0 +1,35 @@ +# This file is part of WarpX. +# +# Authors: Axel Huebl +# License: BSD-3-Clause-LBNL + +from ._libwarpx import libwarpx + + +def load_cupy(): + """ + This is a helper for + https://docs.cupy.dev/en/stable/user_guide/basic.html + """ + amr = libwarpx.amr + status = None + + if amr.Config.have_gpu: + try: + import cupy as cp + xp = cp + # Note: found and will use cupy + except ImportError: + status = "Warning: GPU found but cupy not available! Trying managed memory in numpy..." + import numpy as np + xp = np + if amr.Config.gpu_backend == "SYCL": + status = "Warning: SYCL GPU backend not yet implemented for Python" + import numpy as np + xp = np + + else: + import numpy as np + xp = np + # Note: found and will use numpy + return xp, status diff --git a/Python/pywarpx/WarpX.py b/Python/pywarpx/WarpX.py index cfa55641427..3fa59285f26 100644 --- a/Python/pywarpx/WarpX.py +++ b/Python/pywarpx/WarpX.py @@ -98,7 +98,7 @@ def init(self, mpi_comm=None, **kw): libwarpx.initialize(argv, mpi_comm=mpi_comm) def evolve(self, nsteps=-1): - libwarpx.evolve(nsteps) + libwarpx.warpx.evolve(nsteps) def finalize(self, finalize_mpi=1): libwarpx.finalize(finalize_mpi) diff --git a/Python/pywarpx/__init__.py b/Python/pywarpx/__init__.py index 8b57f60fb79..f89926399ca 100644 --- a/Python/pywarpx/__init__.py +++ b/Python/pywarpx/__init__.py @@ -34,6 +34,7 @@ from .HybridPICModel import hybridpicmodel from .Interpolation import interpolation from .Lasers import lasers +from .LoadThirdParty import load_cupy from .PSATD import psatd from .Particles import newspecies, particles from .WarpX import warpx diff --git a/Python/pywarpx/_libwarpx.py b/Python/pywarpx/_libwarpx.py index bf3ec0ded1a..fa2044d4240 100755 --- a/Python/pywarpx/_libwarpx.py +++ b/Python/pywarpx/_libwarpx.py @@ -14,7 +14,6 @@ import atexit import os -import sys import numpy as np @@ -114,46 +113,6 @@ def load_library(self): self.__version__ = self.libwarpx_so.__version__ - def getNProcs(self): - ''' - - Get the number of processors - - ''' - return self.libwarpx_so.getNProcs() - - def getMyProc(self): - ''' - - Get the number of the processor - - ''' - return self.libwarpx_so.getMyProc() - - def get_nattr(self): - ''' - - Get the number of extra particle attributes. - - ''' - # --- The -3 is because the comps include the velocites - return self.libwarpx_so.warpx_nComps() - 3 - - def get_nattr_species(self, species_name): - ''' - Get the number of real attributes for the given species. - - Parameters - ---------- - - species_name: str - Name of the species - ''' - warpx = self.libwarpx_so.get_instance() - mpc = warpx.multi_particle_container() - pc = mpc.get_particle_container_from_name(species_name) - return pc.num_real_comps() - def amrex_init(self, argv, mpi_comm=None): if mpi_comm is None: # or MPI is None: self.libwarpx_so.amrex_init(argv) @@ -162,9 +121,7 @@ def amrex_init(self, argv, mpi_comm=None): def initialize(self, argv=None, mpi_comm=None): ''' - Initialize WarpX and AMReX. Must be called before doing anything else. - ''' if argv is None: argv = sys.argv @@ -180,9 +137,7 @@ def initialize(self, argv=None, mpi_comm=None): def finalize(self, finalize_mpi=1): ''' - Call finalize for WarpX and AMReX. Registered to run at program exit. - ''' # TODO: simplify, part of pyAMReX already if self.initialized: @@ -194,193 +149,4 @@ def finalize(self, finalize_mpi=1): from pywarpx import callbacks callbacks.clear_all() - def getistep(self, level=0): - ''' - Get the current time step number for the specified level - - Parameter - --------- - - level : int - The refinement level to reference - ''' - - return self.warpx.getistep(level) - - def gett_new(self, level=0): - ''' - - Get the next time for the specified level. - - Parameters - ---------- - - level : int - The refinement level to reference - ''' - - return self.warpx.gett_new(level) - - def evolve(self, num_steps=-1): - ''' - Evolve the simulation for num_steps steps. If num_steps=-1, - the simulation will be run until the end as specified in the - inputs file. - - Parameters - ---------- - - num_steps: int - The number of steps to take - ''' - - self.warpx.evolve(num_steps) - - def getProbLo(self, direction, level=0): - ''' - Get the values of the lower domain boundary. - - Parameters - ---------- - - direction : int - Direction of interest - ''' - - assert 0 <= direction < self.dim, 'Inappropriate direction specified' - return self.warpx.Geom(level).ProbLo(direction) - - def getProbHi(self, direction, level=0): - ''' - Get the values of the upper domain boundary. - - Parameters - ---------- - - direction : int - Direction of interest - ''' - - assert 0 <= direction < self.dim, 'Inappropriate direction specified' - return self.warpx.Geom(level).ProbHi(direction) - - def getCellSize(self, direction, level=0): - ''' - Get the cell size in the given direction and on the given level. - - Parameters - ---------- - - direction : int - Direction of interest - - level : int - The refinement level to reference - ''' - - assert 0 <= direction < 3, 'Inappropriate direction specified' - assert 0 <= level and level <= self.libwarpx_so.warpx_finestLevel(), 'Inappropriate level specified' - return self.libwarpx_so.warpx_getCellSize(direction, level) - - #def get_sigma(self, direction): - # ''' - # - # Return the 'sigma' PML coefficients for the electric field - # in a given direction. - # - # ''' - # - # size = ctypes.c_int(0) - # data = self.libwarpx_so.warpx_getPMLSigma(direction, ctypes.byref(size)) - # arr = np.ctypeslib.as_array(data, (size.value,)) - # arr.setflags(write=1) - # return arr - # - # - #def get_sigma_star(self, direction): - # ''' - # - # Return the 'sigma*' PML coefficients for the magnetic field - # in the given direction. - # - # ''' - # - # size = ctypes.c_int(0) - # data = self.libwarpx_so.warpx_getPMLSigmaStar(direction, ctypes.byref(size)) - # arr = np.ctypeslib.as_array(data, (size.value,)) - # arr.setflags(write=1) - # return arr - # - # - #def compute_pml_factors(self, lev, dt): - # ''' - # - # This recomputes the PML coefficients for a given level, using the - # time step dt. This needs to be called after modifying the coefficients - # from Python. - # - # ''' - # - # self.libwarpx_so.warpx_ComputePMLFactors(lev, dt) - - - - def depositChargeDensity(self, species_name, level, clear_rho=True, sync_rho=True): - ''' - Deposit the specified species' charge density in rho_fp in order to - access that data via pywarpx.fields.RhoFPWrapper(). - - Parameters - ---------- - - species_name : str - The species name that will be deposited. - - level : int - Which AMR level to retrieve scraped particle data from. - - clear_rho : bool - If True, zero out rho_fp before deposition. - - sync_rho : bool - If True, perform MPI exchange and properly set boundary cells for rho_fp. - ''' - rho_fp = self.warpx.multifab(f'rho_fp[level={level}]') - - if rho_fp is None: - raise RuntimeError("Multifab `rho_fp` is not allocated.") - # ablastr::warn_manager::WMRecordWarning( - # "WarpXWrappers", "rho_fp is not allocated", - # ablastr::warn_manager::WarnPriority::low - # ); - # return - - if clear_rho: - rho_fp.set_val(0.0) - - # deposit the charge density from the desired species - mypc = self.warpx.multi_particle_container() - myspc = mypc.get_particle_container_from_name(species_name) - myspc.deposit_charge(rho_fp, level) - - if self.geometry_dim == 'rz': - self.warpx.apply_inverse_volume_scaling_to_charge_density(rho_fp, level) - - if sync_rho: - self.warpx.sync_rho() - - - def set_potential_EB(self, potential): - """ - Set the expression string for the embedded boundary potential - - Parameters - ---------- - - potential : str - The expression string - """ - self.warpx.set_potential_on_eb(potential) - - libwarpx = LibWarpX() diff --git a/Python/pywarpx/callbacks.py b/Python/pywarpx/callbacks.py index b8824d0ef16..8f3c1dc5e2c 100644 --- a/Python/pywarpx/callbacks.py +++ b/Python/pywarpx/callbacks.py @@ -1,66 +1,79 @@ -# Copyright 2017-2022 The WarpX Community +# Copyright 2017-2023 The WarpX Community # # This file is part of WarpX. # -# Authors: David Grote, Roelof Groenewald +# Authors: David Grote, Roelof Groenewald, Axel Huebl # # License: BSD-3-Clause-LBNL -"""call back operations -===================== +"""Callback Locations +------------------ These are the functions which allow installing user created functions so that they are called at various places along the time step. The following three functions allow the user to install, uninstall and verify the different call back types. - - installcallback: Installs a function to be called at that specified time - - uninstallcallback: Uninstalls the function (so it won't be called anymore) - - isinstalled: Checks if the function is installed + +* :py:func:`installcallback`: Installs a function to be called at that specified time +* :py:func:`uninstallcallback`: Uninstalls the function (so it won't be called anymore) +* :py:func:`isinstalled`: Checks if the function is installed These functions all take a callback location name (string) and function or instance method as an argument. Note that if an instance method is used, an extra reference to the method's object is saved. -The install can be done using a decorator, which has the prefix "callfrom". See -example below. - Functions can be called at the following times: - - beforeInitEsolve: before the initial solve for the E fields (i.e. before the PIC loop starts) - - afterinit: immediately after the init is complete - - beforeEsolve: before the solve for E fields - - poissonsolver: In place of the computePhi call but only in an electrostatic simulation - - afterEsolve: after the solve for E fields - - beforedeposition: before the particle deposition (for charge and/or current) - - afterdeposition: after particle deposition (for charge and/or current) - - beforestep: before the time step - - afterstep: after the time step - - afterdiagnostics: after diagnostic output - - oncheckpointsignal: on a checkpoint signal - - onbreaksignal: on a break signal. These callbacks will be the last ones executed before the simulation ends. - - particlescraper: just after the particle boundary conditions are applied - but before lost particles are processed - - particleloader: at the time that the standard particle loader is called - - particleinjection: called when particle injection happens, after the position - advance and before deposition is called, allowing a user - defined particle distribution to be injected each time step - - appliedfields: allows directly specifying any fields to be applied to the particles - during the advance - -To use a decorator, the syntax is as follows. This will install the function -``myplots`` to be called after each step. - -@callfromafterstep -def myplots(): - ppzx() - -This is equivalent to the following: - -def myplots(): - ppzx() -installcallback('afterstep', myplots) +* ``beforeInitEsolve``: before the initial solve for the E fields (i.e. before the PIC loop starts) +* ``afterinit``: immediately after the init is complete +* ``beforeEsolve``: before the solve for E fields +* ``poissonsolver``: In place of the computePhi call but only in an electrostatic simulation +* ``afterEsolve``: after the solve for E fields +* ``beforedeposition``: before the particle deposition (for charge and/or current) +* ``afterdeposition``: after particle deposition (for charge and/or current) +* ``beforestep``: before the time step +* ``afterstep``: after the time step +* ``afterdiagnostics``: after diagnostic output +* ``oncheckpointsignal``: on a checkpoint signal +* ``onbreaksignal``: on a break signal. These callbacks will be the last ones executed before the simulation ends. +* ``particlescraper``: just after the particle boundary conditions are applied + but before lost particles are processed +* ``particleloader``: at the time that the standard particle loader is called +* ``particleinjection``: called when particle injection happens, after the position + advance and before deposition is called, allowing a user + defined particle distribution to be injected each time step + +Example that calls the Python function ``myplots`` after each step: + +.. code-block:: python3 + + from pywarpx.callbacks import installcallback + + def myplots(): + # do something here + + installcallback('afterstep', myplots) + + # run simulation + sim.step(nsteps=100) + +The install can also be done using a `Python decorator `__, which has the prefix ``callfrom``. +To use a decorator, the syntax is as follows. This will install the function ``myplots`` to be called after each step. +The above example is quivalent to the following: + +.. code-block:: python3 + + from pywarpx.callbacks import callfromafterstep + + @callfromafterstep + def myplots(): + # do something here + + # run simulation + sim.step(nsteps=100) """ + import copy import sys import time @@ -81,7 +94,7 @@ class CallbackFunctions(object): original reference. This is good since the user does not need to keep the reference to the object (for example it can be created using a local variable in a function). It may be bad if the user thinks an object was - deleted, but it actually isn't since it had (unkown to the user) + deleted, but it actually isn't since it had (unknown to the user) installed a method in one of the call back lists. """ @@ -282,15 +295,20 @@ def callfuncsinlist(self,*args,**kw): callback_instances[key] = CallbackFunctions(name=key, **val) def installcallback(name, f): - "Adds a function to the list of functions called by this callback" + """Installs a function to be called at that specified time. + + Adds a function to the list of functions called by this callback. + """ callback_instances[name].installfuncinlist(f) def uninstallcallback(name, f): - "Removes the function from the list of functions called by this callback" + """Uninstalls the function (so it won't be called anymore). + + Removes the function from the list of functions called by this callback.""" callback_instances[name].uninstallfuncinlist(f) def isinstalled(name, f): - "Checks if the function is called by this callback" + """Checks if a function is installed for this callback.""" return callback_instances[name].isinstalledfuncinlist(f) def clear_all(): diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index ab66a464628..dc0de9d9491 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -65,9 +65,14 @@ class _MultiFABWrapper(object): Parameters ---------- + mf: MultiFab + The Multifab that is wrapped. + mf_name: string The name of the MultiFab to be accessed, the tag specified when the - MultiFab is allocated + MultiFab is allocated. The Multifab will be accessed anew from WarpX + everytime it is called if this argument is given instead of directly + providing the Multifab. level: int The refinement level @@ -77,7 +82,8 @@ class _MultiFABWrapper(object): Note that when True, the first n-ghost negative indices will refer to the lower ghost cells. """ - def __init__(self, mf_name, level, include_ghosts=False): + def __init__(self, mf=None, mf_name=None, level=0, include_ghosts=False): + self._mf = mf self.mf_name = mf_name self.level = level self.include_ghosts = include_ghosts @@ -99,10 +105,13 @@ def __iter__(self): @property def mf(self): - # Always fetch this anew in case the C++ MultiFab is recreated - warpx = libwarpx.libwarpx_so.get_instance() - # All MultiFab names have the level suffix - return warpx.multifab(f'{self.mf_name}[level={self.level}]') + if self._mf is not None: + return self._mf + else: + # Always fetch this anew in case the C++ MultiFab is recreated + warpx = libwarpx.libwarpx_so.get_instance() + # All MultiFab names have the level suffix + return warpx.multifab(f'{self.mf_name}[level={self.level}]') @property def shape(self): @@ -111,7 +120,7 @@ def shape(self): min_box = self.mf.box_array().minimal_box() shape = list(min_box.size - min_box.small_end) if self.include_ghosts: - nghosts = self.mf.n_grow_vect() + nghosts = self.mf.n_grow_vect shape = [shape[i] + 2*nghosts[i] for i in range(self.dim)] shape.append(self.mf.nComp) return tuple(shape) @@ -150,16 +159,9 @@ def mesh(self, direction): if self.include_ghosts: # The ghost cells are added to the upper and lower end of the global domain. - nghosts = self.mf.n_grow_vect() - ilo = list(ilo) - ihi = list(ihi) - min_box = self.mf.box_array().minimal_box() - imax = min_box.big_end - for i in range(self.dim): - if ilo[i] == 0: - ilo[i] -= nghosts[i] - if ihi[i] == imax[i]: - ihi[i] += nghosts[i] + nghosts = self.mf.n_grow_vect + ilo -= nghosts[idir] + ihi += nghosts[idir] # Cell size in the direction warpx = libwarpx.libwarpx_so.get_instance() @@ -197,7 +199,7 @@ def _get_indices(self, index, missing): def _get_n_ghosts(self): """Return the list of number of ghosts. This includes the component dimension.""" - nghosts = list(self._get_indices(self.mf.n_grow_vect(), 0)) + nghosts = list(self._get_indices(self.mf.n_grow_vect, 0)) # The components always has nghosts = 0 nghosts.append(0) return nghosts @@ -206,7 +208,7 @@ def _get_min_indices(self): """Returns the minimum indices, expanded to length 3""" min_box = self.mf.box_array().minimal_box() if self.include_ghosts: - min_box.grow(self.mf.n_grow_vect()) + min_box.grow(self.mf.n_grow_vect) imin = self._get_indices(min_box.small_end, 0) return imin @@ -215,7 +217,7 @@ def _get_max_indices(self): """ min_box = self.mf.box_array().minimal_box() if self.include_ghosts: - min_box.grow(self.mf.n_grow_vect()) + min_box.grow(self.mf.n_grow_vect) imax = self._get_indices(min_box.big_end, 0) return imax @@ -289,10 +291,14 @@ def _get_field(self, mfi): # self.mf.array(mfi) is in C ordering. # Note: transposing creates a view and not a copy. device_arr4 = self.mf.array(mfi) - if cp is not None: - device_arr = cp.array(device_arr4, copy=False).T + if libwarpx.libwarpx_so.Config.have_gpu: + if cp is not None: + device_arr = device_arr4.to_cupy(copy=False) + else: + # Relies on managed memory + device_arr = device_arr4.to_numpy(copy=False) else: - device_arr = np.array(device_arr4, copy=False).T + device_arr = device_arr4.to_numpy(copy=False) if not self.include_ghosts: nghosts = self._get_n_ghosts() device_arr = device_arr[tuple([slice(ng, -ng) for ng in nghosts[:self.dim]])] @@ -335,7 +341,7 @@ def _get_intersect_slice(self, mfi, starts, stops, icstart, icstop): """ box = mfi.tilebox() if self.include_ghosts: - box.grow(self.mf.n_grow_vect()) + box.grow(self.mf.n_grow_vect) ilo = self._get_indices(box.small_end, 0) ihi = self._get_indices(box.big_end, 0) @@ -403,7 +409,7 @@ def __getitem__(self, index): ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax+1, 0) iystart, iystop = self._find_start_stop(ii[1], iymin, iymax+1, 1) izstart, izstop = self._find_start_stop(ii[2], izmin, izmax+1, 2) - icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp(), 3) + icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp, 3) # Gather the data to be included in a list to be sent to other processes starts = [ixstart, iystart, izstart] @@ -414,11 +420,10 @@ def __getitem__(self, index): if global_slices is not None: # Note that the array will always have 4 dimensions. device_arr = self._get_field(mfi) - if cp is not None: - # Copy the data from the device to the host - slice_arr = cp.asnumpy(device_arr[block_slices]) - else: - slice_arr = device_arr[block_slices] + slice_arr = device_arr[block_slices] + if (cp is not None) and (type(slice_arr) is cp.ndarray): + # Copy data from host to device using cupy syntax + slice_arr = slice_arr.get() datalist.append((global_slices, slice_arr)) # Gather the data from all processors @@ -494,7 +499,7 @@ def __setitem__(self, index, value): ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax+1, 0) iystart, iystop = self._find_start_stop(ii[1], iymin, iymax+1, 1) izstart, izstop = self._find_start_stop(ii[2], izmin, izmax+1, 2) - icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp(), 3) + icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp, 3) if isinstance(value, np.ndarray): # Expand the shape of the input array to match the shape of the global array @@ -519,9 +524,9 @@ def __setitem__(self, index, value): mf_arr = self._get_field(mfi) if isinstance(value, np.ndarray): slice_value = value3d[global_slices] - if cp is not None: + if libwarpx.libwarpx_so.Config.have_gpu: # Copy data from host to device - slice_value = cp.asarray(value3d[global_slices]) + slice_value = cp.asarray(slice_value) mf_arr[block_slices] = slice_value else: mf_arr[block_slices] = value @@ -544,143 +549,144 @@ def max_index(self, *args): def norm0(self, *args): return self.mf.norm0(*args) + def ExWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_aux[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_aux[x]', level=level, include_ghosts=include_ghosts) def EyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_aux[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_aux[y]', level=level, include_ghosts=include_ghosts) def EzWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_aux[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_aux[z]', level=level, include_ghosts=include_ghosts) def BxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_aux[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_aux[x]', level=level, include_ghosts=include_ghosts) def ByWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_aux[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_aux[y]', level=level, include_ghosts=include_ghosts) def BzWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_aux[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_aux[z]', level=level, include_ghosts=include_ghosts) def JxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[x]', level=level, include_ghosts=include_ghosts) def JyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[y]', level=level, include_ghosts=include_ghosts) def JzWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[z]', level=level, include_ghosts=include_ghosts) def ExFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_fp[x]', level=level, include_ghosts=include_ghosts) def EyFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_fp[y]', level=level, include_ghosts=include_ghosts) def EzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_fp[z]', level=level, include_ghosts=include_ghosts) def BxFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_fp[x]', level=level, include_ghosts=include_ghosts) def ByFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_fp[y]', level=level, include_ghosts=include_ghosts) def BzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_fp[z]', level=level, include_ghosts=include_ghosts) def JxFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[x]', level=level, include_ghosts=include_ghosts) def JyFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[y]', level=level, include_ghosts=include_ghosts) def JzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp[z]', level=level, include_ghosts=include_ghosts) def RhoFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'rho_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='rho_fp', level=level, include_ghosts=include_ghosts) def PhiFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'phi_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='phi_fp', level=level, include_ghosts=include_ghosts) def FFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'F_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='F_fp', level=level, include_ghosts=include_ghosts) def GFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'G_fp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='G_fp', level=level, include_ghosts=include_ghosts) def AxFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'vector_potential_fp_nodal[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='vector_potential_fp_nodal[x]', level=level, include_ghosts=include_ghosts) def AyFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'vector_potential_fp_nodal[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='vector_potential_fp_nodal[y]', level=level, include_ghosts=include_ghosts) def AzFPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'vector_potential_fp_nodal[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='vector_potential_fp_nodal[z]', level=level, include_ghosts=include_ghosts) def ExCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_cp[x]', level=level, include_ghosts=include_ghosts) def EyCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_cp[y]', level=level, include_ghosts=include_ghosts) def EzCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Efield_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Efield_cp[z]', level=level, include_ghosts=include_ghosts) def BxCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_cp[x]', level=level, include_ghosts=include_ghosts) def ByCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_cp[y]', level=level, include_ghosts=include_ghosts) def BzCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'Bfield_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='Bfield_cp[z]', level=level, include_ghosts=include_ghosts) def JxCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_cp[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_cp[x]', level=level, include_ghosts=include_ghosts) def JyCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_cp[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_cp[y]', level=level, include_ghosts=include_ghosts) def JzCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_cp[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_cp[z]', level=level, include_ghosts=include_ghosts) def RhoCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'rho_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='rho_cp', level=level, include_ghosts=include_ghosts) def FCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'F_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='F_cp', level=level, include_ghosts=include_ghosts) def GCPWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'G_cp', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='G_cp', level=level, include_ghosts=include_ghosts) def EdgeLengthsxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_edge_lengths[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_edge_lengths[x]', level=level, include_ghosts=include_ghosts) def EdgeLengthsyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_edge_lengths[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_edge_lengths[y]', level=level, include_ghosts=include_ghosts) def EdgeLengthszWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_edge_lengths[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_edge_lengths[z]', level=level, include_ghosts=include_ghosts) def FaceAreasxWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_face_areas[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_face_areas[x]', level=level, include_ghosts=include_ghosts) def FaceAreasyWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_face_areas[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_face_areas[y]', level=level, include_ghosts=include_ghosts) def FaceAreaszWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'm_face_areas[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='m_face_areas[z]', level=level, include_ghosts=include_ghosts) def JxFPAmpereWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp_ampere[x]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp_ampere[x]', level=level, include_ghosts=include_ghosts) def JyFPAmpereWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp_ampere[y]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp_ampere[y]', level=level, include_ghosts=include_ghosts) def JzFPAmpereWrapper(level=0, include_ghosts=False): - return _MultiFABWrapper(mf_name=f'current_fp_ampere[z]', level=level, include_ghosts=include_ghosts) + return _MultiFABWrapper(mf_name='current_fp_ampere[z]', level=level, include_ghosts=include_ghosts) def ExFPPMLWrapper(level=0, include_ghosts=False): return _MultiFABWrapper(mf_name='pml_E_fp[x]', level=level, include_ghosts=include_ghosts) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index aed05121668..273a981f4bd 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -2,12 +2,13 @@ # # This file is part of WarpX. # -# Authors: David Grote, Roelof Groenewald +# Authors: David Grote, Roelof Groenewald, Axel Huebl # # License: BSD-3-Clause-LBNL import numpy as np +from .LoadThirdParty import load_cupy from ._libwarpx import libwarpx @@ -28,6 +29,7 @@ def __init__(self, species_name): mypc = libwarpx.warpx.multi_particle_container() self.particle_container = mypc.get_particle_container_from_name(self.name) + def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, uz=None, w=None, unique_particles=True, **kwargs): ''' @@ -122,7 +124,7 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, built_in_attrs += 1 # --- The number of extra attributes (including the weight) - nattr = self.particle_container.num_real_comps() - built_in_attrs + nattr = self.particle_container.num_real_comps - built_in_attrs attr = np.zeros((maxlen, nattr)) attr[:,0] = w @@ -148,6 +150,7 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, nattr, attr, nattr_int, attr_int, unique_particles ) + def get_particle_count(self, local=False): ''' Get the number of particles of this species in the simulation. @@ -168,6 +171,7 @@ def get_particle_count(self, local=False): return self.particle_container.total_number_of_particles(True, local) nps = property(get_particle_count) + def add_real_comp(self, pid_name, comm=True): ''' Add a real component to the particle data array. @@ -183,40 +187,65 @@ def add_real_comp(self, pid_name, comm=True): ''' self.particle_container.add_real_comp(pid_name, comm) - def get_particle_structs(self, level): + + def get_particle_structs(self, level, copy_to_host=False): ''' - This returns a list of numpy arrays containing the particle struct data + This returns a list of numpy or cupy arrays containing the particle struct data on each tile for this process. The particle data is represented as a structured - numpy array and contains the particle 'x', 'y', 'z', and 'idcpu'. + array and contains the particle 'x', 'y', 'z', and 'idcpu'. + + Unless copy_to_host is specified, the data for the structs are + not copied, but share the underlying memory buffer with WarpX. The + arrays are fully writeable. - The data for the numpy arrays are not copied, but share the underlying - memory buffer with WarpX. The numpy arrays are fully writeable. + Note that cupy does not support structs: + https://github.com/cupy/cupy/issues/2031 + and will return arrays of binary blobs for the AoS (DP: ``"|V24"``). If copied + to host via copy_to_host, we correct for the right numpy AoS type. Parameters ---------- level : int - The refinement level to reference + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. Returns ------- - List of numpy arrays + List of arrays The requested particle struct data ''' particle_data = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): - aos_arr = np.array(pti.aos(), copy=False) - particle_data.append(aos_arr) + if copy_to_host: + particle_data.append(pti.aos().to_numpy(copy=True)) + else: + if libwarpx.amr.Config.have_gpu: + libwarpx.amr.Print( + "get_particle_structs: cupy does not yet support structs. " + "https://github.com/cupy/cupy/issues/2031" + "Did you mean copy_to_host=True?" + ) + xp, cupy_status = load_cupy() + if cupy_status is not None: + libwarpx.amr.Print(cupy_status) + aos_arr = xp.array(pti.aos(), copy=False) # void blobs for cupy + particle_data.append(aos_arr) return particle_data - def get_particle_arrays(self, comp_name, level): + + def get_particle_arrays(self, comp_name, level, copy_to_host=False): ''' - This returns a list of numpy arrays containing the particle array data + This returns a list of numpy or cupy arrays containing the particle array data on each tile for this process. - The data for the numpy arrays are not copied, but share the underlying - memory buffer with WarpX. The numpy arrays are fully writeable. + Unless copy_to_host is specified, the data for the arrays are not + copied, but share the underlying memory buffer with WarpX. The + arrays are fully writeable. Parameters ---------- @@ -224,13 +253,17 @@ def get_particle_arrays(self, comp_name, level): comp_name : str The component of the array data that will be returned - level : int - The refinement level to reference + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. Returns ------- - List of numpy arrays + List of arrays The requested particle array data ''' comp_idx = self.particle_container.get_comp_index(comp_name) @@ -238,101 +271,233 @@ def get_particle_arrays(self, comp_name, level): data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): soa = pti.soa() - data_array.append(np.array(soa.GetRealData(comp_idx), copy=False)) + idx = soa.GetRealData(comp_idx) + if copy_to_host: + data_array.append(idx.to_numpy(copy=True)) + else: + xp, cupy_status = load_cupy() + if cupy_status is not None: + libwarpx.amr.Print(cupy_status) + + data_array.append(xp.array(idx, copy=False)) + return data_array - def get_particle_id(self, level=0): - ''' - Return a list of numpy arrays containing the particle 'id' + def get_particle_id(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle 'id' numbers on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle ids ''' - structs = self.get_particle_structs(level) + structs = self.get_particle_structs(level, copy_to_host) return [libwarpx.amr.unpack_ids(struct['cpuid']) for struct in structs] - def get_particle_cpu(self, level=0): - ''' - Return a list of numpy arrays containing the particle 'cpu' + def get_particle_cpu(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle 'cpu' numbers on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle cpus ''' - structs = self.get_particle_structs(level) + structs = self.get_particle_structs(level, copy_to_host) return [libwarpx.amr.unpack_cpus(struct['cpuid']) for struct in structs] - def get_particle_x(self, level=0): - ''' - Return a list of numpy arrays containing the particle 'x' + def get_particle_x(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle 'x' positions on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle x position ''' - structs = self.get_particle_structs(level) + if copy_to_host: + # Use the numpy version of cosine + xp = np + else: + xp, cupy_status = load_cupy() + + structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == '3d' or libwarpx.geometry_dim == '2d': return [struct['x'] for struct in structs] elif libwarpx.geometry_dim == 'rz': - return [struct['x']*np.cos(theta) for struct, theta in zip(structs, self.get_particle_theta())] + theta = self.get_particle_theta(level, copy_to_host) + return [struct['x']*xp.cos(theta) for struct, theta in zip(structs, theta)] elif libwarpx.geometry_dim == '1d': raise Exception('get_particle_x: There is no x coordinate with 1D Cartesian') xp = property(get_particle_x) - def get_particle_y(self, level=0): - ''' - Return a list of numpy arrays containing the particle 'y' + def get_particle_y(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle 'y' positions on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle y position ''' - structs = self.get_particle_structs(level) + if copy_to_host: + # Use the numpy version of sine + xp = np + else: + xp, cupy_status = load_cupy() + + structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == '3d': return [struct['y'] for struct in structs] elif libwarpx.geometry_dim == 'rz': - return [struct['x']*np.sin(theta) for struct, theta in zip(structs, self.get_particle_theta())] + theta = self.get_particle_theta(level, copy_to_host) + return [struct['x']*xp.sin(theta) for struct, theta in zip(structs, theta)] elif libwarpx.geometry_dim == '1d' or libwarpx.geometry_dim == '2d': raise Exception('get_particle_y: There is no y coordinate with 1D or 2D Cartesian') yp = property(get_particle_y) - def get_particle_r(self, level=0): - ''' - Return a list of numpy arrays containing the particle 'r' + def get_particle_r(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle 'r' positions on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle r position ''' - structs = self.get_particle_structs(level) + xp, cupy_status = load_cupy() + + structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == 'rz': return [struct['x'] for struct in structs] elif libwarpx.geometry_dim == '3d': - return [np.sqrt(struct['x']**2 + struct['y']**2) for struct in structs] + return [xp.sqrt(struct['x']**2 + struct['y']**2) for struct in structs] elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_r: There is no r coordinate with 1D or 2D Cartesian') rp = property(get_particle_r) - def get_particle_theta(self, level=0): - ''' - Return a list of numpy arrays containing the particle + def get_particle_theta(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle theta on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle theta position ''' + xp, cupy_status = load_cupy() + if libwarpx.geometry_dim == 'rz': - return self.get_particle_arrays('theta', level) + return self.get_particle_arrays('theta', level, copy_to_host) elif libwarpx.geometry_dim == '3d': - structs = self.get_particle_structs(level) - return [np.arctan2(struct['y'], struct['x']) for struct in structs] + structs = self.get_particle_structs(level, copy_to_host) + return [xp.arctan2(struct['y'], struct['x']) for struct in structs] elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_theta: There is no theta coordinate with 1D or 2D Cartesian') thetap = property(get_particle_theta) - def get_particle_z(self, level=0): - ''' - Return a list of numpy arrays containing the particle 'z' + def get_particle_z(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle 'z' positions on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle z position ''' - structs = self.get_particle_structs(level) + structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == '3d': return [struct['z'] for struct in structs] elif libwarpx.geometry_dim == 'rz' or libwarpx.geometry_dim == '2d': @@ -341,47 +506,108 @@ def get_particle_z(self, level=0): return [struct['x'] for struct in structs] zp = property(get_particle_z) - def get_particle_weight(self, level=0): - ''' - Return a list of numpy arrays containing the particle + def get_particle_weight(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle weight on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle weight ''' - return self.get_particle_arrays('w', level) + return self.get_particle_arrays('w', level, copy_to_host=copy_to_host) wp = property(get_particle_weight) - def get_particle_ux(self, level=0): - ''' - Return a list of numpy arrays containing the particle + def get_particle_ux(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle x momentum on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle x momentum ''' - return self.get_particle_arrays('ux', level) + return self.get_particle_arrays('ux', level, copy_to_host=copy_to_host) uxp = property(get_particle_ux) - def get_particle_uy(self, level=0): - ''' - Return a list of numpy arrays containing the particle + def get_particle_uy(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle y momentum on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle y momentum ''' - return self.get_particle_arrays('uy', level) + return self.get_particle_arrays('uy', level, copy_to_host=copy_to_host) uyp = property(get_particle_uy) - def get_particle_uz(self, level=0): - ''' - Return a list of numpy arrays containing the particle + def get_particle_uz(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle z momentum on each tile. + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle z momentum ''' - return self.get_particle_arrays('uz', level) + return self.get_particle_arrays('uz', level, copy_to_host=copy_to_host) uzp = property(get_particle_uz) + def get_species_charge_sum(self, local=False): ''' Returns the total charge in the simulation due to the given species. @@ -392,36 +618,73 @@ def get_species_charge_sum(self, local=False): local : bool If True return total charge per processor ''' - raise NotImplementedError() - return self.libwarpx_so.warpx_sumParticleCharge( - ctypes.c_char_p(species_name.encode('utf-8')), local - ) + return self.particle_container.sum_particle_charge(local) + def getex(self): raise NotImplementedError('Particle E fields not supported') ex = property(getex) + def getey(self): raise NotImplementedError('Particle E fields not supported') ey = property(getey) + def getez(self): raise NotImplementedError('Particle E fields not supported') ez = property(getez) + def getbx(self): raise NotImplementedError('Particle B fields not supported') bx = property(getbx) + def getby(self): raise NotImplementedError('Particle B fields not supported') by = property(getby) + def getbz(self): raise NotImplementedError('Particle B fields not supported') bz = property(getbz) + def deposit_charge_density(self, level, clear_rho=True, sync_rho=True): + """ + Deposit this species' charge density in rho_fp in order to + access that data via pywarpx.fields.RhoFPWrapper(). + + Parameters + ---------- + species_name : str + The species name that will be deposited. + level : int + Which AMR level to retrieve scraped particle data from. + clear_rho : bool + If True, zero out rho_fp before deposition. + sync_rho : bool + If True, perform MPI exchange and properly set boundary cells for rho_fp. + """ + rho_fp = libwarpx.warpx.multifab(f'rho_fp[level={level}]') + + if rho_fp is None: + raise RuntimeError("Multifab `rho_fp` is not allocated.") + + if clear_rho: + rho_fp.set_val(0.0) + + # deposit the charge density from the desired species + self.particle_container.deposit_charge(rho_fp, level) + + if libwarpx.geometry_dim == 'rz': + libwarpx.warpx.apply_inverse_volume_scaling_to_charge_density(rho_fp, level) + + if sync_rho: + libwarpx.warpx.sync_rho() + + class ParticleBoundaryBufferWrapper(object): """Wrapper around particle boundary buffer containers. This provides a convenient way to query data in the particle boundary @@ -431,6 +694,7 @@ class ParticleBoundaryBufferWrapper(object): def __init__(self): self.particle_buffer = libwarpx.warpx.get_particle_boundary_buffer() + def get_particle_boundary_buffer_size(self, species_name, boundary, local=False): ''' This returns the number of particles that have been scraped so far in the simulation @@ -455,15 +719,24 @@ def get_particle_boundary_buffer_size(self, species_name, boundary, local=False) local=local ) - def get_particle_boundary_buffer_structs(self, species_name, boundary, level): + + def get_particle_boundary_buffer_structs( + self, species_name, boundary, level, copy_to_host=False + ): ''' - This returns a list of numpy arrays containing the particle struct data + This returns a list of numpy or cupy arrays containing the particle struct data for a species that has been scraped by a specific simulation boundary. The - particle data is represented as a structured numpy array and contains the + particle data is represented as a structured array and contains the particle 'x', 'y', 'z', and 'idcpu'. - The data for the numpy arrays are not copied, but share the underlying - memory buffer with WarpX. The numpy arrays are fully writeable. + Unless copy_to_host is specified, the data for the structs are + not copied, but share the underlying memory buffer with WarpX. The + arrays are fully writeable. + + Note that cupy does not support structs: + https://github.com/cupy/cupy/issues/2031 + and will return arrays of binary blobs for the AoS (DP: ``"|V24"``). If copied + to host via copy_to_host, we correct for the right numpy AoS type. Parameters ---------- @@ -476,35 +749,48 @@ def get_particle_boundary_buffer_structs(self, species_name, boundary, level): form x/y/z_hi/lo or eb. level : int - Which AMR level to retrieve scraped particle data from. - ''' + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- - particles_per_tile = _LP_c_int() - num_tiles = ctypes.c_int(0) - data = self.libwarpx_so.warpx_getParticleBoundaryBufferStructs( - ctypes.c_char_p(species_name.encode('utf-8')), - self._get_boundary_number(boundary), level, - ctypes.byref(num_tiles), ctypes.byref(particles_per_tile) + List of arrays + The requested particle struct data + ''' + particle_container = self.particle_buffer.get_particle_container( + species_name, self._get_boundary_number(boundary) ) particle_data = [] - for i in range(num_tiles.value): - if particles_per_tile[i] == 0: - continue - arr = self._array1d_from_pointer(data[i], self._p_dtype, particles_per_tile[i]) - particle_data.append(arr) - - _libc.free(particles_per_tile) - _libc.free(data) + for pti in libwarpx.libwarpx_so.BoundaryBufferParIter(particle_container, level): + if copy_to_host: + particle_data.append(pti.aos().to_numpy(copy=True)) + else: + if libwarpx.amr.Config.have_gpu: + libwarpx.amr.Print( + "get_particle_structs: cupy does not yet support structs. " + "https://github.com/cupy/cupy/issues/2031" + "Did you mean copy_to_host=True?" + ) + xp, cupy_status = load_cupy() + if cupy_status is not None: + libwarpx.amr.Print(cupy_status) + aos_arr = xp.array(pti.aos(), copy=False) # void blobs for cupy + particle_data.append(aos_arr) return particle_data + def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level): ''' - This returns a list of numpy arrays containing the particle array data + This returns a list of numpy or cupy arrays containing the particle array data for a species that has been scraped by a specific simulation boundary. - The data for the numpy arrays are not copied, but share the underlying - memory buffer with WarpX. The numpy arrays are fully writeable. + The data for the arrays are not copied, but share the underlying + memory buffer with WarpX. The arrays are fully writeable. Parameters ---------- @@ -524,26 +810,27 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) level : int Which AMR level to retrieve scraped particle data from. ''' + xp, cupy_status = load_cupy() + part_container = self.particle_buffer.get_particle_container( species_name, self._get_boundary_number(boundary) ) data_array = [] if comp_name == 'step_scraped': # the step scraped is always the final integer component - comp_idx = part_container.num_int_comps() - 1 + comp_idx = part_container.num_int_comps - 1 for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() - data_array.append(np.array(soa.GetIntData(comp_idx), copy=False)) + data_array.append(xp.array(soa.GetIntData(comp_idx), copy=False)) else: - mypc = libwarpx.warpx.multi_particle_container() - sim_part_container_wrapper = mypc.get_particle_container_from_name(species_name) - comp_idx = sim_part_container_wrapper.get_comp_index(comp_name) + container_wrapper = ParticleContainerWrapper(species_name) + comp_idx = container_wrapper.particle_container.get_comp_index(comp_name) for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() - data_array.append(np.array(soa.GetRealData(comp_idx), copy=False)) - + data_array.append(xp.array(soa.GetRealData(comp_idx), copy=False)) return data_array + def clear_buffer(self): ''' @@ -552,6 +839,7 @@ def clear_buffer(self): ''' self.particle_buffer.clear_particles() + def _get_boundary_number(self, boundary): ''' diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index ce20724950d..7d7e150f111 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -8,6 +8,7 @@ """Classes following the PICMI standard """ +from dataclasses import dataclass import os import re @@ -44,7 +45,7 @@ class constants: class Species(picmistandard.PICMI_Species): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -78,7 +79,7 @@ class Species(picmistandard.PICMI_Species): Whether or not to push this species warpx_do_not_gather: bool, default=False - Whether or not to gahter the fields from grids for this species + Whether or not to gather the fields from grids for this species warpx_random_theta: bool, default=True Whether or not to add random angle to the particles in theta @@ -270,7 +271,21 @@ def initialize_inputs(self, layout, pywarpx.Particles.particles_list.append(self.species) if self.initial_distribution is not None: - self.initial_distribution.initialize_inputs(self.species_number, layout, self.species, self.density_scale) + distributions_is_list = np.iterable(self.initial_distribution) + layout_is_list = np.iterable(layout) + if not distributions_is_list and not layout_is_list: + self.initial_distribution.initialize_inputs(self.species_number, layout, self.species, self.density_scale, '') + elif distributions_is_list and (layout_is_list or layout is None): + assert layout is None or (len(self.initial_distribution) == len(layout)),\ + Exception('The initial distribution and layout lists must have the same lenth') + source_names = [f'dist{i}' for i in range(len(self.initial_distribution))] + self.species.injection_sources = source_names + for i, dist in enumerate(self.initial_distribution): + layout_i = layout[i] if layout is not None else None + dist.initialize_inputs(self.species_number, layout_i, self.species, + self.density_scale, source_names[i]) + else: + raise Exception('The initial distribution and layout must both be scalars or both be lists') if injection_plane_position is not None: if injection_plane_normal_vector is not None: @@ -299,17 +314,17 @@ def init(self, kw): self.do_symmetrize = kw.pop('warpx_do_symmetrize', None) self.symmetrization_order = kw.pop('warpx_symmetrization_order', None) - def initialize_inputs(self, species_number, layout, species, density_scale): - species.injection_style = "gaussian_beam" - species.x_m = self.centroid_position[0] - species.y_m = self.centroid_position[1] - species.z_m = self.centroid_position[2] - species.x_rms = self.rms_bunch_size[0] - species.y_rms = self.rms_bunch_size[1] - species.z_rms = self.rms_bunch_size[2] + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): + species.add_new_group_attr(source_name, 'injection_style', "gaussian_beam") + species.add_new_group_attr(source_name, 'x_m', self.centroid_position[0]) + species.add_new_group_attr(source_name, 'y_m', self.centroid_position[1]) + species.add_new_group_attr(source_name, 'z_m', self.centroid_position[2]) + species.add_new_group_attr(source_name, 'x_rms', self.rms_bunch_size[0]) + species.add_new_group_attr(source_name, 'y_rms', self.rms_bunch_size[1]) + species.add_new_group_attr(source_name, 'z_rms', self.rms_bunch_size[2]) # --- Only PseudoRandomLayout is supported - species.npart = layout.n_macroparticles + species.add_new_group_attr(source_name, 'npart', layout.n_macroparticles) # --- Calculate the total charge. Note that charge might be a string instead of a number. charge = species.charge @@ -317,9 +332,9 @@ def initialize_inputs(self, species_number, layout, species, density_scale): charge = constants.q_e elif charge == '-q_e': charge = -constants.q_e - species.q_tot = self.n_physical_particles*charge + species.add_new_group_attr(source_name, 'q_tot', self.n_physical_particles*charge) if density_scale is not None: - species.q_tot *= density_scale + species.add_new_group_attr(source_name, 'q_tot', density_scale) # --- The PICMI standard doesn't yet have a way of specifying these values. # --- They should default to the size of the domain. They are not typically @@ -333,26 +348,26 @@ def initialize_inputs(self, species_number, layout, species, density_scale): # --- Note that WarpX takes gamma*beta as input if np.any(np.not_equal(self.velocity_divergence, 0.)): - species.momentum_distribution_type = "radial_expansion" - species.u_over_r = self.velocity_divergence[0]/constants.c - #species.u_over_y = self.velocity_divergence[1]/constants.c - #species.u_over_z = self.velocity_divergence[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "radial_expansion") + species.add_new_group_attr(source_name, 'u_over_r', self.velocity_divergence[0]/constants.c) + #species.add_new_group_attr(source_name, 'u_over_y', self.velocity_divergence[1]/constants.c) + #species.add_new_group_attr(source_name, 'u_over_z', self.velocity_divergence[2]/constants.c) elif np.any(np.not_equal(self.rms_velocity, 0.)): - species.momentum_distribution_type = "gaussian" - species.ux_m = self.centroid_velocity[0]/constants.c - species.uy_m = self.centroid_velocity[1]/constants.c - species.uz_m = self.centroid_velocity[2]/constants.c - species.ux_th = self.rms_velocity[0]/constants.c - species.uy_th = self.rms_velocity[1]/constants.c - species.uz_th = self.rms_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussian") + species.add_new_group_attr(source_name, 'ux_m', self.centroid_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_m', self.centroid_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_m', self.centroid_velocity[2]/constants.c) + species.add_new_group_attr(source_name, 'ux_th', self.rms_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_th', self.rms_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_th', self.rms_velocity[2]/constants.c) else: - species.momentum_distribution_type = "constant" - species.ux = self.centroid_velocity[0]/constants.c - species.uy = self.centroid_velocity[1]/constants.c - species.uz = self.centroid_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "constant") + species.add_new_group_attr(source_name, 'ux', self.centroid_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy', self.centroid_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz', self.centroid_velocity[2]/constants.c) - species.do_symmetrize = self.do_symmetrize - species.symmetrization_order = self.symmetrization_order + species.add_new_group_attr(source_name, 'do_symmetrize', self.do_symmetrize) + species.add_new_group_attr(source_name, 'symmetrization_order', self.symmetrization_order) class DensityDistributionBase(object): @@ -368,124 +383,144 @@ def set_mangle_dict(self): # times self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) - def set_species_attributes(self, species, layout): + def set_species_attributes(self, species, layout, source_name): if isinstance(layout, GriddedLayout): # --- Note that the grid attribute of GriddedLayout is ignored - species.injection_style = "nuniformpercell" - species.num_particles_per_cell_each_dim = layout.n_macroparticle_per_cell + species.add_new_group_attr(source_name, 'injection_style', "nuniformpercell") + species.add_new_group_attr(source_name, 'num_particles_per_cell_each_dim', layout.n_macroparticle_per_cell) elif isinstance(layout, PseudoRandomLayout): assert (layout.n_macroparticles_per_cell is not None), Exception('WarpX only supports n_macroparticles_per_cell for the PseudoRandomLayout with this distribution') - species.injection_style = "nrandompercell" - species.num_particles_per_cell = layout.n_macroparticles_per_cell + species.add_new_group_attr(source_name, 'injection_style', "nrandompercell") + species.add_new_group_attr(source_name, 'num_particles_per_cell', layout.n_macroparticles_per_cell) else: raise Exception('WarpX does not support the specified layout for this distribution') - species.xmin = self.lower_bound[0] - species.xmax = self.upper_bound[0] - species.ymin = self.lower_bound[1] - species.ymax = self.upper_bound[1] - species.zmin = self.lower_bound[2] - species.zmax = self.upper_bound[2] + species.add_new_group_attr(source_name, 'xmin', self.lower_bound[0]) + species.add_new_group_attr(source_name, 'xmax', self.upper_bound[0]) + species.add_new_group_attr(source_name, 'ymin', self.lower_bound[1]) + species.add_new_group_attr(source_name, 'ymax', self.upper_bound[1]) + species.add_new_group_attr(source_name, 'zmin', self.lower_bound[2]) + species.add_new_group_attr(source_name, 'zmax', self.upper_bound[2]) if self.fill_in: - species.do_continuous_injection = 1 + species.add_new_group_attr(source_name, 'do_continuous_injection', 1) # --- Note that WarpX takes gamma*beta as input - if (hasattr(self, "momentum_expressions") + if (hasattr(self, "momentum_spread_expressions") + and np.any(np.not_equal(self.momentum_spread_expressions, None)) + ): + species.momentum_distribution_type = 'gaussian_parse_momentum_function' + self.setup_parse_momentum_functions(species, source_name, self.momentum_expressions, '_m', self.directed_velocity) + self.setup_parse_momentum_functions(species, source_name, self.momentum_spread_expressions, '_th', [0.,0.,0.]) + elif (hasattr(self, "momentum_expressions") and np.any(np.not_equal(self.momentum_expressions, None)) ): - species.momentum_distribution_type = 'parse_momentum_function' - self.setup_parse_momentum_functions(species) + species.add_new_group_attr(source_name, 'momentum_distribution_type', 'parse_momentum_function') + self.setup_parse_momentum_functions(species, source_name, self.momentum_expressions, '', self.directed_velocity) elif np.any(np.not_equal(self.rms_velocity, 0.)): - species.momentum_distribution_type = "gaussian" - species.ux_m = self.directed_velocity[0]/constants.c - species.uy_m = self.directed_velocity[1]/constants.c - species.uz_m = self.directed_velocity[2]/constants.c - species.ux_th = self.rms_velocity[0]/constants.c - species.uy_th = self.rms_velocity[1]/constants.c - species.uz_th = self.rms_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussian") + species.add_new_group_attr(source_name, 'ux_m', self.directed_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_m', self.directed_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_m', self.directed_velocity[2]/constants.c) + species.add_new_group_attr(source_name, 'ux_th', self.rms_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_th', self.rms_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_th', self.rms_velocity[2]/constants.c) else: - species.momentum_distribution_type = "constant" - species.ux = self.directed_velocity[0]/constants.c - species.uy = self.directed_velocity[1]/constants.c - species.uz = self.directed_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "constant") + species.add_new_group_attr(source_name, 'ux', self.directed_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy', self.directed_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz', self.directed_velocity[2]/constants.c) - def setup_parse_momentum_functions(self, species): + def setup_parse_momentum_functions(self, species, source_name, expressions, suffix, defaults): for sdir, idir in zip(['x', 'y', 'z'], [0, 1, 2]): - if self.momentum_expressions[idir] is not None: - expression = pywarpx.my_constants.mangle_expression(self.momentum_expressions[idir], self.mangle_dict) + if expressions[idir] is not None: + expression = pywarpx.my_constants.mangle_expression(expressions[idir], self.mangle_dict) else: - expression = f'{self.directed_velocity[idir]}' - species.__setattr__(f'momentum_function_u{sdir}(x,y,z)', f'({expression})/{constants.c}') + expression = f'{defaults[idir]}' + species.add_new_group_attr(source_name, f'momentum_function_u{sdir}{suffix}(x,y,z)', f'({expression})/{constants.c}') class UniformFluxDistribution(picmistandard.PICMI_UniformFluxDistribution, DensityDistributionBase): - def initialize_inputs(self, species_number, layout, species, density_scale): + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.fill_in = False self.set_mangle_dict() - self.set_species_attributes(species, layout) + self.set_species_attributes(species, layout, source_name) - species.flux_profile = "constant" - species.flux = self.flux + species.add_new_group_attr(source_name, 'flux_profile', "constant") + species.add_new_group_attr(source_name, 'flux', self.flux) if density_scale is not None: - species.flux *= density_scale - species.flux_normal_axis = self.flux_normal_axis - species.surface_flux_pos = self.surface_flux_position - species.flux_direction = self.flux_direction - species.flux_tmin = self.flux_tmin - species.flux_tmax = self.flux_tmax + species.add_new_group_attr(source_name, 'flux', density_scale) + species.add_new_group_attr(source_name, 'flux_normal_axis', self.flux_normal_axis) + species.add_new_group_attr(source_name, 'surface_flux_pos', self.surface_flux_position) + species.add_new_group_attr(source_name, 'flux_direction', self.flux_direction) + species.add_new_group_attr(source_name, 'flux_tmin', self.flux_tmin) + species.add_new_group_attr(source_name, 'flux_tmax', self.flux_tmax) # --- Use specific attributes for flux injection - species.injection_style = "nfluxpercell" + species.add_new_group_attr(source_name, 'injection_style', "nfluxpercell") assert (isinstance(layout, PseudoRandomLayout)), Exception('UniformFluxDistribution only supports the PseudoRandomLayout in WarpX') if self.gaussian_flux_momentum_distribution: - species.momentum_distribution_type = "gaussianflux" + species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussianflux") class UniformDistribution(picmistandard.PICMI_UniformDistribution, DensityDistributionBase): - def initialize_inputs(self, species_number, layout, species, density_scale): + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.set_mangle_dict() - self.set_species_attributes(species, layout) + self.set_species_attributes(species, layout, source_name) # --- Only constant density is supported by this class - species.profile = "constant" - species.density = self.density + species.add_new_group_attr(source_name, 'profile', "constant") + species.add_new_group_attr(source_name, 'density', self.density) if density_scale is not None: - species.density *= density_scale + species.add_new_group_attr(source_name, 'density', density_scale) class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDistributionBase): - def initialize_inputs(self, species_number, layout, species, density_scale): + """ + Parameters + ---------- + + warpx_momentum_spread_expressions: list of string + Analytic expressions describing the gamma*velocity spread for each axis [m/s]. + Expressions should be in terms of the position, written as 'x', 'y', and 'z'. + Parameters can be used in the expression with the values given as keyword arguments. + For any axis not supplied (set to None), zero will be used. + + """ + def init(self, kw): + self.momentum_spread_expressions = kw.pop('warpx_momentum_spread_expressions', [None, None, None]) + + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.set_mangle_dict() - self.set_species_attributes(species, layout) + self.set_species_attributes(species, layout, source_name) - species.profile = "parse_density_function" + species.add_new_group_attr(source_name, 'profile', "parse_density_function") expression = pywarpx.my_constants.mangle_expression(self.density_expression, self.mangle_dict) if density_scale is None: - species.__setattr__('density_function(x,y,z)', expression) + species.add_new_group_attr(source_name, 'density_function(x,y,z)', expression) else: - species.__setattr__('density_function(x,y,z)', "{}*({})".format(density_scale, expression)) + species.add_new_group_attr(source_name, 'density_function(x,y,z)', "{}*({})".format(density_scale, expression)) class ParticleListDistribution(picmistandard.PICMI_ParticleListDistribution): def init(self, kw): pass - def initialize_inputs(self, species_number, layout, species, density_scale): + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): - species.injection_style = "multipleparticles" - species.multiple_particles_pos_x = self.x - species.multiple_particles_pos_y = self.y - species.multiple_particles_pos_z = self.z - species.multiple_particles_ux = np.array(self.ux)/constants.c - species.multiple_particles_uy = np.array(self.uy)/constants.c - species.multiple_particles_uz = np.array(self.uz)/constants.c - species.multiple_particles_weight = self.weight + species.add_new_group_attr(source_name, 'injection_style', "multipleparticles") + species.add_new_group_attr(source_name, 'multiple_particles_pos_x', self.x) + species.add_new_group_attr(source_name, 'multiple_particles_pos_y', self.y) + species.add_new_group_attr(source_name, 'multiple_particles_pos_z', self.z) + species.add_new_group_attr(source_name, 'multiple_particles_ux', np.array(self.ux)/constants.c) + species.add_new_group_attr(source_name, 'multiple_particles_uy', np.array(self.uy)/constants.c) + species.add_new_group_attr(source_name, 'multiple_particles_uz', np.array(self.uz)/constants.c) + species.add_new_group_attr(source_name, 'multiple_particles_weight', self.weight) if density_scale is not None: - species.multiple_particles_weight = self.weight*density_scale + species.add_new_group_attr(source_name, 'multiple_particles_weight', self.weight*density_scale) class ParticleDistributionPlanarInjector(picmistandard.PICMI_ParticleDistributionPlanarInjector): @@ -523,10 +558,10 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): """ This assumes that WarpX was compiled with USE_RZ = TRUE - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -560,6 +595,13 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): warpx_reflect_all_velocities: bool default=False Whether the sign of all of the particle velocities are changed upon reflection on a boundary, or only the velocity normal to the surface + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -577,6 +619,9 @@ def init(self, kw): self.potential_zmax = kw.pop('warpx_potential_hi_z', None) self.reflect_all_velocities = kw.pop('warpx_reflect_all_velocities', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -617,6 +662,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[1]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -630,10 +678,10 @@ def initialize_inputs(self): class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -651,6 +699,13 @@ class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): warpx_potential_hi_z: float, default=0. Electrostatic potential on the upper longitudinal boundary + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -665,6 +720,9 @@ def init(self, kw): self.potential_zmin = kw.pop('warpx_potential_lo_z', None) self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -694,6 +752,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[0]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -706,10 +767,10 @@ def initialize_inputs(self): class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -739,6 +800,13 @@ class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): warpx_potential_hi_z: float, default=0. Electrostatic potential on the upper z boundary + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -755,6 +823,9 @@ def init(self, kw): self.potential_zmin = kw.pop('warpx_potential_lo_z', None) self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -789,6 +860,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[1]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -802,10 +876,10 @@ def initialize_inputs(self): class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -847,6 +921,13 @@ class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): warpx_potential_hi_z: float, default=0. Electrostatic potential on the upper z boundary + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -865,6 +946,9 @@ def init(self, kw): self.potential_zmin = kw.pop('warpx_potential_lo_z', None) self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -904,6 +988,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[2]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -917,7 +1004,7 @@ def initialize_inputs(self): class ElectromagneticSolver(picmistandard.PICMI_ElectromagneticSolver): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -1049,9 +1136,14 @@ class HybridPICSolver(picmistandard.base._ClassWithInit): substeps: int, default=100 Number of substeps to take when updating the B-field. + + Jx/y/z_external_function: str + Function of space and time specifying external (non-plasma) currents. """ def __init__(self, grid, Te=None, n0=None, gamma=None, - n_floor=None, plasma_resistivity=None, substeps=None, **kw): + n_floor=None, plasma_resistivity=None, substeps=None, + Jx_external_function=None, Jy_external_function=None, + Jz_external_function=None, **kw): self.grid = grid self.method = "hybrid" @@ -1063,10 +1155,25 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.substeps = substeps + self.Jx_external_function = Jx_external_function + self.Jy_external_function = Jy_external_function + self.Jz_external_function = Jz_external_function + + # Handle keyword arguments used in expressions + self.user_defined_kw = {} + for k in list(kw.keys()): + self.user_defined_kw[k] = kw[k] + del kw[k] + self.handle_init(kw) def initialize_inputs(self): + # Add the user defined keywords to my_constants + # The keywords are mangled if there is a conflicting variable already + # defined in my_constants with the same name but different value. + self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) + self.grid.initialize_inputs() pywarpx.algo.maxwell_solver = self.method @@ -1076,14 +1183,27 @@ def initialize_inputs(self): pywarpx.hybridpicmodel.gamma = self.gamma pywarpx.hybridpicmodel.n_floor = self.n_floor pywarpx.hybridpicmodel.__setattr__( - 'plasma_resistivity(rho)', self.plasma_resistivity + 'plasma_resistivity(rho)', + pywarpx.my_constants.mangle_expression(self.plasma_resistivity, self.mangle_dict) ) pywarpx.hybridpicmodel.substeps = self.substeps + pywarpx.hybridpicmodel.__setattr__( + 'Jx_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jx_external_function, self.mangle_dict) + ) + pywarpx.hybridpicmodel.__setattr__( + 'Jy_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jy_external_function, self.mangle_dict) + ) + pywarpx.hybridpicmodel.__setattr__( + 'Jz_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jz_external_function, self.mangle_dict) + ) class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -1091,7 +1211,7 @@ class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): Whether to use the relativistic solver or lab frame solver warpx_absolute_tolerance: float, default=0. - Absolute tolerance on the lab fram solver + Absolute tolerance on the lab frame solver warpx_self_fields_verbosity: integer, default=2 Level of verbosity for the lab frame solver @@ -1212,9 +1332,11 @@ def initialize_inputs(self): class AnalyticInitialField(picmistandard.PICMI_AnalyticAppliedField): def init(self, kw): self.mangle_dict = None + self.maxlevel_extEMfield_init = kw.pop('warpx_maxlevel_extEMfield_init', None); def initialize_inputs(self): # Note that lower and upper_bound are not used by WarpX + pywarpx.warpx.maxlevel_extEMfield_init = self.maxlevel_extEMfield_init; if self.mangle_dict is None: # Only do this once so that the same variables are used in this distribution @@ -1315,7 +1437,7 @@ def initialize_inputs(self): class CoulombCollisions(picmistandard.base._ClassWithInit): """ - Custom class to handle setup of binary Coulmb collisions in WarpX. If + Custom class to handle setup of binary Coulomb collisions in WarpX. If collision initialization is added to picmistandard this can be changed to inherit that functionality. @@ -1423,6 +1545,49 @@ def initialize_inputs(self): collision.add_new_attr(process+'_'+key, val) +class DSMCCollisions(picmistandard.base._ClassWithInit): + """ + Custom class to handle setup of DSMC collisions in WarpX. If collision + initialization is added to picmistandard this can be changed to inherit + that functionality. + + Parameters + ---------- + name: string + Name of instance (used in the inputs file) + + species: species instance + The species involved in the collision + + scattering_processes: dictionary + The scattering process to use and any needed information + + ndt: integer, optional + The collisions will be applied every "ndt" steps. Must be 1 or larger. + """ + + def __init__(self, name, species, scattering_processes, ndt=None, **kw): + self.name = name + self.species = species + self.scattering_processes = scattering_processes + self.ndt = ndt + + self.handle_init(kw) + + def initialize_inputs(self): + collision = pywarpx.Collisions.newcollision(self.name) + collision.type = 'dsmc' + collision.species = [species.name for species in self.species] + collision.ndt = self.ndt + + collision.scattering_processes = self.scattering_processes.keys() + for process, kw in self.scattering_processes.items(): + for key, val in kw.items(): + if key == 'species': + val = val.name + collision.add_new_attr(process+'_'+key, val) + + class EmbeddedBoundary(picmistandard.base._ClassWithInit): """ Custom class to handle set up of embedded boundaries specific to WarpX. @@ -1542,13 +1707,13 @@ class PlasmaLens(picmistandard.base._ClassWithInit): The field that is applied depends on the transverse position of the particle, (x,y) - - Ex = x*stengths_E + - Ex = x*strengths_E - - Ey = y*stengths_E + - Ey = y*strengths_E - - Bx = +y*stengths_B + - Bx = +y*strengths_B - - By = -x*stengths_B + - By = -x*strengths_B """ def __init__(self, period, starts, lengths, strengths_E=None, strengths_B=None, **kw): @@ -1576,7 +1741,7 @@ def initialize_inputs(self): class Simulation(picmistandard.PICMI_Simulation): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -1680,6 +1845,9 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_amrex_the_arena_init_size: long int, optional The amount of memory in bytes to allocate in the Arena. + warpx_amrex_use_gpu_aware_mpi: bool, optional + Whether to use GPU-aware MPI communications + warpx_zmax_plasma_to_compute_max_step: float, optional Sets the simulation run time based on the maximum z value @@ -1704,6 +1872,30 @@ class Simulation(picmistandard.PICMI_Simulation): The domain will be chopped into the exact number of pieces in each dimension as specified by this parameter. https://warpx.readthedocs.io/en/latest/usage/parameters.html#distribution-across-mpi-ranks-and-parallelization https://warpx.readthedocs.io/en/latest/usage/domain_decomposition.html#simple-method + + warpx_sort_intervals: string, optional (defaults: -1 on CPU; 4 on GPU) + Using the Intervals parser syntax, this string defines the timesteps at which particles are sorted. If <=0, do not sort particles. + It is turned on on GPUs for performance reasons (to improve memory locality). + + warpx_sort_particles_for_deposition: bool, optional (default: true for the CUDA backend, otherwise false) + This option controls the type of sorting used if particle sorting is turned on, i.e. if sort_intervals is not <=0. + If `true`, particles will be sorted by cell to optimize deposition with many particles per cell, in the order `x` -> `y` -> `z` -> `ppc`. + If `false`, particles will be sorted by bin, using the sort_bin_size parameter below, in the order `ppc` -> `x` -> `y` -> `z`. + `true` is recommended for best performance on NVIDIA GPUs, especially if there are many particles per cell. + + warpx_sort_idx_type: list of int, optional (default: 0 0 0) + This controls the type of grid used to sort the particles when sort_particles_for_deposition is true. + Possible values are: + + * idx_type = {0, 0, 0}: Sort particles to a cell centered grid, + * idx_type = {1, 1, 1}: Sort particles to a node centered grid, + * idx_type = {2, 2, 2}: Compromise between a cell and node centered grid. + + In 2D (XZ and RZ), only the first two elements are read. In 1D, only the first element is read. + + warpx_sort_bin_size: list of int, optional (default 1 1 1) + If `sort_intervals` is activated and `sort_particles_for_deposition` is false, particles are sorted in bins of `sort_bin_size` cells. + In 2D, only the first two elements are read. """ # Set the C++ WarpX interface (see _libwarpx.LibWarpX) as an extension to @@ -1739,8 +1931,13 @@ def init(self, kw): self.amr_restart = kw.pop('warpx_amr_restart', None) self.amrex_the_arena_is_managed = kw.pop('warpx_amrex_the_arena_is_managed', None) self.amrex_the_arena_init_size = kw.pop('warpx_amrex_the_arena_init_size', None) + self.amrex_use_gpu_aware_mpi = kw.pop('warpx_amrex_use_gpu_aware_mpi', None) self.zmax_plasma_to_compute_max_step = kw.pop('warpx_zmax_plasma_to_compute_max_step', None) self.compute_max_step_from_btd = kw.pop('warpx_compute_max_step_from_btd', None) + self.sort_intervals = kw.pop('warpx_sort_intervals', None) + self.sort_particles_for_deposition = kw.pop('warpx_sort_particles_for_deposition', None) + self.sort_idx_type = kw.pop('warpx_sort_idx_type', None) + self.sort_bin_size = kw.pop('warpx_sort_bin_size', None) self.collisions = kw.pop('warpx_collisions', None) self.embedded_boundary = kw.pop('warpx_embedded_boundary', None) @@ -1769,6 +1966,11 @@ def initialize_inputs(self): pywarpx.warpx.zmax_plasma_to_compute_max_step = self.zmax_plasma_to_compute_max_step pywarpx.warpx.compute_max_step_from_btd = self.compute_max_step_from_btd + pywarpx.warpx.sort_intervals = self.sort_intervals + pywarpx.warpx.sort_particles_for_deposition = self.sort_particles_for_deposition + pywarpx.warpx.sort_idx_type = self.sort_idx_type + pywarpx.warpx.sort_bin_size = self.sort_bin_size + pywarpx.algo.current_deposition = self.current_deposition_algo pywarpx.algo.charge_deposition = self.charge_deposition_algo pywarpx.algo.field_gathering = self.field_gathering_algo @@ -1869,6 +2071,9 @@ def initialize_inputs(self): if self.amrex_the_arena_init_size is not None: pywarpx.amrex.the_arena_init_size = self.amrex_the_arena_init_size + if self.amrex_use_gpu_aware_mpi is not None: + pywarpx.amrex.use_gpu_aware_mpi = self.amrex_use_gpu_aware_mpi + def initialize_warpx(self, mpi_comm=None): if self.warpx_initialized: return @@ -1934,9 +2139,38 @@ def set_write_dir(self): self.diagnostic.file_prefix = os.path.join(write_dir, file_prefix) +@dataclass(frozen=True) +class ParticleFieldDiagnostic: + """ + Class holding particle field diagnostic information to be processed in FieldDiagnostic below. + + Parameters + ---------- + name: str + Name of particle field diagnostic. If a component of a vector field, for the openPMD viewer + to treat it as a vector, the coordinate (i.e x, y, z) should be the last character. + + func: parser str + Parser function to be calculated for each particle per cell. Should be of the form + f(x,y,z,ux,uy,uz) + + do_average: (0 or 1) optional, default 1 + Whether the diagnostic is averaged by the sum of particle weights in each cell + + filter: parser str, optional + Parser function returning a boolean for whether to include a particle in the diagnostic. + If not specified, all particles will be included. The function arguments are the same + as the `func` above. + """ + name: str + func: str + do_average: int = 1 + filter: str = None + + class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -1946,9 +2180,6 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): warpx_plot_raw_fields_guards: bool, optional Flag whether the raw fields should include the guard cells - warpx_write_species: bool, optional - Flag whether to output particle data with the diagnostic - warpx_format: {plotfile, checkpoint, openpmd, ascent, sensei}, optional Diagnostic file format @@ -1964,13 +2195,14 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): warpx_dump_rz_modes: bool, optional Flag whether to dump the data for all RZ modes - warpx_lower_bound: vector of floats, optional - Lower corner of output fields - that is passed to .lower_bound + warpx_particle_fields_to_plot: list of ParticleFieldDiagnostics + List of ParticleFieldDiagnostic classes to install in the simulation. Error + checking is handled in the class itself. - warpx_upper_bound: vector of floats, optional - Upper corner of output fields - that is passed to .upper_bound + warpx_particle_fields_species: list of strings, optional + Species for which to calculate particle_fields_to_plot functions. Fields will + be calculated separately for each specified species. If not passed, default is + all of the available particle species. """ def init(self, kw): @@ -1978,14 +2210,13 @@ def init(self, kw): self.plot_raw_fields_guards = kw.pop('warpx_plot_raw_fields_guards', None) self.plot_finepatch = kw.pop('warpx_plot_finepatch', None) self.plot_crsepatch = kw.pop('warpx_plot_crsepatch', None) - self.write_species = kw.pop('warpx_write_species', None) self.format = kw.pop('warpx_format', 'plotfile') self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.dump_rz_modes = kw.pop('warpx_dump_rz_modes', None) - self.lower_bound = kw.pop('warpx_lower_bound', None) - self.upper_bound = kw.pop('warpx_upper_bound', None) + self.particle_fields_to_plot = kw.pop('warpx_particle_fields_to_plot', []) + self.particle_fields_species = kw.pop('warpx_particle_fields_species', None) def initialize_inputs(self): @@ -2061,14 +2292,35 @@ def initialize_inputs(self): # --- is the same on all processors. fields_to_plot = list(fields_to_plot) fields_to_plot.sort() - self.diagnostic.fields_to_plot = fields_to_plot + self.diagnostic.set_or_replace_attr('fields_to_plot', fields_to_plot) + + particle_fields_to_plot_names = list() + for pfd in self.particle_fields_to_plot: + if pfd.name in particle_fields_to_plot_names: + raise Exception('A particle fields name can not be repeated.') + particle_fields_to_plot_names.append(pfd.name) + self.diagnostic.__setattr__( + f'particle_fields.{pfd.name}(x,y,z,ux,uy,uz)', pfd.func + ) + self.diagnostic.__setattr__( + f'particle_fields.{pfd.name}.do_average', pfd.do_average + ) + self.diagnostic.__setattr__( + f'particle_fields.{pfd.name}.filter(x,y,z,ux,uy,uz)', pfd.filter + ) + + # --- Convert to a sorted list so that the order + # --- is the same on all processors. + particle_fields_to_plot_names.sort() + self.diagnostic.particle_fields_to_plot = particle_fields_to_plot_names + self.diagnostic.particle_fields_species = self.particle_fields_species self.diagnostic.plot_raw_fields = self.plot_raw_fields self.diagnostic.plot_raw_fields_guards = self.plot_raw_fields_guards self.diagnostic.plot_finepatch = self.plot_finepatch self.diagnostic.plot_crsepatch = self.plot_crsepatch - self.diagnostic.write_species = self.write_species - + if 'write_species' not in self.diagnostic.argvattrs: + self.diagnostic.write_species = False self.set_write_dir() @@ -2079,7 +2331,7 @@ class Checkpoint(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): """ Sets up checkpointing of the simulation, allowing for later restarts - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -2118,7 +2370,7 @@ def initialize_inputs(self): class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -2173,7 +2425,9 @@ def initialize_inputs(self): self.diagnostic.openpmd_backend = self.openpmd_backend self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.intervals = self.period - + self.diagnostic.set_or_replace_attr('write_species', True) + if 'fields_to_plot' not in self.diagnostic.argvattrs: + self.diagnostic.fields_to_plot = 'none' self.set_write_dir() # --- Use a set to ensure that fields don't get repeated. @@ -2206,24 +2460,26 @@ def initialize_inputs(self): variables.sort() # species list - if np.iterable(self.species): - species_list = self.species + if self.species is None: + species_names = pywarpx.particles.species_names + elif np.iterable(self.species): + species_names = [specie.name for specie in self.species] else: - species_list = [self.species] + species_names = [self.species.name] if self.mangle_dict is None: # Only do this once so that the same variables are used in this distribution # is used multiple times self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) - for specie in species_list: - diag = pywarpx.Bucket.Bucket(self.name + '.' + specie.name, + for name in species_names: + diag = pywarpx.Bucket.Bucket(self.name + '.' + name, variables = variables, random_fraction = self.random_fraction, uniform_stride = self.uniform_stride) expression = pywarpx.my_constants.mangle_expression(self.plot_filter_function, self.mangle_dict) diag.__setattr__('plot_filter_function(t,x,y,z,ux,uy,uz)', expression) - self.diagnostic._species_dict[specie.name] = diag + self.diagnostic._species_dict[name] = diag # ---------------------------- # Lab frame diagnostics @@ -2233,11 +2489,9 @@ def initialize_inputs(self): class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ + See `Input Parameters `__ for more information. - This will by default write out both field and particle data. This can be changed by setting warpx_write_species. - Parameters ---------- warpx_format: string, optional @@ -2264,11 +2518,10 @@ class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, warpx_upper_bound: vector of floats, optional Passed to .upper_bound - - warpx_write_species: bool, optional, default=True - Whether the species will also be written out. """ def init(self, kw): + """The user is using the new BTD""" + self.format = kw.pop('warpx_format', None) self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) self.file_prefix = kw.pop('warpx_file_prefix', None) @@ -2277,7 +2530,6 @@ def init(self, kw): self.buffer_size = kw.pop('warpx_buffer_size', None) self.lower_bound = kw.pop('warpx_lower_bound', None) self.upper_bound = kw.pop('warpx_upper_bound', None) - self.write_species = kw.pop('warpx_write_species', None) def initialize_inputs(self): @@ -2290,7 +2542,7 @@ def initialize_inputs(self): self.diagnostic.diag_lo = self.lower_bound self.diagnostic.diag_hi = self.upper_bound - self.diagnostic.do_back_transformed_fields = 1 + self.diagnostic.do_back_transformed_fields = True self.diagnostic.dt_snapshots_lab = self.dt_snapshots self.diagnostic.buffer_size = self.buffer_size @@ -2300,8 +2552,6 @@ def initialize_inputs(self): else: self.diagnostic.num_snapshots_lab = self.num_snapshots - self.diagnostic.do_back_transformed_particles = self.write_species - # --- Use a set to ensure that fields don't get repeated. fields_to_plot = set() @@ -2338,19 +2588,19 @@ def initialize_inputs(self): # --- is the same on all processors. fields_to_plot = list(fields_to_plot) fields_to_plot.sort() - self.diagnostic.fields_to_plot = fields_to_plot + self.diagnostic.set_or_replace_attr('fields_to_plot', fields_to_plot) + if 'write_species' not in self.diagnostic.argvattrs: + self.diagnostic.write_species = False self.set_write_dir() class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ + See `Input Parameters `__ for more information. - This will by default write out both field and particle data. This can be changed by setting warpx_write_fields. - Parameters ---------- warpx_format: string, optional @@ -2371,9 +2621,6 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, warpx_buffer_size: integer, optional Passed to .buffer_size - - warpx_write_fields: bool, optional, default=True - Whether the fields will also be written out. """ def init(self, kw): self.format = kw.pop('warpx_format', None) @@ -2382,7 +2629,6 @@ def init(self, kw): self.intervals = kw.pop('warpx_intervals', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.buffer_size = kw.pop('warpx_buffer_size', None) - self.write_fields = kw.pop('warpx_write_fields', None) def initialize_inputs(self): @@ -2393,7 +2639,7 @@ def initialize_inputs(self): self.diagnostic.openpmd_backend = self.openpmd_backend self.diagnostic.file_min_digits = self.file_min_digits - self.diagnostic.do_back_transformed_particles = 1 + self.diagnostic.do_back_transformed_particles = True self.diagnostic.dt_snapshots_lab = self.dt_snapshots self.diagnostic.buffer_size = self.buffer_size @@ -2403,7 +2649,11 @@ def initialize_inputs(self): else: self.diagnostic.num_snapshots_lab = self.num_snapshots - self.diagnostic.do_back_transformed_fields = self.write_fields + self.diagnostic.do_back_transformed_fields = False + + self.diagnostic.set_or_replace_attr('write_species', True) + if 'fields_to_plot' not in self.diagnostic.argvattrs: + self.diagnostic.fields_to_plot = 'none' self.set_write_dir() @@ -2437,22 +2687,24 @@ def initialize_inputs(self): variables.sort() # species list - if np.iterable(self.species): - species_list = self.species + if self.species is None: + species_names = pywarpx.particles.species_names + elif np.iterable(self.species): + species_names = [specie.name for specie in self.species] else: - species_list = [self.species] + species_names = [species.name] - for specie in species_list: - diag = pywarpx.Bucket.Bucket(self.name + '.' + specie.name, + for name in species_names: + diag = pywarpx.Bucket.Bucket(self.name + '.' + name, variables = variables) - self.diagnostic._species_dict[specie.name] = diag + self.diagnostic._species_dict[name] = diag class ReducedDiagnostic(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): """ Sets up a reduced diagnostic in the simulation. - See `Input Parameters `_ + See `Input Parameters `__ for more information. Parameters @@ -2508,7 +2760,7 @@ class ReducedDiagnostic(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): reduction_type: {'Maximum', 'Minimum', or 'Integral'} For diagnostic type 'FieldReduction', the type of reduction - probe_geometry: {'Point', 'Line', 'Plane'}, defaut='Point' + probe_geometry: {'Point', 'Line', 'Plane'}, default='Point' For diagnostic type 'FieldProbe', the geometry of the probe integrate: bool, default=false diff --git a/Python/setup.py b/Python/setup.py index 99f79cd1d18..47028642cce 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,12 +54,12 @@ package_data = {} setup(name = 'pywarpx', - version = '23.09', + version = '23.12', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", package_data = package_data, - install_requires = ['numpy', 'picmistandard==0.26.0', 'periodictable'], + install_requires = ['numpy', 'picmistandard==0.28.0', 'periodictable'], python_requires = '>=3.8', zip_safe=False ) diff --git a/README.md b/README.md index ef0551405a0..87cf2b012c0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Code Status development](https://dev.azure.com/ECP-WarpX/WarpX/_apis/build/status/ECP-WarpX.WarpX?branchName=development)](https://dev.azure.com/ECP-WarpX/WarpX/_build/latest?definitionId=1&branchName=development) [![Nightly Installation Tests](https://dev.azure.com/ECP-WarpX/WarpX/_apis/build/status/ECP-WarpX.Nightly?branchName=nightly&label=nightly%20packages)](https://dev.azure.com/ECP-WarpX/WarpX/_build?definitionId=2) -[![Documentation Status](https://readthedocs.org/projects/warpx/badge/?version=latest)](https://warpx.readthedocs.io/en/latest/?badge=latest) +[![Documentation Status](https://readthedocs.org/projects/warpx/badge/?version=latest)](https://warpx.readthedocs.io) [![Spack Version](https://img.shields.io/spack/v/warpx)](https://spack.readthedocs.io/en/latest/package_list.html#warpx) [![Conda Version](https://img.shields.io/conda/vn/conda-forge/warpx)](https://anaconda.org/conda-forge/warpx) [![Discussions](https://img.shields.io/badge/chat-discussions-turquoise.svg)](https://github.com/ECP-WarpX/WarpX/discussions) diff --git a/Regression/Checksum/benchmarks_json/BTD_rz.json b/Regression/Checksum/benchmarks_json/BTD_rz.json index 3110120b70a..01e4c687292 100644 --- a/Regression/Checksum/benchmarks_json/BTD_rz.json +++ b/Regression/Checksum/benchmarks_json/BTD_rz.json @@ -1,13 +1,13 @@ { "lev=0": { - "Br": 1.8705552174367742e-08, - "Bt": 2380179.567792259, - "Bz": 2.4079075035536954e-08, + "Br": 1.8705552264208163e-08, + "Bt": 2380179.5677922587, + "Bz": 2.4079077116116535e-08, "Er": 497571119514841.25, - "Et": 7.048225282653208, - "Ez": 137058870936728.17, + "Et": 7.048225464720808, + "Ez": 137058870936728.28, "jr": 0.0, "jt": 0.0, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json new file mode 100644 index 00000000000..da2a8500b53 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json @@ -0,0 +1,128 @@ +{ + "lev=0": { + "Bx": 970235841795.1099, + "By": 970175630167.5084, + "Bz": 51789226.68307851, + "Ex": 2.90874251210462e+20, + "Ey": 2.908934769466038e+20, + "Ez": 1.70819200999153e+17, + "rho_beam1": 7.969271809437626e+16, + "rho_beam2": 7.969216746820773e+16, + "rho_ele": 0.0, + "rho_ele_nlbw1": 299758753127495.56, + "rho_ele_nlbw2": 289860995480960.4, + "rho_pos": 613963689581.3057, + "rho_pos_nlbw1": 287891375746148.75, + "rho_pos_nlbw2": 295214367940598.1 + }, + "beam1": { + "particle_opticalDepthQSR": 104868.583557123, + "particle_position_x": 0.0015001423941487055, + "particle_position_y": 0.001500178877546805, + "particle_position_z": 0.004965525054740317, + "particle_momentum_x": 6.20719343794678e-15, + "particle_momentum_y": 6.1639864036830354e-15, + "particle_momentum_z": 6.807872002761295e-12, + "particle_weight": 635864683.4991333 + }, + "beam2": { + "particle_opticalDepthQSR": 104166.96636860794, + "particle_position_x": 0.001500093343835549, + "particle_position_y": 0.0015001802698114421, + "particle_position_z": 0.00496555337137147, + "particle_momentum_x": 6.1976310171196696e-15, + "particle_momentum_y": 6.1844493116189144e-15, + "particle_momentum_z": 6.796731846337045e-12, + "particle_weight": 635863931.615053 + }, + "ele": { + "particle_opticalDepthQSR": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "ele_nlbw1": { + "particle_opticalDepthQSR": 405.16545575146574, + "particle_position_x": 5.119886480641899e-06, + "particle_position_y": 5.266875857997872e-06, + "particle_position_z": 1.855770689791573e-05, + "particle_momentum_x": 5.744620543080891e-18, + "particle_momentum_y": 5.463133845399354e-18, + "particle_momentum_z": 2.562592228511742e-15, + "particle_weight": 2359597.3527386785 + }, + "ele_nlbw2": { + "particle_opticalDepthQSR": 354.8472581769572, + "particle_position_x": 5.235909912617143e-06, + "particle_position_y": 5.068613695636293e-06, + "particle_position_z": 1.6665840940849017e-05, + "particle_momentum_x": 4.47916023711595e-18, + "particle_momentum_y": 4.092494634513803e-18, + "particle_momentum_z": 2.2277004588526963e-15, + "particle_weight": 2354612.144268994 + }, + "pho": { + "particle_opticalDepthBW": 462.76829758918615, + "particle_position_x": 6.163512168886876e-06, + "particle_position_y": 5.4614856451855956e-06, + "particle_position_z": 2.0316590334121627e-05, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 2923343.7532989895 + }, + "pho1": { + "particle_opticalDepthBW": 9769.560457892825, + "particle_position_x": 0.00013930069652083352, + "particle_position_y": 0.00014046343589264903, + "particle_position_z": 0.00046498427057467117, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 59881947.934441805 + }, + "pho2": { + "particle_opticalDepthBW": 9986.841892753177, + "particle_position_x": 0.000143478808886347, + "particle_position_y": 0.00014235781306129165, + "particle_position_z": 0.0004760406640177353, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 60412035.77539255 + }, + "pos": { + "particle_opticalDepthQSR": 1.0913353138611541, + "particle_position_x": 1.297022467502343e-08, + "particle_position_y": 6.533761995664651e-09, + "particle_position_z": 5.270388297177834e-08, + "particle_momentum_x": 9.474214808949084e-21, + "particle_momentum_y": 3.664691658234474e-21, + "particle_momentum_z": 5.559027877774493e-20, + "particle_weight": 4777.4993625771185 + }, + "pos_nlbw1": { + "particle_opticalDepthQSR": 353.1921715878159, + "particle_position_x": 5.413239275792192e-06, + "particle_position_y": 5.599083716468022e-06, + "particle_position_z": 1.729829030925741e-05, + "particle_momentum_x": 5.130129014453181e-18, + "particle_momentum_y": 5.151236001734455e-18, + "particle_momentum_z": 2.7592435303859743e-15, + "particle_weight": 2298018.034763882 + }, + "pos_nlbw2": { + "particle_opticalDepthQSR": 350.113008933412, + "particle_position_x": 4.925666132330728e-06, + "particle_position_y": 4.677928261139806e-06, + "particle_position_z": 1.7528432992559524e-05, + "particle_momentum_x": 5.3410973872204896e-18, + "particle_momentum_y": 5.0088980554153576e-18, + "particle_momentum_z": 2.4803849492900593e-15, + "particle_weight": 2379194.375363683 + } +} diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json index 382c6010c1c..23231eab789 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json @@ -1,23 +1,8 @@ { - "deuterium_1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 - }, - "deuterium_2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 3.356807324363973e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240779905e+29 + "lev=0": { + "rho": 0.0 }, - "helium4_1": { + "neutron_1": { "particle_momentum_x": 1.7519716491839538e-15, "particle_momentum_y": 1.7523289312260283e-15, "particle_momentum_z": 1.7480231586369996e-15, @@ -26,19 +11,25 @@ "particle_position_z": 325970.4138010667, "particle_weight": 4.421535775967805e-28 }, - "helium4_2": { - "particle_momentum_x": 1.5330942227771018e-15, - "particle_momentum_y": 1.5328473121602395e-15, - "particle_momentum_z": 1.7635828326228758e-15, - "particle_position_x": 137011.89739173267, - "particle_position_y": 136605.24328988983, - "particle_position_z": 290143.4673994485, - "particle_weight": 5.756530048087129e+18 + "tritium_1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.8875978729147693e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 }, - "lev=0": { - "rho": 0.0 + "deuterium_2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 3.356220762633467e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503, + "particle_weight": 1.0240001137713503e+30 }, - "neutron_1": { + "helium4_1": { "particle_momentum_x": 1.7519716491839538e-15, "particle_momentum_y": 1.7523289312260283e-15, "particle_momentum_z": 1.7480231586369996e-15, @@ -47,24 +38,33 @@ "particle_position_z": 325970.4138010667, "particle_weight": 4.421535775967805e-28 }, - "neutron_2": { - "particle_momentum_x": 1.5330942227771018e-15, - "particle_momentum_y": 1.5328473121602395e-15, - "particle_momentum_z": 1.549297051563983e-15, - "particle_position_x": 137011.89739173267, - "particle_position_y": 136605.24328988983, - "particle_position_z": 290143.4673994485, - "particle_weight": 5.756530048087129e+18 - }, - "tritium_1": { + "deuterium_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, "particle_weight": 1024.000000000021 }, + "neutron_2": { + "particle_momentum_x": 1.538345593941914e-15, + "particle_momentum_y": 1.536969107959402e-15, + "particle_momentum_z": 1.5535605933245691e-15, + "particle_position_x": 137088.2335716813, + "particle_position_y": 136370.55273167047, + "particle_position_z": 290148.26756873244, + "particle_weight": 6.427760042646557e+18 + }, + "helium4_2": { + "particle_momentum_x": 1.538345593941914e-15, + "particle_momentum_y": 1.536969107959402e-15, + "particle_momentum_z": 1.769309377628039e-15, + "particle_position_x": 137088.2335716813, + "particle_position_y": 136370.55273167047, + "particle_position_z": 290148.26756873244, + "particle_weight": 6.427760042646557e+18 + }, "tritium_2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -72,6 +72,6 @@ "particle_position_x": 409798.0158217681, "particle_position_y": 409670.9858143465, "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999424347e+29 + "particle_weight": 1.0239999999357226e+29 } } diff --git a/Regression/Checksum/benchmarks_json/FluxInjection3D.json b/Regression/Checksum/benchmarks_json/FluxInjection3D.json index b2b3733737e..aa30f988f68 100644 --- a/Regression/Checksum/benchmarks_json/FluxInjection3D.json +++ b/Regression/Checksum/benchmarks_json/FluxInjection3D.json @@ -1,21 +1,39 @@ { - "electron": { - "particle_momentum_x": 1.1192116199394354e-18, - "particle_momentum_y": 2.238114590066897e-18, - "particle_momentum_z": 1.1156457989239732e-18, - "particle_position_x": 102495.14197173176, - "particle_position_y": 188132.22608016344, - "particle_position_z": 102423.13701045913, + "lev=0": {}, + "electron_negative": { + "particle_momentum_x": 1.1222699783863554e-18, + "particle_momentum_y": 1.1202176725070554e-18, + "particle_momentum_z": 1.3925955132362978e-18, + "particle_position_x": 102352.09026544492, + "particle_position_y": 102418.88243172191, + "particle_position_z": 194298.7949373403, "particle_weight": 8.959999999999998e-07 }, - "lev=0": {}, "proton": { - "particle_momentum_x": 3.835423016604918e-15, - "particle_momentum_y": 2.0468371931479925e-15, - "particle_momentum_z": 2.055186547721331e-15, - "particle_position_x": 189256.1546041931, - "particle_position_y": 102293.00576740496, - "particle_position_z": 102314.93877691089, + "particle_momentum_x": 3.8338884590187296e-15, + "particle_momentum_y": 2.0442156829943128e-15, + "particle_momentum_z": 2.045804260395492e-15, + "particle_position_x": 189238.69249885075, + "particle_position_y": 102242.91543133644, + "particle_position_z": 102297.92915049737, + "particle_weight": 8.959999999999998e-07 + }, + "electron": { + "particle_momentum_x": 1.1150196665556376e-18, + "particle_momentum_y": 2.2311586451156107e-18, + "particle_momentum_z": 1.1115069298757383e-18, + "particle_position_x": 102477.68142952351, + "particle_position_y": 188137.20906834095, + "particle_position_z": 102443.44417709563, + "particle_weight": 8.959999999999998e-07 + }, + "proton_negative": { + "particle_momentum_x": 2.051429985038282e-15, + "particle_momentum_y": 2.053711846305655e-15, + "particle_momentum_z": 2.7212135003240815e-15, + "particle_position_x": 102307.73649638034, + "particle_position_y": 102475.13878406698, + "particle_position_z": 193638.89296895845, "particle_weight": 8.959999999999998e-07 } } diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json new file mode 100644 index 00000000000..cd4120002be --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 3730.0029376363264, + "By": 1593.5906541698305, + "Bz": 0.0, + "Ex": 797541065253.7858, + "Ey": 981292393404.0359, + "Ez": 3528134993266.091, + "divE": 2.0829069134855788e+21, + "jx": 7.639291641209293e+17, + "jy": 1.4113587963237038e+18, + "jz": 1.3506587033985085e+18, + "rho": 18442449008.581665 + }, + "protons": { + "particle_momentum_x": 5.231105747759245e-19, + "particle_momentum_y": 5.367982834807453e-19, + "particle_momentum_z": 5.253213507906386e-19, + "particle_position_x": 0.00010628272743703996, + "particle_weight": 5.314093261582036e+22 + }, + "electrons": { + "particle_momentum_x": 1.196379551301037e-20, + "particle_momentum_y": 1.2271443795645239e-20, + "particle_momentum_z": 1.2277752539495415e-20, + "particle_position_x": 0.00010649569055433632, + "particle_weight": 5.314093261582036e+22 + } +} diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_1D.json b/Regression/Checksum/benchmarks_json/Langmuir_fluid_1D.json new file mode 100644 index 00000000000..454afc73bcc --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_fluid_1D.json @@ -0,0 +1,7 @@ +{ + "lev=0": { + "Ez": 123800860242.32724, + "jz": 48549778373280.2, + "rho": 344278.3437340355 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_2D.json b/Regression/Checksum/benchmarks_json/Langmuir_fluid_2D.json new file mode 100644 index 00000000000..8afc645639d --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_fluid_2D.json @@ -0,0 +1,9 @@ +{ + "lev=0": { + "Ex": 3790984950257.2373, + "Ez": 3790984950257.2373, + "jx": 1.0093528745824196e+16, + "jz": 1.0093528745824196e+16, + "rho": 21098347.344494 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_RZ.json b/Regression/Checksum/benchmarks_json/Langmuir_fluid_RZ.json new file mode 100644 index 00000000000..2613f971f91 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_fluid_RZ.json @@ -0,0 +1,10 @@ +{ + "lev=0": { + "Bt": 3.64571737083816, + "Er": 1300317001574.1577, + "Ez": 1815076954086.807, + "jr": 244937533603400.0, + "jz": 342537403264912.5, + "rho": 10096982.130957149 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_fluid_multi.json b/Regression/Checksum/benchmarks_json/Langmuir_fluid_multi.json new file mode 100644 index 00000000000..1e5d41f6677 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_fluid_multi.json @@ -0,0 +1,15 @@ +{ + "lev=0": { + "Bx": 20.814308179643078, + "By": 20.814308179643234, + "Bz": 20.81430817964295, + "Ex": 82994788022201.23, + "Ey": 82994788022201.23, + "Ez": 82994788022201.23, + "jx": 6.343089944595606e+16, + "jy": 6.3430899445956056e+16, + "jz": 6.3430899445956056e+16, + "part_per_cell": 0.0, + "rho": 694435081.2275198 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_momentum_conserving.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_momentum_conserving.json new file mode 100644 index 00000000000..72c11245e82 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_momentum_conserving.json @@ -0,0 +1,40 @@ +{ + "electrons": { + "particle_momentum_x": 4.2339125386796835e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 4.2339186493637336e-20, + "particle_position_x": 0.6553603030161138, + "particle_position_y": 0.6553603026454923, + "particle_weight": 3200000000000000.5 + }, + "lev=0": { + "Bx": 0.0, + "By": 44.73189518299763, + "Bz": 0.0, + "Ex": 7583674122515.894, + "Ey": 0.0, + "Ez": 7583672042436.007, + "jx": 7283798122345162.0, + "jy": 0.0, + "jz": 7283804678733656.0 + }, + "lev=1": { + "Bx": 0.0, + "By": 467.8271651312273, + "Bz": 0.0, + "Ex": 7609732451490.526, + "Ey": 0.0, + "Ez": 7604891944279.171, + "jx": 7103859890231833.0, + "jy": 0.0, + "jz": 7099873215674540.0 + }, + "positrons": { + "particle_momentum_x": 4.233907500710132e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 4.2339135644675305e-20, + "particle_position_x": 0.6553596934061604, + "particle_position_y": 0.6553596937869579, + "particle_weight": 3200000000000000.5 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json index 4f8c01586cd..12f1f37dfd7 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json @@ -1,30 +1,29 @@ { - "electrons": { - "particle_momentum_x": 3.059978628132309e-36, - "particle_momentum_y": 2.003371642665396e-20, - "particle_momentum_z": 2.785324114261674e-20, - "particle_position_x": 0.5290000114309512, - "particle_position_y": 0.5888000690527484, - "particle_theta": 92488.48772168353, - "particle_weight": 81147583679.15044 + "lev=0": { + "Bt": 6.633814882712189, + "Er": 478973170490.2528, + "Ez": 662610386678.0768, + "divE": 4.354742261016806e+17, + "jr": 893973344247781.6, + "jz": 1251505325555575.2, + "rho": 3855770.585538005 }, "ions": { - "particle_momentum_x": 1.4925140714935799e-36, - "particle_momentum_y": 1.4765381278752215e-21, - "particle_momentum_z": 1.9505526940463635e-21, - "particle_position_x": 0.5290000097195765, - "particle_position_y": 0.5887999999975628, + "particle_momentum_x": 9.303284812749536e-37, + "particle_momentum_y": 1.475435436402189e-21, + "particle_momentum_z": 1.9507023826482256e-21, + "particle_position_x": 0.5290000097194892, + "particle_position_y": 0.5887999999995832, "particle_theta": 92488.48772168353, "particle_weight": 81147583679.15044 }, - "lev=0": { - "Bt": 6.632261581203373, - "Er": 478937338832.87415, - "Ez": 662607529111.8489, - "divE": 4.355808435163123e+17, - "jr": 893968704248372.6, - "jz": 1251505252231323.8, - "rho": 3856714.5961512737 + "electrons": { + "particle_momentum_x": 2.5301631422412464e-36, + "particle_momentum_y": 2.0033141618079855e-20, + "particle_momentum_z": 2.785324108949854e-20, + "particle_position_x": 0.5290000111160095, + "particle_position_y": 0.5888000013680398, + "particle_theta": 92488.48772168353, + "particle_weight": 81147583679.15044 } } - diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json index ee19b84c7af..94e243f21a2 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 3.535284585563231e-19, - "particle_momentum_y": 4.403094613346061e-19, - "particle_momentum_z": 5.658013779496569e-17, - "particle_position_x": 0.008317876520240174, - "particle_position_y": 1.1704335094514386, - "particle_weight": 62415090744.60765 + "lev=0": { + "Bx": 4818955.485214876, + "By": 1752.8017794063862, + "Bz": 14516.212849468406, + "Ex": 2366115511749.6064, + "Ey": 1446112026972328.0, + "Ez": 21864189477873.78, + "jx": 1996366361598696.5, + "jy": 5.312583836700398e+16, + "jz": 2.0491352591140016e+16, + "rho": 68443961.6079968 }, "electrons": { - "particle_momentum_x": 2.2135959939611405e-23, - "particle_momentum_y": 2.822519730011994e-22, - "particle_momentum_z": 5.260625039372931e-22, - "particle_position_x": 0.010800577787577741, - "particle_position_y": 0.21115060628258137, + "particle_momentum_x": 2.2135944672847805e-23, + "particle_momentum_y": 2.82245592458273e-22, + "particle_momentum_z": 5.260626007649988e-22, + "particle_position_x": 0.010800577787630073, + "particle_position_y": 0.21115060628317733, "particle_weight": 4.121554826246186e+16 }, "ions": { - "particle_momentum_x": 6.248472008953412e-23, - "particle_momentum_y": 4.449200926395666e-22, - "particle_momentum_z": 5.768167708374496e-22, - "particle_position_x": 0.010800001678510793, - "particle_position_y": 0.21114947608115497, + "particle_momentum_x": 6.24847236820344e-23, + "particle_momentum_y": 4.449097670697282e-22, + "particle_momentum_z": 5.768168724446374e-22, + "particle_position_x": 0.010800001678510515, + "particle_position_y": 0.21114947608115428, "particle_weight": 4.121554826246186e+16 }, - "lev=0": { - "Bx": 4818965.813977506, - "By": 1752.781451346485, - "Bz": 14516.29947347909, - "Ex": 2366115950699.717, - "Ey": 1446115205306590.0, - "Ez": 21864183756771.12, - "jx": 1996366451911408.0, - "jy": 5.312642483130767e+16, - "jz": 2.049134468340688e+16, - "rho": 68443935.06252655 + "beam": { + "particle_momentum_x": 3.5357352601344873e-19, + "particle_momentum_y": 4.363147101327531e-19, + "particle_momentum_z": 5.658494028187168e-17, + "particle_position_x": 0.008314957057032625, + "particle_position_y": 1.1704335719922687, + "particle_weight": 62415090744.60765 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json index 976a3bdee69..1602058dbdc 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json @@ -1,64 +1,64 @@ { - "beam": { - "particle_momentum_x": 3.880115499392648e-20, - "particle_momentum_y": 5.07820548292446e-20, - "particle_momentum_z": 1.3503614921828295e-17, - "particle_position_x": 6.242131246165356e-05, - "particle_position_y": 0.0026764363296957524, - "particle_theta": 151.40798444325364, - "particle_weight": 6241509.074460764 + "lev=0": { + "Br": 104965.61239547684, + "Br_0_real": 0.2747311066289638, + "Br_1_imag": 104.10451752777048, + "Br_1_real": 104965.62478916143, + "Bt": 1296.2723277779253, + "Btheta_0_real": 1297.3296772773874, + "Btheta_1_imag": 105725.25398122355, + "Btheta_1_real": 141.2554810196911, + "Bz": 5076.74476238461, + "Bz_0_real": 0.4603819957306249, + "Bz_1_imag": 1.0073608129672305, + "Bz_1_real": 5076.632351216658, + "Er": 273570493775.55588, + "Er_0_real": 271974091013.7082, + "Er_1_imag": 39530787242966.72, + "Er_1_real": 42616886403.53069, + "Et": 39016542462571.68, + "Etheta_0_real": 112249482.93975669, + "Etheta_1_imag": 33602799006.874825, + "Etheta_1_real": 39016517454881.91, + "Ez": 511653064863.048, + "Ez_0_real": 496845125310.57947, + "Ez_1_imag": 1245709520854.488, + "Ez_1_real": 24849976994.356266, + "Jr_0_real": 1264417193503.146, + "Jr_1_imag": 2.3356633334993878e+17, + "Jr_1_real": 2272922066062.586, + "Jtheta_0_real": 475304056513.7001, + "Jtheta_1_imag": 1028929701334.8286, + "Jtheta_1_real": 2.1766379427377802e+17, + "Jz_0_real": 1832468408628522.8, + "Jz_1_imag": 556484600934089.9, + "Jz_1_real": 602703622358902.4, + "jr": 1749985661974.403, + "jt": 2.1766379408857264e+17, + "jz": 1954078684852864.2, + "rho": 39314210.931097575, + "rho_0_real": 38889615.839967206, + "rho_1_imag": 21546499.619336277, + "rho_1_real": 2012888.5658883736 }, "electrons": { - "particle_momentum_x": 1.886388623495423e-24, - "particle_momentum_y": 3.989021920324841e-22, - "particle_momentum_z": 1.2499427438675522e-23, + "particle_momentum_x": 1.3203417227476271e-24, + "particle_momentum_y": 4.0070625287284664e-22, + "particle_momentum_z": 1.2493391415020162e-23, "particle_orig_x": 0.026508328457558912, "particle_orig_z": 0.04789125000000001, - "particle_position_x": 0.041602500066712574, - "particle_position_y": 0.047891250456211995, - "particle_theta": 7325.121562757762, + "particle_position_x": 0.04160250006680718, + "particle_position_y": 0.04789125046517409, + "particle_theta": 7325.160426984388, "particle_weight": 813672305.532158 }, - "lev=0": { - "Br_0_real": 0.27473108145012964, - "Br_1_imag": 104.10424416504374, - "Br_1_real": 104965.62622212195, - "Btheta_0_real": 1297.3299824026299, - "Btheta_1_imag": 105725.25637121125, - "Btheta_1_real": 141.25524413452112, - "Br": 104965.6138283532, - "Bt": 1296.2727613183374, - "Bz": 5076.743764997268, - "Bz_0_real": 0.46038147543152824, - "Bz_1_imag": 1.007452397747621, - "Bz_1_real": 5076.631353934757, - "Er_0_real": 271974182110.8858, - "Er_1_imag": 39530787290253.98, - "Er_1_real": 42616765306.284, - "Etheta_0_real": 112249661.08828562, - "Etheta_1_imag": 33602739100.133934, - "Etheta_1_real": 39016517445019.95, - "Er": 273570655229.63535, - "Et": 39016542452492.57, - "Ez": 511653044133.8529, - "Ez_0_real": 496845145101.94775, - "Ez_1_imag": 1245709559726.1033, - "Ez_1_real": 24849961919.57713, - "Jr_0_real": 1264766566288.467, - "Jr_1_imag": 2.335663089152921e+17, - "Jr_1_real": 2273346021690.177, - "Jtheta_0_real": 475301266327.4687, - "Jtheta_1_imag": 1028946515774.2778, - "Jtheta_1_real": 2.176637922654668e+17, - "Jz_0_real": 1832469154489130.8, - "Jz_1_imag": 556484676782123.3, - "Jz_1_real": 602703174081284.9, - "jr": 1750423139263.8418, - "jt": 2.176637920803457e+17, - "jz": 1954078914516329.2, - "rho": 39314213.383921936, - "rho_0_real": 38889619.16177814, - "rho_1_imag": 21546507.958223887, - "rho_1_real": 2012888.0198535046 + "beam": { + "particle_momentum_x": 3.8798910618915136e-20, + "particle_momentum_y": 5.078287631744985e-20, + "particle_momentum_z": 1.3503610556163801e-17, + "particle_position_x": 6.242134237822313e-05, + "particle_position_y": 0.0026764363296840257, + "particle_theta": 151.40797316586125, + "particle_weight": 6241509.074460764 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid.json new file mode 100644 index 00000000000..2843e49ce22 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid.json @@ -0,0 +1,15 @@ +{ + "lev=0": { + "Bx": 14264658.38987597, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 4276056420659746.5, + "Ez": 762168740318568.1, + "jx": 0.0, + "jy": 7.47674123799233e+16, + "jz": 4.817762115932484e+17, + "rho": 1609691680.1267354 + } +} + diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid_boosted.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid_boosted.json new file mode 100644 index 00000000000..85919660834 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_1d_fluid_boosted.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "Bx": 7268241.144300916, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 2146970168314705.0, + "Ez": 370041408451418.1, + "jx": 0.0, + "jy": 3.4818196503065956e+16, + "jz": 2.3196405062669472e+17, + "rho": 775945622.8993922 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json index 6b30064365c..dd224516c5c 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json @@ -1,32 +1,32 @@ { "lev=0": { - "Bx": 497087553.4743837, - "By": 252364344.72213668, - "Bz": 16986109.62608196, - "Ex": 7.304419665394173e+16, - "Ey": 1.4583459400150573e+17, - "Ez": 2.261309965998215e+16, - "jx": 8.273582439836952e+17, - "jy": 2.1044181577417728e+18, - "jz": 2.84208498868846e+19, - "rho": 91637594416.50476 + "Bx": 497756265.44724107, + "By": 252197580.117894, + "Bz": 16988475.5444833, + "Ex": 7.299911104263803e+16, + "Ey": 1.460116488178734e+17, + "Ez": 2.26033100286114e+16, + "jx": 8.27644503757345e+17, + "jy": 2.1062078409959675e+18, + "jz": 2.8491727096438305e+19, + "rho": 91863764144.41415 }, "electrons": { - "particle_momentum_x": 5.922677819206403e-20, - "particle_momentum_y": 2.7778096010261295e-19, - "particle_momentum_z": 4.183846407463124e-19, - "particle_position_x": 0.0025865980636989934, - "particle_position_y": 0.002652673793955211, - "particle_position_z": 0.18411922027879662, - "particle_weight": 1731649095408.5505 + "particle_momentum_x": 5.76165226700654e-20, + "particle_momentum_y": 2.7567389504898156e-19, + "particle_momentum_z": 4.134562048099117e-19, + "particle_position_x": 0.0025269863605945427, + "particle_position_y": 0.0024538321295153346, + "particle_position_z": 0.17818421763751244, + "particle_weight": 1675789447169.5652 }, "beam": { - "particle_momentum_x": 1.1925686969448298e-17, - "particle_momentum_y": 2.570810132551278e-17, - "particle_momentum_z": 5.156190278524037e-14, - "particle_position_x": 0.0005608477913384702, - "particle_position_y": 0.0008431191912437461, - "particle_position_z": 2.9800048756186395, + "particle_momentum_x": 1.1926313055043134e-17, + "particle_momentum_y": 2.7056205218547404e-17, + "particle_momentum_z": 5.1562131494813424e-14, + "particle_position_x": 0.000560821518739447, + "particle_position_y": 0.000800729816549036, + "particle_position_z": 2.9800048650570097, "particle_weight": 62415.090744607616 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json index 793d8c11cb3..30eef0cedb7 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json @@ -1,25 +1,25 @@ { + "lev=0": { + "Bx": 5863879.051452597, + "By": 2411.5124004699082, + "Bz": 116025.40178726945, + "Ex": 6267725953308.365, + "Ey": 1670763222566621.8, + "Ez": 104345977762699.77, + "jx": 555688154318027.4, + "jy": 1595897074125252.8, + "jz": 1045267363178542.9, + "rho": 2205684400.3678846 + }, "electrons": { "particle_initialenergy": 0.0, - "particle_momentum_x": 1.7921229236407606e-20, - "particle_momentum_y": 7.225825184653885e-20, - "particle_momentum_z": 4.231731488281653e-20, - "particle_position_x": 0.7139122621535502, - "particle_position_y": 0.7150340889556129, - "particle_position_z": 1.3175770602802896, "particle_regionofinterest": 1936.0, + "particle_position_x": 0.7139122620952513, + "particle_position_y": 0.7150340902640349, + "particle_position_z": 1.317577060220835, + "particle_momentum_x": 1.7921232385614662e-20, + "particle_momentum_y": 7.22582607277354e-20, + "particle_momentum_z": 4.231730764749506e-20, "particle_weight": 12926557617.187498 - }, - "lev=0": { - "Bx": 5863879.02030842, - "By": 2411.501904737579, - "Bz": 116025.42462998998, - "Ex": 6267728094111.663, - "Ey": 1670763233105822.0, - "Ez": 104345989831222.4, - "jx": 555687912108453.8, - "jy": 1595896363359136.0, - "jz": 1045267552192496.5, - "rho": 2205684400.307701 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json index 0d7afc38099..242cf00f0c3 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json @@ -1,12 +1,12 @@ { "lev=0": { - "Br": 100278934.99628444, - "Bz": 2509040.7710191337, - "Er": 3.258814166960793, - "Et": 3.0045384450930932e+16, - "Ez": 0.24012250267165877, - "jr": 50.13336345238631, - "jt": 4.3064420556391546e+17, + "Br": 100278645.72225758, + "Bz": 2509008.1146699786, + "Er": 3.263343388037509, + "Et": 3.0045297157982412e+16, + "Ez": 0.2422526411295923, + "jr": 49.67802097760541, + "jt": 4.3051397628943917e+17, "jz": 0.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json index 008080d9f5e..8d58f5c9551 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json @@ -1,12 +1,12 @@ { "lev=0": { - "Br": 193619956.9879392, - "Bz": 4691018.480978214, - "Er": 197265445300168.38, - "Et": 5.972016823892645e+16, - "Ez": 1383831581199501.0, - "jr": 18581252675.28125, - "jt": 1.194996054262225e+18, + "Br": 193609031.89343664, + "Bz": 4689634.787639907, + "Er": 197280940689075.1, + "Et": 5.9716806429474664e+16, + "Ez": 1383812184076636.0, + "jr": 18582087468.75, + "jt": 1.193738117372394e+18, "jz": 0.0 } } diff --git a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json index 36b2bc66aed..1678e794683 100644 --- a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json +++ b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json @@ -1,34 +1,33 @@ { - "electrons": { - "particle_momentum_x": 3.856058278845215e-19, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.633111056849423e-18, - "particle_position_x": 0.008155341094488198, - "particle_position_y": 0.03087362289643631, - "particle_weight": 2.6462205036771795e+17 - }, - "hydrogen": { - "particle_momentum_x": 2.237944099185357e-18, - "particle_momentum_z": 1.0744000122946915e-18, - "particle_orig_x": 0.007768349609375002, - "particle_orig_z": 0.035127407226562504, - "particle_position_x": 0.007767975241766832, - "particle_position_y": 0.03512562609400349, - "particle_weight": 2.68584931046923e+17 - }, "lev=0": { "Bx": 0.0, - "By": 11399442.711372176, + "By": 11411047.898351602, "Bz": 0.0, - "Ex": 2033057399920464.5, + "Ex": 2031641076084948.5, "Ey": 0.0, - "Ez": 339244862686685.9, - "jx": 1.6477362358927364e+19, + "Ez": 334777106874158.8, + "jx": 1.6602050255725978e+19, "jy": 0.0, - "jz": 9.633815033523364e+18, - "rho": 68121082972.17873, - "rho_electrons": 17448735668294.348, - "rho_hydrogen": 17441797017887.941 + "jz": 9.545451466990307e+18, + "rho": 67237998613.86053, + "rho_electrons": 17449705826002.385, + "rho_hydrogen": 17441792654743.146 + }, + "hydrogen": { + "particle_momentum_x": 2.2575255298569512e-18, + "particle_momentum_z": 1.0773391029868316e-18, + "particle_orig_x": 0.007763427734375002, + "particle_orig_z": 0.0351975439453125, + "particle_position_x": 0.007763034484095496, + "particle_position_y": 0.035195761508116416, + "particle_weight": 2.6792730619156992e+17 + }, + "electrons": { + "particle_momentum_x": 3.8463518636930147e-19, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.6287398567676136e-18, + "particle_position_x": 0.008139313907538123, + "particle_position_y": 0.0308605164193468, + "particle_weight": 2.647983436428149e+17 } -} - +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/PlasmaMirror.json b/Regression/Checksum/benchmarks_json/PlasmaMirror.json index 9a686e78d56..861f42f2d74 100644 --- a/Regression/Checksum/benchmarks_json/PlasmaMirror.json +++ b/Regression/Checksum/benchmarks_json/PlasmaMirror.json @@ -1,29 +1,29 @@ { - "electrons": { - "particle_momentum_x": 2.000457748960032e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.446670640741138e-17, - "particle_position_x": 0.37688793289418476, - "particle_position_y": 0.1714047288635171, - "particle_weight": 2.904173860599889e+18 - }, - "ions": { - "particle_momentum_x": 2.805690782833416e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 9.605283484426823e-17, - "particle_position_x": 0.3828171295926821, - "particle_position_y": 0.17237242725257168, - "particle_weight": 2.9212132379167017e+18 - }, "lev=0": { "Bx": 0.0, - "By": 75294856.83091721, + "By": 75293687.71417463, "Bz": 0.0, - "Ex": 2.081697252453305e+16, + "Ex": 2.0817302986029584e+16, "Ey": 0.0, - "Ez": 2.350511387513211e+16, - "jx": 5.496779405360711e+19, + "Ez": 2.350518662171605e+16, + "jx": 5.496864993586538e+19, "jy": 0.0, - "jz": 7.159348923050661e+19 + "jz": 7.159302467888838e+19 + }, + "ions": { + "particle_momentum_x": 2.8056907319288714e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 9.605283524169195e-17, + "particle_position_x": 0.382817129592504, + "particle_position_y": 0.1723724272525192, + "particle_weight": 2.9212132379167017e+18 + }, + "electrons": { + "particle_momentum_x": 2.000448978774197e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.446668081246458e-17, + "particle_position_x": 0.376887964713013, + "particle_position_y": 0.17140472970668535, + "particle_weight": 2.9041738605998894e+18 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index 979cffc0aa7..7f359f76e94 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -1,117 +1,117 @@ { - "alpha1": { - "particle_momentum_x": 4.714227948839551e-15, - "particle_momentum_y": 4.676078701959904e-15, - "particle_momentum_z": 4.676142266462326e-15, - "particle_position_x": 463834.9497057527, - "particle_position_y": 977592.6452214213, - "particle_weight": 2.940000093807323e-28 + "lev=0": { + "rho": 0.0 }, "alpha2": { - "particle_momentum_x": 4.081538674143691e-15, - "particle_momentum_y": 4.122349693618236e-15, - "particle_momentum_z": 4.201699189448955e-15, + "particle_momentum_x": 4.0815251136381854e-15, + "particle_momentum_y": 4.122335955939925e-15, + "particle_momentum_z": 4.201726051973563e-15, "particle_position_x": 410664.0477768091, "particle_position_y": 868193.1483606595, - "particle_weight": 2.959852107088932e+18 + "particle_weight": 2.947401325355292e+18 }, "alpha3": { - "particle_momentum_x": 4.934249084589244e-16, - "particle_momentum_y": 4.791039564460439e-16, - "particle_momentum_z": 4.698462130524534e-16, - "particle_position_x": 52309.63968324501, - "particle_position_y": 104322.0547863817, - "particle_weight": 1.491736282762796e+27 - }, - "alpha4": { - "particle_momentum_x": 2.342240913445794e-14, - "particle_momentum_y": 2.34177318709557e-14, - "particle_momentum_z": 2.353174771675334e-14, - "particle_position_x": 2457367.458278153, - "particle_position_y": 4915112.044373058, - "particle_weight": 384.0000000000002 - }, - "alpha5": { - "particle_momentum_x": 2.330088308142745e-14, - "particle_momentum_y": 2.344976927616691e-14, - "particle_momentum_z": 2.361827647381584e-14, - "particle_position_x": 2457556.857163842, - "particle_position_y": 4914659.635379327, - "particle_weight": 3.839999999999998e-19 + "particle_momentum_x": 4.925995964082186e-16, + "particle_momentum_y": 4.780894181553476e-16, + "particle_momentum_z": 4.688680302272935e-16, + "particle_position_x": 52240.870920562535, + "particle_position_y": 103873.30862578771, + "particle_weight": 1.4784234459335885e+27 }, - "boron1": { + "proton1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 81921136.14476715, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983792, + "particle_position_y": 81919772.69310114, "particle_weight": 128.00000000000261 }, - "boron2": { + "alpha4": { + "particle_momentum_x": 2.3422553081132076e-14, + "particle_momentum_y": 2.3416860254545166e-14, + "particle_momentum_z": 2.3532380279126124e-14, + "particle_position_x": 2457367.4582781536, + "particle_position_y": 4915112.044373058, + "particle_weight": 384.0000000000002 + }, + "proton4": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.015821768, - "particle_position_y": 819270.9858143466, - "particle_weight": 1.2799999999016446e+28 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 819198.7077771134, + "particle_weight": 1.2800000000000004e+37 }, "boron3": { "particle_momentum_x": 9.277692671587846e-15, "particle_momentum_y": 9.268409636965691e-15, "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.166422465, - "particle_position_y": 8192499.706038672, - "particle_weight": 6.399502754572407e+30 + "particle_position_x": 4096178.1664224654, + "particle_position_y": 8192499.7060386725, + "particle_weight": 6.399507192184686e+30 }, - "boron5": { + "boron2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_position_x": 409547.3312927569, - "particle_position_y": 819118.5558814355, - "particle_weight": 127.99999999999999 - }, - "lev=0": { - "rho": 0.0 + "particle_position_x": 409798.015821768, + "particle_position_y": 819270.9858143466, + "particle_weight": 1.2799999999017534e+28 }, - "proton1": { + "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40960140.72983792, - "particle_position_y": 81919772.69310114, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 81921136.14476715, "particle_weight": 128.00000000000261 }, - "proton2": { + "proton3": { + "particle_momentum_x": 1.684673285246867e-15, + "particle_momentum_y": 1.6827557106531144e-15, + "particle_momentum_z": 1.6802642612723895e-15, + "particle_position_x": 2457259.537951346, + "particle_position_y": 4914248.771185745, + "particle_weight": 1.2795071921846893e+30 + }, + "proton5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.371679089706304e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512789516445e+28 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.2877618571, + "particle_position_y": 819101.3225783394, + "particle_weight": 1.2800000000000004e+37 }, - "proton3": { - "particle_momentum_x": 1.684214433067055e-15, - "particle_momentum_y": 1.682290595962152e-15, - "particle_momentum_z": 1.679806121802876e-15, - "particle_position_x": 2457258.407562205, - "particle_position_y": 4914234.891662369, - "particle_weight": 1.279502754572413e+30 + "alpha1": { + "particle_momentum_x": 4.714228774936194e-15, + "particle_momentum_y": 4.676079523757908e-15, + "particle_momentum_z": 4.676143086393951e-15, + "particle_position_x": 463834.9497057527, + "particle_position_y": 977592.6452214213, + "particle_weight": 2.9280275175431764e-28 }, - "proton4": { + "alpha5": { + "particle_momentum_x": 2.3300446729308862e-14, + "particle_momentum_y": 2.3449941485701153e-14, + "particle_momentum_z": 2.3618372938672865e-14, + "particle_position_x": 2457556.8571638414, + "particle_position_y": 4914659.635379325, + "particle_weight": 3.839999999999998e-19 + }, + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 819198.7077771134, - "particle_weight": 1.2800000000000004e+37 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512789517532e+28 }, - "proton5": { + "boron5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409638.2877618571, - "particle_position_y": 819101.3225783394, - "particle_weight": 1.2800000000000004e+37 + "particle_momentum_z": 0.0, + "particle_position_x": 409547.3312927569, + "particle_position_y": 819118.5558814355, + "particle_weight": 127.99999999999999 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index b510fb10ff9..c3517c32b5b 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -1,131 +1,131 @@ { - "alpha1": { - "particle_momentum_x": 4.659631949650719e-15, - "particle_momentum_y": 4.617308377231923e-15, - "particle_momentum_z": 4.617389330519975e-15, - "particle_position_x": 453219.3436004627, - "particle_position_y": 457393.42114332493, - "particle_position_z": 970481.0580153911, - "particle_weight": 1.9043802455895325e-27 - }, - "alpha2": { - "particle_momentum_x": 4.071678402460901e-15, - "particle_momentum_y": 4.1196410276880726e-15, - "particle_momentum_z": 4.181784710058622e-15, - "particle_position_x": 405848.1200755021, - "particle_position_y": 408011.53191288817, - "particle_position_z": 866330.5923068221, - "particle_weight": 1.9261737121655108e+19 - }, - "alpha3": { - "particle_momentum_x": 4.853582081563116e-16, - "particle_momentum_y": 4.797297548379446e-16, - "particle_momentum_z": 4.759501504671248e-16, - "particle_position_x": 51342.57223394147, - "particle_position_y": 50913.6612890173, - "particle_position_z": 101306.34059008301, - "particle_weight": 1.0360147595748265e+28 - }, - "alpha4": { - "particle_momentum_x": 2.3373908106523292e-14, - "particle_momentum_y": 2.3439419725501755e-14, - "particle_momentum_z": 2.3559411845892612e-14, - "particle_position_x": 2457367.4582781526, - "particle_position_y": 2457512.044373058, - "particle_position_z": 4914475.776513073, - "particle_weight": 3072.000000000002 - }, - "alpha5": { - "particle_momentum_x": 2.333624251961068e-14, - "particle_momentum_y": 2.3455939695798916e-14, - "particle_momentum_z": 2.3574526216382372e-14, - "particle_position_x": 2457556.857163842, - "particle_position_y": 2457059.6353793237, - "particle_position_z": 4915847.043341331, - "particle_weight": 3.0719999999999984e-18 - }, - "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 - }, - "boron2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999357942e+29 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.119654661746806e+31 - }, - "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.33129275695, - "particle_position_y": 409518.5558814353, - "particle_position_z": 819306.5006950963, - "particle_weight": 1023.9999999999999 - }, - "lev=0": { - "rho": 0.0 - }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 - }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3738870507189214e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240713489e+29 - }, - "proton3": { - "particle_momentum_x": 1.6843510515198723e-15, - "particle_momentum_y": 1.6823818968683368e-15, - "particle_momentum_z": 1.6799961456539653e-15, - "particle_position_x": 2457338.899376694, - "particle_position_y": 2457069.647393952, - "particle_position_z": 4914642.288898885, - "particle_weight": 1.0236546617468092e+31 - }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 409598.7077771135, - "particle_position_z": 818958.0399127571, - "particle_weight": 1.0240000000000003e+38 - }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409638.28776185703, - "particle_position_y": 409501.32257833943, - "particle_position_z": 819309.1804186807, - "particle_weight": 1.0240000000000003e+38 - } + "lev=0": { + "rho": 0.0 + }, + "proton3": { + "particle_momentum_x": 1.684809640261926e-15, + "particle_momentum_y": 1.6828399494797829e-15, + "particle_momentum_z": 1.6804535487104095e-15, + "particle_position_x": 2457338.899376694, + "particle_position_y": 2457069.647393952, + "particle_position_z": 4914642.288898885, + "particle_weight": 1.0236580897818594e+31 + }, + "alpha5": { + "particle_momentum_x": 2.3336421176154494e-14, + "particle_momentum_y": 2.345583787205298e-14, + "particle_momentum_z": 2.3574859187827772e-14, + "particle_position_x": 2457556.857163842, + "particle_position_y": 2457059.6353793233, + "particle_position_z": 4915847.043341331, + "particle_weight": 3.0719999999999984e-18 + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.119658089781855e+31 + }, + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 + }, + "alpha4": { + "particle_momentum_x": 2.3374367002622734e-14, + "particle_momentum_y": 2.3439200330576348e-14, + "particle_momentum_z": 2.3559200621107925e-14, + "particle_position_x": 2457367.4582781536, + "particle_position_y": 2457512.044373057, + "particle_position_z": 4914475.776513074, + "particle_weight": 3072.000000000002 + }, + "alpha3": { + "particle_momentum_x": 4.857656981301579e-16, + "particle_momentum_y": 4.811177053359803e-16, + "particle_momentum_z": 4.766749863500389e-16, + "particle_position_x": 51282.9136300544, + "particle_position_y": 51045.39533309579, + "particle_position_z": 101578.82008857065, + "particle_weight": 1.0257306544231767e+28 + }, + "alpha2": { + "particle_momentum_x": 4.071664828713061e-15, + "particle_momentum_y": 4.119627260272026e-15, + "particle_momentum_z": 4.181812341443065e-15, + "particle_position_x": 405848.1200755021, + "particle_position_y": 408011.53191288817, + "particle_position_z": 866330.5923068221, + "particle_weight": 1.9180757241165136e+19 + }, + "proton2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.3745333755307162e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503005, + "particle_weight": 1.0227810240716198e+29 + }, + "proton4": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 409598.7077771135, + "particle_position_z": 818958.0399127571, + "particle_weight": 1.0240000000000003e+38 + }, + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 + }, + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.28776185703, + "particle_position_y": 409501.32257833943, + "particle_position_z": 819309.1804186807, + "particle_weight": 1.0240000000000003e+38 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.33129275695, + "particle_position_y": 409518.5558814353, + "particle_position_z": 819306.5006950963, + "particle_weight": 1023.9999999999999 + }, + "alpha1": { + "particle_momentum_x": 4.659632773478301e-15, + "particle_momentum_y": 4.617309192789389e-15, + "particle_momentum_z": 4.6173901462667306e-15, + "particle_position_x": 453219.3436004627, + "particle_position_y": 457393.42114332493, + "particle_position_z": 970481.0580153911, + "particle_weight": 1.8966442168729786e-27 + }, + "boron2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409798.0158217681, + "particle_position_y": 409670.9858143465, + "particle_position_z": 819255.8152412223, + "particle_weight": 1.023999999936064e+29 + } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json index 0b0355b7cb6..08969db023e 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json @@ -1,32 +1,32 @@ { + "lev=0": { + "Bx": 5866866.518334419, + "By": 11175.108378013305, + "Bz": 116026.79919117832, + "Ex": 8178060925462.576, + "Ey": 1671615340558021.2, + "Ez": 106548534464553.72, + "jx": 555903248104584.56, + "jy": 1595980426561245.2, + "jz": 1366292265497407.5, + "rho": 2206755250.226452 + }, "beam": { - "particle_momentum_x": 4.707586336874016e-20, - "particle_momentum_y": 4.4850722108112576e-20, - "particle_momentum_z": 1.36054441043288e-17, - "particle_position_x": 4.058764306495361e-05, - "particle_position_y": 3.7888695722549883e-05, - "particle_position_z": 0.00019656701118308398, + "particle_momentum_x": 4.7075864374777216e-20, + "particle_momentum_y": 4.4875025179072e-20, + "particle_momentum_z": 1.3605444179112827e-17, + "particle_position_x": 4.058764328440382e-05, + "particle_position_y": 3.788866110052658e-05, + "particle_position_z": 0.00019656700944015084, "particle_weight": 6241509.074460764 }, "electrons": { - "particle_momentum_x": 1.7921232203945004e-20, - "particle_momentum_y": 7.225819894813053e-20, - "particle_momentum_z": 4.231725460173154e-20, - "particle_position_x": 0.7139122621161993, - "particle_position_y": 0.7150340887578637, - "particle_position_z": 1.3175770600690966, + "particle_momentum_x": 1.7921232210868553e-20, + "particle_momentum_y": 7.225819896136567e-20, + "particle_momentum_z": 4.2317254599358777e-20, + "particle_position_x": 0.713912262116188, + "particle_position_y": 0.7150340887578024, + "particle_position_z": 1.31757706006908, "particle_weight": 12926557617.187498 - }, - "lev=0": { - "Bx": 5866866.85492377, - "By": 11177.920546471447, - "Bz": 116026.93444649166, - "Ex": 8178548880638.266, - "Ey": 1671614207789070.8, - "Ez": 106548168484665.61, - "jx": 555903247963958.4, - "jy": 1595974150308405.2, - "jz": 1366292284444382.5, - "rho": 2206755250.226321 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json index 1c75b69dc9b..720148888e2 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json @@ -1,62 +1,62 @@ { - "beam": { - "particle_momentum_x": 3.880109055649298e-20, - "particle_momentum_y": 5.0781930103830196e-20, - "particle_momentum_z": 1.3503608494680855e-17, - "particle_position_x": 6.242131236443886e-05, - "particle_position_y": 0.0026764363296979446, - "particle_theta": 151.4079870868123, - "particle_weight": 6241509.074460764 + "lev=0": { + "Br": 142258.01161290894, + "Br_0_real": 0.36356680791855334, + "Br_1_imag": 115.41915541351706, + "Br_1_real": 142258.0180774376, + "Bt": 1301.5691003159159, + "Btheta_0_real": 1299.8813681794945, + "Btheta_1_imag": 143318.04184449295, + "Btheta_1_real": 155.37799428725944, + "Bz": 5993.641173367065, + "Bz_0_real": 0.4737418612272665, + "Bz_1_imag": 1.1408917538312353, + "Bz_1_real": 5993.529100973486, + "Er": 278531478379.8872, + "Er_0_real": 276179480674.09875, + "Er_1_imag": 47911368149173.86, + "Er_1_real": 46900731766.219345, + "Et": 47328876349791.85, + "Etheta_0_real": 135868018.2930996, + "Etheta_1_imag": 36802947213.299324, + "Etheta_1_real": 47328835574237.43, + "Ez": 514006688162.3537, + "Ez_0_real": 499008057521.7594, + "Ez_1_imag": 1565161964565.9724, + "Ez_1_real": 28898941539.846146, + "Jr_0_real": 1458783813352.7048, + "Jr_1_imag": 2.335663323286376e+17, + "Jr_1_real": 2725720908168.6816, + "Jtheta_0_real": 499386783042.5499, + "Jtheta_1_imag": 1179203035377.2214, + "Jtheta_1_real": 2.1766371791828432e+17, + "Jz_0_real": 1832469778455967.8, + "Jz_1_imag": 621923937700173.0, + "Jz_1_real": 660910016274555.4, + "jr": 2108641839684.7007, + "jt": 2.1766371088502803e+17, + "jz": 1954236547059862.2, + "rho": 39480728.08547403, + "rho_0_real": 39055923.162689775, + "rho_1_imag": 21660762.696674515, + "rho_1_real": 2131498.5888879234 }, "electrons": { - "particle_momentum_x": 1.787201778017949e-24, - "particle_momentum_y": 3.9234822345143987e-22, - "particle_momentum_z": 1.0100062552925791e-23, - "particle_position_x": 0.041602500069929174, - "particle_position_y": 0.047891250477036906, - "particle_theta": 7325.1193688944695, + "particle_momentum_x": 1.237777638089911e-24, + "particle_momentum_y": 3.9465600403988994e-22, + "particle_momentum_z": 1.0097949593654349e-23, + "particle_position_x": 0.04160250006989259, + "particle_position_y": 0.047891250488567585, + "particle_theta": 7325.162679147719, "particle_weight": 813672305.532158 }, - "lev=0": { - "Br_0_real": 0.36356649135193925, - "Br_1_imag": 115.41886748920795, - "Br_1_real": 142258.01965536995, - "Btheta_0_real": 1299.8816721733124, - "Btheta_1_imag": 143318.04456658955, - "Btheta_1_real": 155.37774833024366, - "Br": 142258.01319076555, - "Bt": 1301.5695263567557, - "Bz": 5993.640969075834, - "Bz_0_real": 0.4737412745527051, - "Bz_1_imag": 1.1409956493384725, - "Bz_1_real": 5993.528898267216, - "Er_0_real": 276179575540.0639, - "Er_1_imag": 47911367858371.875, - "Er_1_real": 46900598536.03668, - "Etheta_0_real": 135868121.7945822, - "Etheta_1_imag": 36802874200.20133, - "Etheta_1_real": 47328835452079.97, - "Er": 278531658643.55005, - "Et": 47328876227481.125, - "Ez": 514006664374.9789, - "Ez_0_real": 499008075334.7451, - "Ez_1_imag": 1565161989236.6174, - "Ez_1_real": 28898922272.75169, - "Jr_0_real": 1459118139844.9216, - "Jr_1_imag": 2.3356630589200717e+17, - "Jr_1_real": 2726204346551.3022, - "Jtheta_0_real": 499384029970.2145, - "Jtheta_1_imag": 1179215927404.282, - "Jtheta_1_real": 2.17663715880068e+17, - "Jz_0_real": 1832470462501306.8, - "Jz_1_imag": 621924149855721.0, - "Jz_1_real": 660909646259030.1, - "jr": 2109207014985.481, - "jt": 2.1766370884715638e+17, - "jz": 1954236712029783.0, - "rho": 39480730.556067616, - "rho_0_real": 39055926.50167212, - "rho_1_imag": 21660770.34248945, - "rho_1_real": 2131498.060778751 + "beam": { + "particle_momentum_x": 3.8798818966213747e-20, + "particle_momentum_y": 5.078274625117952e-20, + "particle_momentum_z": 1.3503604116563051e-17, + "particle_position_x": 6.242134249270767e-05, + "particle_position_y": 0.0026764363296859625, + "particle_theta": 151.40797595010355, + "particle_weight": 6241509.074460764 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json new file mode 100644 index 00000000000..4d30f741bfb --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -0,0 +1,27 @@ +{ + "lev=0": { + "rho_electrons": 0.004436602398896733, + "rho_he_ions": 0.0052003262664415285 + }, + "he_ions": { + "particle_momentum_x": 2.7735512966774165e-19, + "particle_momentum_y": 2.7574111491186894e-19, + "particle_momentum_z": 3.620520352986572e-19, + "particle_position_x": 2201.236370518716, + "particle_weight": 17190734375000.002 + }, + "electrons": { + "particle_momentum_x": 3.50212700099208e-20, + "particle_momentum_y": 3.5368926859820716e-20, + "particle_momentum_z": 1.2588108956625115e-19, + "particle_position_x": 2139.6498177543617, + "particle_weight": 14582968750000.002 + }, + "neutrals": { + "particle_momentum_x": 1.405588503355727e-19, + "particle_momentum_y": 1.408077689882847e-19, + "particle_momentum_z": 1.4024616940779626e-19, + "particle_position_x": 1121.2330379095083, + "particle_weight": 6.4588e+19 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/Python_ionization.json index 35aa2c07a60..31f426aa362 100644 --- a/Regression/Checksum/benchmarks_json/Python_ionization.json +++ b/Regression/Checksum/benchmarks_json/Python_ionization.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bx": 0.0, + "By": 26296568.434868, + "Bz": 0.0, + "Ex": 7878103122971888.0, + "Ey": 0.0, + "Ez": 3027.995293554466, + "jx": 1.2111358330750162e+16, + "jy": 0.0, + "jz": 1.3483401471475687e-07 + }, "electrons": { - "particle_momentum_x": 4.410992240916769e-18, + "particle_momentum_x": 4.4206237143449475e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.637306716013194e-18, - "particle_position_x": 0.1095060155214851, - "particle_position_y": 0.6411875726789248, - "particle_weight": 3.44453125e-10 + "particle_momentum_z": 2.6361297302081026e-18, + "particle_position_x": 0.11009154442846772, + "particle_position_y": 0.6414658436421568, + "particle_weight": 3.4450781249999996e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.761324019342538e-18, + "particle_momentum_x": 1.76132401934254e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.644887006212893e-23, - "particle_position_x": 0.03199998810579664, - "particle_position_y": 0.12800000462171646, + "particle_momentum_z": 3.644887053263054e-23, + "particle_position_x": 0.03200001189420337, + "particle_position_y": 0.1280000046901387, "particle_weight": 9.999999999999999e-11 - }, - "lev=0": { - "Bx": 0.0, - "By": 26296568.43487075, - "Bz": 0.0, - "Ex": 7878103122973058.0, - "Ey": 0.0, - "Ez": 3027.995293554496, - "jx": 1.2111358333819302e+16, - "jy": 0.0, - "jz": 1.355517269126987e-07 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json new file mode 100644 index 00000000000..870e4a82180 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json @@ -0,0 +1,12 @@ +{ + "lev=0": {}, + "ions": { + "particle_momentum_x": 5.043873837579827e-17, + "particle_momentum_y": 5.04445113937769e-17, + "particle_momentum_z": 5.051930416996439e-17, + "particle_position_x": 143164.4268324243, + "particle_position_y": 143166.5184643828, + "particle_theta": 2573261.7450408577, + "particle_weight": 8.12868064536689e+18 + } +} diff --git a/Regression/Checksum/benchmarks_json/Python_prev_positions.json b/Regression/Checksum/benchmarks_json/Python_prev_positions.json index e6e39d2272b..55c886775d5 100644 --- a/Regression/Checksum/benchmarks_json/Python_prev_positions.json +++ b/Regression/Checksum/benchmarks_json/Python_prev_positions.json @@ -6,9 +6,9 @@ "Ex": 3710588.849989976, "Ey": 0.0, "Ez": 646727.8074440088, - "jx": 4745.078379617619, - "jy": 368.4331779923921, - "jz": 632.1508106460103 + "jx": 15259.034603501308, + "jy": 650.139263398662, + "jz": 943.0244062246846 }, "electrons": { "particle_momentum_x": 8.78764082600202e-23, diff --git a/Regression/Checksum/benchmarks_json/RefinedInjection.json b/Regression/Checksum/benchmarks_json/RefinedInjection.json index 8be545f3887..43cc8e3072c 100644 --- a/Regression/Checksum/benchmarks_json/RefinedInjection.json +++ b/Regression/Checksum/benchmarks_json/RefinedInjection.json @@ -1,11 +1,27 @@ { - "beam": { - "particle_momentum_x": 4.5603842543853726e-20, - "particle_momentum_y": 4.191542263916011e-20, - "particle_momentum_z": 1.363947302075425e-17, - "particle_position_x": 4.672031050423284e-05, - "particle_position_y": 0.00024387147640533396, - "particle_weight": 12483018148921.525 + "lev=0": { + "Bx": 26338425.87401078, + "By": 160257.72293376247, + "Bz": 694883.1248007242, + "Ex": 56513906294889.77, + "Ey": 7806060318610865.0, + "Ez": 65816740142650.89, + "jx": 6873283794395257.0, + "jy": 4286847636945161.5, + "jz": 7506503405703726.0, + "rho": 278060758.9118884 + }, + "lev=1": { + "Bx": 41971351.93872842, + "By": 347834.73215718823, + "Bz": 1093501.9056767717, + "Ex": 136247073074384.92, + "Ey": 1.2435868091440666e+16, + "Ez": 92449081296414.34, + "jx": 1.6499002361434856e+16, + "jy": 1568934856430368.5, + "jz": 2.2190671920159212e+16, + "rho": 298322648.1883917 }, "electrons": { "particle_momentum_x": 4.599605609558194e-19, @@ -15,28 +31,12 @@ "particle_position_y": 0.3695766840020302, "particle_weight": 216500976562500.75 }, - "lev=0": { - "Bx": 26338374.3731115, - "By": 160257.72168459874, - "Bz": 694868.2285976049, - "Ex": 56513906233209.63, - "Ey": 7806074074131210.0, - "Ez": 65816739650512.98, - "jx": 6873283793354275.0, - "jy": 4287548168426209.0, - "jz": 7506499988368183.0, - "rho": 278060758.9118884 - }, - "lev=1": { - "Bx": 41971136.54330983, - "By": 347834.7278931723, - "Bz": 1093489.2683699469, - "Ex": 136247072743280.97, - "Ey": 1.2435931966910504e+16, - "Ez": 92449079327862.72, - "jx": 1.6499002357270928e+16, - "jy": 1571736982354559.0, - "jz": 2.219065825081704e+16, - "rho": 298322648.1883917 + "beam": { + "particle_momentum_x": 4.560382242970367e-20, + "particle_momentum_y": 4.2263074230751857e-20, + "particle_momentum_z": 1.3639472455787962e-17, + "particle_position_x": 4.672031008809431e-05, + "particle_position_y": 0.0002438714470005561, + "particle_weight": 12483018148921.525 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json new file mode 100644 index 00000000000..2c9859b037d --- /dev/null +++ b/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 3559.0541122456157, + "By": 1685.942868827529, + "Bz": 0.0, + "Ex": 796541204346.5195, + "Ey": 961740397927.6577, + "Ez": 3528140764527.8877, + "divE": 2.0829159085076083e+21, + "jx": 7.683674095607745e+17, + "jy": 1.4132459141738875e+18, + "jz": 1.350650739310074e+18, + "rho": 18442528652.19583 + }, + "protons": { + "particle_momentum_x": 5.231109104020749e-19, + "particle_momentum_y": 5.367985047933474e-19, + "particle_momentum_z": 5.253213505843665e-19, + "particle_position_x": 0.00010628272743703473, + "particle_weight": 5.314093261582036e+22 + }, + "electrons": { + "particle_momentum_x": 1.196357181461066e-20, + "particle_momentum_y": 1.2271903040162696e-20, + "particle_momentum_z": 1.2277743615209627e-20, + "particle_position_x": 0.0001064956905491333, + "particle_weight": 5.314093261582036e+22 + } +} diff --git a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json index d62cd008670..c8571869cd5 100644 --- a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 6.875024052604206e-19, - "particle_momentum_y": 4.375444487966637e-19, - "particle_momentum_z": 6.432593176909762e-18, - "particle_position_x": 0.0012933598223665212, - "particle_position_y": 0.35872101659459354, - "particle_weight": 3120754537230.3823 - }, - "electrons": { - "particle_momentum_x": 7.058217306102507e-19, - "particle_momentum_y": 2.2042314720139012e-18, - "particle_momentum_z": 2.5305214299582585e-16, - "particle_position_x": 1.5006580327952221, - "particle_position_y": 16.454388306646475, - "particle_weight": 1.234867020725368e+18 + "lev=0": { + "Bx": 1118808.3734374465, + "By": 3248957.1122179274, + "Bz": 280612.78289064515, + "Ex": 975532732112639.6, + "Ey": 402861836732114.4, + "Ez": 159049610399317.66, + "jx": 2.9997053130250828e+16, + "jy": 8.866654890775573e+16, + "jz": 3.163974708948631e+17, + "rho": 1059977729.0184418 }, "ions": { - "particle_momentum_x": 1.6150569264030943e-18, - "particle_momentum_y": 2.2334239793537862e-18, - "particle_momentum_z": 4.279249530683602e-13, - "particle_position_x": 1.4883816864868449, - "particle_position_y": 16.452386504130835, + "particle_momentum_x": 1.6150569180478943e-18, + "particle_momentum_y": 2.2334266828401142e-18, + "particle_momentum_z": 4.2792495306800117e-13, + "particle_position_x": 1.4883816864865955, + "particle_position_y": 16.45238650413084, "particle_weight": 1.234867369440658e+18 }, - "lev=0": { - "Bx": 1118823.5061574595, - "By": 3248957.084212374, - "Bz": 280624.78102759377, - "Ex": 975532719061463.4, - "Ey": 402851207946673.1, - "Ez": 159049601047523.06, - "jx": 2.9997052529118484e+16, - "jy": 8.866616130905646e+16, - "jz": 3.163974567840701e+17, - "rho": 1059977703.1166165 + "electrons": { + "particle_momentum_x": 7.05821754019506e-19, + "particle_momentum_y": 2.2042393263043917e-18, + "particle_momentum_z": 2.5305214316289944e-16, + "particle_position_x": 1.5006580331362074, + "particle_position_y": 16.45438830674347, + "particle_weight": 1.234867020725368e+18 + }, + "beam": { + "particle_momentum_x": 6.874634694077579e-19, + "particle_momentum_y": 4.374677735660533e-19, + "particle_momentum_z": 6.432600800266472e-18, + "particle_position_x": 0.0012933700124436584, + "particle_position_y": 0.358720803656086, + "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/divb_cleaning_3d.json b/Regression/Checksum/benchmarks_json/divb_cleaning_3d.json index cbb43a81004..17d56dcc906 100644 --- a/Regression/Checksum/benchmarks_json/divb_cleaning_3d.json +++ b/Regression/Checksum/benchmarks_json/divb_cleaning_3d.json @@ -1,13 +1,13 @@ { "lev=0": { - "Bx": 30128649.582633037, - "By": 30128649.58263304, - "Bz": 30128649.582633033, - "Ex": 137776481658696.53, - "Ey": 137776481658695.72, - "Ez": 137776481658736.47, - "G": 7382627392167406.0, - "divB": 6914882882637.787, - "divE": 60880563.38401385 + "Bx": 30110529.73244452, + "By": 30110529.73244452, + "Bz": 30110529.732444517, + "Ex": 137103615680453.66, + "Ey": 137103615680454.5, + "Ez": 137103615680494.48, + "G": 7413121805692223.0, + "divB": 6944252335584.075, + "divE": 60882624.59445304 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json index cfd4bc83a85..03cf06b3158 100644 --- a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 7.005046981277183e-19, - "particle_momentum_y": 4.375209352729912e-19, - "particle_momentum_z": 6.175459223479959e-18, - "particle_position_x": 0.001602571749982411, - "particle_position_y": 0.35897924403061005, - "particle_weight": 3120754537230.3823 + "lev=0": { + "Bx": 1086729.9983020213, + "By": 2886537.292820136, + "Bz": 264259.5410465989, + "Ex": 867382749986595.4, + "Ey": 392666737316432.8, + "Ez": 146897959667920.22, + "jx": 2.7028854305928356e+16, + "jy": 8.615938520657634e+16, + "jz": 2.7328721771319978e+17, + "rho": 915931445.5917195 }, "electrons": { - "particle_momentum_x": 6.240989852249656e-19, - "particle_momentum_y": 1.5790301798509742e-18, - "particle_momentum_z": 2.5064352637575283e-16, - "particle_position_x": 1.501413662801212, - "particle_position_y": 16.523781706919216, + "particle_momentum_x": 6.2409905560361695e-19, + "particle_momentum_y": 1.5790611507526441e-18, + "particle_momentum_z": 2.506435264953701e-16, + "particle_position_x": 1.5014136629221837, + "particle_position_y": 16.52378170697116, "particle_weight": 1.2372401466086835e+18 }, "ions": { - "particle_momentum_x": 1.4394968631660078e-18, - "particle_momentum_y": 1.5967458174525868e-18, - "particle_momentum_z": 4.2873406586841774e-13, - "particle_position_x": 1.4911814217840753, - "particle_position_y": 16.52196497877435, + "particle_momentum_x": 1.4394967936107095e-18, + "particle_momentum_y": 1.5967629122067375e-18, + "particle_momentum_z": 4.287340658682592e-13, + "particle_position_x": 1.4911814217840271, + "particle_position_y": 16.521964978774346, "particle_weight": 1.2372405194129536e+18 }, - "lev=0": { - "Bx": 1086731.2285090145, - "By": 2886537.258822082, - "Bz": 264280.80501631316, - "Ex": 867382744066315.0, - "Ey": 392669698675093.25, - "Ez": 146897949570681.72, - "jx": 2.7028852724044516e+16, - "jy": 8.615686606708202e+16, - "jz": 2.7328720244417827e+17, - "rho": 915931431.8889401 + "beam": { + "particle_momentum_x": 7.0050974522936195e-19, + "particle_momentum_y": 4.374915691594075e-19, + "particle_momentum_z": 6.1754662837862646e-18, + "particle_position_x": 0.0016025830388130494, + "particle_position_y": 0.35897909980539305, + "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json b/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json index eddc22348f0..c412bebafb0 100644 --- a/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json +++ b/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bt": 0.0001546688348134161, + "Er": 46301.81902926975, + "Et": 198512.5412869523, + "Ez": 4269.5944463710985, + "divE": 56474.76102581718, + "jr": 5.550914333033371, + "jz": 149.7445144478365, + "rho": 5.000381408055163e-07 + }, "electrons": { - "particle_momentum_x": 7.188314912300234e-22, - "particle_momentum_y": 1.0394436737288987e-23, - "particle_momentum_z": 8.903837852634527e-17, - "particle_position_x": 633733.136187464, - "particle_position_y": 7862152.0078902915, - "particle_theta": 51471.82060104632, + "particle_momentum_x": 7.188316348440827e-22, + "particle_momentum_y": 1.0394552237458006e-23, + "particle_momentum_z": 8.903837851755449e-17, + "particle_position_x": 633733.136190813, + "particle_position_y": 7862152.007883845, + "particle_theta": 51471.82060123148, "particle_weight": 1.0261080645329302e+20 }, "ions": { - "particle_momentum_x": 1.3139399186198985e-18, - "particle_momentum_y": 1.0393368162404926e-23, - "particle_momentum_z": 1.6348806630609593e-13, - "particle_position_x": 633733.1365361367, - "particle_position_y": 7862152.007286081, - "particle_theta": 51471.609916929985, + "particle_momentum_x": 1.3139399189530324e-18, + "particle_momentum_y": 1.0393490021212813e-23, + "particle_momentum_z": 1.6348806630610471e-13, + "particle_position_x": 633733.1365361346, + "particle_position_y": 7862152.007286085, + "particle_theta": 51471.609916929934, "particle_weight": 1.0261080645329302e+20 - }, - "lev=0": { - "Bt": 0.00015467138145729178, - "Er": 46302.55104771345, - "Et": 198512.61435989104, - "Ez": 4269.500411629442, - "divE": 56473.85900924703, - "jr": 5.550691097640847, - "jz": 149.74203244538995, - "rho": 5.000301541815611e-07 } } diff --git a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json index 80a2dc87964..f8186748f2c 100644 --- a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json +++ b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json @@ -6,9 +6,9 @@ "Ex": 9.882421146615367e-06, "Ey": 1.0440261046714249e-05, "Ez": 1.003739697324731e-05, - "jx": 2.9148662809570633e-10, - "jy": 8.46582291468749e-19, - "jz": 3.823492756863969e-08 + "jx": 2.914866280957325e-10, + "jy": 8.46605718473121e-19, + "jz": 3.82349275686397e-08 }, "electron": { "particle_momentum_x": 2.0819392991319055e-25, diff --git a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles_moving.json b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles_moving.json index 813eead7374..05b398e4292 100644 --- a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles_moving.json +++ b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles_moving.json @@ -6,9 +6,9 @@ "Ex": 6.0282565190090465e-05, "Ey": 6.38479659567398e-05, "Ez": 7.880459213065183e-05, - "jx": 1.1659465170956616e-09, - "jy": 2.6115239381823616e-17, - "jz": 1.5293971084510288e-07 + "jx": 1.1659465170956563e-09, + "jy": 1.3057688751494639e-17, + "jz": 1.5293971084510282e-07 }, "electron": { "particle_momentum_x": 2.0819392998019267e-25, diff --git a/Regression/Checksum/benchmarks_json/initial_distribution.json b/Regression/Checksum/benchmarks_json/initial_distribution.json index d251f1dc93e..0366526e6c6 100644 --- a/Regression/Checksum/benchmarks_json/initial_distribution.json +++ b/Regression/Checksum/benchmarks_json/initial_distribution.json @@ -1,54 +1,54 @@ { "lev=0": { - "Bx": 1.4391873150708752e-11, - "By": 2.649275763336405e-12, - "Bz": 2.264858284005838e-12, - "Ex": 116126.69618118233, - "Ey": 2721568.897218672, - "Ez": 13311634.798029516, - "jx": 213561086649.1517, - "jy": 5005061111647.613, - "jz": 24480565503287.883 + "Bx": 1.1440874094834086e-11, + "By": 5.118726025680838e-12, + "Bz": 2.2926252330134977e-12, + "Ex": 485137.7295298721, + "Ey": 2693064.232411463, + "Ez": 13309483.565569244, + "jx": 892185381139.6952, + "jy": 4952640028546.276, + "jz": 24476609310982.766 }, "gaussian": { - "particle_momentum_x": 1.114869485442012e-18, - "particle_momentum_y": 1.113950484707249e-18, - "particle_momentum_z": 1.1145095462051466e-18, + "particle_momentum_x": 1.1148694854425342e-18, + "particle_momentum_y": 1.113950484707441e-18, + "particle_momentum_z": 1.1145095462059476e-18, "particle_position_x": 256080.8791703085, "particle_position_y": 255990.74838430836, "particle_position_z": 255985.3138786131, "particle_weight": 8e+21 }, "velocity_constant": { - "particle_momentum_x": 3.525652955891248e-21, - "particle_momentum_y": 2.854131883579884e-17, - "particle_momentum_z": 3.529207120167883e-21, + "particle_momentum_x": 3.525652955494033e-21, + "particle_momentum_y": 2.854131883579772e-17, + "particle_momentum_z": 3.529207119583616e-21, "particle_position_x": 256041.594025335, "particle_position_y": 255990.80243810173, "particle_position_z": 256058.6711995684, "particle_weight": 8e+21 }, "velocity_parser": { - "particle_momentum_x": 3.5275715705676264e-21, - "particle_momentum_y": 2.85413213756175e-17, - "particle_momentum_z": 3.524858764586134e-21, + "particle_momentum_x": 3.527571570555911e-21, + "particle_momentum_y": 2.8541321375838457e-17, + "particle_momentum_z": 3.5248587639885176e-21, "particle_position_x": 256005.32957331865, "particle_position_y": 256024.073334694, "particle_position_z": 255884.2303468903, "particle_weight": 8e+21 }, "maxwell_boltzmann": { - "particle_momentum_x": 1.115726254239367e-18, - "particle_momentum_y": 1.1158627328262547e-18, - "particle_momentum_z": 1.116611006192975e-18, + "particle_momentum_x": 1.1157262542392384e-18, + "particle_momentum_y": 1.115862732826051e-18, + "particle_momentum_z": 1.1166110061936783e-18, "particle_position_x": 255961.1556903974, "particle_position_y": 255863.7685374788, "particle_position_z": 255905.39310238374, "particle_weight": 8e+21 }, "maxwell_juttner_parser": { - "particle_momentum_x": 3.244003577912503e-16, - "particle_momentum_y": 3.25040141276013e-16, + "particle_momentum_x": 3.2440035779125025e-16, + "particle_momentum_y": 3.2504014127601276e-16, "particle_momentum_z": 3.2451492327611594e-16, "particle_position_x": 255982.1948847503, "particle_position_y": 255914.12054794806, @@ -56,9 +56,9 @@ "particle_weight": 8e+21 }, "uniform": { - "particle_momentum_x": 1.8165231916763963e-17, + "particle_momentum_x": 1.8165231916763772e-17, "particle_momentum_y": 6.992054835330534e-18, - "particle_momentum_z": 1.482077848751643e-15, + "particle_momentum_z": 1.4820778487516442e-15, "particle_position_x": 256022.44604413788, "particle_position_y": 256023.176965898, "particle_position_z": 255947.8223255734, @@ -66,21 +66,29 @@ }, "maxwell_juttner": { "particle_momentum_x": 2.2134518777013892e-16, - "particle_momentum_y": 2.2161036396263055e-16, - "particle_momentum_z": 2.2128980139002375e-16, + "particle_momentum_y": 2.2161036396263068e-16, + "particle_momentum_z": 2.2128980139002385e-16, "particle_position_x": 256016.2534041072, "particle_position_y": 255955.27567227924, "particle_position_z": 255979.3536929577, "particle_weight": 8e+21 }, "beam": { - "particle_momentum_x": 1.0656686508664945e-16, - "particle_momentum_y": 1.0652904561298736e-16, - "particle_momentum_z": 1.0646689144385907e-16, + "particle_momentum_x": 1.065668650866494e-16, + "particle_momentum_y": 1.065290456129872e-16, + "particle_momentum_z": 1.0646689144385903e-16, "particle_position_x": 97438.20096731099, "particle_position_y": 97358.839485784, "particle_position_z": 88428.30909066088, "particle_weight": 0.05958446885576042 + }, + "gaussian_parser": { + "particle_momentum_x": 1.250319517250506e-17, + "particle_momentum_y": 1.3593384741126328e-17, + "particle_momentum_z": 1.4688563133041566e-17, + "particle_position_x": 255966.4923850546, + "particle_position_y": 255982.45790441945, + "particle_position_z": 256101.68177582137, + "particle_weight": 8e+21 } } - diff --git a/Regression/Checksum/benchmarks_json/ionization_boost.json b/Regression/Checksum/benchmarks_json/ionization_boost.json index cb02eeba5b0..35b2db84a1a 100644 --- a/Regression/Checksum/benchmarks_json/ionization_boost.json +++ b/Regression/Checksum/benchmarks_json/ionization_boost.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bx": 0.0, + "By": 18263123.342891, + "Bz": 0.0, + "Ex": 5472992180428804.0, + "Ey": 0.0, + "Ez": 922.5589707939612, + "jx": 12440856508004.96, + "jy": 0.0, + "jz": 78616.0000011086 + }, "electrons": { - "particle_momentum_x": 2.137334032699983e-17, + "particle_momentum_x": 2.1386770170623736e-17, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.729179869336611e-17, - "particle_position_x": 0.11036860836114047, - "particle_position_y": 1.7121959960266107, - "particle_weight": 3.075501431906104e-09 + "particle_momentum_z": 1.7287241262743654e-17, + "particle_position_x": 0.11064981928849067, + "particle_position_y": 1.7121826057017473, + "particle_weight": 3.0755014319061045e-09 }, "ions": { "particle_ionizationLevel": 52741.0, - "particle_momentum_x": 3.63061972838759e-18, + "particle_momentum_x": 3.630619728387593e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 1.0432995297946715e-13, - "particle_position_x": 0.021440031727258925, + "particle_position_x": 0.021439968272741083, "particle_position_y": 0.4742770090248555, "particle_weight": 5.000948082142308e-10 - }, - "lev=0": { - "Bx": 0.0, - "By": 18263123.342890996, - "Bz": 0.0, - "Ex": 5472992180428804.0, - "Ey": 0.0, - "Ez": 922.6036749671132, - "jx": 12440856508004.96, - "jy": 0.0, - "jz": 78616.00000110848 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/ionization_lab.json index 3fafb968f29..82984141b59 100644 --- a/Regression/Checksum/benchmarks_json/ionization_lab.json +++ b/Regression/Checksum/benchmarks_json/ionization_lab.json @@ -1,30 +1,31 @@ { - "electrons": { - "particle_momentum_x": 4.407898469197755e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.642991174223682e-18, - "particle_position_x": 0.1095015206652257, - "particle_position_y": 0.6413864600981052, - "particle_weight": 3.443203125e-10 + "lev=0": { + "Bx": 0.0, + "By": 26296568.434868, + "Bz": 0.0, + "Ex": 7878103122971888.0, + "Ey": 0.0, + "Ez": 3027.995293554467, + "jx": 1.2111358330750162e+16, + "jy": 0.0, + "jz": 1.3575651149270143e-07 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.761324005206226e-18, + "particle_momentum_x": 1.7613240052056494e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.618696690270335e-23, - "particle_position_x": 0.03199998819289686, + "particle_momentum_z": 3.6186967375178554e-23, + "particle_position_x": 0.0320000118071032, "particle_position_y": 0.12800000462171646, "particle_weight": 9.999999999999999e-11 }, - "lev=0": { - "Bx": 0.0, - "By": 26296568.43487075, - "Bz": 0.0, - "Ex": 7878103122973058.0, - "Ey": 0.0, - "Ez": 3027.995293554496, - "jx": 1.2111358333819302e+16, - "jy": 0.0, - "jz": 1.346772722412443e-07 + "electrons": { + "particle_momentum_x": 4.428135211584547e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.6627518442038486e-18, + "particle_orig_z": 0.43043207622230534, + "particle_position_x": 0.11023346490802505, + "particle_position_y": 0.6417814148540429, + "particle_weight": 3.44453125e-10 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/pml_x_galilean.json b/Regression/Checksum/benchmarks_json/pml_x_galilean.json new file mode 100644 index 00000000000..b32eb7a6518 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/pml_x_galilean.json @@ -0,0 +1,12 @@ +{ + "lev=0": { + "Bx": 9.111505955244123e-09, + "By": 1.743610723263872e-08, + "Bz": 8.692076437162089e-09, + "Ex": 4.8377358061392215, + "Ey": 3.8096703194246215, + "Ez": 4.747325170136437, + "divE": 480170.26408400893, + "rho": 1.7297829145620754e-06 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart.json b/Regression/Checksum/benchmarks_json/restart.json index 9ed02d0faad..d8aa0788997 100644 --- a/Regression/Checksum/benchmarks_json/restart.json +++ b/Regression/Checksum/benchmarks_json/restart.json @@ -1,12 +1,15 @@ { - "beam": { - "particle_momentum_x": 4.178482505909375e-19, - "particle_momentum_y": 4.56492260137707e-19, - "particle_momentum_z": 2.733972888170628e-17, - "particle_position_x": 0.0003995213395426269, - "particle_position_y": 0.0004148795632360405, - "particle_position_z": 1.9019426942919677, - "particle_weight": 3120754537.230381 + "lev=0": { + "Bx": 115361.74185283793, + "By": 242516.32397638055, + "Bz": 62078.1602361236, + "Ex": 119701543932824.69, + "Ey": 20420953176049.12, + "Ez": 53561498853753.336, + "jx": 2.1550943713704252e+16, + "jy": 328452566832977.2, + "jz": 4525238578330174.0, + "rho": 22112877.392750103 }, "driver": { "particle_momentum_x": 4.700436405078562e+21, @@ -26,34 +29,31 @@ "particle_position_z": 0.4899808854005956, "particle_weight": 6241509074.460762 }, - "lev=0": { - "Bx": 115361.85363382133, - "By": 242516.41036288123, - "Bz": 62078.12299476949, - "Ex": 119701563770527.33, - "Ey": 20420746160110.2, - "Ez": 53561518084714.88, - "jx": 2.155094272131813e+16, - "jy": 328438528912277.4, - "jz": 4525204093100645.0, - "rho": 22113102.212642767 + "beam": { + "particle_momentum_x": 4.178482505909375e-19, + "particle_momentum_y": 4.56492260137707e-19, + "particle_momentum_z": 2.733972888170628e-17, + "particle_position_x": 0.0003995213395426269, + "particle_position_y": 0.0004148795632360405, + "particle_position_z": 1.9019426942919677, + "particle_weight": 3120754537.230381 }, - "plasma_e": { - "particle_momentum_x": 2.6421618201433585e-19, - "particle_momentum_y": 1.3141433018232061e-20, - "particle_momentum_z": 2.6326692402728787e-17, - "particle_position_x": 0.49916042906025937, - "particle_position_y": 0.4991834641855549, - "particle_position_z": 0.45626372602154797, + "plasma_p": { + "particle_momentum_x": 2.6392309174575904e-19, + "particle_momentum_y": 5.301543151656233e-21, + "particle_momentum_z": 4.829600619848831e-14, + "particle_position_x": 0.4991250033821341, + "particle_position_y": 0.49912499879083855, + "particle_position_z": 0.4563845472726038, "particle_weight": 33067341227104.594 }, - "plasma_p": { - "particle_momentum_x": 2.639231047445633e-19, - "particle_momentum_y": 5.302522423284582e-21, - "particle_momentum_z": 4.829600619851657e-14, - "particle_position_x": 0.4991250033821394, - "particle_position_y": 0.49912499879083844, - "particle_position_z": 0.45638454727260425, + "plasma_e": { + "particle_momentum_x": 2.6421607111722265e-19, + "particle_momentum_y": 1.3141424232991868e-20, + "particle_momentum_z": 2.6326692443085376e-17, + "particle_position_x": 0.49916042919135184, + "particle_position_y": 0.49918346422905135, + "particle_position_z": 0.4562637258155577, "particle_weight": 33067341227104.594 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd.json b/Regression/Checksum/benchmarks_json/restart_psatd.json index 81df3c64c4c..d22fb5b57e7 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd.json @@ -1,4 +1,16 @@ { + "lev=0": { + "Bx": 116102.13010406512, + "By": 245467.5412026496, + "Bz": 65305.35287114741, + "Ex": 118565481099326.73, + "Ey": 18640100573642.96, + "Ez": 51777969821120.38, + "jx": 2.1177246490642464e+16, + "jy": 351544815339900.3, + "jz": 4573466562652713.0, + "rho": 22294533.587530266 + }, "beam": { "particle_momentum_x": 4.178482505909375e-19, "particle_momentum_y": 4.56492260137707e-19, @@ -26,34 +38,22 @@ "particle_position_z": 0.4899808854005956, "particle_weight": 6241509074.460762 }, - "lev=0": { - "Bx": 116102.28415732287, - "By": 245468.57438100342, - "Bz": 65305.38781838063, - "Ex": 118565257880521.4, - "Ey": 18640027127418.004, - "Ez": 51777964618631.914, - "jx": 2.1177240176169996e+16, - "jy": 351529204246424.9, - "jz": 4573419547907262.0, - "rho": 22294773.912022192 - }, "plasma_e": { - "particle_momentum_x": 2.0912370354448323e-19, - "particle_momentum_y": 1.3510736736168866e-20, - "particle_momentum_z": 2.633607024821822e-17, - "particle_position_x": 0.4991733347875822, - "particle_position_y": 0.49919828067421096, - "particle_position_z": 0.4562416755755165, + "particle_momentum_x": 2.0912364079892767e-19, + "particle_momentum_y": 1.3510804924903876e-20, + "particle_momentum_z": 2.6336070297356917e-17, + "particle_position_x": 0.49917333523960167, + "particle_position_y": 0.49919828074154127, + "particle_position_z": 0.45624167532314724, "particle_weight": 33067341227104.625 }, "plasma_p": { - "particle_momentum_x": 2.105303160755537e-19, - "particle_momentum_y": 3.3281768098792752e-21, - "particle_momentum_z": 4.829599953650214e-14, - "particle_position_x": 0.4991250029759827, - "particle_position_y": 0.49912499825258744, - "particle_position_z": 0.4563845470979585, + "particle_momentum_x": 2.1053025446547977e-19, + "particle_momentum_y": 3.3272838305044046e-21, + "particle_momentum_z": 4.829599953646669e-14, + "particle_position_x": 0.49912500297599893, + "particle_position_y": 0.49912499825258866, + "particle_position_z": 0.45638454709795806, "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json index 344495f8144..50971f0d45e 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json @@ -1,4 +1,25 @@ { + "lev=0": { + "Bx": 115622.1878941125, + "By": 243403.66715242947, + "Bz": 63602.66529020351, + "Ex": 117118934931289.56, + "Ey": 18448377440588.27, + "Ez": 50821967818817.24, + "jx": 2.1044522674691452e+16, + "jy": 329314843111847.94, + "jz": 4524787623275627.0, + "rho": 22128883.53068064 + }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, "beam": { "particle_momentum_x": 4.178482505909375e-19, "particle_momentum_y": 4.56492260137707e-19, @@ -8,6 +29,24 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, + "plasma_e": { + "particle_momentum_x": 1.9905288525315664e-19, + "particle_momentum_y": 1.2685401564810352e-20, + "particle_momentum_z": 2.6334746597885943e-17, + "particle_position_x": 0.4991702885212871, + "particle_position_y": 0.4991949978513417, + "particle_position_z": 0.4562491611716817, + "particle_weight": 33067341227104.625 + }, + "plasma_p": { + "particle_momentum_x": 2.002897995377179e-19, + "particle_momentum_y": 2.87423099731012e-21, + "particle_momentum_z": 4.8296000849128737e-14, + "particle_position_x": 0.49912500259846493, + "particle_position_y": 0.4991249979476612, + "particle_position_z": 0.45638454712861487, + "particle_weight": 33067341227104.625 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, @@ -16,44 +55,5 @@ "particle_position_y": 0.0016212373149699414, "particle_position_z": 0.30417779498653386, "particle_weight": 6241509074.460762 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 - }, - "lev=0": { - "Bx": 115622.38124168929, - "By": 243405.25199746038, - "Bz": 63602.713209078014, - "Ex": 117118723119070.72, - "Ey": 18448325716024.688, - "Ez": 50821980097172.29, - "jx": 2.104451599585796e+16, - "jy": 329301545706958.4, - "jz": 4524740919108259.0, - "rho": 22129124.901803844 - }, - "plasma_e": { - "particle_momentum_x": 1.9905296213155052e-19, - "particle_momentum_y": 1.2685326003720766e-20, - "particle_momentum_z": 2.6334746549552728e-17, - "particle_position_x": 0.4991702881158205, - "particle_position_y": 0.4991949977965001, - "particle_position_z": 0.45624916139143146, - "particle_weight": 33067341227104.625 - }, - "plasma_p": { - "particle_momentum_x": 2.0028987523417718e-19, - "particle_momentum_y": 2.875037979316222e-21, - "particle_momentum_z": 4.8296000849162443e-14, - "particle_position_x": 0.49912500259844994, - "particle_position_y": 0.4991249979476608, - "particle_position_z": 0.4563845471286154, - "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/checksum.py b/Regression/Checksum/checksum.py index cc3c53a24c2..454edfc2606 100644 --- a/Regression/Checksum/checksum.py +++ b/Regression/Checksum/checksum.py @@ -11,6 +11,8 @@ from benchmark import Benchmark import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c import yt yt.funcs.mylog.setLevel(50) @@ -20,19 +22,22 @@ class Checksum: """Class for checksum comparison of one test. """ - def __init__(self, test_name, plotfile, do_fields=True, do_particles=True): + def __init__(self, test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): """ Checksum constructor. - Store test_name and plotfile name, and compute checksum - from plotfile and store it in self.data. + Store test_name, output file name and format, compute checksum + from output file and store it in self.data. Parameters ---------- test_name: string Name of test, as found between [] in .ini file. - plotfile: string - Plotfile from which the checksum is computed. + output_file: string + Output file from which the checksum is computed. + + output_format: string + Format of the output file (plotfile, openpmd). do_fields: bool, default=True Whether to compare fields in the checksum. @@ -42,83 +47,142 @@ def __init__(self, test_name, plotfile, do_fields=True, do_particles=True): """ self.test_name = test_name - self.plotfile = plotfile - self.data = self.read_plotfile(do_fields=do_fields, - do_particles=do_particles) + self.output_file = output_file + self.output_format = output_format + self.data = self.read_output_file(do_fields=do_fields, + do_particles=do_particles) - def read_plotfile(self, do_fields=True, do_particles=True): + def read_output_file(self, do_fields=True, do_particles=True): """ - Get checksum from plotfile. - Read an AMReX plotfile with yt, compute 1 checksum per field and return - all checksums in a dictionary. + Get checksum from output file. + Read an AMReX plotfile with yt or an openPMD file with openPMD viewer, + compute 1 checksum per field and return all checksums in a dictionary. The checksum of quantity Q is max(abs(Q)). Parameters ---------- do_fields: bool, default=True - Whether to read fields from the plotfile. + Whether to read fields from the output file. do_particles: bool, default=True - Whether to read particles from the plotfile. + Whether to read particles from the output file. """ - ds = yt.load(self.plotfile) - # yt 4.0+ has rounding issues with our domain data: - # RuntimeError: yt attempted to read outside the boundaries - # of a non-periodic domain along dimension 0. - if 'force_periodicity' in dir(ds): ds.force_periodicity() - grid_fields = [item for item in ds.field_list if item[0] == 'boxlib'] - - # "fields"/"species" we remove: - # - nbody: added by yt by default, unused by us - species_list = set([item[0] for item in ds.field_list if - item[1][:9] == 'particle_' and - item[0] != 'all' and - item[0] != 'nbody']) - - data = {} - - # Compute checksum for field quantities - if do_fields: - for lev in range(ds.max_level+1): - data_lev = {} - lev_grids = [grid for grid in ds.index.grids - if grid.Level == lev] - # Warning: For now, we assume all levels are rectangular - LeftEdge = np.min( - np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0) - all_data_level = ds.covering_grid( - level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions) - for field in grid_fields: - Q = all_data_level[field].v.squeeze() - data_lev[field[1]] = np.sum(np.abs(Q)) - data['lev=' + str(lev)] = data_lev - - # Compute checksum for particle quantities - if do_particles: - ad = ds.all_data() - for species in species_list: - # properties we remove: - # - particle_cpu/id: they depend on the parallelism: MPI-ranks and - # on-node acceleration scheme, thus not portable - # and irrelevant for physics benchmarking - part_fields = [item[1] for item in ds.field_list - if item[0] == species and - item[1] != 'particle_cpu' and - item[1] != 'particle_id' - ] - data_species = {} - for field in part_fields: - Q = ad[(species, field)].v - data_species[field] = np.sum(np.abs(Q)) - data[species] = data_species + if self.output_format == 'plotfile': + ds = yt.load(self.output_file) + # yt 4.0+ has rounding issues with our domain data: + # RuntimeError: yt attempted to read outside the boundaries + # of a non-periodic domain along dimension 0. + if 'force_periodicity' in dir(ds): ds.force_periodicity() + grid_fields = [item for item in ds.field_list if item[0] == 'boxlib'] + + # "fields"/"species" we remove: + # - nbody: added by yt by default, unused by us + species_list = set([item[0] for item in ds.field_list if + item[1][:9] == 'particle_' and + item[0] != 'all' and + item[0] != 'nbody']) + + data = {} + + # Compute checksum for field quantities + if do_fields: + for lev in range(ds.max_level+1): + data_lev = {} + lev_grids = [grid for grid in ds.index.grids + if grid.Level == lev] + # Warning: For now, we assume all levels are rectangular + LeftEdge = np.min( + np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0) + all_data_level = ds.covering_grid( + level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions) + for field in grid_fields: + Q = all_data_level[field].v.squeeze() + data_lev[field[1]] = np.sum(np.abs(Q)) + data['lev=' + str(lev)] = data_lev + + # Compute checksum for particle quantities + if do_particles: + ad = ds.all_data() + for species in species_list: + # properties we remove: + # - particle_cpu/id: they depend on the parallelism: MPI-ranks and + # on-node acceleration scheme, thus not portable + # and irrelevant for physics benchmarking + part_fields = [item[1] for item in ds.field_list + if item[0] == species and + item[1] != 'particle_cpu' and + item[1] != 'particle_id' + ] + data_species = {} + for field in part_fields: + Q = ad[(species, field)].v + data_species[field] = np.sum(np.abs(Q)) + data[species] = data_species + + elif self.output_format == 'openpmd': + # Load time series + ts = OpenPMDTimeSeries(self.output_file) + data = {} + # Compute number of MR levels + # TODO This calculation of nlevels assumes that the last element + # of level_fields is by default on the highest MR level. + level_fields = [field for field in ts.avail_fields if 'lvl' in field] + nlevels = 0 if level_fields == [] else int(level_fields[-1][-1]) + # Compute checksum for field quantities + if do_fields: + for lev in range(nlevels+1): + # Create list of fields specific to level lev + grid_fields = [] + if lev == 0: + grid_fields = [field for field in ts.avail_fields if 'lvl' not in field] + else: + grid_fields = [field for field in ts.avail_fields if f'lvl{lev}' in field] + data_lev = {} + for field in grid_fields: + vector_components = ts.fields_metadata[field]['avail_components'] + if vector_components != []: + for coord in vector_components: + Q, info = ts.get_field(field=field, iteration=ts.iterations[-1], coord=coord) + # key stores strings composed of field name and vector components + # (e.g., field='B' or field='B_lvl1' + coord='y' results in key='By') + key = field.replace(f'_lvl{lev}', '') + coord + data_lev[key] = np.sum(np.abs(Q)) + else: # scalar field + Q, info = ts.get_field(field=field, iteration=ts.iterations[-1]) + data_lev[field] = np.sum(np.abs(Q)) + data[f'lev={lev}'] = data_lev + # Compute checksum for particle quantities + if do_particles: + species_list = [] + if ts.avail_record_components is not None: + species_list = ts.avail_record_components.keys() + for species in species_list: + data_species = {} + part_fields = [item for item in ts.avail_record_components[species] + if item != 'id' and item != 'charge' and item != 'mass'] + # Convert the field name to the name used in plotfiles + for field in part_fields: + Q = ts.get_particle(var_list=[field], species=species, iteration=ts.iterations[-1]) + if field in ['x', 'y', 'z']: + field_name = 'particle_position_' + field + elif field in ['ux', 'uy', 'uz']: + field_name = 'particle_momentum_' + field[-1] + m, = ts.get_particle(['mass'], species=species, iteration=ts.iterations[-1]) + Q *= m*c + elif field in ['w']: + field_name = 'particle_weight' + else: + field_name = 'particle_' + field + data_species[field_name] = np.sum(np.abs(Q)) + data[species] = data_species return data def evaluate(self, rtol=1.e-9, atol=1.e-40): """ - Compare plotfile checksum with benchmark. - Read checksum from input plotfile, read benchmark + Compare output file checksum with benchmark. + Read checksum from output file, read benchmark corresponding to test_name, and assert that they are equal. Almost all the body of this functions is for user-readable print statements. @@ -136,10 +200,10 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): # Dictionaries have same outer keys (levels, species)? if (self.data.keys() != ref_benchmark.data.keys()): - print("ERROR: Benchmark and plotfile checksum " + print("ERROR: Benchmark and output file checksum " "have different outer keys:") print("Benchmark: %s" % ref_benchmark.data.keys()) - print("Plotfile : %s" % self.data.keys()) + print("Test file: %s" % self.data.keys()) print("\n----------------\nNew file for " + self.test_name + ":") print(json.dumps(self.data, indent=2)) print("----------------") @@ -148,12 +212,12 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): # Dictionaries have same inner keys (field and particle quantities)? for key1 in ref_benchmark.data.keys(): if (self.data[key1].keys() != ref_benchmark.data[key1].keys()): - print("ERROR: Benchmark and plotfile checksum have " + print("ERROR: Benchmark and output file checksum have " "different inner keys:") print("Common outer keys: %s" % ref_benchmark.data.keys()) print("Benchmark inner keys in %s: %s" % (key1, ref_benchmark.data[key1].keys())) - print("Plotfile inner keys in %s: %s" + print("Test file inner keys in %s: %s" % (key1, self.data[key1].keys())) print("\n----------------\nNew file for " + self.test_name + ":") print(json.dumps(self.data, indent=2)) @@ -168,11 +232,11 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): ref_benchmark.data[key1][key2], rtol=rtol, atol=atol) if not passed: - print("ERROR: Benchmark and plotfile checksum have " + print("ERROR: Benchmark and output file checksum have " "different value for key [%s,%s]" % (key1, key2)) print("Benchmark: [%s,%s] %.15e" % (key1, key2, ref_benchmark.data[key1][key2])) - print("Plotfile : [%s,%s] %.15e" + print("Test file: [%s,%s] %.15e" % (key1, key2, self.data[key1][key2])) checksums_differ = True # Print absolute and relative error for each failing key diff --git a/Regression/Checksum/checksumAPI.py b/Regression/Checksum/checksumAPI.py index c45b6e233f4..cc13ceefa28 100755 --- a/Regression/Checksum/checksumAPI.py +++ b/Regression/Checksum/checksumAPI.py @@ -23,23 +23,23 @@ Example: add these lines to a WarpX CI Python analysis script to run the checksum test. > import checksumAPI - > checksumAPI.evaluate_checksum(test_name, plotfile) + > checksumAPI.evaluate_checksum(test_name, output_file, output_format) - As a script, to evaluate or to reset a benchmark: * Evaluate a benchmark. From a bash terminal: - $ ./checksumAPI.py --evaluate --plotfile \ - --test-name + $ ./checksumAPI.py --evaluate --output-file \ + --output-format 'plotfile' --test-name * Reset a benchmark. From a bash terminal: - $ ./checksumAPI.py --reset-benchmark --plotfile \ - --test-name + $ ./checksumAPI.py --reset-benchmark --output-file \ + --output-format 'openpmd' --test-name """ -def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, +def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e-9, atol=1.e-40, do_fields=True, do_particles=True): """ - Compare plotfile checksum with benchmark. - Read checksum from input plotfile, read benchmark + Compare output file checksum with benchmark. + Read checksum from output file, read benchmark corresponding to test_name, and assert their equality. Parameters @@ -47,8 +47,11 @@ def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, test_name: string Name of test, as found between [] in .ini file. - plotfile : string - Plotfile from which the checksum is computed. + output_file : string + Output file from which the checksum is computed. + + output_format : string + Format of the output file (plotfile, openpmd). rtol: float, default=1.e-9 Relative tolerance for the comparison. @@ -62,24 +65,27 @@ def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, do_particles: bool, default=True Whether to compare particles in the checksum. """ - test_checksum = Checksum(test_name, plotfile, do_fields=do_fields, - do_particles=do_particles) + test_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) test_checksum.evaluate(rtol=rtol, atol=atol) -def reset_benchmark(test_name, plotfile, do_fields=True, do_particles=True): +def reset_benchmark(test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): """ Update the benchmark (overwrites reference json file). Overwrite value of benchmark corresponding to - test_name with checksum read from input plotfile. + test_name with checksum read from output file. Parameters ---------- test_name: string Name of test, as found between [] in .ini file. - plotfile: string - Plotfile from which the checksum is computed. + output_file: string + Output file from which the checksum is computed. + + output_format: string + Format of the output file (plotfile, openpmd). do_fields: bool, default=True Whether to write field checksums in the benchmark. @@ -87,34 +93,37 @@ def reset_benchmark(test_name, plotfile, do_fields=True, do_particles=True): do_particles: bool, default=True Whether to write particles checksums in the benchmark. """ - ref_checksum = Checksum(test_name, plotfile, do_fields=do_fields, - do_particles=do_particles) + ref_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) ref_benchmark = Benchmark(test_name, ref_checksum.data) ref_benchmark.reset() -def reset_all_benchmarks(path_to_all_plotfiles): +def reset_all_benchmarks(path_to_all_output_files, output_format='plotfile'): """ Update all benchmarks (overwrites reference json files) - found in path_to_all_plotfiles + found in path_to_all_output_files Parameters ---------- - path_to_all_plotfiles: string - Path to all plotfiles for which the benchmarks - are to be reset. The plotfiles should be named _plt, which is + path_to_all_output_files: string + Path to all output files for which the benchmarks + are to be reset. The output files should be named _plt, which is what regression_testing.regtests.py does, provided we're careful enough. + + output_format: string + Format of the output files (plotfile, openpmd). """ - # Get list of plotfiles in path_to_all_plotfiles - plotfile_list = glob.glob(path_to_all_plotfiles + '*_plt*[0-9]', - recursive=True) - plotfile_list.sort() + # Get list of output files in path_to_all_output_files + output_file_list = glob.glob(path_to_all_output_files + '*_plt*[0-9]', + recursive=True) + output_file_list.sort() - # Loop over plotfiles and reset the corresponding benchmark - for plotfile in plotfile_list: - test_name = os.path.split(plotfile)[1][:-9] - reset_benchmark(test_name, plotfile) + # Loop over output files and reset the corresponding benchmark + for output_file in output_file_list: + test_name = os.path.split(output_file)[1][:-9] + reset_benchmark(test_name, output_file, output_format) if __name__ == '__main__': @@ -131,11 +140,14 @@ def reset_all_benchmarks(path_to_all_plotfiles): required='--evaluate' in sys.argv or '--reset-benchmark' in sys.argv, help='Name of the test (as in WarpX-tests.ini)') - parser.add_argument('--plotfile', dest='plotfile', type=str, default='', + parser.add_argument('--output-file', dest='output_file', type=str, default='', required='--evaluate' in sys.argv or '--reset-benchmark' in sys.argv, - help='Name of WarpX plotfile') - + help='Name of WarpX output file') + parser.add_argument('--output-format', dest='output_format', type=str, default='plotfile', + required='--evaluate' in sys.argv or + '--reset-benchmark' in sys.argv, + help='Format of the output file (plotfile, openpmd)') parser.add_argument('--skip-fields', dest='do_fields', default=True, action='store_false', help='If used, do not read/write field checksums') @@ -143,7 +155,7 @@ def reset_all_benchmarks(path_to_all_plotfiles): default=True, action='store_false', help='If used, do not read/write particle checksums') - # Fields and/or particles are read from plotfile/written to benchmark? + # Fields and/or particles are read from output file/written to benchmark? parser.add_argument('--rtol', dest='rtol', type=float, default=1.e-9, help='relative tolerance for comparison') @@ -155,26 +167,27 @@ def reset_all_benchmarks(path_to_all_plotfiles): parser.add_argument('--reset-all-benchmarks', dest='reset_all_benchmarks', action='store_true', default=False, help='Reset all benchmarks.') - parser.add_argument('--path-to-all-plotfiles', - dest='path_to_all_plotfiles', type=str, default='', + parser.add_argument('--path-to-all-output-files', + dest='path_to_all_output_files', type=str, default='', required='--reset-all-benchmarks' in sys.argv, - help='Directory containing all benchmark plotfiles, \ + help='Directory containing all benchmark output files, \ typically WarpX-benchmarks generated by \ regression_testing/regtest.py') args = parser.parse_args() if args.reset_benchmark: - reset_benchmark(args.test_name, args.plotfile, + reset_benchmark(args.test_name, args.output_file, args.output_format, do_fields=args.do_fields, do_particles=args.do_particles) if args.evaluate: - evaluate_checksum(args.test_name, args.plotfile, rtol=args.rtol, - atol=args.atol, do_fields=args.do_fields, - do_particles=args.do_particles) + evaluate_checksum(args.test_name, args.output_file, args.output_format, + rtol=args.rtol, atol=args.atol, + do_fields=args.do_fields, do_particles=args.do_particles) if args.reset_all_benchmarks: - # WARNING: this mode does not support skip-fields/particles - # and tolerances - reset_all_benchmarks(args.path_to_all_plotfiles) + if args.output_format == 'openpmd': + sys.exit('Option --reset-all-benchmarks does not work with openPMD format') + # WARNING: this mode does not support skip-fields/particles and tolerances + reset_all_benchmarks(args.path_to_all_output_files, args.output_format) diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 0e033f186b3..0566184848b 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = b98bdae9fb67e5d9aafc488de92c53001bd323ec +branch = 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 48364535760..974615ad725 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = b98bdae9fb67e5d9aafc488de92c53001bd323ec +branch = 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 [source] dir = /home/regtester/AMReX_RegTesting/warpx @@ -317,7 +317,7 @@ analysisRoutine = Examples/analysis_default_regression.py [Deuterium_Deuterium_Fusion_3D] buildDir = . inputFile = Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -333,7 +333,7 @@ analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [Deuterium_Deuterium_Fusion_3D_intraspecies] buildDir = . inputFile = Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -349,7 +349,7 @@ analysisRoutine = Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_ [Deuterium_Tritium_Fusion_3D] buildDir = . inputFile = Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -357,7 +357,7 @@ restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 -numthreads = 2 +numthreads = 1 compileTest = 0 doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py @@ -365,7 +365,7 @@ analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [Deuterium_Tritium_Fusion_RZ] buildDir = . inputFile = Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_rz -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 warpx.abort_on_warning_threshold=high +runtime_params = warpx.abort_on_warning_threshold=high dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ @@ -426,7 +426,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -runtime_params = warpx.do_dynamic_scheduling=0 geometry.dims=2 +runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/dive_cleaning/analysis.py analysisOutputImage = Comparison.png @@ -444,7 +444,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = analysisRoutine = Examples/Tests/dive_cleaning/analysis.py analysisOutputImage = Comparison.png @@ -887,7 +887,7 @@ analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd] buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -1013,7 +1013,7 @@ analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [initial_distribution] buildDir = . inputFile = Examples/Tests/initial_distribution/inputs -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -1080,7 +1080,7 @@ analysisRoutine = Examples/Tests/ion_stopping/analysis_ion_stopping.py [Langmuir_multi] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -1096,6 +1096,78 @@ particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png +[Langmuir_fluid_1D] +buildDir = . +inputFile = Examples/Tests/langmuir_fluids/inputs_1d +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/langmuir_fluids/analysis_1d.py +analysisOutputImage = langmuir_fluid_multi_1d_analysis.png + +[Langmuir_fluid_RZ] +buildDir = . +inputFile = Examples/Tests/langmuir_fluids/inputs_rz +runtime_params = +dim = 2 +addToCompileString = USE_RZ=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/langmuir_fluids/analysis_rz.py +analysisOutputImage = langmuir_fluid_rz_analysis.png + +[Langmuir_fluid_2D] +buildDir = . +inputFile = Examples/Tests/langmuir_fluids/inputs_2d +runtime_params = +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 -DCMAKE_BUILD_TYPE=Debug +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/langmuir_fluids/analysis_2d.py +analysisOutputImage = langmuir_fluid_multi_2d_analysis.png + +[Langmuir_fluid_multi] +buildDir = . +inputFile = Examples/Tests/langmuir_fluids/inputs_3d +runtime_params = +dim = 3 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=3 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/langmuir_fluids/analysis_3d.py +analysisOutputImage = langmuir_fluid_multi_analysis.png + [Langmuir_multi_1d] buildDir = . inputFile = Examples/Tests/langmuir/inputs_1d @@ -1153,6 +1225,25 @@ particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_MR.png +[Langmuir_multi_2d_MR_momentum_conserving] +buildDir = . +inputFile = Examples/Tests/langmuir/inputs_2d +runtime_params = algo.maxwell_solver=ckc warpx.use_filter=1 amr.max_level=1 amr.ref_ratio=4 warpx.fine_tag_lo=-10.e-6 -10.e-6 warpx.fine_tag_hi=10.e-6 10.e-6 algo.field_gathering=momentum-conserving diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons positrons +analysisRoutine = Examples/Tests/langmuir/analysis_2d.py +analysisOutputImage = Langmuir_multi_2d_MR_momentum_conserving.png + [Langmuir_multi_2d_MR_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d @@ -1365,7 +1456,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = warpx.do_dynamic_scheduling=0 warpx.grid_type=collocated algo.current_deposition=direct +runtime_params = warpx.grid_type=collocated algo.current_deposition=direct dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -1517,7 +1608,7 @@ analysisOutputImage = Langmuir_multi_psatd_multiJ_nodal.png [Langmuir_multi_psatd_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = algo.maxwell_solver=psatd warpx.do_dynamic_scheduling=0 warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON @@ -1673,7 +1764,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_single_precision] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = dim = 3 addToCompileString = PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PRECISION=SINGLE @@ -1721,7 +1812,8 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons -analysisRoutine = Examples/analysis_default_regression.py +outputFile = LaserAcceleration_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserAcceleration_1d] buildDir = . @@ -1741,10 +1833,46 @@ compareParticles = 1 particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py +[LaserAcceleration_1d_fluid] +buildDir = . +inputFile = Examples/Physics_applications/laser_acceleration/inputs_1d_fluids +runtime_params = +dim = 1 +addToCompileString = USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons ions +analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py + +[LaserAcceleration_1d_fluid_boosted] +buildDir = . +inputFile = Examples/Physics_applications/laser_acceleration/inputs_1d_fluids_boosted +runtime_params = +dim = 1 +addToCompileString = USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons ions +analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py + [LaserAccelerationBoost] buildDir = . inputFile = Examples/Physics_applications/laser_acceleration/inputs_2d_boost -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 amr.n_cell=64 512 max_step=300 +runtime_params = amr.n_cell=64 512 max_step=300 dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1846,7 +1974,8 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons -analysisRoutine = Examples/analysis_default_regression.py +outputFile = LaserAcceleration_single_precision_comms_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserInjection] buildDir = . @@ -1869,7 +1998,7 @@ analysisOutputImage = laser_analysis.png [LaserInjection_1d] buildDir = . inputFile = Examples/Tests/laser_injection/inputs_1d_rt -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 1 addToCompileString = USE_OPENPMD=TRUE QED=FALSE cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF @@ -1877,7 +2006,7 @@ restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 -numthreads = 2 +numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 @@ -1886,7 +2015,7 @@ analysisRoutine = Examples/Tests/laser_injection/analysis_1d.py [LaserInjection_2d] buildDir = . inputFile = Examples/Tests/laser_injection/inputs_2d_rt -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1905,7 +2034,7 @@ buildDir = . inputFile = Examples/Tests/laser_injection_from_file/analysis_2d_binary.py aux1File = Examples/Tests/laser_injection_from_file/inputs.2d_test_binary customRunCmd = ./analysis_2d_binary.py -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1924,7 +2053,7 @@ buildDir = . inputFile = Examples/Tests/laser_injection_from_file/analysis_3d.py aux1File = Examples/Tests/laser_injection_from_file/inputs.3d_test customRunCmd = ./analysis_3d.py -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -1943,7 +2072,7 @@ buildDir = . inputFile = Examples/Tests/laser_injection_from_file/analysis_1d.py aux1File = Examples/Tests/laser_injection_from_file/inputs.1d_test customRunCmd = ./analysis_1d.py -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = dim = 1 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=1 @@ -1962,7 +2091,7 @@ buildDir = . inputFile = Examples/Tests/laser_injection_from_file/analysis_1d_boost.py aux1File = Examples/Tests/laser_injection_from_file/inputs.1d_boost_test customRunCmd = ./analysis_1d_boost.py -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = dim = 1 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=1 @@ -1981,7 +2110,7 @@ buildDir = . inputFile = Examples/Tests/laser_injection_from_file/analysis_2d.py aux1File = Examples/Tests/laser_injection_from_file/inputs.2d_test customRunCmd = ./analysis_2d.py -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -2000,7 +2129,7 @@ buildDir = . inputFile = Examples/Tests/laser_injection_from_file/analysis_RZ.py aux1File = Examples/Tests/laser_injection_from_file/inputs.RZ_test customRunCmd = ./analysis_RZ.py -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ @@ -2019,7 +2148,7 @@ buildDir = . inputFile = Examples/Tests/laser_injection_from_file/analysis_from_RZ_file.py aux1File = Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test customRunCmd = ./analysis_from_RZ_file.py -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ @@ -2035,8 +2164,8 @@ doVis = 0 [LaserIonAcc2d] buildDir = . -inputFile = Examples/Physics_applications/laser_ion/inputs -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 amr.n_cell=384 512 max_step=100 +inputFile = Examples/Physics_applications/laser_ion/inputs_2d +runtime_params = amr.n_cell=384 512 max_step=100 dim = 2 addToCompileString = USE_OPENPMD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON @@ -2136,7 +2265,7 @@ analysisRoutine = Examples/Tests/maxwell_hybrid_qed/analysis_Maxwell_QED_Hybrid. [momentum-conserving-gather] buildDir = . inputFile = Examples/Physics_applications/plasma_acceleration/inputs_2d -runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=400 warpx.serialize_initial_conditions=1 warpx.do_dynamic_scheduling=0 algo.field_gathering=momentum-conserving +runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=400 algo.field_gathering=momentum-conserving dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -2154,7 +2283,7 @@ analysisRoutine = Examples/analysis_default_regression.py [multi_J_rz_psatd] buildDir = . inputFile = Examples/Tests/multi_j/inputs_rz -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 warpx.abort_on_warning_threshold=medium psatd.J_in_time=linear +runtime_params = warpx.abort_on_warning_threshold=medium psatd.J_in_time=linear dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -2217,7 +2346,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = analysisRoutine = Examples/Tests/initial_plasma_profile/analysis.py [particle_absorption] @@ -2259,7 +2388,7 @@ analysisRoutine = Examples/Tests/boundaries/analysis.py buildDir = . inputFile = Examples/Tests/particle_fields_diags/inputs aux1File = Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 3 addToCompileString = USE_OPENPMD=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON @@ -2277,7 +2406,7 @@ analysisRoutine = Examples/Tests/particle_fields_diags/analysis_particle_diags.p buildDir = . inputFile = Examples/Tests/particle_fields_diags/inputs aux1File = Examples/Tests/particle_fields_diags/analysis_particle_diags_impl.py -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 3 addToCompileString = PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE USE_OPENPMD=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PRECISION=SINGLE -DWarpX_OPENPMD=ON @@ -2574,7 +2703,7 @@ analysisRoutine = Examples/Tests/photon_pusher/analysis_photon_pusher.py [PlasmaAccelerationBoost2d] buildDir = . inputFile = Examples/Physics_applications/plasma_acceleration/inputs_2d_boost -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 amr.n_cell=64 256 max_step=20 +runtime_params = amr.n_cell=64 256 max_step=20 dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -2590,7 +2719,7 @@ analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationBoost3d] buildDir = . inputFile = Examples/Physics_applications/plasma_acceleration/inputs_3d_boost -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 amr.n_cell=64 64 128 max_step=5 +runtime_params = amr.n_cell=64 64 128 max_step=5 dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -2606,7 +2735,7 @@ analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationBoost3d_hybrid] buildDir = . inputFile = Examples/Physics_applications/plasma_acceleration/inputs_3d_boost -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 amr.n_cell=64 64 128 max_step=25 warpx.grid_type=hybrid warpx.do_current_centering=0 +runtime_params = amr.n_cell=64 64 128 max_step=25 warpx.grid_type=hybrid warpx.do_current_centering=0 dim = 3 addToCompileString = restartTest = 0 @@ -2621,7 +2750,7 @@ analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationMR] buildDir = . inputFile = Examples/Physics_applications/plasma_acceleration/inputs_2d -runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=400 warpx.serialize_initial_conditions=1 warpx.do_dynamic_scheduling=0 +runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=400 dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -2693,7 +2822,7 @@ analysisRoutine = Examples/Tests/plasma_lens/analysis.py [PlasmaMirror] buildDir = . inputFile = Examples/Physics_applications/plasma_mirror/inputs_2d -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 amr.n_cell=256 128 max_step=20 +runtime_params = amr.n_cell=256 128 max_step=20 dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -2725,7 +2854,7 @@ analysisRoutine = Examples/analysis_default_regression.py [pml_psatd_rz] buildDir = . inputFile = Examples/Tests/pml/inputs_rz -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 warpx.cfl=0.7 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = warpx.cfl=0.7 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -2741,7 +2870,7 @@ analysisRoutine = Examples/Tests/pml/analysis_pml_psatd_rz.py [pml_x_ckc] buildDir = . inputFile = Examples/Tests/pml/inputs_2d -runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_solver=ckc +runtime_params = algo.maxwell_solver=ckc dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -2754,10 +2883,26 @@ compileTest = 0 doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py +[pml_x_galilean] +buildDir = . +inputFile = Examples/Tests/pml/inputs_2d +runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz rho divE warpx.cfl=0.7071067811865475 warpx.do_pml_dive_cleaning=1 warpx.do_pml_divb_cleaning=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium psatd.v_galilean=0. 0. 0.99 warpx.grid_type=collocated +dim = 2 +addToCompileString = USE_PSATD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py + [pml_x_psatd] buildDir = . inputFile = Examples/Tests/pml/inputs_2d -runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 warpx.do_dynamic_scheduling=0 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE warpx.cfl = 0.7071067811865475 warpx.do_pml_dive_cleaning=0 warpx.do_pml_divb_cleaning=0 chk.file_prefix=pml_x_psatd_chk chk.file_min_digits=5 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE warpx.cfl = 0.7071067811865475 warpx.do_pml_dive_cleaning=0 warpx.do_pml_divb_cleaning=0 chk.file_prefix=pml_x_psatd_chk chk.file_min_digits=5 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -2774,7 +2919,7 @@ analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py [pml_x_yee] buildDir = . inputFile = Examples/Tests/pml/inputs_2d -runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_solver=yee chk.file_prefix=pml_x_yee_chk chk.file_min_digits=5 +runtime_params = algo.maxwell_solver=yee chk.file_prefix=pml_x_yee_chk chk.file_min_digits=5 dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -2791,7 +2936,7 @@ analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py [pml_x_yee_eb] buildDir = . inputFile = Examples/Tests/pml/inputs_2d -runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_solver=yee chk.file_prefix=pml_x_yee_eb_chk chk.file_min_digits=5 +runtime_params = algo.maxwell_solver=yee chk.file_prefix=pml_x_yee_eb_chk chk.file_min_digits=5 dim = 2 addToCompileString = USE_EB=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_EB=ON @@ -2808,7 +2953,7 @@ analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py [Proton_Boron_Fusion_2D] buildDir = . inputFile = Examples/Tests/nuclear_fusion/inputs_proton_boron_2d -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -2816,7 +2961,7 @@ restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 -numthreads = 2 +numthreads = 1 compileTest = 0 doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -2824,7 +2969,7 @@ analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py [Proton_Boron_Fusion_3D] buildDir = . inputFile = Examples/Tests/nuclear_fusion/inputs_proton_boron_3d -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -2832,7 +2977,7 @@ restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 -numthreads = 2 +numthreads = 1 compileTest = 0 doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -2933,6 +3078,25 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py +[Python_dsmc_1d] +buildDir = . +inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc +dim = 1 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py + [Python_ElectrostaticSphereEB] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py @@ -3288,6 +3452,25 @@ doVis = 0 compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis.py +[Python_ohms_law_solver_EM_modes_rz] +buildDir = . +inputFile = Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +runtime_params = warpx.abort_on_warning_threshold = medium +customRunCmd = python3 PICMI_inputs_rz.py --test +dim = 1 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE USE_RZ=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_APP=OFF -DWarpX_QED=OFF -DWarpX_PYTHON=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis_rz.py + [Python_ohms_law_solver_ion_beam_1d] buildDir = . inputFile = Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -3544,7 +3727,7 @@ analysisRoutine = Examples/analysis_default_regression.py [Python_reduced_diags_loadbalancecosts_timers] buildDir = . inputFile = Examples/Tests/reduced_diags/PICMI_inputs_loadbalancecosts.py -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 algo.load_balance_costs_update=Timers +runtime_params = algo.load_balance_costs_update=Timers customRunCmd = python3 PICMI_inputs_loadbalancecosts.py dim = 3 addToCompileString = USE_PYTHON_MAIN=TRUE @@ -3813,7 +3996,7 @@ analysisRoutine = Examples/Tests/radiation_reaction/test_const_B_analytical/ana buildDir = . inputFile = Examples/Tests/reduced_diags/inputs aux1File = Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -3830,7 +4013,7 @@ analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags.py [reduced_diags_loadbalancecosts_heuristic] buildDir = . inputFile = Examples/Tests/reduced_diags/inputs_loadbalancecosts -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 algo.load_balance_costs_update=Heuristic +runtime_params = algo.load_balance_costs_update=Heuristic dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -3847,7 +4030,7 @@ analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalanc [reduced_diags_loadbalancecosts_timers] buildDir = . inputFile = Examples/Tests/reduced_diags/inputs_loadbalancecosts -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 algo.load_balance_costs_update=Timers +runtime_params = algo.load_balance_costs_update=Timers dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -3864,7 +4047,7 @@ analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalanc [reduced_diags_loadbalancecosts_timers_psatd] buildDir = . inputFile = Examples/Tests/reduced_diags/inputs_loadbalancecosts -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 algo.load_balance_costs_update=Timers +runtime_params = algo.load_balance_costs_update=Timers dim = 3 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON @@ -3882,7 +4065,7 @@ analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalanc buildDir = . inputFile = Examples/Tests/reduced_diags/inputs aux1File = Examples/Tests/reduced_diags/analysis_reduced_diags_impl.py -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 3 addToCompileString = PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PRECISION=SINGLE @@ -3928,7 +4111,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = analysisRoutine = Examples/Tests/relativistic_space_charge_initialization/analysis.py analysisOutputImage = Comparison.png @@ -4008,7 +4191,7 @@ analysisRoutine = Examples/Tests/restart/analysis_restart.py [RigidInjection_BTD] buildDir = . inputFile = Examples/Tests/rigid_injection/inputs_2d_BoostedFrame -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 2 addToCompileString = USE_OPENPMD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON @@ -4026,7 +4209,7 @@ analysisRoutine = Examples/Tests/rigid_injection/analysis_rigid_injection_Booste [RigidInjection_lab] buildDir = . inputFile = Examples/Tests/rigid_injection/inputs_2d_LabFrame -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1 +runtime_params = dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -4057,6 +4240,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz.py +[scraping_filter] +buildDir = . +inputFile = Examples/Tests/scraping/inputs_rz_filter +runtime_params = warpx.abort_on_warning_threshold = medium +dim = 2 +addToCompileString = USE_EB=TRUE USE_RZ=TRUE USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/scraping/analysis_rz_filter.py + [silver_mueller_1d] buildDir = . inputFile = Examples/Tests/silver_mueller/inputs_1d @@ -4135,7 +4335,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -runtime_params = warpx.do_dynamic_scheduling=0 +runtime_params = analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py analysisOutputImage = Comparison.png @@ -4153,14 +4353,14 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -runtime_params = warpx.do_dynamic_scheduling=0 geometry.dims=2 +runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py analysisOutputImage = Comparison.png [subcyclingMR] buildDir = . inputFile = Examples/Tests/subcycling/inputs_2d -runtime_params = warpx.serialize_initial_conditions=1 warpx.do_dynamic_scheduling=0 +runtime_params = dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -4273,9 +4473,60 @@ cmakeSetupOpts = -DWarpX_DIMS=3 restartTest = 0 useMPI = 1 numprocs = 1 -useOMP = 0 +useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/nodal_electrostatic/analysis_3d.py + +[BeamBeamCollision] +buildDir = . +inputFile = Examples/Physics_applications/beam-beam_collision/inputs +runtime_params = warpx.abort_on_warning_threshold=high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +outputFile = BeamBeamCollision_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py + +[ImplicitPicard_1d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_1d +runtime_params = warpx.abort_on_warning_threshold=high +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_1d.py + +[SemiImplicitPicard_1d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_1d_semiimplicit +runtime_params = warpx.abort_on_warning_threshold=high +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_1d.py diff --git a/Regression/prepare_file_ci.py b/Regression/prepare_file_ci.py index 7ec0af427d2..d52d05b1139 100644 --- a/Regression/prepare_file_ci.py +++ b/Regression/prepare_file_ci.py @@ -7,7 +7,7 @@ import os # This script modifies `WarpX-test.ini` (which is used for nightly builds) -# and creates the file `ci-test.ini` (which is used for continous +# and creates the file `ci-test.ini` (which is used for continuous # integration) # The subtests that are selected are controlled by WARPX_TEST_DIM # The architecture (CPU/GPU) is selected by WARPX_TEST_ARCH @@ -62,7 +62,7 @@ text = re.sub('USE_PSATD=FALSE', '', text) -# Ccache +# CCache if ci_ccache: text = re.sub('addToCompileString =', 'addToCompileString = USE_CCACHE=TRUE ', text) @@ -77,6 +77,14 @@ 'warpx.always_warn_immediately=1 warpx.abort_on_warning_threshold=low', text) +# Add runtime options for CPU: +# > serialize initial conditions and no dynamic scheduling in OpenMP +if arch == 'CPU': + text = re.sub('runtime_params =', + 'runtime_params = '+ + 'warpx.do_dynamic_scheduling=0 warpx.serialize_initial_conditions=1', + text) + # Use less/more cores for compiling, e.g. public CI only provides 2 cores if ci_num_make_jobs is not None: text = re.sub( 'numMakeJobs = \d+', 'numMakeJobs = {}'.format(ci_num_make_jobs), text ) diff --git a/Source/AcceleratorLattice/AcceleratorLattice.H b/Source/AcceleratorLattice/AcceleratorLattice.H index 67611743586..59527b734cb 100644 --- a/Source/AcceleratorLattice/AcceleratorLattice.H +++ b/Source/AcceleratorLattice/AcceleratorLattice.H @@ -11,6 +11,8 @@ #include "LatticeElements/Drift.H" #include "LatticeElements/HardEdgedQuadrupole.H" #include "LatticeElements/HardEdgedPlasmaLens.H" +#include "LatticeElements/Solenoid.H" +#include "LatticeElements/SolenoidRF.H" #include #include @@ -66,12 +68,15 @@ public: * @param[in] a_pti the grid where the finder is needed * @param[in] a_offset the particle offset since the finded needs information about the particles as well */ - LatticeElementFinderDevice GetFinderDeviceInstance (WarpXParIter const& a_pti, int a_offset) const; + [[nodiscard]] LatticeElementFinderDevice + GetFinderDeviceInstance (WarpXParIter const& a_pti, int a_offset) const; /* All of the available lattice element types */ Drift h_drift; HardEdgedQuadrupole h_quad; HardEdgedPlasmaLens h_plasmalens; + Solenoid h_solenoid; + SolenoidRF h_solenoidrf; }; diff --git a/Source/AcceleratorLattice/AcceleratorLattice.cpp b/Source/AcceleratorLattice/AcceleratorLattice.cpp index e05167c261d..a521d44310b 100644 --- a/Source/AcceleratorLattice/AcceleratorLattice.cpp +++ b/Source/AcceleratorLattice/AcceleratorLattice.cpp @@ -11,6 +11,8 @@ #include "LatticeElements/Drift.H" #include "LatticeElements/HardEdgedQuadrupole.H" #include "LatticeElements/HardEdgedPlasmaLens.H" +#include "LatticeElements/Solenoid.H" +#include "LatticeElements/SolenoidRF.H" #include @@ -28,6 +30,8 @@ AcceleratorLattice::AcceleratorLattice () h_quad.WriteToDevice(); h_plasmalens.WriteToDevice(); + h_solenoid.WriteToDevice(); + h_solenoidrf.WriteToDevice(); } void @@ -65,6 +69,12 @@ AcceleratorLattice::ReadLattice (std::string const & root_name, amrex::ParticleR else if (element_type == "plasmalens") { h_plasmalens.AddElement(pp_element, z_location); } + else if (element_type == "solenoid") { + h_solenoid.AddElement(pp_element, z_location); + } + else if (element_type == "solenoidrf") { + h_solenoidrf.AddElement(pp_element, z_location); + } else if (element_type == "line") { ReadLattice(element_name, z_location); } diff --git a/Source/AcceleratorLattice/LatticeElementFinder.H b/Source/AcceleratorLattice/LatticeElementFinder.H index 43e97980774..b1db4bc503d 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.H +++ b/Source/AcceleratorLattice/LatticeElementFinder.H @@ -9,6 +9,8 @@ #include "LatticeElements/HardEdgedQuadrupole.H" #include "LatticeElements/HardEdgedPlasmaLens.H" +#include "LatticeElements/Solenoid.H" +#include "LatticeElements/SolenoidRF.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/WarpXParticleContainer.H" @@ -78,6 +80,8 @@ struct LatticeElementFinder /* The index lookup tables for each lattice element type */ amrex::Gpu::DeviceVector d_quad_indices; amrex::Gpu::DeviceVector d_plasmalens_indices; + amrex::Gpu::DeviceVector d_solenoid_indices; + amrex::Gpu::DeviceVector d_solenoidrf_indices; /** * \brief Fill in the index lookup tables @@ -89,52 +93,7 @@ struct LatticeElementFinder */ void setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, amrex::Gpu::DeviceVector const & ze, - amrex::Gpu::DeviceVector & indices) - { - - using namespace amrex::literals; - - const auto nelements = static_cast(zs.size()); - amrex::ParticleReal const * zs_arr = zs.data(); - amrex::ParticleReal const * ze_arr = ze.data(); - int * indices_arr = indices.data(); - - amrex::Real const zmin = m_zmin; - amrex::Real const dz = m_dz; - - amrex::ParticleReal const gamma_boost = m_gamma_boost; - amrex::ParticleReal const uz_boost = m_uz_boost; - amrex::Real const time = m_time; - - amrex::ParallelFor( m_nz, - [=] AMREX_GPU_DEVICE (int iz) { - - // Get the location of the grid node - amrex::Real z_node = zmin + iz*dz; - - if (gamma_boost > 1._prt) { - // Transform to lab frame - z_node = gamma_boost*z_node + uz_boost*time; - } - - // Find the index to the element that is closest to the grid cell. - // For now, this assumes that there is no overlap among elements of the same type. - for (int ie = 0 ; ie < nelements ; ie++) { - // Find the mid points between element ie and the ones before and after it. - // The first and last element need special handling. - const amrex::ParticleReal zcenter_left = (ie == 0)? - (std::numeric_limits::lowest()) : (0.5_prt*(ze_arr[ie-1] + zs_arr[ie])); - const amrex::ParticleReal zcenter_right = (ie < nelements - 1)? - (0.5_prt*(ze_arr[ie] + zs_arr[ie+1])) : (std::numeric_limits::max()); - if (zcenter_left <= z_node && z_node < zcenter_right) { - indices_arr[iz] = ie; - } - - } - } - ); - } - + amrex::Gpu::DeviceVector & indices); }; /** @@ -167,7 +126,7 @@ struct LatticeElementFinderDevice amrex::ParticleReal m_uz_boost; amrex::Real m_time; - GetParticlePosition m_get_position; + GetParticlePosition m_get_position; const amrex::ParticleReal* AMREX_RESTRICT m_ux = nullptr; const amrex::ParticleReal* AMREX_RESTRICT m_uy = nullptr; const amrex::ParticleReal* AMREX_RESTRICT m_uz = nullptr; @@ -175,10 +134,14 @@ struct LatticeElementFinderDevice /* Device level instances for each lattice element type */ HardEdgedQuadrupoleDevice d_quad; HardEdgedPlasmaLensDevice d_plasmalens; + SolenoidDevice d_solenoid; + SolenoidRFDevice d_solenoidrf; /* Device level index lookup tables for each element type */ int const* d_quad_indices_arr = nullptr; int const* d_plasmalens_indices_arr = nullptr; + int const* d_solenoid_indices_arr = nullptr; + int const* d_solenoidrf_indices_arr = nullptr; /** * \brief Gather the field for the particle from the lattice elements @@ -219,10 +182,10 @@ struct LatticeElementFinderDevice amrex::ParticleReal Ex_sum = 0._prt; amrex::ParticleReal Ey_sum = 0._prt; - const amrex::ParticleReal Ez_sum = 0._prt; + amrex::ParticleReal Ez_sum = 0._prt; amrex::ParticleReal Bx_sum = 0._prt; amrex::ParticleReal By_sum = 0._prt; - const amrex::ParticleReal Bz_sum = 0._prt; + amrex::ParticleReal Bz_sum = 0._prt; if (d_quad.nelements > 0) { if (d_quad_indices_arr[iz] > -1) { @@ -248,6 +211,34 @@ struct LatticeElementFinderDevice } } + if (d_solenoid.nelements > 0) { + if (d_solenoid_indices_arr[iz] > -1) { + const auto ielement = d_solenoid_indices_arr[iz]; + amrex::ParticleReal Ex, Ey, Ez, Bx, By, Bz; + d_solenoid.get_field(ielement, x, y, z, Ex, Ey, Ez, Bx, By, Bz); + Ex_sum += Ex; + Ey_sum += Ey; + Ez_sum += Ez; + Bx_sum += Bx; + By_sum += By; + Bz_sum += Bz; + } + } + + if (d_solenoidrf.nelements > 0) { + if (d_solenoidrf_indices_arr[iz] > -1) { + const auto ielement = d_solenoidrf_indices_arr[iz]; + amrex::ParticleReal Ex, Ey, Ez, Bx, By, Bz; + d_solenoidrf.get_field(ielement, x, y, z, m_time, Ex, Ey, Ez, Bx, By, Bz); + Ex_sum += Ex; + Ey_sum += Ey; + Ez_sum += Ez; + Bx_sum += Bx; + By_sum += By; + Bz_sum += Bz; + } + } + if (m_gamma_boost > 1._prt) { // The fields returned from get_field is in the lab frame // Transform the fields to the boosted frame diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index 7e9825c28a6..2350f22183e 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -8,6 +8,8 @@ #include "LatticeElementFinder.H" #include "LatticeElements/HardEdgedQuadrupole.H" #include "LatticeElements/HardEdgedPlasmaLens.H" +#include "LatticeElements/Solenoid.H" +#include "LatticeElements/SolenoidRF.H" #include #include @@ -48,6 +50,14 @@ LatticeElementFinder::AllocateIndices (AcceleratorLattice const& accelerator_lat if (accelerator_lattice.h_plasmalens.nelements > 0) { d_plasmalens_indices.resize(m_nz); } + + if (accelerator_lattice.h_solenoid.nelements > 0) { + d_solenoid_indices.resize(m_nz); + } + + if (accelerator_lattice.h_solenoidrf.nelements > 0) { + d_solenoidrf_indices.resize(m_nz); + } } void @@ -74,6 +84,18 @@ LatticeElementFinder::UpdateIndices (int const lev, amrex::MFIter const& a_mfi, accelerator_lattice.h_plasmalens.d_ze, d_plasmalens_indices); } + + if (accelerator_lattice.h_solenoid.nelements > 0) { + setup_lattice_indices(accelerator_lattice.h_solenoid.d_zs, + accelerator_lattice.h_solenoid.d_ze, + d_solenoid_indices); + } + + if (accelerator_lattice.h_solenoidrf.nelements > 0) { + setup_lattice_indices(accelerator_lattice.h_solenoidrf.d_zs, + accelerator_lattice.h_solenoidrf.d_ze, + d_solenoidrf_indices); + } } LatticeElementFinderDevice @@ -85,7 +107,6 @@ LatticeElementFinder::GetFinderDeviceInstance (WarpXParIter const& a_pti, int co return result; } - void LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& a_pti, int const a_offset, AcceleratorLattice const& accelerator_lattice, @@ -96,8 +117,8 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& int const lev = a_pti.GetLevel(); - m_get_position = GetParticlePosition(a_pti, a_offset); - auto& attribs = a_pti.GetAttribs(); + m_get_position = GetParticlePosition(a_pti, a_offset); + const auto& attribs = a_pti.GetAttribs(); m_ux = attribs[PIdx::ux].dataPtr() + a_offset; m_uy = attribs[PIdx::uy].dataPtr() + a_offset; m_uz = attribs[PIdx::uz].dataPtr() + a_offset; @@ -120,4 +141,61 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& d_plasmalens_indices_arr = h_finder.d_plasmalens_indices.data(); } + if (accelerator_lattice.h_solenoid.nelements > 0) { + d_solenoid = accelerator_lattice.h_solenoid.GetDeviceInstance(); + d_solenoid_indices_arr = h_finder.d_solenoid_indices.data(); + } + + if (accelerator_lattice.h_solenoidrf.nelements > 0) { + d_solenoidrf = accelerator_lattice.h_solenoidrf.GetDeviceInstance(); + d_solenoidrf_indices_arr = h_finder.d_solenoidrf_indices.data(); + } + +} + +void +LatticeElementFinder::setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, + amrex::Gpu::DeviceVector const & ze, + amrex::Gpu::DeviceVector & indices) +{ + + using namespace amrex::literals; + + const auto nelements = static_cast(zs.size()); + amrex::ParticleReal const * zs_arr = zs.data(); + amrex::ParticleReal const * ze_arr = ze.data(); + int * indices_arr = indices.data(); + + amrex::Real const zmin = m_zmin; + amrex::Real const dz = m_dz; + + amrex::ParticleReal const gamma_boost = m_gamma_boost; + amrex::ParticleReal const uz_boost = m_uz_boost; + amrex::Real const time = m_time; + + amrex::ParallelFor( m_nz, + [=] AMREX_GPU_DEVICE (int iz) { + + // Get the location of the grid node + amrex::Real z_node = zmin + iz*dz; + + if (gamma_boost > 1._prt) { + // Transform to lab frame + z_node = gamma_boost*z_node + uz_boost*time; + } + + // Find the index to the element that is closest to the grid cell. + // For now, this assumes that there is no overlap among elements of the same type. + for (int ie = 0 ; ie < nelements ; ie++) { + // Find the mid points between element ie and the ones before and after it. + // The first and last element need special handling. + const amrex::ParticleReal zcenter_left = (ie == 0)? + (std::numeric_limits::lowest()) : (0.5_prt*(ze_arr[ie-1] + zs_arr[ie])); + const amrex::ParticleReal zcenter_right = (ie < nelements - 1)? + (0.5_prt*(ze_arr[ie] + zs_arr[ie+1])) : (std::numeric_limits::max()); + if (zcenter_left <= z_node && z_node < zcenter_right) { + indices_arr[iz] = ie; + } + } + }); } diff --git a/Source/AcceleratorLattice/LatticeElements/CMakeLists.txt b/Source/AcceleratorLattice/LatticeElements/CMakeLists.txt index 2ffd74c2e77..ae1190424a5 100644 --- a/Source/AcceleratorLattice/LatticeElements/CMakeLists.txt +++ b/Source/AcceleratorLattice/LatticeElements/CMakeLists.txt @@ -6,5 +6,7 @@ foreach(D IN LISTS WarpX_DIMS) Drift.cpp HardEdgedQuadrupole.cpp HardEdgedPlasmaLens.cpp + Solenoid.cpp + SolenoidRF.cpp ) endforeach() diff --git a/Source/AcceleratorLattice/LatticeElements/Drift.H b/Source/AcceleratorLattice/LatticeElements/Drift.H index a9067a09caa..7ad36df8dba 100644 --- a/Source/AcceleratorLattice/LatticeElements/Drift.H +++ b/Source/AcceleratorLattice/LatticeElements/Drift.H @@ -17,8 +17,6 @@ struct Drift Drift (); - ~Drift () = default; - /** * \brief Read in an element and add it to the lists * diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.H b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.H index 43009c0c4ed..2d15dfa3bd4 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.H +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.H @@ -27,8 +27,6 @@ struct HardEdgedPlasmaLens HardEdgedPlasmaLens (); - ~HardEdgedPlasmaLens () = default; - /** * \brief Read in an element and add it to the lists * @@ -55,7 +53,7 @@ struct HardEdgedPlasmaLens /** * \brief Returns the device level instance with the lattice information */ - HardEdgedPlasmaLensDevice GetDeviceInstance () const; + [[nodiscard]] HardEdgedPlasmaLensDevice GetDeviceInstance () const; }; diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp index 9c8726a9c4b..21b1f164777 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp @@ -58,7 +58,7 @@ HardEdgedPlasmaLensDevice::InitHardEdgedPlasmaLensDevice (HardEdgedPlasmaLens co nelements = h_plasmalens.nelements; - if (nelements == 0) return; + if (nelements == 0) { return; } d_zs_arr = h_plasmalens.d_zs.data(); d_ze_arr = h_plasmalens.d_ze.data(); diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.H b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.H index 82f1e860a40..5ac4f7fe7c1 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.H +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.H @@ -27,8 +27,6 @@ struct HardEdgedQuadrupole HardEdgedQuadrupole (); - ~HardEdgedQuadrupole () = default; - /** * \brief Read in an element and add it to the lists * @@ -55,8 +53,8 @@ struct HardEdgedQuadrupole /** * \brief Returns the device level instance with the lattice information */ - HardEdgedQuadrupoleDevice GetDeviceInstance () const; - + [[nodiscard]] HardEdgedQuadrupoleDevice + GetDeviceInstance () const; }; diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp index 51bcf7a2497..bcf50c55666 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp @@ -58,7 +58,7 @@ HardEdgedQuadrupoleDevice::InitHardEdgedQuadrupoleDevice (HardEdgedQuadrupole co nelements = h_quad.nelements; - if (nelements == 0) return; + if (nelements == 0) { return; } d_zs_arr = h_quad.d_zs.data(); d_ze_arr = h_quad.d_ze.data(); diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdged_K.H b/Source/AcceleratorLattice/LatticeElements/HardEdged_K.H index 35e2dde7a8f..6be88deb04d 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdged_K.H +++ b/Source/AcceleratorLattice/LatticeElements/HardEdged_K.H @@ -34,14 +34,14 @@ amrex::ParticleReal hard_edged_fraction(const amrex::ParticleReal z, amrex::ParticleReal const zl = std::min(z, zpvdt); amrex::ParticleReal const zr = std::max(z, zpvdt); - // Calculate the residence correction - // frac will be 1 if the step is completely inside the lens, between 0 and 1 - // when entering or leaving the lens, and otherwise 0. - // This accounts for the case when particles step over the element without landing in it. - // This assumes that vzp != 0. - amrex::ParticleReal const zl_bounded = std::min(std::max(zl, zs), ze); - amrex::ParticleReal const zr_bounded = std::min(std::max(zr, zs), ze); - const amrex::ParticleReal frac = (zr_bounded - zl_bounded)/(zr - zl); + // Calculate the residence correction + // frac will be 1 if the step is completely inside the lens, between 0 and 1 + // when entering or leaving the lens, and otherwise 0. + // This accounts for the case when particles step over the element without landing in it. + // This assumes that vzp != 0. + amrex::ParticleReal const zl_bounded = std::min(std::max(zl, zs), ze); + amrex::ParticleReal const zr_bounded = std::min(std::max(zr, zs), ze); + const amrex::ParticleReal frac = (zr_bounded - zl_bounded)/(zr - zl); return frac; } diff --git a/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.H b/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.H index ad72eeef910..15cb72435bd 100644 --- a/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.H +++ b/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.H @@ -25,8 +25,6 @@ struct LatticeElementBase */ LatticeElementBase (std::string const& element_name); - ~LatticeElementBase () = default; - /** * \brief Read in an element base data and add it to the lists * diff --git a/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.cpp b/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.cpp index 248db59aaf6..19afd8cd617 100644 --- a/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.cpp +++ b/Source/AcceleratorLattice/LatticeElements/LatticeElementBase.cpp @@ -12,10 +12,8 @@ #include -LatticeElementBase::LatticeElementBase (std::string const& element_name) -{ - m_element_name = element_name; -} +LatticeElementBase::LatticeElementBase (std::string const& element_name): + m_element_name{element_name}{} void LatticeElementBase::AddElementBase (amrex::ParmParse & pp_element, amrex::ParticleReal & z_location) diff --git a/Source/AcceleratorLattice/LatticeElements/Make.package b/Source/AcceleratorLattice/LatticeElements/Make.package index 2d55e10d68b..3e89b03049c 100644 --- a/Source/AcceleratorLattice/LatticeElements/Make.package +++ b/Source/AcceleratorLattice/LatticeElements/Make.package @@ -2,5 +2,7 @@ CEXE_sources += LatticeElementBase.cpp CEXE_sources += Drift.cpp CEXE_sources += HardEdgedQuadrupole.cpp CEXE_sources += HardEdgedPlasmaLens.cpp +CEXE_sources += Solenoid.cpp +CEXE_sources += SolenoidRF.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/AcceleratorLattice/LatticeElements diff --git a/Source/AcceleratorLattice/LatticeElements/Solenoid.H b/Source/AcceleratorLattice/LatticeElements/Solenoid.H new file mode 100644 index 00000000000..63a602975a1 --- /dev/null +++ b/Source/AcceleratorLattice/LatticeElements/Solenoid.H @@ -0,0 +1,145 @@ +/* Copyright 2023 David Grote and David Bizzozero + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_ACCELERATORLATTICE_LATTICEELEMENTS_SOLENOID_H_ +#define WARPX_ACCELERATORLATTICE_LATTICEELEMENTS_SOLENOID_H_ + +#include "LatticeElementBase.H" +#include "HardEdged_K.H" +#include "Utils/WarpXConst.H" + +#include +#include +#include + +#include +#include + +// Specifies a field that models the field generated by a solenoidal field, +// an axisymmetric field expansion + +struct SolenoidDevice; + +struct Solenoid + : LatticeElementBase +{ + + Solenoid (); + + ~Solenoid () = default; + + /** + * \brief Read in an element and add it to the lists + * + * @param[in] pp_element The ParmParse instance to read in the data + * @param[inout] z_location The current z location in the lattice + */ + void + AddElement (amrex::ParmParse & pp_element, amrex::ParticleReal & z_location); + + /** + * \brief Write the element information to the device + */ + void + WriteToDevice (); + + // Host variables read from input file + std::vector h_scale; + std::vector> h_b_coef; + + // Device variable read from input files + amrex::Gpu::DeviceVector d_scale; + amrex::Gpu::DeviceVector *> d_b_coef; + + /** + * \brief Returns the device level instance with the lattice information + */ + SolenoidDevice GetDeviceInstance () const; + + +}; + +// Instance that is trivially copyable to the device. + +struct SolenoidDevice +{ + + /** + * \brief Initializes the data and pointer needed to reference the lattice element info + * + * @param[in] h_solenoid host level instance that this is associated with + */ + void InitSolenoidDevice (Solenoid const& h_solenoid); + + int nelements = 0; + + const amrex::ParticleReal* AMREX_RESTRICT d_zs_arr; + const amrex::ParticleReal* AMREX_RESTRICT d_ze_arr; + const amrex::ParticleReal* AMREX_RESTRICT d_scale_arr; + unsigned long* AMREX_RESTRICT d_num_b_coef; + const amrex::ParticleReal** AMREX_RESTRICT d_b_coef_arr; + + /** + * \brief Fetch the field of the specified element at the given location + * + * @param[in] ielement the element number + * @param[in] x, y, z the particle position in the lab frame + * @param[in] m_time the current time in the lab frame + * @param[out] Ex, Ey, Ez, Bx, By, Bz the fetched field in the lab frame + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void get_field (const int ielement, + const amrex::ParticleReal x, + const amrex::ParticleReal y, + const amrex::ParticleReal z, + amrex::ParticleReal& Ex, + amrex::ParticleReal& Ey, + amrex::ParticleReal& Ez, + amrex::ParticleReal& Bx, + amrex::ParticleReal& By, + amrex::ParticleReal& Bz) const + { + + const amrex::ParticleReal zs = d_zs_arr[ielement]; + const amrex::ParticleReal ze = d_ze_arr[ielement]; + const amrex::ParticleReal scale = d_scale_arr[ielement]; + const amrex::ParticleReal* b_coef = d_b_coef_arr[ielement]; + + constexpr amrex::ParticleReal pi = MathConst::pi; // Everyone's favorite circle ratio + + const int n_coef = (d_num_b_coef[ielement]-1) / 2; // Number of coef. is half the length of the list: zero and odd inds are cosine, even inds are sine + const amrex::ParticleReal zlen = ze - zs; // Length of element + + amrex::ParticleReal zz = z - ze + zlen/2.0; // Shifted coordinate to midpoint of element (zz = 0 is midpoint of element) + amrex::ParticleReal bz = b_coef[0]/2.0; // A_0 cosine fourier coef. is halved (sine coef. is trivially zero) + amrex::ParticleReal bzp = 0.0; // Variable to store Bz'(zz) (z-derivative of on-axis longitudinal field at z = zz) + amrex::ParticleReal bzpp = 0.0; // Variable to store Bz''(zz) + amrex::ParticleReal bzppp = 0.0; // Variable to store Bz'''(zz) + for (int k = 1; k < n_coef; ++k){ + bz = bz + b_coef[2*k-1]*cos(2*pi*k*zz/zlen) + b_coef[2*k]*sin(2.0*pi*k*zz/zlen); + bzp = bzp - (2.0*pi*k*zz/zlen)*b_coef[2*k-1]*sin(2.0*pi*k*zz/zlen) + (2.0*pi*k*zz/zlen)*b_coef[2*k]*cos(2.0*pi*k*zz/zlen); + bzpp = bzpp - pow(2.0*pi*k*zz/zlen,2)*b_coef[2*k-1]*cos(2.0*pi*k*zz/zlen) - pow(2.0*pi*k*zz/zlen,2)*b_coef[2*k]*sin(2.0*pi*k*zz/zlen); + bzppp = bzppp + pow(2.0*pi*k*zz/zlen,3)*b_coef[2*k-1]*sin(2.0*pi*k*zz/zlen) - pow(2.0*pi*k*zz/zlen,3)*b_coef[2*k]*cos(2.0*pi*k*zz/zlen); + } + bz = bz*scale; // Scale fields and derivatives by user-provided scaling factor (from input file) + bzp = bzp*scale; + bzpp = bzpp*scale; + bzppp = bzppp*scale; + + amrex::ParticleReal r2 = x*x + y*y; + + Ex = 0.0; // Update E and B fields from Fourier coefficients and scaling factors + Ey = 0.0; + Ez = 0.0; + Bx = -(bzp*x/2) + (bzppp*x*r2/16); + By = -(bzp*y/2) + (bzppp*y*r2/16); + Bz = bz - (bzpp*r2/4); + + } + +}; + +#endif // WARPX_ACCELERATORLATTICE_LATTICEELEMENTS_SOLENOID_H_ diff --git a/Source/AcceleratorLattice/LatticeElements/Solenoid.cpp b/Source/AcceleratorLattice/LatticeElements/Solenoid.cpp new file mode 100644 index 00000000000..2ca1e27e570 --- /dev/null +++ b/Source/AcceleratorLattice/LatticeElements/Solenoid.cpp @@ -0,0 +1,78 @@ +/* Copyright 2023 David Grote and David Bizzozero + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "Solenoid.H" +#include "Utils/Parser/ParserUtils.H" + +#include +#include + +#include + +Solenoid::Solenoid () + : LatticeElementBase("solenoid") +{ +} + +void +Solenoid::AddElement (amrex::ParmParse & pp_element, amrex::ParticleReal & z_location) +{ + using namespace amrex::literals; + + AddElementBase(pp_element, z_location); + + amrex::ParticleReal scale = 0._prt; + amrex::Vector b_coef = {}; + + utils::parser::queryWithParser(pp_element, "scale", scale); + utils::parser::queryArrWithParser(pp_element, "b_coef", b_coef); + + h_scale.push_back(scale); + h_b_coef.push_back(b_coef); + +} + +void +Solenoid::WriteToDevice () +{ + WriteToDeviceBase(); + + d_scale.resize(h_scale.size()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scale.begin(), h_scale.end(), d_scale.begin()); + + d_b_coef.resize(h_b_coef.size()); + for (auto i = 0lu; i < h_b_coef.size(); ++i) { + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_b_coef[i].begin(), h_b_coef[i].end(), d_b_coef[i].begin()); + } + +} + +SolenoidDevice +Solenoid::GetDeviceInstance () const +{ + SolenoidDevice result; + result.InitSolenoidDevice(*this); + return result; +} + +void +SolenoidDevice::InitSolenoidDevice (Solenoid const& h_solenoid) +{ + + nelements = h_solenoid.nelements; + + if (nelements == 0) return; + + // remember pointers to device data in simple data structures + d_zs_arr = h_solenoid.d_zs.data(); + d_ze_arr = h_solenoid.d_ze.data(); + d_scale_arr = h_solenoid.d_scale.data(); + for (auto i = 0lu; i < nelements; ++i) { + d_num_b_coef[i] = h_solenoid.d_b_coef[i].size(); + d_b_coef_arr[i] = h_solenoid.d_b_coef[i].data(); + } + +} diff --git a/Source/AcceleratorLattice/LatticeElements/SolenoidRF.H b/Source/AcceleratorLattice/LatticeElements/SolenoidRF.H new file mode 100644 index 00000000000..953cfb7a93b --- /dev/null +++ b/Source/AcceleratorLattice/LatticeElements/SolenoidRF.H @@ -0,0 +1,161 @@ +/* Copyright 2023 David Grote and David Bizzozero + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_ACCELERATORLATTICE_LATTICEELEMENTS_SOLENOIDRF_H_ +#define WARPX_ACCELERATORLATTICE_LATTICEELEMENTS_SOLENOIDRF_H_ + +#include "LatticeElementBase.H" +#include "HardEdged_K.H" +#include "Utils/WarpXConst.H" + +#include +#include +#include + +#include +#include + +// Specifies a field that models the field generated by a solenoidal field, +// an axisymmetric field expansion + +struct SolenoidRFDevice; + +struct SolenoidRF + : LatticeElementBase +{ + + SolenoidRF (); + + ~SolenoidRF () = default; + + /** + * \brief Read in an element and add it to the lists + * + * @param[in] pp_element The ParmParse instance to read in the data + * @param[inout] z_location The current z location in the lattice + */ + void + AddElement (amrex::ParmParse & pp_element, amrex::ParticleReal & z_location); + + /** + * \brief Write the element information to the device + */ + void + WriteToDevice (); + + // Host variables read from input file + std::vector h_scale; + std::vector h_freq; + std::vector h_theta; + std::vector> h_e_coef; + + + // Device variable read from input files + amrex::Gpu::DeviceVector d_scale; + amrex::Gpu::DeviceVector d_freq; + amrex::Gpu::DeviceVector d_theta; + amrex::Vector> d_e_coef; + + /** + * \brief Returns the device level instance with the lattice information + */ + SolenoidRFDevice GetDeviceInstance () const; + + +}; + +// Instance that is trivially copyable to the device. + +struct SolenoidRFDevice +{ + + /** + * \brief Initializes the data and pointer needed to reference the lattice element info + * + * @param[in] h_solenoidrf host level instance that this is associated with + */ + void InitSolenoidRFDevice (SolenoidRF const& h_solenoidrf); + + int nelements = 0; + + const amrex::ParticleReal* AMREX_RESTRICT d_zs_arr; + const amrex::ParticleReal* AMREX_RESTRICT d_ze_arr; + const amrex::ParticleReal* AMREX_RESTRICT d_scale_arr; + const amrex::ParticleReal* AMREX_RESTRICT d_freq_arr; + const amrex::ParticleReal* AMREX_RESTRICT d_theta_arr; + unsigned long* AMREX_RESTRICT d_num_e_coef; + const amrex::ParticleReal** AMREX_RESTRICT d_e_coef_arr; + + /** + * \brief Fetch the field of the specified element at the given location + * + * @param[in] ielement the element number + * @param[in] x, y, z the particle position in the lab frame + * @param[in] m_time the current time in the lab frame + * @param[out] Ex, Ey, Ez, Bx, By, Bz the fetched field in the lab frame + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void get_field (const int ielement, + const amrex::ParticleReal x, + const amrex::ParticleReal y, + const amrex::ParticleReal z, + const amrex::ParticleReal m_time, + amrex::ParticleReal& Ex, + amrex::ParticleReal& Ey, + amrex::ParticleReal& Ez, + amrex::ParticleReal& Bx, + amrex::ParticleReal& By, + amrex::ParticleReal& Bz) const + { + + const amrex::ParticleReal zs = d_zs_arr[ielement]; + const amrex::ParticleReal ze = d_ze_arr[ielement]; + const amrex::ParticleReal scale = d_scale_arr[ielement]; + const amrex::ParticleReal freq = d_freq_arr[ielement]; + const amrex::ParticleReal theta = d_theta_arr[ielement]; + const amrex::ParticleReal* e_coef = d_e_coef_arr[ielement]; + + constexpr amrex::ParticleReal pi = MathConst::pi; // Everyone's favorite circle ratio + constexpr amrex::ParticleReal c = PhysConst::c; + + const int n_coef = (d_num_e_coef[ielement]-1) / 2; // Number of coef. is half the length of the list: zero and odd inds are cosine, even inds are sine + const amrex::ParticleReal zlen = ze - zs; // Length of element + const amrex::ParticleReal xl = 2.0*pi*freq/c; // Wavelength of RF frequency + const amrex::ParticleReal tmpcos = cos((2.0*pi*freq*m_time) + theta); // Cosine factor based on time and phase shift + const amrex::ParticleReal tmpsin = sin((2.0*pi*freq*m_time) + theta); // Sine factor based on time and phase shift + + amrex::ParticleReal zz = z - ze + zlen/2.0; // Shifted coordinate to midpoint of element (zz = 0 is midpoint of element) + amrex::ParticleReal ez = e_coef[0]/2.0; // A_0 cosine fourier coef. is halved (sine coef. is trivially zero) + amrex::ParticleReal ezp = 0.0; // Variable to store Ez'(zz) (z-derivative of on-axis longitudinal field at z = zz) + amrex::ParticleReal ezpp = 0.0; // Variable to store Ez''(zz) + amrex::ParticleReal ezppp = 0.0; // Variable to store Ez'''(zz) + for (int k = 1; k < n_coef; ++k){ + ez = ez + e_coef[2*k-1]*cos(2*pi*k*zz/zlen) + e_coef[2*k]*sin(2.0*pi*k*zz/zlen); + ezp = ezp - (2.0*pi*k*zz/zlen)*e_coef[2*k-1]*sin(2.0*pi*k*zz/zlen) + (2.0*pi*k*zz/zlen)*e_coef[2*k]*cos(2.0*pi*k*zz/zlen); + ezpp = ezpp - pow(2.0*pi*k*zz/zlen,2)*e_coef[2*k-1]*cos(2.0*pi*k*zz/zlen) - pow(2.0*pi*k*zz/zlen,2)*e_coef[2*k]*sin(2.0*pi*k*zz/zlen); + ezppp = ezppp + pow(2.0*pi*k*zz/zlen,3)*e_coef[2*k-1]*sin(2.0*pi*k*zz/zlen) - pow(2.0*pi*k*zz/zlen,3)*e_coef[2*k]*cos(2.0*pi*k*zz/zlen); + } + ez = ez*scale; // Scale fields and derivatives by user-provided scaling factor (from input file) + ezp = ezp*scale; + ezpp = ezpp*scale; + ezppp = ezppp*scale; + + amrex::ParticleReal f1 = -(ezpp + ez*xl*xl)/4.0; + amrex::ParticleReal f1p = -(ezppp + ezp*xl*xl)/4.0; + amrex::ParticleReal r2 = x*x + y*y; + + Ex = -x*(ezp/2.0 + f1p*r2/4.0)*tmpcos; // Update E and B fields from Fourier coefficients and scaling factors + Ey = -y*(ezp/2.0 + f1p*r2/4.0)*tmpcos; + Ez = (ez + f1*r2)*tmpcos; + Bx = y*xl/c*(ez/2.0 + f1*r2/4.0)*tmpsin; + By = -x*xl/c*(ez/2.0 + f1*r2/4.0)*tmpsin; + Bz = 0.0; + + } + +}; + +#endif // WARPX_ACCELERATORLATTICE_LATTICEELEMENTS_SOLENOIDRF_H_ diff --git a/Source/AcceleratorLattice/LatticeElements/SolenoidRF.cpp b/Source/AcceleratorLattice/LatticeElements/SolenoidRF.cpp new file mode 100644 index 00000000000..b100cb8c5c4 --- /dev/null +++ b/Source/AcceleratorLattice/LatticeElements/SolenoidRF.cpp @@ -0,0 +1,91 @@ +/* Copyright 2023 David Grote and David Bizzozero + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "SolenoidRF.H" +#include "Utils/Parser/ParserUtils.H" + +#include +#include + +#include + +SolenoidRF::SolenoidRF () + : LatticeElementBase("solenoidrf") +{ +} + +void +SolenoidRF::AddElement (amrex::ParmParse & pp_element, amrex::ParticleReal & z_location) +{ + using namespace amrex::literals; + + AddElementBase(pp_element, z_location); + + amrex::ParticleReal scale = 0._prt; + amrex::ParticleReal freq = 0._prt; + amrex::ParticleReal theta = 0._prt; + amrex::Vector e_coef = {}; + amrex::Vector b_coef = {}; + + utils::parser::queryWithParser(pp_element, "scale", scale); + utils::parser::queryWithParser(pp_element, "freq", freq); + utils::parser::queryWithParser(pp_element, "theta", theta); + utils::parser::queryArrWithParser(pp_element, "e_coef", e_coef); + + h_scale.push_back(scale); + h_freq.push_back(freq); + h_theta.push_back(theta); + h_e_coef.push_back(e_coef); + +} + +void +SolenoidRF::WriteToDevice () +{ + WriteToDeviceBase(); + + d_scale.resize(h_scale.size()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scale.begin(), h_scale.end(), d_scale.begin()); + d_freq.resize(h_freq.size()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_freq.begin(), h_freq.end(), d_freq.begin()); + d_theta.resize(h_theta.size()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_theta.begin(), h_theta.end(), d_theta.begin()); + + d_e_coef.resize(h_e_coef.size()); + for (auto i = 0lu; i < h_e_coef.size(); ++i) { + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_e_coef[i].begin(), h_e_coef[i].end(), d_e_coef[i].begin()); + } + +} + +SolenoidRFDevice +SolenoidRF::GetDeviceInstance () const +{ + SolenoidRFDevice result; + result.InitSolenoidRFDevice(*this); + return result; +} + +void +SolenoidRFDevice::InitSolenoidRFDevice (SolenoidRF const& h_solenoidrf) +{ + + nelements = h_solenoidrf.nelements; + + if (nelements == 0) return; + + // remember pointers to device data in simple data structures + d_zs_arr = h_solenoidrf.d_zs.data(); + d_ze_arr = h_solenoidrf.d_ze.data(); + d_scale_arr = h_solenoidrf.d_scale.data(); + d_freq_arr = h_solenoidrf.d_freq.data(); + d_theta_arr = h_solenoidrf.d_theta.data(); + for (auto i = 0lu; i < nelements; ++i) { + d_num_e_coef[i] = h_solenoidrf.d_e_coef[i].size(); + d_e_coef_arr[i] = h_solenoidrf.d_e_coef[i].data(); + } + +} diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index a925cd6c9ef..5f41ecbd706 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -29,13 +29,14 @@ #include #include +#include #include #include struct Sigma : amrex::Gpu::DeviceVector { - int lo() const { return m_lo; } - int hi() const { return m_hi; } + [[nodiscard]] int lo() const { return m_lo; } + [[nodiscard]] int hi() const { return m_hi; } int m_lo, m_hi; }; @@ -80,7 +81,7 @@ public: const amrex::IntVect& ncell, const amrex::IntVect& delta, const amrex::Box& regular_domain, const amrex::Real v_sigma_sb) : m_grids(grid_ba), m_dx(dx), m_ncell(ncell), m_delta(delta), m_regdomain(regular_domain), m_v_sigma_sb(v_sigma_sb) {} - virtual ~SigmaBoxFactory () = default; + ~SigmaBoxFactory () override = default; SigmaBoxFactory (const SigmaBoxFactory&) = default; SigmaBoxFactory (SigmaBoxFactory&&) noexcept = default; @@ -89,15 +90,23 @@ public: SigmaBoxFactory& operator= (const SigmaBoxFactory&) = delete; SigmaBoxFactory& operator= (SigmaBoxFactory&&) = delete; - virtual SigmaBox* create (const amrex::Box& box, int /*ncomps*/, - const amrex::FabInfo& /*info*/, int /*box_index*/) const final - { return new SigmaBox(box, m_grids, m_dx, m_ncell, m_delta, m_regdomain, m_v_sigma_sb); } - virtual void destroy (SigmaBox* fab) const final { + [[nodiscard]] SigmaBox* create (const amrex::Box& box, int /*ncomps*/, + const amrex::FabInfo& /*info*/, int /*box_index*/) const final + { + return new SigmaBox(box, m_grids, m_dx, m_ncell, m_delta, m_regdomain, m_v_sigma_sb); + } + + void destroy (SigmaBox* fab) const final + { delete fab; } - virtual SigmaBoxFactory* clone () const final { + + [[nodiscard]] SigmaBoxFactory* + clone () const final + { return new SigmaBoxFactory(*this); } + private: const amrex::BoxArray& m_grids; const amrex::Real* m_dx; @@ -160,11 +169,15 @@ public: amrex::MultiFab* GetG_fp (); amrex::MultiFab* GetG_cp (); - const MultiSigmaBox& GetMultiSigmaBox_fp () const - { return *sigba_fp; } + [[nodiscard]] const MultiSigmaBox& GetMultiSigmaBox_fp () const + { + return *sigba_fp; + } - const MultiSigmaBox& GetMultiSigmaBox_cp () const - { return *sigba_cp; } + [[nodiscard]] const MultiSigmaBox& GetMultiSigmaBox_cp () const + { + return *sigba_cp; + } #ifdef WARPX_USE_PSATD void PushPSATD (int lev); @@ -187,25 +200,18 @@ public: void ExchangeG (amrex::MultiFab* G_fp, amrex::MultiFab* G_cp, int do_pml_in_domain); void ExchangeG (PatchType patch_type, amrex::MultiFab* Gp, int do_pml_in_domain); - void FillBoundary (); - void FillBoundaryE (); - void FillBoundaryB (); - void FillBoundaryF (); - void FillBoundaryG (); - void FillBoundaryE (PatchType patch_type); - void FillBoundaryB (PatchType patch_type); - void FillBoundaryF (PatchType patch_type); - void FillBoundaryG (PatchType patch_type); + void FillBoundaryE (PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundaryB (PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundaryF (PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundaryG (PatchType patch_type, std::optional nodal_sync=std::nullopt); - bool ok () const { return m_ok; } + [[nodiscard]] bool ok () const { return m_ok; } void CheckPoint (const std::string& dir) const; void Restart (const std::string& dir); static void Exchange (amrex::MultiFab& pml, amrex::MultiFab& reg, const amrex::Geometry& geom, int do_pml_in_domain); - ~PML () = default; - private: bool m_ok; @@ -247,9 +253,11 @@ private: // Factory for field data std::unique_ptr > pml_field_factory; - amrex::FabFactory const& fieldFactory () const noexcept { + [[nodiscard]] amrex::FabFactory const& fieldFactory () const noexcept + { return *pml_field_factory; } + #ifdef AMREX_USE_EB amrex::EBFArrayBoxFactory const& fieldEBFactory () const noexcept { return static_cast(*pml_field_factory); diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index d34a2f73222..1f0b6f09a33 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -513,7 +513,7 @@ MultiSigmaBox::MultiSigmaBox (const BoxArray& ba, const DistributionMapping& dm, void MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) { - if (dt == dt_B) return; + if (dt == dt_B) { return; } dt_B = dt; @@ -529,7 +529,7 @@ MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) void MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) { - if (dt == dt_E) return; + if (dt == dt_E) { return; } dt_E = dt; @@ -594,11 +594,13 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri } // Define the number of guard cells in each direction, for E, B, and F - IntVect nge = IntVect(AMREX_D_DECL(2, 2, 2)); - IntVect ngb = IntVect(AMREX_D_DECL(2, 2, 2)); + auto nge = IntVect(AMREX_D_DECL(2, 2, 2)); + auto ngb = IntVect(AMREX_D_DECL(2, 2, 2)); int ngf_int = 0; - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) ngf_int = std::max( ngf_int, 1 ); - IntVect ngf = IntVect(AMREX_D_DECL(ngf_int, ngf_int, ngf_int)); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) { + ngf_int = std::max( ngf_int, 1 ); + } + auto ngf = IntVect(AMREX_D_DECL(ngf_int, ngf_int, ngf_int)); if (do_moving_window) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(lev <= 1, @@ -622,11 +624,11 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri utils::parser::queryWithParser(pp_psatd, "nz_guard", ngFFt_z); #if defined(WARPX_DIM_3D) - IntVect ngFFT = IntVect(ngFFt_x, ngFFt_y, ngFFt_z); + auto ngFFT = IntVect(ngFFt_x, ngFFt_y, ngFFt_z); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - IntVect ngFFT = IntVect(ngFFt_x, ngFFt_z); + auto ngFFT = IntVect(ngFFt_x, ngFFt_z); #elif defined(WARPX_DIM_1D_Z) - IntVect ngFFT = IntVect(ngFFt_z); + auto ngFFT = IntVect(ngFFt_z); #endif // Set the number of guard cells to the maximum of each field @@ -738,11 +740,11 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const RealVect dx{AMREX_D_DECL(geom->CellSize(0), geom->CellSize(1), geom->CellSize(2))}; // Get the cell-centered box, with guard cells BoxArray realspace_ba = ba; // Copy box - amrex::Vector const v_galilean_zero = {0., 0., 0.}; + amrex::Vector const v_galilean = WarpX::GetInstance().m_v_galilean; amrex::Vector const v_comoving_zero = {0., 0., 0.}; realspace_ba.enclosedCells().grow(nge); // cell-centered + guard cells spectral_solver_fp = std::make_unique(lev, realspace_ba, dm, - nox_fft, noy_fft, noz_fft, grid_type, v_galilean_zero, + nox_fft, noy_fft, noz_fft, grid_type, v_galilean, v_comoving_zero, dx, dt, in_pml, periodic_single_box, update_with_rho, fft_do_time_averaging, psatd_solution_type, J_in_time, rho_in_time, m_dive_cleaning, m_divb_cleaning); #endif @@ -846,11 +848,11 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const RealVect cdx{AMREX_D_DECL(cgeom->CellSize(0), cgeom->CellSize(1), cgeom->CellSize(2))}; // Get the cell-centered box, with guard cells BoxArray realspace_cba = cba; // Copy box - amrex::Vector const v_galilean_zero = {0., 0., 0.}; + amrex::Vector const v_galilean = WarpX::GetInstance().m_v_galilean; amrex::Vector const v_comoving_zero = {0., 0., 0.}; realspace_cba.enclosedCells().grow(nge); // cell-centered + guard cells spectral_solver_cp = std::make_unique(lev, realspace_cba, cdm, - nox_fft, noy_fft, noz_fft, grid_type, v_galilean_zero, + nox_fft, noy_fft, noz_fft, grid_type, v_galilean, v_comoving_zero, cdx, dt, in_pml, periodic_single_box, update_with_rho, fft_do_time_averaging, psatd_solution_type, J_in_time, rho_in_time, m_dive_cleaning, m_divb_cleaning); #endif @@ -877,7 +879,8 @@ PML::MakeBoxArray_single (const amrex::Box& regular_domain, const amrex::BoxArra const amrex::IntVect& do_pml_Hi) { BoxList bl; - for (int i = 0, N = grid_ba.size(); i < N; ++i) { + const auto grid_ba_size = static_cast(grid_ba.size()); + for (int i = 0; i < grid_ba_size; ++i) { Box const& b = grid_ba[i]; for (OrientationIter oit; oit.isValid(); ++oit) { // In 3d, a Box has 6 faces. This iterates over the 6 faces. @@ -926,7 +929,8 @@ PML::MakeBoxArray_multiple (const amrex::Geometry& geom, const amrex::BoxArray& } } BoxList bl; - for (int i = 0, N = grid_ba.size(); i < N; ++i) + const auto grid_ba_size = static_cast(grid_ba.size()); + for (int i = 0; i < grid_ba_size; ++i) { const Box& grid_bx = grid_ba[i]; const IntVect& grid_bx_sz = grid_bx.size(); @@ -1072,9 +1076,9 @@ void PML::Exchange (const std::array& mf_pml, const int do_pml_in_domain) { const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; - if (mf_pml[0] && mf[0]) Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); - if (mf_pml[1] && mf[1]) Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); - if (mf_pml[2] && mf[2]) Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); + if (mf_pml[0] && mf[0]) { Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); } + if (mf_pml[1] && mf[1]) { Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); } + if (mf_pml[2] && mf[2]) { Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); } } void @@ -1229,103 +1233,66 @@ PML::CopyToPML (MultiFab& pml, MultiFab& reg, const Geometry& geom) } void -PML::FillBoundary () -{ - FillBoundaryE(); - FillBoundaryB(); - FillBoundaryF(); - FillBoundaryG(); -} - -void -PML::FillBoundaryE () -{ - FillBoundaryE(PatchType::fine); - FillBoundaryE(PatchType::coarse); -} - -void -PML::FillBoundaryE (PatchType patch_type) +PML::FillBoundaryE (PatchType patch_type, std::optional nodal_sync) { if (patch_type == PatchType::fine && pml_E_fp[0] && pml_E_fp[0]->nGrowVect().max() > 0) { const auto& period = m_geom->periodicity(); const Vector mf{pml_E_fp[0].get(),pml_E_fp[1].get(),pml_E_fp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } else if (patch_type == PatchType::coarse && pml_E_cp[0] && pml_E_cp[0]->nGrowVect().max() > 0) { const auto& period = m_cgeom->periodicity(); const Vector mf{pml_E_cp[0].get(),pml_E_cp[1].get(),pml_E_cp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } void -PML::FillBoundaryB () -{ - FillBoundaryB(PatchType::fine); - FillBoundaryB(PatchType::coarse); -} - -void -PML::FillBoundaryB (PatchType patch_type) +PML::FillBoundaryB (PatchType patch_type, std::optional nodal_sync) { if (patch_type == PatchType::fine && pml_B_fp[0]) { const auto& period = m_geom->periodicity(); const Vector mf{pml_B_fp[0].get(),pml_B_fp[1].get(),pml_B_fp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } else if (patch_type == PatchType::coarse && pml_B_cp[0]) { const auto& period = m_cgeom->periodicity(); const Vector mf{pml_B_cp[0].get(),pml_B_cp[1].get(),pml_B_cp[2].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } void -PML::FillBoundaryF () -{ - FillBoundaryF(PatchType::fine); - FillBoundaryF(PatchType::coarse); -} - -void -PML::FillBoundaryF (PatchType patch_type) +PML::FillBoundaryF (PatchType patch_type, std::optional nodal_sync) { if (patch_type == PatchType::fine && pml_F_fp && pml_F_fp->nGrowVect().max() > 0) { const auto& period = m_geom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_F_fp, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(*pml_F_fp, WarpX::do_single_precision_comms, period, nodal_sync); } else if (patch_type == PatchType::coarse && pml_F_cp && pml_F_cp->nGrowVect().max() > 0) { const auto& period = m_cgeom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_F_cp, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(*pml_F_cp, WarpX::do_single_precision_comms, period, nodal_sync); } } void -PML::FillBoundaryG () -{ - FillBoundaryG(PatchType::fine); - FillBoundaryG(PatchType::coarse); -} - -void -PML::FillBoundaryG (PatchType patch_type) +PML::FillBoundaryG (PatchType patch_type, std::optional nodal_sync) { if (patch_type == PatchType::fine && pml_G_fp && pml_G_fp->nGrowVect().max() > 0) { const auto& period = m_geom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_G_fp, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(*pml_G_fp, WarpX::do_single_precision_comms, period, nodal_sync); } else if (patch_type == PatchType::coarse && pml_G_cp && pml_G_cp->nGrowVect().max() > 0) { const auto& period = m_cgeom->periodicity(); - ablastr::utils::communication::FillBoundary(*pml_G_cp, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(*pml_G_cp, WarpX::do_single_precision_comms, period, nodal_sync); } } diff --git a/Source/BoundaryConditions/PMLComponent.H b/Source/BoundaryConditions/PMLComponent.H index 6fa4f8af784..83dca1e8b9a 100644 --- a/Source/BoundaryConditions/PMLComponent.H +++ b/Source/BoundaryConditions/PMLComponent.H @@ -9,7 +9,7 @@ /* In WarpX, the split fields of the PML (e.g. Eyx, Eyz) are stored as * components of a MultiFab (e.g. component 0 and 1 of the MultiFab for Ey) - * The correspondance between the component index (0,1) and its meaning + * The correspondence between the component index (0,1) and its meaning * (yx, yz, etc.) is defined in the present file */ struct PMLComp { diff --git a/Source/BoundaryConditions/PML_RZ.H b/Source/BoundaryConditions/PML_RZ.H index 72c901bf480..3a5a51770f8 100644 --- a/Source/BoundaryConditions/PML_RZ.H +++ b/Source/BoundaryConditions/PML_RZ.H @@ -22,6 +22,7 @@ #include #include +#include #include enum struct PatchType : int; @@ -45,14 +46,12 @@ public: void FillBoundaryE (); void FillBoundaryB (); - void FillBoundaryE (PatchType patch_type); - void FillBoundaryB (PatchType patch_type); + void FillBoundaryE (PatchType patch_type, std::optional nodal_sync=std::nullopt); + void FillBoundaryB (PatchType patch_type, std::optional nodal_sync=std::nullopt); void CheckPoint (const std::string& dir) const; void Restart (const std::string& dir); - ~PML_RZ () = default; - private: const int m_ncell; diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 98ed12e7d67..7eb011a6bb2 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -95,7 +95,7 @@ PML_RZ::ApplyDamping (amrex::MultiFab* Et_fp, amrex::MultiFab* Ez_fp, amrex::ParallelFor( tilebox, Et_fp->nComp(), [=] AMREX_GPU_DEVICE (int i, int j, int k, int icomp) { - const amrex::Real rr = static_cast(i - nr_damp_min); + const auto rr = static_cast(i - nr_damp_min); const amrex::Real wr = rr/nr_damp; const amrex::Real damp_factor = std::exp( -4._rt * cdt_over_dr * wr*wr ); @@ -134,13 +134,13 @@ PML_RZ::FillBoundaryE () } void -PML_RZ::FillBoundaryE (PatchType patch_type) +PML_RZ::FillBoundaryE (PatchType patch_type, std::optional nodal_sync) { if (patch_type == PatchType::fine && pml_E_fp[0] && pml_E_fp[0]->nGrowVect().max() > 0) { const amrex::Periodicity& period = m_geom->periodicity(); const Vector mf{pml_E_fp[0].get(),pml_E_fp[1].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } @@ -151,13 +151,13 @@ PML_RZ::FillBoundaryB () } void -PML_RZ::FillBoundaryB (PatchType patch_type) +PML_RZ::FillBoundaryB (PatchType patch_type, std::optional nodal_sync) { if (patch_type == PatchType::fine && pml_B_fp[0]) { const amrex::Periodicity& period = m_geom->periodicity(); const Vector mf{pml_B_fp[0].get(),pml_B_fp[1].get()}; - ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); + ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period, nodal_sync); } } diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 8f3c5a1b49b..af721d70b6d 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -51,13 +51,13 @@ void WarpX::DampPML (const int lev) { DampPML(lev, PatchType::fine); - if (lev > 0) DampPML(lev, PatchType::coarse); + if (lev > 0) { DampPML(lev, PatchType::coarse); } } void WarpX::DampPML (const int lev, PatchType patch_type) { - if (!do_pml) return; + if (!do_pml) { return; } WARPX_PROFILE("WarpX::DampPML()"); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) @@ -228,15 +228,15 @@ void WarpX::DampJPML (int lev) { DampJPML(lev, PatchType::fine); - if (lev > 0) DampJPML(lev, PatchType::coarse); + if (lev > 0) { DampJPML(lev, PatchType::coarse); } } void WarpX::DampJPML (int lev, PatchType patch_type) { - if (!do_pml) return; - if (!do_pml_j_damping) return; - if (!pml[lev]) return; + if (!do_pml) { return; } + if (!do_pml_j_damping) { return; } + if (!pml[lev]) { return; } WARPX_PROFILE("WarpX::DampJPML()"); diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index fc4b563915d..0864c00067b 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -41,6 +41,18 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) } } } + +#ifdef WARPX_DIM_RZ + if (patch_type == PatchType::fine) { + ApplyFieldBoundaryOnAxis(get_pointer_Efield_fp(lev, 0), + get_pointer_Efield_fp(lev, 1), + get_pointer_Efield_fp(lev, 2), lev); + } else { + ApplyFieldBoundaryOnAxis(get_pointer_Efield_cp(lev, 0), + get_pointer_Efield_cp(lev, 1), + get_pointer_Efield_cp(lev, 2), lev); + } +#endif } void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_dt_type) @@ -69,27 +81,123 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d applySilverMueller = true; } } - if(applySilverMueller) m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( + if(applySilverMueller) { m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( Efield_fp[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], WarpX::field_boundary_lo, WarpX::field_boundary_hi); + } } } + +#ifdef WARPX_DIM_RZ + if (patch_type == PatchType::fine) { + ApplyFieldBoundaryOnAxis(get_pointer_Bfield_fp(lev, 0), + get_pointer_Bfield_fp(lev, 1), + get_pointer_Bfield_fp(lev, 2), lev); + } else { + ApplyFieldBoundaryOnAxis(get_pointer_Bfield_cp(lev, 0), + get_pointer_Bfield_cp(lev, 1), + get_pointer_Bfield_cp(lev, 2), lev); + } +#endif } void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) PEC::ApplyPECtoRhofield(rho, lev, patch_type); + if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoRhofield(rho, lev, patch_type); } } void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); + if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); } +} + +#ifdef WARPX_DIM_RZ +// Applies the boundary conditions that are specific to the axis when in RZ. +void +WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) +{ + const amrex::IntVect ngE = get_ng_fieldgather(); + + constexpr int NODE = amrex::IndexType::NODE; + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Er, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Box const & tilebox = mfi.tilebox(); + + // Lower corner of tile box physical domain + // Note that this is done before the tilebox.grow so that + // these do not include the guard cells. + const std::array& xyzmin = LowerCorner(tilebox, lev, 0._rt); + const amrex::Real rmin = xyzmin[0]; + + // Skip blocks that don't touch the axis + if (rmin > 0._rt) { continue; } + + amrex::Array4 const& Er_arr = Er->array(mfi); + amrex::Array4 const& Et_arr = Et->array(mfi); + amrex::Array4 const& Ez_arr = Ez->array(mfi); + + amrex::Box tbr = amrex::convert( tilebox, Er->ixType().toIntVect() ); + amrex::Box tbt = amrex::convert( tilebox, Et->ixType().toIntVect() ); + amrex::Box tbz = amrex::convert( tilebox, Ez->ixType().toIntVect() ); + + // For ishift, 1 means cell centered, 0 means node centered + int const ishift_r = (tbr.type(0) != NODE); + int const ishift_t = (tbt.type(0) != NODE); + int const ishift_z = (tbz.type(0) != NODE); + + // Set tileboxes to only include the axis guard cells + // (including the corners in z). + tbr.setRange(0, -ngE[0], ngE[0]); + tbt.setRange(0, -ngE[0], ngE[0]); + tbz.setRange(0, -ngE[0], ngE[0]); + tbr.grow(1, ngE[1]); + tbt.grow(1, ngE[1]); + tbz.grow(1, ngE[1]); + + const int nmodes = n_rz_azimuthal_modes; + + amrex::ParallelFor(tbr, tbt, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Er_arr(i,j,0,0) = -Er_arr(-i-ishift_r,j,0,0); + + for (int imode=1 ; imode < nmodes ; imode++) { + Er_arr(i,j,0,2*imode-1) = std::pow(-1._rt, imode+1._rt)*Er_arr(-i-ishift_r,j,0,2*imode-1); + Er_arr(i,j,0,2*imode) = std::pow(-1._rt, imode+1._rt)*Er_arr(-i-ishift_r,j,0,2*imode); + } + }, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Et_arr(i,j,0,0) = -Et_arr(-i-ishift_t,j,0,0); + + for (int imode=1 ; imode < nmodes ; imode++) { + Et_arr(i,j,0,2*imode-1) = std::pow(-1._rt, imode+1._rt)*Et_arr(-i-ishift_t,j,0,2*imode-1); + Et_arr(i,j,0,2*imode) = std::pow(-1._rt, imode+1._rt)*Et_arr(-i-ishift_t,j,0,2*imode); + } + }, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Ez_arr(i,j,0,0) = Ez_arr(-i-ishift_z,j,0,0); + + for (int imode=1 ; imode < nmodes ; imode++) { + Ez_arr(i,j,0,2*imode-1) = -std::pow(-1._rt, imode+1._rt)*Ez_arr(-i-ishift_z,j,0,2*imode-1); + Ez_arr(i,j,0,2*imode) = -std::pow(-1._rt, imode+1._rt)*Ez_arr(-i-ishift_z,j,0,2*imode); + } + + }); + } } +#endif void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index 78da9349d0d..62b92ba94b5 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -147,7 +147,7 @@ using namespace amrex; amrex::GpuArray const& fbndry_lo, amrex::GpuArray const& fbndry_hi ) { - // Tangential Efield componentes in guard cells set equal and opposite to cells + // Tangential Efield components in guard cells set equal and opposite to cells // in the mirror locations across the PEC boundary, whereas normal E-field // components are set equal to values in the mirror locations across the PEC // boundary. Here we just initialize it. @@ -191,7 +191,17 @@ using namespace amrex; : (dom_hi[idim] + 1 - ig)); GuardCell = true; // tangential components are inverted across PEC boundary - if (is_tangent_to_PEC) sign *= -1._rt; + if (is_tangent_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drEr/dr = 0. + // This only works for the first guard cell and with + // Er cell centered in r. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif } } // is PEC boundary } // loop over iside @@ -319,7 +329,15 @@ using namespace amrex; : (dom_hi[idim] + 1 - ig)); GuardCell = true; // Sign of the normal component in guard cell is inverted - if (is_normal_to_PEC) sign *= -1._rt; + if (is_normal_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drBr/dr = 0. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif } } // if PEC Boundary } // loop over sides @@ -373,37 +391,38 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } // Get the mirror guard cell index amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - // On the PEC boundary the current density is set to 0 - if (ijk_vec == iv_mirror) field(ijk_vec, n) = 0._rt; + // On the PEC boundary the charge/current density is set to 0 + if (ijk_vec == iv_mirror) { + field(ijk_vec, n) = 0._rt; // otherwise update the internal cell if the mirror guard cell exists - else if (fabbox.contains(iv_mirror)) - { + } else if (fabbox.contains(iv_mirror)) { field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); } } } // 2) The guard cells are updated with the appropriate image - // charge current based on the current in the valid cells + // charge based on the charge/current in the valid cells for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) { - if (tangent_to_bndy[idim]) + if (tangent_to_bndy[idim]) { field(iv_mirror, n) = -field(ijk_vec, n); - else + } else { field(iv_mirror, n) = field(ijk_vec, n); + } } } } @@ -436,7 +455,7 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } // Get the mirror guard cell index amrex::IntVect iv_mirror = ijk_vec; @@ -446,7 +465,7 @@ using namespace amrex; // first value in the domain (nodal fields) if (ijk_vec == iv_mirror) { iv_mirror[idim] += (iside == 0) ? 1 : -1; - if (fabbox.contains(iv_mirror)) field(ijk_vec, n) = field(iv_mirror, n); + if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } } // otherwise set the mirror guard cell equal to the internal cell value else if (fabbox.contains(iv_mirror)) diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index b3986da0645..4692d8e980f 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -18,8 +18,8 @@ using namespace amrex::literals; bool PEC::isAnyBoundaryPEC() { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) return true; - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) return true; + if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) { return true; } + if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) { return true; } } return false; } @@ -250,8 +250,8 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } // rho values inside guard cells are updated the same as tangential // components of the current density @@ -266,16 +266,6 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty } const int nComp = rho->nComp(); -#ifdef WARPX_DIM_RZ - if (is_pec[0][1]) { - ablastr::warn_manager::WMRecordWarning( - "PEC", - "PEC boundary handling is not yet properly implemented for r_max so it is skipped in PEC::ApplyPECtoRhofield", - ablastr::warn_manager::WarnPriority::medium); - is_pec[0][1] = false; - } -#endif - #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -286,7 +276,7 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& rho_array = rho->array(mfi); @@ -352,8 +342,8 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } for (int icomp=0; icomp < 3; ++icomp) { // Set the psign value correctly for each current component for each @@ -396,16 +386,6 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, mirrorfac[2][idim][1] = 2*domain_hi[idim] - (1 - Jz_nodal[idim]); } -#ifdef WARPX_DIM_RZ - if (is_pec[0][1]) { - ablastr::warn_manager::WMRecordWarning( - "PEC", - "PEC boundary handling is not yet properly implemented for r_max so it is skipped in PEC::ApplyPECtoJfield", - ablastr::warn_manager::WarnPriority::medium); - is_pec[0][1] = false; - } -#endif - // Each current component is handled separately below, starting with Jx. #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -418,7 +398,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jx_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jx_array = Jx->array(mfi); @@ -453,7 +433,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jy_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jy_array = Jy->array(mfi); @@ -488,7 +468,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jz_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jz_array = Jz->array(mfi); @@ -540,24 +520,14 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - Pe_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - Pe_nodal[idim]); } const int nComp = Pefield->nComp(); -#ifdef WARPX_DIM_RZ - if (is_pec[0][1]) { - ablastr::warn_manager::WMRecordWarning( - "PEC", - "PEC boundary handling is not yet properly implemented for r_max so it is skipped in PEC::ApplyPECtoPefield", - ablastr::warn_manager::WarnPriority::medium); - is_pec[0][1] = false; - } -#endif - #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -568,7 +538,7 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Pe_array = Pefield->array(mfi); diff --git a/Source/BoundaryConditions/WarpX_PML_kernels.H b/Source/BoundaryConditions/WarpX_PML_kernels.H index 1396613b4f5..7aaf5a57369 100644 --- a/Source/BoundaryConditions/WarpX_PML_kernels.H +++ b/Source/BoundaryConditions/WarpX_PML_kernels.H @@ -278,12 +278,12 @@ void warpx_damp_pml_bx (int i, int j, int k, amrex::Array4 const& B } } - // Bxz - if (sz == 0) { - Bx(i,j,k,PMLComp::xz) *= sigma_star_fac_z[j-zlo]; - } else { - Bx(i,j,k,PMLComp::xz) *= sigma_fac_z[j-zlo]; - } + // Bxz + if (sz == 0) { + Bx(i,j,k,PMLComp::xz) *= sigma_star_fac_z[j-zlo]; + } else { + Bx(i,j,k,PMLComp::xz) *= sigma_fac_z[j-zlo]; + } #elif defined(WARPX_DIM_3D) @@ -302,19 +302,19 @@ void warpx_damp_pml_bx (int i, int j, int k, amrex::Array4 const& B } } - // Bxy - if (sy == 0) { - Bx(i,j,k,PMLComp::xy) *= sigma_star_fac_y[j-ylo]; - } else { - Bx(i,j,k,PMLComp::xy) *= sigma_fac_y[j-ylo]; - } + // Bxy + if (sy == 0) { + Bx(i,j,k,PMLComp::xy) *= sigma_star_fac_y[j-ylo]; + } else { + Bx(i,j,k,PMLComp::xy) *= sigma_fac_y[j-ylo]; + } - // Bxz - if (sz == 0) { - Bx(i,j,k,PMLComp::xz) *= sigma_star_fac_z[k-zlo]; - } else { - Bx(i,j,k,PMLComp::xz) *= sigma_fac_z[k-zlo]; - } + // Bxz + if (sz == 0) { + Bx(i,j,k,PMLComp::xz) *= sigma_star_fac_z[k-zlo]; + } else { + Bx(i,j,k,PMLComp::xz) *= sigma_fac_z[k-zlo]; + } #endif } @@ -345,19 +345,19 @@ void warpx_damp_pml_by (int i, int j, int k, amrex::Array4 const& B const int sx = By_stag[0]; const int sz = By_stag[1]; - // Byx - if (sx == 0) { - By(i,j,k,PMLComp::yx) *= sigma_star_fac_x[i-xlo]; - } else { - By(i,j,k,PMLComp::yx) *= sigma_fac_x[i-xlo]; - } + // Byx + if (sx == 0) { + By(i,j,k,PMLComp::yx) *= sigma_star_fac_x[i-xlo]; + } else { + By(i,j,k,PMLComp::yx) *= sigma_fac_x[i-xlo]; + } - // Byz - if (sz == 0) { - By(i,j,k,PMLComp::yz) *= sigma_star_fac_z[j-zlo]; - } else { - By(i,j,k,PMLComp::yz) *= sigma_fac_z[j-zlo]; - } + // Byz + if (sz == 0) { + By(i,j,k,PMLComp::yz) *= sigma_star_fac_z[j-zlo]; + } else { + By(i,j,k,PMLComp::yz) *= sigma_fac_z[j-zlo]; + } #elif defined(WARPX_DIM_3D) @@ -366,12 +366,12 @@ void warpx_damp_pml_by (int i, int j, int k, amrex::Array4 const& B const int sy = By_stag[1]; const int sz = By_stag[2]; - // Byx - if (sx == 0) { - By(i,j,k,PMLComp::yx) *= sigma_star_fac_x[i-xlo]; - } else { - By(i,j,k,PMLComp::yx) *= sigma_fac_x[i-xlo]; - } + // Byx + if (sx == 0) { + By(i,j,k,PMLComp::yx) *= sigma_star_fac_x[i-xlo]; + } else { + By(i,j,k,PMLComp::yx) *= sigma_fac_x[i-xlo]; + } if (divb_cleaning) { @@ -383,12 +383,12 @@ void warpx_damp_pml_by (int i, int j, int k, amrex::Array4 const& B } } - // Byz - if (sz == 0) { - By(i,j,k,PMLComp::yz) *= sigma_star_fac_z[k-zlo]; - } else { - By(i,j,k,PMLComp::yz) *= sigma_fac_z[k-zlo]; - } + // Byz + if (sz == 0) { + By(i,j,k,PMLComp::yz) *= sigma_star_fac_z[k-zlo]; + } else { + By(i,j,k,PMLComp::yz) *= sigma_fac_z[k-zlo]; + } #endif } @@ -419,12 +419,12 @@ void warpx_damp_pml_bz (int i, int j, int k, amrex::Array4 const& B const int sx = Bz_stag[0]; const int sz = Bz_stag[1]; - // Bzx - if (sx == 0) { - Bz(i,j,k,PMLComp::zx) *= sigma_star_fac_x[i-xlo]; - } else { - Bz(i,j,k,PMLComp::zx) *= sigma_fac_x[i-xlo]; - } + // Bzx + if (sx == 0) { + Bz(i,j,k,PMLComp::zx) *= sigma_star_fac_x[i-xlo]; + } else { + Bz(i,j,k,PMLComp::zx) *= sigma_fac_x[i-xlo]; + } if (divb_cleaning) { @@ -443,19 +443,19 @@ void warpx_damp_pml_bz (int i, int j, int k, amrex::Array4 const& B const int sy = Bz_stag[1]; const int sz = Bz_stag[2]; - // Bzx - if (sx == 0) { - Bz(i,j,k,PMLComp::zx) *= sigma_star_fac_x[i-xlo]; - } else { - Bz(i,j,k,PMLComp::zx) *= sigma_fac_x[i-xlo]; - } + // Bzx + if (sx == 0) { + Bz(i,j,k,PMLComp::zx) *= sigma_star_fac_x[i-xlo]; + } else { + Bz(i,j,k,PMLComp::zx) *= sigma_fac_x[i-xlo]; + } - // Bzy - if (sy == 0) { - Bz(i,j,k,PMLComp::zy) *= sigma_star_fac_y[j-ylo]; - } else { - Bz(i,j,k,PMLComp::zy) *= sigma_fac_y[j-ylo]; - } + // Bzy + if (sy == 0) { + Bz(i,j,k,PMLComp::zy) *= sigma_star_fac_y[j-ylo]; + } else { + Bz(i,j,k,PMLComp::zy) *= sigma_fac_y[j-ylo]; + } if (divb_cleaning) { @@ -495,19 +495,19 @@ void warpx_damp_pml_scalar (int i, int j, int k, amrex::Array4 cons const int sx = arr_stag[0]; const int sz = arr_stag[1]; - // Component along x - if (sx == 0) { - arr(i,j,k,PMLComp::x) *= sigma_star_fac_x[i-xlo]; - } else { - arr(i,j,k,PMLComp::x) *= sigma_fac_x[i-xlo]; - } + // Component along x + if (sx == 0) { + arr(i,j,k,PMLComp::x) *= sigma_star_fac_x[i-xlo]; + } else { + arr(i,j,k,PMLComp::x) *= sigma_fac_x[i-xlo]; + } - // Component along z - if (sz == 0) { - arr(i,j,k,PMLComp::z) *= sigma_star_fac_z[j-zlo]; - } else { - arr(i,j,k,PMLComp::z) *= sigma_fac_z[j-zlo]; - } + // Component along z + if (sz == 0) { + arr(i,j,k,PMLComp::z) *= sigma_star_fac_z[j-zlo]; + } else { + arr(i,j,k,PMLComp::z) *= sigma_fac_z[j-zlo]; + } #elif defined(WARPX_DIM_3D) @@ -516,26 +516,26 @@ void warpx_damp_pml_scalar (int i, int j, int k, amrex::Array4 cons const int sy = arr_stag[1]; const int sz = arr_stag[2]; - // Component along x - if (sx == 0) { - arr(i,j,k,PMLComp::x) *= sigma_star_fac_x[i-xlo]; - } else { - arr(i,j,k,PMLComp::x) *= sigma_fac_x[i-xlo]; - } - - // Component along y - if (sy == 0) { - arr(i,j,k,PMLComp::y) *= sigma_star_fac_y[j-ylo]; - } else { - arr(i,j,k,PMLComp::y) *= sigma_fac_y[j-ylo]; - } - - // Component along z - if (sz == 0) { - arr(i,j,k,PMLComp::z) *= sigma_star_fac_z[k-zlo]; - } else { - arr(i,j,k,PMLComp::z) *= sigma_fac_z[k-zlo]; - } + // Component along x + if (sx == 0) { + arr(i,j,k,PMLComp::x) *= sigma_star_fac_x[i-xlo]; + } else { + arr(i,j,k,PMLComp::x) *= sigma_fac_x[i-xlo]; + } + + // Component along y + if (sy == 0) { + arr(i,j,k,PMLComp::y) *= sigma_star_fac_y[j-ylo]; + } else { + arr(i,j,k,PMLComp::y) *= sigma_fac_y[j-ylo]; + } + + // Component along z + if (sz == 0) { + arr(i,j,k,PMLComp::z) *= sigma_star_fac_z[k-zlo]; + } else { + arr(i,j,k,PMLComp::z) *= sigma_fac_z[k-zlo]; + } #endif } diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index d8038a48f3d..209113dedeb 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -31,48 +31,46 @@ public: * \param[in] Headerfile_path string containing path of Headerfile */ BTDPlotfileHeaderImpl (std::string const& Headerfile_path); - /** Destructor */ - ~BTDPlotfileHeaderImpl () = default; /** Returns the Header file version for plotfile */ - std::string fileVersion () const noexcept {return m_file_version; } + [[nodiscard]] std::string fileVersion () const noexcept {return m_file_version; } /** Returns the number of components written in the Headerfile */ - int ncomp () const noexcept {return m_nComp; } + [[nodiscard]] int ncomp () const noexcept {return m_nComp; } /** Returns the names of components in the Headerfile */ - const amrex::Vector& varnames () const noexcept {return m_varnames; } + [[nodiscard]] const amrex::Vector& varnames () const noexcept {return m_varnames; } /** Returns the number of dimensions in the Headerfile */ - int spaceDim () const noexcept {return m_spacedim; } + [[nodiscard]] int spaceDim () const noexcept {return m_spacedim; } /** Returns the physical time in the simulation in the boosted-frame */ - amrex::Real time () const noexcept {return m_time; } + [[nodiscard]] amrex::Real time () const noexcept {return m_time; } /** Returns finest level output in the Headerfile */ - int finestLevel () const noexcept { return m_finest_level; } + [[nodiscard]] int finestLevel () const noexcept { return m_finest_level; } /** Returns the physical co-ordinates of the lower corner in dimension, idim, * that corresponds the to the respective plotfile data */ - amrex::Real problo (int dim) const noexcept {return m_prob_lo[dim]; } + [[nodiscard]] amrex::Real problo (int dim) const noexcept {return m_prob_lo[dim]; } /** Returns the physical co-ordinates of the upper corner in dimension, idim, * that corresponds the to the respective plotfile data. */ - amrex::Real probhi (int dim) const noexcept {return m_prob_hi[dim]; } + [[nodiscard]] amrex::Real probhi (int dim) const noexcept {return m_prob_hi[dim]; } /** Returns the bounding box of the domain spanned in the plotfile */ - amrex::Box probDomain () const noexcept {return m_prob_domain; } + [[nodiscard]] amrex::Box probDomain () const noexcept {return m_prob_domain; } /** Returns timestep at which the plotfile was written */ - int timestep () const noexcept {return m_timestep; } - int numFabs () const noexcept {return m_numFabs; } + [[nodiscard]] int timestep () const noexcept {return m_timestep; } + [[nodiscard]] int numFabs () const noexcept {return m_numFabs; } /** Returns physical co-ordinates of the lower-corner for the i-th Fab. * \param[in] iFab id of the ith Fab in the list of Multifabs * \returns Array of lower-corner physical co-ordinates corresponding to the ith Fab */ - amrex::Array FabLo (int iFab) const noexcept {return m_glo[iFab]; } + [[nodiscard]] amrex::Array FabLo (int iFab) const noexcept {return m_glo[iFab]; } /** Returns physical co-ordinates of the lower-corner for the i-th Fab. * \param[in] iFab id of the ith Fab in the list of Multifabs * \returns Array of lower-corner physical co-ordinates corresponding to the ith Fab */ - amrex::Array FabHi (int iFab) const noexcept {return m_ghi[iFab]; } + [[nodiscard]] amrex::Array FabHi (int iFab) const noexcept {return m_ghi[iFab]; } /** Returns path to location of multifabs */ - std::string CellPath () const noexcept {return m_CellPath; } + [[nodiscard]] std::string CellPath () const noexcept {return m_CellPath; } /** Reads the Header file data for BTD */ void ReadHeaderData (); @@ -166,7 +164,7 @@ class BTDMultiFabHeaderImpl * \param[in] Headerfile_path string containing path of Headerfile */ BTDMultiFabHeaderImpl (std::string const& Headerfile_path); - ~BTDMultiFabHeaderImpl () = default; + /** Reads the Multifab Header file and stores its data. */ void ReadMultiFabHeader (); /** Writes the meta-data of the Multifab in a header file, with path, m_Header_path. */ @@ -249,7 +247,7 @@ public: * \param[in] species_name string containing species name */ BTDSpeciesHeaderImpl (std::string const& Headerfile_path, std::string const& species_name); - ~BTDSpeciesHeaderImpl () = default; + /** Reads the Header file for BTD species*/ void ReadHeader (); /** Writes the meta-data of species Header file, with path, m_Header_path*/ @@ -314,8 +312,7 @@ public: * \param[in] Headerfile_path containing path of Headerfile */ BTDParticleDataHeaderImpl (std::string const& Headerfile_path); - /** Destructor */ - ~BTDParticleDataHeaderImpl () = default; + /** Reads the particle header file at m_Header_path and stores its data*/ void ReadHeader (); /** Writes the meta-data of particle box array in header file, with path, m_Header_path*/ diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp index 9f4976c20cb..8486a487f40 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp @@ -118,7 +118,7 @@ BTDPlotfileHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); @@ -241,7 +241,7 @@ BTDMultiFabHeaderImpl::ReadMultiFabHeader () m_minval[ifab].resize(m_ncomp); for (int icomp = 0; icomp < m_ncomp; ++icomp) { is >> m_minval[ifab][icomp] >> ch; - if( ch != ',' ) amrex::Error("Expected a ',' got something else"); + if( ch != ',' ) { amrex::Error("Expected a ',' got something else"); } } } ablastr::utils::text::goto_next_line(is); @@ -251,7 +251,7 @@ BTDMultiFabHeaderImpl::ReadMultiFabHeader () m_maxval[ifab].resize(m_ncomp); for (int icomp = 0; icomp < m_ncomp; ++icomp) { is >> m_maxval[ifab][icomp] >> ch; - if( ch != ',' ) amrex::Error("Expected a ',' got something else"); + if( ch != ',' ) { amrex::Error("Expected a ',' got something else"); } } } @@ -266,9 +266,9 @@ BTDMultiFabHeaderImpl::WriteMultiFabHeader () } std::ofstream FabHeaderFile; FabHeaderFile.open(m_Header_path.c_str(), std::ofstream::out | - std::ofstream::trunc | - std::ofstream::binary); - if ( !FabHeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + std::ofstream::trunc | + std::ofstream::binary); + if ( !FabHeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } FabHeaderFile.precision(17); @@ -421,7 +421,7 @@ BTDSpeciesHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); @@ -525,7 +525,7 @@ BTDParticleDataHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); m_ba.writeOn(HeaderFile); diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index 9356fdbdaa2..8940a89711a 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -24,8 +24,7 @@ #include #include -class -BTDiagnostics final : public Diagnostics +class BTDiagnostics final : public Diagnostics { public: @@ -47,7 +46,7 @@ private: * can be defined and implemented in Diagnostics.H, such that, * the function call to flush out buffer data for * FullDiagnostics and BTDiagnostics is the same */ - void Flush (int i_buffer) override; + void Flush (int i_buffer, bool force_flush) override; /** whether to write output files at this time step * The data is flushed when the buffer is full and/or * when the simulation ends or when forced. @@ -381,8 +380,8 @@ private: /** Interleave meta-data of the buffer multifabs to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveFabArrayHeader( std::string Buffer_FabHeaderFilename, - std::string snapshot_FabHeaderFilename, + void InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, + std::string snapshot_FabHeader_path, std::string newsnapshot_FabFilename); /** Interleave lab-frame metadata of the species header file in the buffers to * be consistent with the merged plotfile lab-frame data diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index b79132e83db..fd939b0370b 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -112,8 +112,9 @@ void BTDiagnostics::DerivedInitData () const amrex::ParmParse pp_diag_name(m_diag_name); int write_species = 1; pp_diag_name.query("write_species", write_species); - if ((m_output_species_names.empty()) && (write_species == 1)) + if ((m_output_species_names.empty()) && (write_species == 1)) { m_output_species_names = mpc.GetSpeciesNames(); + } m_do_back_transformed_particles = ((!m_output_species_names.empty()) && (write_species == 1)); @@ -233,7 +234,7 @@ BTDiagnostics::ReadParameters () pp_diag_name.query("do_back_transformed_fields", m_do_back_transformed_fields); pp_diag_name.query("do_back_transformed_particles", m_do_back_transformed_particles); AMREX_ALWAYS_ASSERT(m_do_back_transformed_fields or m_do_back_transformed_particles); - if (!m_do_back_transformed_fields) m_varnames.clear(); + if (!m_do_back_transformed_fields) { m_varnames.clear(); } std::vector intervals_string_vec = {"0"}; @@ -348,7 +349,7 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) amrex::RealBox diag_dom; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim ) { // Setting lo-coordinate for the diag domain by taking the max of user-defined - // lo-cordinate and lo-coordinat of the simulation domain at level, lev + // lo-cordinate and lo-coordinate of the simulation domain at level, lev diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); // Setting hi-coordinate for the diag domain by taking the max of user-defined // hi-cordinate and hi-coordinate of the simulation domain at level, lev @@ -394,7 +395,7 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) diag_dom.setLo( idim, warpx.Geom(lev).ProbLo(idim) + diag_ba.getCellCenteredBox(0).smallEnd(idim) * warpx.Geom(lev).CellSize(idim)); diag_dom.setHi( idim, warpx.Geom(lev).ProbLo(idim) + - (diag_ba.getCellCenteredBox( diag_ba.size()-1 ).bigEnd(idim) + 1) * warpx.Geom(lev).CellSize(idim)); + (diag_ba.getCellCenteredBox( static_cast(diag_ba.size()-1) ).bigEnd(idim) + 1) * warpx.Geom(lev).CellSize(idim)); } // Define buffer_domain in lab-frame for the i^th snapshot. @@ -423,12 +424,12 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) // For the z-dimension, number of cells in the lab-frame is // computed using the coarsened cell-size in the lab-frame obtained using // the ref_ratio at level, lev-1. - amrex::IntVect ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); + auto ref_ratio = amrex::IntVect(1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } // Number of lab-frame cells in z-direction at level, lev const int num_zcells_lab = static_cast( std::floor ( ( zmax_buffer_lab - zmin_buffer_lab) - / dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]) ) ); + / dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]))); // Take the max of 0 and num_zcells_lab const int Nz_lab = std::max( 0, num_zcells_lab ); #if (AMREX_SPACEDIM >= 2) @@ -502,7 +503,7 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) void BTDiagnostics::DefineCellCenteredMultiFab(int lev) { - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } // Creating MultiFab to store cell-centered data in boosted-frame for the entire-domain // This MultiFab will store all the user-requested fields in the boosted-frame auto & warpx = WarpX::GetInstance(); @@ -524,7 +525,7 @@ void BTDiagnostics::InitializeFieldFunctors (int lev) { // Initialize fields functors only if do_back_transformed_fields is selected - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } #ifdef WARPX_DIM_RZ // For RZ, initialize field functors RZ for openpmd @@ -561,7 +562,9 @@ BTDiagnostics::InitializeFieldFunctors (int lev) // Fill vector of cell-center functors for all field-components, namely, // Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, and rho are included in the // cell-center functors for BackTransform Diags - for (int comp=0, n=m_cell_center_functors.at(lev).size(); comp( + m_cell_center_functors.at(lev).size()); + for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ey" ){ @@ -600,24 +603,25 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () const bool update_varnames = true; if (update_varnames) { - const int n_rz = ncomp * m_varnames_fields.size(); + const auto n_rz = ncomp * static_cast(m_varnames_fields.size()); m_varnames.clear(); m_varnames.reserve(n_rz); } // AddRZ modes to output names for the back-transformed data if (update_varnames) { - for (int comp=0, n=m_varnames_fields.size(); comp(m_varnames_fields.size()); + for (int comp=0; comp(m_cellcenter_varnames.size()); m_cellcenter_varnames.clear(); m_cellcenter_varnames.reserve(n_rz); - for (int comp=0, n=m_cellcenter_varnames_fields.size(); comp(m_cellcenter_varnames_fields.size()); + for (int comp=0; comp(m_cell_center_functors.at(lev).size()); + for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Et" ){ @@ -765,10 +771,11 @@ BTDiagnostics::UpdateBufferData () for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer ) { const bool ZSliceInDomain = GetZSliceInDomainFlag (i_buffer, lev); - if (ZSliceInDomain) ++m_buffer_counter[i_buffer]; + if (ZSliceInDomain) { ++m_buffer_counter[i_buffer]; } // when the z-index is equal to the smallEnd of the snapshot box, then set lastValidZSlice to 1 - if (k_index_zlab(i_buffer, lev) == m_snapshot_box[i_buffer].smallEnd(m_moving_window_dir)) + if (k_index_zlab(i_buffer, lev) == m_snapshot_box[i_buffer].smallEnd(m_moving_window_dir)) { m_lastValidZSlice[i_buffer] = 1; + } } } } @@ -778,15 +785,16 @@ void BTDiagnostics::PrepareFieldDataForOutput () { // Initialize fields functors only if do_back_transformed_fields is selected - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } auto & warpx = WarpX::GetInstance(); // In this function, we will get cell-centered data for every level, lev, - // using the cell-center functors and their respective opeators() + // using the cell-center functors and their respective operators() // Call m_cell_center_functors->operator for (int lev = 0; lev < nmax_lev; ++lev) { int icomp_dst = 0; - for (int icomp = 0, n=m_cell_center_functors.at(lev).size(); icomp(m_cell_center_functors.at(lev).size()); + for (int icomp = 0; icomp 0; --lev) { ablastr::coarsen::sample::Coarsen(*m_cell_centered_data[lev - 1], *m_cell_centered_data[lev], 0, 0, - m_cellcenter_varnames.size(), 0, WarpX::RefRatio(lev-1) ); + static_cast(m_cellcenter_varnames.size()), 0, WarpX::RefRatio(lev-1) ); } const int num_BT_functors = 1; @@ -865,9 +873,9 @@ BTDiagnostics::k_index_zlab (int i_buffer, int lev) { auto & warpx = WarpX::GetInstance(); const amrex::Real prob_domain_zmin_lab = m_snapshot_domain_lab[i_buffer].lo( m_moving_window_dir ); - amrex::IntVect ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); - const int k_lab = static_cast(floor ( + auto ref_ratio = amrex::IntVect(1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } + const int k_lab = static_cast(std::floor ( ( m_current_z_lab[i_buffer] - (prob_domain_zmin_lab ) ) / dz_lab( warpx.getdt(lev), ref_ratio[m_moving_window_dir] ) @@ -878,17 +886,16 @@ BTDiagnostics::k_index_zlab (int i_buffer, int lev) void BTDiagnostics::SetSnapshotFullStatus (const int i_buffer) { - if (m_snapshot_full[i_buffer] == 1) return; - // if the last valid z-index of the snapshot, which is 0, is filled, then - // set the snapshot full integer to 1 - if (m_lastValidZSlice[i_buffer] == 1) m_snapshot_full[i_buffer] = 1; - + if (m_snapshot_full[i_buffer] == 1) { return; } + // if the last valid z-index of the snapshot, which is 0, is filled, then + // set the snapshot full integer to 1 + if (m_lastValidZSlice[i_buffer] == 1) { m_snapshot_full[i_buffer] = 1; } } void BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) { - if (m_field_buffer_multifab_defined[i_buffer] == 1) return; + if (m_field_buffer_multifab_defined[i_buffer] == 1) { return; } auto & warpx = WarpX::GetInstance(); const int hi_k_lab = m_buffer_k_index_hi[i_buffer]; @@ -901,11 +908,11 @@ BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) // Unlike FullDiagnostics, "m_format == sensei" option is not included here. const int ngrow = 0; m_mf_output[i_buffer][lev] = amrex::MultiFab( buffer_ba, buffer_dmap, - m_varnames.size(), ngrow ); + static_cast(m_varnames.size()), ngrow ); m_mf_output[i_buffer][lev].setVal(0.); - amrex::IntVect ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); + auto ref_ratio = amrex::IntVect(1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::Real cellsize = (idim < WARPX_ZINDEX)? warpx.Geom(lev).CellSize(idim): @@ -915,7 +922,7 @@ BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) - m_snapshot_box[i_buffer].smallEnd(idim) ) * cellsize; const amrex::Real buffer_hi = m_snapshot_domain_lab[i_buffer].lo(idim) - + ( buffer_ba.getCellCenteredBox( buffer_ba.size()-1 ).bigEnd(idim) + + ( buffer_ba.getCellCenteredBox( static_cast(buffer_ba.size()-1) ).bigEnd(idim) - m_snapshot_box[i_buffer].smallEnd(idim) + 1 ) * cellsize; m_buffer_domain_lab[i_buffer].setLo(idim, buffer_lo); @@ -948,7 +955,7 @@ BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) void BTDiagnostics::DefineSnapshotGeometry (const int i_buffer, const int lev) { - if (m_snapshot_geometry_defined[i_buffer] == 1) return; + if (m_snapshot_geometry_defined[i_buffer] == 1) { return; } if (lev == 0) { // Default non-periodic geometry for diags @@ -994,7 +1001,7 @@ BTDiagnostics::GetKIndexInSnapshotBoxFlag (const int i_buffer, const int lev) } void -BTDiagnostics::Flush (int i_buffer) +BTDiagnostics::Flush (int i_buffer, bool force_flush) { auto & warpx = WarpX::GetInstance(); std::string file_name = m_file_prefix; @@ -1003,7 +1010,7 @@ BTDiagnostics::Flush (int i_buffer) file_name = file_name+"/buffer"; } SetSnapshotFullStatus(i_buffer); - const bool isLastBTDFlush = ( m_snapshot_full[i_buffer] == 1 ); + const bool isLastBTDFlush = ( m_snapshot_full[i_buffer] == 1 ) || force_flush; bool const use_pinned_pc = true; bool const isBTD = true; double const labtime = m_t_lab[i_buffer]; @@ -1145,17 +1152,20 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) // Create directory only when the first buffer is flushed out. if (m_buffer_flush_counter[i_snapshot] == 0 || m_first_flush_after_restart[i_snapshot] == 1) { // Create Level_0 directory to store all Cell_D and Cell_H files - if (!amrex::UtilCreateDirectory(snapshot_Level0_path, permission_flag_rwxrxrx) ) + if (!amrex::UtilCreateDirectory(snapshot_Level0_path, permission_flag_rwxrxrx) ) { amrex::CreateDirectoryFailed(snapshot_Level0_path); + } // Create directory for each species selected for diagnostic for (int i = 0; i < m_particles_buffer[i_snapshot].size(); ++i) { const std::string snapshot_species_path = snapshot_path + "/" + m_output_species_names[i]; - if ( !amrex::UtilCreateDirectory(snapshot_species_path, permission_flag_rwxrxrx)) + if ( !amrex::UtilCreateDirectory(snapshot_species_path, permission_flag_rwxrxrx)) { amrex::CreateDirectoryFailed(snapshot_species_path); + } // Create Level_0 directory for particles to store Particle_H and DATA files const std::string species_Level0_path = snapshot_species_path + "/Level_0"; - if ( !amrex::UtilCreateDirectory(species_Level0_path, permission_flag_rwxrxrx)) + if ( !amrex::UtilCreateDirectory(species_Level0_path, permission_flag_rwxrxrx)) { amrex::CreateDirectoryFailed(species_Level0_path); + } } const std::string buffer_WarpXHeader_path = recent_Buffer_filepath + "/WarpXHeader"; const std::string snapshot_WarpXHeader_path = snapshot_path + "/WarpXHeader"; @@ -1248,7 +1258,7 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_species_Header.c_str(), snapshot_species_Header.c_str()) == 0, std::string("Renaming ").append(recent_species_Header).append(" to ").append(snapshot_species_Header).append(" has failed")); - if (BufferSpeciesHeader.m_total_particles == 0) continue; + if (BufferSpeciesHeader.m_total_particles == 0) { continue; } // if finite number of particles in the output, copy ParticleHdr and Data file WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, @@ -1259,7 +1269,7 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) } else { InterleaveSpeciesHeader(recent_species_Header,snapshot_species_Header, m_output_species_names[i], m_buffer_flush_counter[i_snapshot]); - if (BufferSpeciesHeader.m_total_particles == 0) continue; + if (BufferSpeciesHeader.m_total_particles == 0) { continue; } if (m_totalParticles_flushed_already[i_snapshot][i]==0) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, @@ -1326,9 +1336,9 @@ BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_pat void -BTDiagnostics::InterleaveFabArrayHeader(std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename) +BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, + std::string snapshot_FabHeader_path, + std::string newsnapshot_FabFilename) { BTDMultiFabHeaderImpl snapshot_FabHeader(snapshot_FabHeader_path); snapshot_FabHeader.ReadMultiFabHeader(); @@ -1450,8 +1460,9 @@ BTDiagnostics::PrepareParticleDataForOutput() { // Check if the zslice is in domain const bool ZSliceInDomain = GetZSliceInDomainFlag (i_buffer, lev); - if (ZSliceInDomain) { - if ( m_totalParticles_in_buffer[i_buffer][i] == 0) { + const bool kindexInSnapshotBox = GetKIndexInSnapshotBoxFlag (i_buffer, lev); + if (kindexInSnapshotBox) { + if ( buffer_empty(i_buffer) ) { if (!m_do_back_transformed_fields || m_varnames_fields.empty()) { if ( m_buffer_flush_counter[i_buffer] == 0) { DefineSnapshotGeometry(i_buffer, lev); @@ -1481,7 +1492,8 @@ void BTDiagnostics::UpdateTotalParticlesFlushed(int i_buffer) { for (int isp = 0; isp < m_totalParticles_flushed_already[i_buffer].size(); ++isp) { - m_totalParticles_flushed_already[i_buffer][isp] += m_particles_buffer[i_buffer][isp]->TotalNumberOfParticles(); + m_totalParticles_flushed_already[i_buffer][isp] += static_cast( + m_particles_buffer[i_buffer][isp]->TotalNumberOfParticles()); } } diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.H b/Source/Diagnostics/BoundaryScrapingDiagnostics.H index 71ee7bfbed8..60d184c30e2 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.H +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.H @@ -14,8 +14,7 @@ /** collect the particles that are absorbed at the embedded boundary, throughout the simulation */ -class -BoundaryScrapingDiagnostics final : public Diagnostics +class BoundaryScrapingDiagnostics final : public Diagnostics { public: @@ -34,7 +33,7 @@ private: utils::parser::IntervalsParser m_intervals; /** \brief Flush data to file. */ - void Flush (int i_buffer) override; + void Flush (int i_buffer, bool /* force_flush */) override; /** \brief Return whether to dump data to file at this time step. * (i.e. whether to call Flush) * @@ -52,7 +51,7 @@ private: * * This is not used for BoundaryScrapingDiagnostics: no field to output */ - void InitializeBufferData (int i_buffer, int lev, bool restart=false) override; + void InitializeBufferData (int i_buffer, int lev, bool restart=false) override; /** Initialize functors that point to the fields requested by the user. * * This is not used for BoundaryScrapingDiagnostics: no field to output diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index a9b75481bb7..c85dbd6b226 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -105,7 +105,7 @@ BoundaryScrapingDiagnostics::InitializeParticleBuffer () // Initialize total number of particles flushed m_totalParticles_flushed_already.resize(m_num_buffers); for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - int const n_species = m_output_species_names.size(); + int const n_species = static_cast(m_output_species_names.size()); m_totalParticles_flushed_already[i_buffer].resize(n_species); for (int i_species=0; i_species m_get_position; /** Z coordinate in boosted frame that corresponds to a give snapshot*/ amrex::Real m_current_z_boost; /** Previous Z coordinate in boosted frame that corresponds to a give snapshot*/ @@ -154,7 +154,7 @@ struct LorentzTransformParticles dst.m_aos[i_dst].pos(0) = xp; dst.m_aos[i_dst].pos(1) = zp; amrex::ignore_unused(yp); -#elif defined (WARPX_DIM_1D) +#elif defined (WARPX_DIM_1D_Z) dst.m_aos[i_dst].pos(0) = zp; amrex::ignore_unused(xp, yp); #else @@ -166,7 +166,7 @@ struct LorentzTransformParticles dst.m_rdata[PIdx::uz][i_dst] = uzp; } - GetParticlePosition m_get_position; + GetParticlePosition m_get_position; amrex::ParticleReal* AMREX_RESTRICT m_xpold = nullptr; amrex::ParticleReal* AMREX_RESTRICT m_ypold = nullptr; @@ -195,15 +195,14 @@ struct LorentzTransformParticles * \brief BackTransform functor to select particles and Lorentz Transform them * and store in particle buffers */ -class -BackTransformParticleFunctor final : public ComputeParticleDiagFunctor +class BackTransformParticleFunctor final : public ComputeParticleDiagFunctor { public: BackTransformParticleFunctor(WarpXParticleContainer *pc_src, std::string species_name, int num_buffers); /** Computes the Lorentz transform of source particles to obtain lab-frame data in pc_dst*/ void operator () (PinnedMemoryParticleContainer& pc_dst, int &TotalParticleCounter, int i_buffer) const override; void InitData() override; - /** \brief Prepare data required to back-transform particle attribtutes for lab-frame snapshot, i_buffer + /** \brief Prepare data required to back-transform particle attributes for lab-frame snapshot, i_buffer * * \param[in] i_buffer index of the snapshot * \param[in] z_slice_in_domain if the z-slice at current_z_boost is within the bounds of diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp index 1bee13181df..8a6f0765664 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp @@ -20,7 +20,7 @@ SelectParticles::SelectParticles (const WarpXParIter& a_pti, TmpParticles& tmp_p int a_offset) : m_current_z_boost(current_z_boost), m_old_z_boost(old_z_boost) { - m_get_position = GetParticlePosition(a_pti, a_offset); + m_get_position = GetParticlePosition(a_pti, a_offset); const auto lev = a_pti.GetLevel(); const auto index = a_pti.GetPairIndex(); @@ -37,10 +37,10 @@ LorentzTransformParticles::LorentzTransformParticles ( const WarpXParIter& a_pti { using namespace amrex::literals; - if (tmp_particle_data.empty()) return; - m_get_position = GetParticlePosition(a_pti, a_offset); + if (tmp_particle_data.empty()) { return; } + m_get_position = GetParticlePosition(a_pti, a_offset); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_wpnew = attribs[PIdx::w].dataPtr(); m_uxpnew = attribs[PIdx::ux].dataPtr(); m_uypnew = attribs[PIdx::uy].dataPtr(); @@ -80,7 +80,7 @@ BackTransformParticleFunctor::BackTransformParticleFunctor ( void BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst, int &totalParticleCounter, int i_buffer) const { - if (m_perform_backtransform[i_buffer] == 0) return; + if (m_perform_backtransform[i_buffer] == 0) { return; } auto &warpx = WarpX::GetInstance(); // get particle slice const int nlevs = std::max(0, m_pc_src->finestLevel()+1); @@ -142,14 +142,15 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE(int i) { - if (Flag[i] == 1) GetParticleLorentzTransform(dst_data, src_data, i, + if (Flag[i] == 1) { GetParticleLorentzTransform(dst_data, src_data, i, old_size + IndexLocation[i]); + } }); amrex::Gpu::synchronize(); } } } - totalParticleCounter = pc_dst.TotalNumberOfParticles(); + totalParticleCounter = static_cast(pc_dst.TotalNumberOfParticles()); } @@ -171,5 +172,5 @@ BackTransformParticleFunctor::PrepareFunctorData ( int i_buffer, bool z_slice_in m_current_z_boost.at(i_buffer) = current_z_boost; m_t_lab.at(i_buffer) = t_lab; m_perform_backtransform.at(i_buffer) = 0; - if (z_slice_in_domain && (snapshot_full == 0)) m_perform_backtransform.at(i_buffer) = 1; + if (z_slice_in_domain && (snapshot_full == 0)) { m_perform_backtransform.at(i_buffer) = 1; } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H index 84ea079c63c..dd5bb239ecf 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF and store result in mf_out. */ -class -CellCenterFunctor final : public ComputeDiagFunctor +class CellCenterFunctor final : public ComputeDiagFunctor { public: /** Constructor. @@ -34,7 +33,7 @@ public: * \param[in] dcomp first component of mf_dst in which cell-centered * data is stored */ - virtual void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: /** pointer to source multifab (can be multi-component) */ amrex::MultiFab const * const m_mf_src = nullptr; diff --git a/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H index b4c413b3aa1..c254a341122 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H @@ -13,14 +13,20 @@ * \brief Functor to compute a diagnostic and store the result in existing * MultiFab */ -class -ComputeDiagFunctor +class ComputeDiagFunctor { public: ComputeDiagFunctor( int ncomp, amrex::IntVect crse_ratio) : m_ncomp(ncomp), m_crse_ratio(crse_ratio) {} //** Virtual Destructor to handle clean destruction of derived classes */ virtual ~ComputeDiagFunctor() = default; + + // Default move and copy operations + ComputeDiagFunctor(const ComputeDiagFunctor&) = default; + ComputeDiagFunctor& operator=(const ComputeDiagFunctor&) = default; + ComputeDiagFunctor(ComputeDiagFunctor&&) = default; + ComputeDiagFunctor& operator=(ComputeDiagFunctor&&) = default; + /** Compute a field and store the result in mf_dst * * \param[out] mf_dst output MultiFab where the result is written @@ -31,7 +37,7 @@ public: virtual void operator() (amrex::MultiFab& mf_dst, int dcomp, int i_buffer = 0) const = 0; /** Number of component from the input multifab to write to the output * multifab */ - int nComp () const { return m_ncomp; } + [[nodiscard]] int nComp () const { return m_ncomp; } /** \brief Prepare data required to process fields in the operator() * Note that this function has parameters that are specific to diff --git a/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H index e14e777e066..6856c4c90a2 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H @@ -15,16 +15,21 @@ /** * \brief Functor to compute a diagnostic and store the result in existing ParticleContainer. */ -class -ComputeParticleDiagFunctor +class ComputeParticleDiagFunctor { public: - ComputeParticleDiagFunctor( ) {} + ComputeParticleDiagFunctor( ) = default; /** Virtual Destructor to handle clean destruction of derived classes */ virtual ~ComputeParticleDiagFunctor() = default; - /** \brief Prepare data required to back-transform particle attribtutes for + /** Default assignment and copy operations*/ + ComputeParticleDiagFunctor(const ComputeParticleDiagFunctor&) = default; + ComputeParticleDiagFunctor& operator=(const ComputeParticleDiagFunctor&) = default; + ComputeParticleDiagFunctor(ComputeParticleDiagFunctor&&) = default; + ComputeParticleDiagFunctor& operator=(ComputeParticleDiagFunctor&&) = default; + + /** \brief Prepare data required to back-transform particle attributes for * lab-frame snapshot, with index i_buffer. * Note that this function has parameters that are specific to * back-transformed diagnostics, that are unused for regular diagnostics. diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H index f6a297741b9..3d04a56742b 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H @@ -10,8 +10,7 @@ /** * \brief Functor to compute divB into mf_out. */ -class -DivBFunctor final : public ComputeDiagFunctor +class DivBFunctor final : public ComputeDiagFunctor { public: /** Constructor. @@ -32,7 +31,7 @@ public: * \param[in] dcomp first component of mf_dst in which cell-centered * data is stored */ - virtual void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer*/) const override; + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer*/) const override; private: /** Vector of pointer to source multifab Bx, By, Bz */ std::array m_arr_mf_src; diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H index c7d3badcf0d..312ccaa5cd6 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H @@ -10,8 +10,7 @@ /** * \brief Functor to compute divE into mf_out. */ -class -DivEFunctor final : public ComputeDiagFunctor +class DivEFunctor final : public ComputeDiagFunctor { public: /** Constructor. @@ -31,7 +30,7 @@ public: * \param[in] dcomp first component of mf_dst in which cell-centered * data is stored */ - virtual void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: /** Vector of pointer to source multifab Bx, By, Bz */ std::array m_arr_mf_src; diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H index db0e0331148..3080b500712 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF for current density and store result in mf_out. */ -class -JFunctor final : public ComputeDiagFunctor +class JFunctor final : public ComputeDiagFunctor { public: /** Constructor. @@ -35,7 +34,7 @@ public: * \param[in] dcomp first component of mf_dst in which cell-centered * data is stored */ - virtual void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: /** direction of the current density to save */ const int m_dir; diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 4663a5e1a70..b21bbded011 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -36,16 +36,16 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ amrex::Vector, 3 > > current_fp_temp; current_fp_temp.resize(1); - auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); + const auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); current_fp_temp[0][0] = std::make_unique( current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() ); - auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); + const auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); current_fp_temp[0][1] = std::make_unique( current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() ); - auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); + const auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); current_fp_temp[0][2] = std::make_unique( current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() ); diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H index 7e435b3675e..1b8785af7b7 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF and store result in mf_out. */ -class -PartPerCellFunctor final : public ComputeDiagFunctor +class PartPerCellFunctor final : public ComputeDiagFunctor { public: /** Constructor. @@ -29,7 +28,7 @@ public: * \param[in] dcomp first component of mf_dst in which cell-centered * data is stored */ - virtual void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: int const m_lev; /**< level on which mf_src is defined */ }; diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H index d5e0c45b956..9718c9c7163 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF and store result in mf_out. */ -class -PartPerGridFunctor final : public ComputeDiagFunctor +class PartPerGridFunctor final : public ComputeDiagFunctor { public: /** Constructor. @@ -29,7 +28,7 @@ public: * \param[in] dcomp first component of mf_dst in which cell-centered * data is stored */ - virtual void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: int const m_lev; /**< level on which mf_src is defined */ }; diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H index 09b54726211..56c4372fad8 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H @@ -12,8 +12,7 @@ /** * \brief Functor to calculate per-cell averages of particle properties. */ -class -ParticleReductionFunctor final : public ComputeDiagFunctor +class ParticleReductionFunctor final : public ComputeDiagFunctor { public: /** Constructor. @@ -42,7 +41,7 @@ public: * \param[in] dcomp first component of mf_dst in which cell-centered * data is stored */ - virtual void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; private: int const m_lev; /**< level on which mf_src is defined */ int const m_ispec; /**< index of species to average over */ diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index d3932472c3b..feb25c11183 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -78,7 +78,7 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const amrex::ParticleReal x = p.pos(0); const amrex::Real lx = (x - plo[0]) * dxi[0]; ii = static_cast(amrex::Math::floor(lx)); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) const amrex::ParticleReal y = p.pos(1); const amrex::Real ly = (y - plo[1]) * dxi[1]; jj = static_cast(amrex::Math::floor(ly)); @@ -135,8 +135,8 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const amrex::ParticleReal uy = p.rdata(PIdx::uy) / PhysConst::c; const amrex::ParticleReal uz = p.rdata(PIdx::uz) / PhysConst::c; amrex::Real filter; - if ((do_filter) && (!filter_fn(xw, yw, zw, ux, uy, uz))) filter = 0._rt; - else filter = 1._rt; + if ((do_filter) && (filter_fn(xw, yw, zw, ux, uy, uz) == 0._rt)) { filter = 0._rt; + } else { filter = 1._rt; } amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 0), (amrex::Real)(p.rdata(PIdx::w) * filter)); }); // Divide value by number of particles for average. Set average to zero if there are no particles @@ -147,8 +147,8 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, amrex::Array4 const& a_ppc = ppc_mf.const_array(mfi); amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - if (a_ppc(i,j,k,0) == 0) a_red(i,j,k,0) = 0; - else a_red(i,j,k,0) = a_red(i,j,k,0) / a_ppc(i,j,k,0); + if (a_ppc(i,j,k,0) == 0) { a_red(i,j,k,0) = 0; + } else { a_red(i,j,k,0) = a_red(i,j,k,0) / a_ppc(i,j,k,0); } }); } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H index e359bb2f6a2..31641fe23e9 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to compute charge density rho into mf_out */ -class -RhoFunctor final : public ComputeDiagFunctor +class RhoFunctor final : public ComputeDiagFunctor { public: @@ -38,7 +37,7 @@ public: * \param[out] mf_dst output MultiFab where the result is written * \param[in] dcomp first component of mf_dst in which cell-centered data are stored */ - virtual void operator() ( amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/ ) const override; + void operator() ( amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/ ) const override; private: diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp index 73feef4ed00..c36e31c0f93 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -7,6 +7,8 @@ #include "Utils/WarpXAlgorithmSelection.H" #endif #include "Particles/MultiParticleContainer.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Particles/WarpXParticleContainer.H" #include "WarpX.H" @@ -41,6 +43,10 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i if (m_species_index == -1) { auto& mypc = warpx.GetPartContainer(); rho = mypc.GetChargeDensity(m_lev, true); + if (warpx.DoFluidSpecies()) { + auto& myfl = warpx.GetFluidContainer(); + myfl.DepositCharge(m_lev, *rho); + } } // Dump rho per species else { diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 75441d6d9e8..21d5ff699d0 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -40,6 +40,11 @@ public: /** Virtual Destructor to handle clean destruction of derived classes */ virtual ~Diagnostics (); + Diagnostics (Diagnostics const &) = delete; + Diagnostics& operator= (Diagnostics const & ) = delete; + Diagnostics(Diagnostics&& ) = default; + Diagnostics& operator=(Diagnostics&& ) = default; + /** Pack (stack) all fields in the cell-centered output MultiFab m_mf_output. * * Fields are computed (e.g., cell-centered or back-transformed) @@ -58,8 +63,10 @@ public: * multiple times yet. * When these are fixed, the implementation of Flush should be in Diagnostics.cpp * \param[in] i_buffer index of the buffer data to be flushed. + * \param[in] force_flush only used for BTD, whether to do a complete flush of the data + * (including metadata listing the total number of particles) even if the snapshot is incomplete */ - virtual void Flush (int i_buffer) = 0; + virtual void Flush (int i_buffer, bool force_flush) = 0; /** Initialize pointers to main fields and allocate output multifab m_mf_output. */ void InitData (); void InitDataBeforeRestart (); @@ -68,7 +75,7 @@ public: * * Derived classes MUST implement this function, and it must allocate m_all_field_functors * and fill it with ComputeDiagFunctor objects. - * The functions is called at intiailization and when the domain is decomposed + * The functions is called at initialization and when the domain is decomposed * during the simulation to load-balance. * \param[in] lev level on which the vector of unique_ptrs to field functors is initialized. */ @@ -105,7 +112,7 @@ public: */ void FilterComputePackFlush (int step, bool force_flush=false); /** Whether the last timestep is always dumped */ - bool DoDumpLastTimestep () const {return m_dump_last_timestep;} + [[nodiscard]] bool DoDumpLastTimestep () const {return m_dump_last_timestep;} /** Returns the number of snapshots used in BTD. For Full-Diagnostics, the value is 1*/ int getnumbuffers() {return m_num_buffers;} /** Time in lab-frame associated with the ith snapshot @@ -232,13 +239,13 @@ protected: amrex::Vector< std::string > m_pfield_varnames; /** Names of species for which to output particle field diagnostics */ std::vector< std::string > m_pfield_species; - /** Whether to do averaging for each of the particle fied diagnostics */ + /** Whether to do averaging for each of the particle field diagnostics */ std::vector< bool > m_pfield_do_average; /** Species indices corresponding to elements of m_pfield_varnames */ std::vector< int > m_pfield_species_index; /** List of the parser strings for the particle field diagnostics */ std::vector< std::string > m_pfield_strings; - /** Whether to use a filter function on particles before calculating particle fied diagnostics */ + /** Whether to use a filter function on particles before calculating particle field diagnostics */ std::vector< bool> m_pfield_dofilter; /** List of parser strings for pre-average filtering for the particle field diagnostics */ std::vector< std::string > m_pfield_filter_strings; diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index 7eb574aa0ef..3b5daabaffa 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -115,11 +115,15 @@ Diagnostics::BaseReadParameters () if (!pfield_varnames_specified){ m_pfield_varnames = {}; } + #ifdef WARPX_DIM_RZ - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - m_pfield_varnames.empty(), - "Input error: cannot use particle_fields_to_plot with RZ" - ); + if (pfield_varnames_specified){ + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + "Particle field diagnostics will output the 0th mode only.", + ablastr::warn_manager::WarnPriority::low + ); + } #endif // Get parser strings for particle fields and generate map of parsers @@ -127,6 +131,7 @@ Diagnostics::BaseReadParameters () std::string filter_parser_str; const amrex::ParmParse pp_diag_pfield(m_diag_name + ".particle_fields"); for (const auto& var : m_pfield_varnames) { + bool do_average = true; pp_diag_pfield.query((var + ".do_average").c_str(), do_average); m_pfield_do_average.push_back(do_average); @@ -330,7 +335,7 @@ Diagnostics::InitDataAfterRestart () InitializeParticleBuffer(); InitializeParticleFunctors(); } - if (write_species == 0) { + if (write_species == 0) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( m_format != "checkpoint", "For checkpoint format, write_species flag must be 1." @@ -569,9 +574,7 @@ Diagnostics::FilterComputePackFlush (int step, bool force_flush) } for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - if ( !DoDump (step, i_buffer, force_flush) ) continue; - Flush(i_buffer); + if ( !DoDump (step, i_buffer, force_flush) ) { continue; } + Flush(i_buffer, force_flush); } - - } diff --git a/Source/Diagnostics/FieldIO.H b/Source/Diagnostics/FieldIO.H index 1f8bfe5346a..138a1317e08 100644 --- a/Source/Diagnostics/FieldIO.H +++ b/Source/Diagnostics/FieldIO.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_FielIO_H_ -#define WARPX_FielIO_H_ +#ifndef WARPX_FieldIO_H_ +#define WARPX_FieldIO_H_ #include @@ -53,4 +53,4 @@ getReversedVec( const amrex::IntVect& v ); std::vector getReversedVec( const amrex::Real* v ); -#endif // WARPX_FielIO_H_ +#endif // WARPX_FieldIO_H_ diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H index d8820d35a12..403e9df7857 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormat.H +++ b/Source/Diagnostics/FlushFormats/FlushFormat.H @@ -12,7 +12,7 @@ class FlushFormat public: /** Flush fields and particles to file */ virtual void WriteToFile ( - amrex::Vector varnames, + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, @@ -27,7 +27,13 @@ public: bool isLastBTDFlush = false, const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const = 0; - virtual ~FlushFormat() {} + FlushFormat () = default; + virtual ~FlushFormat() = default; + + FlushFormat ( FlushFormat const &) = default; + FlushFormat& operator= ( FlushFormat const & ) = default; + FlushFormat ( FlushFormat&& ) = default; + FlushFormat& operator= ( FlushFormat&& ) = default; }; #endif // WARPX_FLUSHFORMAT_H_ diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H index b15b7432dd5..228e4bc5cf6 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H @@ -28,8 +28,8 @@ class FlushFormatAscent : public FlushFormat { public: /** Do in-situ visualization for field and particle data */ - virtual void WriteToFile ( - amrex::Vector varnames, + void WriteToFile ( + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, @@ -53,7 +53,13 @@ public: void WriteParticles(const amrex::Vector& particle_diags, conduit::Node& a_bp_mesh) const; #endif - ~FlushFormatAscent() {} + FlushFormatAscent () = default; + ~FlushFormatAscent() override = default; + + FlushFormatAscent ( FlushFormatAscent const &) = default; + FlushFormatAscent& operator= ( FlushFormatAscent const & ) = default; + FlushFormatAscent ( FlushFormatAscent&& ) = default; + FlushFormatAscent& operator= ( FlushFormatAscent&& ) = default; }; #endif // WARPX_FLUSHFORMATASCENT_H_ diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp index 8fe25bef32b..980047e3b46 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp @@ -11,7 +11,7 @@ using namespace amrex; void FlushFormatAscent::WriteToFile ( - const amrex::Vector varnames, + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H index 5131d6f4234..f6aad226d75 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H @@ -15,8 +15,8 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile { /** Flush fields and particles to plotfile */ - virtual void WriteToFile ( - amrex::Vector varnames, + void WriteToFile ( + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, @@ -29,7 +29,7 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override final; + const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const final; void CheckpointParticles (const std::string& dir, const amrex::Vector& particle_diags) const; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 6608123bb33..5f59cd723da 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -27,7 +27,7 @@ namespace void FlushFormatCheckpoint::WriteToFile ( - const amrex::Vector /*varnames*/, + const amrex::Vector& /*varnames*/, const amrex::Vector& /*mf*/, amrex::Vector& geom, const amrex::Vector iteration, const double /*time*/, @@ -172,7 +172,7 @@ FlushFormatCheckpoint::CheckpointParticles ( const std::string& dir, const amrex::Vector& particle_diags) const { - for (auto& part_diag: particle_diags) { + for (const auto& part_diag: particle_diags) { WarpXParticleContainer* pc = part_diag.getParticleContainer(); Vector real_names; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H index 4da45c00bbf..88380407f5e 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H @@ -28,7 +28,7 @@ public: /** Flush fields and particles to plotfile */ void WriteToFile ( - amrex::Vector varnames, + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, @@ -45,6 +45,11 @@ public: ~FlushFormatOpenPMD () override = default; + FlushFormatOpenPMD ( FlushFormatOpenPMD const &) = delete; + FlushFormatOpenPMD& operator= ( FlushFormatOpenPMD const & ) = delete; + FlushFormatOpenPMD ( FlushFormatOpenPMD&& ) = default; + FlushFormatOpenPMD& operator= ( FlushFormatOpenPMD&& ) = default; + private: /** This is responsible for dumping to file */ std::unique_ptr< WarpXOpenPMDPlot > m_OpenPMDPlotWriter; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index 3832bf5e9c0..3b7006243e7 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -27,8 +27,9 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) std::string openpmd_backend {"default"}; pp_diag_name.query("openpmd_backend", openpmd_backend); // pick first available backend if default is chosen - if( openpmd_backend == "default" ) + if( openpmd_backend == "default" ) { openpmd_backend = WarpXOpenPMDFileType(); + } pp_diag_name.add("openpmd_backend", openpmd_backend); @@ -37,82 +38,85 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) const bool encodingDefined = pp_diag_name.query("openpmd_encoding", openpmd_encoding); openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased; - if ( openpmd_encoding == "v" ) - encoding = openPMD::IterationEncoding::variableBased; - else if ( openpmd_encoding == "g" ) - encoding = openPMD::IterationEncoding::groupBased; - else if ( openpmd_encoding == "f" ) - encoding = openPMD::IterationEncoding::fileBased; + if ( openpmd_encoding == "v" ) { + encoding = openPMD::IterationEncoding::variableBased; + } else if ( openpmd_encoding == "g" ) { + encoding = openPMD::IterationEncoding::groupBased; + } else if ( openpmd_encoding == "f" ) { + encoding = openPMD::IterationEncoding::fileBased; + } std::string diag_type_str; pp_diag_name.get("diag_type", diag_type_str); if (diag_type_str == "BackTransformed") { - if ( ( openPMD::IterationEncoding::fileBased != encoding ) && - ( openPMD::IterationEncoding::groupBased != encoding ) ) - { - const std::string warnMsg = diag_name+" Unable to support BTD with streaming. Using GroupBased "; - ablastr::warn_manager::WMRecordWarning("Diagnostics", warnMsg); - encoding = openPMD::IterationEncoding::groupBased; - } + if ( ( openPMD::IterationEncoding::fileBased != encoding ) && + ( openPMD::IterationEncoding::groupBased != encoding ) ) + { + const std::string warnMsg = diag_name+" Unable to support BTD with streaming. Using GroupBased "; + ablastr::warn_manager::WMRecordWarning("Diagnostics", warnMsg); + encoding = openPMD::IterationEncoding::groupBased; + } } - // - // if no encoding is defined, then check to see if tspf is defined. - // (backward compatibility) - // - if ( !encodingDefined ) + // + // if no encoding is defined, then check to see if tspf is defined. + // (backward compatibility) + // + if ( !encodingDefined ) { - bool openpmd_tspf = false; - const bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf); - if ( tspfDefined && openpmd_tspf ) - encoding = openPMD::IterationEncoding::fileBased; + bool openpmd_tspf = false; + const bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf); + if ( tspfDefined && openpmd_tspf ) { + encoding = openPMD::IterationEncoding::fileBased; + } } - // ADIOS2 operator type & parameters - std::string operator_type; - pp_diag_name.query("adios2_operator.type", operator_type); - std::string const prefix = diag_name + ".adios2_operator.parameters"; - const ParmParse pp; - auto entr = amrex::ParmParse::getEntries(prefix); - - std::map< std::string, std::string > operator_parameters; - auto const prefix_len = prefix.size() + 1; - for (std::string k : entr) { - std::string v; - pp.get(k.c_str(), v); - k.erase(0, prefix_len); - operator_parameters.insert({k, v}); - } - - // ADIOS2 engine type & parameters - std::string engine_type; - pp_diag_name.query("adios2_engine.type", engine_type); - std::string const engine_prefix = diag_name + ".adios2_engine.parameters"; - const ParmParse ppe; - auto eng_entr = amrex::ParmParse::getEntries(engine_prefix); - - std::map< std::string, std::string > engine_parameters; - auto const prefixlen = engine_prefix.size() + 1; - for (std::string k : eng_entr) { - std::string v; - ppe.get(k.c_str(), v); - k.erase(0, prefixlen); - engine_parameters.insert({k, v}); - } - - auto & warpx = WarpX::GetInstance(); - m_OpenPMDPlotWriter = std::make_unique( - encoding, openpmd_backend, - operator_type, operator_parameters, - engine_type, engine_parameters, - warpx.getPMLdirections() - ); + // ADIOS2 operator type & parameters + std::string operator_type; + pp_diag_name.query("adios2_operator.type", operator_type); + std::string const prefix = diag_name + ".adios2_operator.parameters"; + const ParmParse pp; + auto entr = amrex::ParmParse::getEntries(prefix); + + std::map< std::string, std::string > operator_parameters; + auto const prefix_len = prefix.size() + 1; + for (std::string k : entr) { + std::string v; + pp.get(k.c_str(), v); + k.erase(0, prefix_len); + operator_parameters.insert({k, v}); + } + + // ADIOS2 engine type & parameters + std::string engine_type; + pp_diag_name.query("adios2_engine.type", engine_type); + std::string const engine_prefix = diag_name + ".adios2_engine.parameters"; + const ParmParse ppe; + auto eng_entr = amrex::ParmParse::getEntries(engine_prefix); + + std::map< std::string, std::string > engine_parameters; + auto const prefixlen = engine_prefix.size() + 1; + for (std::string k : eng_entr) { + std::string v; + ppe.get(k.c_str(), v); + k.erase(0, prefixlen); + engine_parameters.insert({k, v}); + } + + auto & warpx = WarpX::GetInstance(); + m_OpenPMDPlotWriter = std::make_unique( + encoding, openpmd_backend, + operator_type, operator_parameters, + engine_type, engine_parameters, + warpx.getPMLdirections(), + warpx.GetAuthors() + ); } void FlushFormatOpenPMD::WriteToFile ( - const amrex::Vector varnames, + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, @@ -128,15 +132,15 @@ FlushFormatOpenPMD::WriteToFile ( const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits); if (!isBTD) { - amrex::Print() << Utils::TextMsg::Info("Writing openPMD file " + filename); + amrex::Print() << Utils::TextMsg::Info("Writing openPMD file " + filename); } else { - amrex::Print() << Utils::TextMsg::Info("Writing buffer " + std::to_string(bufferID+1) + " of " + std::to_string(numBuffers) - + " to snapshot " + std::to_string(snapshotID) + " to openPMD BTD " + prefix); - if (isLastBTDFlush) - { - amrex::Print() << Utils::TextMsg::Info("Finished writing snapshot " + std::to_string(snapshotID) + " in openPMD BTD " + prefix); - } + amrex::Print() << Utils::TextMsg::Info("Writing buffer " + std::to_string(bufferID+1) + " of " + std::to_string(numBuffers) + + " to snapshot " + std::to_string(snapshotID) + " to openPMD BTD " + prefix); + if (isLastBTDFlush) + { + amrex::Print() << Utils::TextMsg::Info("Finished writing snapshot " + std::to_string(snapshotID) + " in openPMD BTD " + prefix); + } } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -147,18 +151,20 @@ FlushFormatOpenPMD::WriteToFile ( int output_iteration = iteration[0]; // in backtransformed diagnostics (BTD), we dump into a series of labframe // snapshots - if( isBTD ) + if( isBTD ) { output_iteration = snapshotID; + } // Set step and output directory name. m_OpenPMDPlotWriter->SetStep(output_iteration, prefix, file_min_digits, isBTD); // fields: only dumped for coarse level m_OpenPMDPlotWriter->WriteOpenPMDFieldsAll( - varnames, mf, geom, output_levels, output_iteration, time, isBTD, full_BTD_snapshot); + varnames, mf, geom, output_levels, output_iteration, static_cast(time), isBTD, full_BTD_snapshot); // particles: all (reside only on locally finest level) - m_OpenPMDPlotWriter->WriteOpenPMDParticles(particle_diags, time, use_pinned_pc, isBTD, isLastBTDFlush, totalParticlesFlushedAlready); + m_OpenPMDPlotWriter->WriteOpenPMDParticles( + particle_diags, static_cast(time), use_pinned_pc, isBTD, isLastBTDFlush, totalParticlesFlushedAlready); // signal that no further updates will be written to this iteration m_OpenPMDPlotWriter->CloseStep(isBTD, isLastBTDFlush); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H index b8461e51f23..486dcc3b5ee 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H @@ -22,8 +22,8 @@ class FlushFormatPlotfile : public FlushFormat { public: /** Flush fields and particles to plotfile */ - virtual void WriteToFile ( - amrex::Vector varnames, + void WriteToFile ( + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, @@ -46,17 +46,23 @@ public: const std::string& plotfilename, bool plot_raw_fields_guards) const; /** \brief Write particles data to file. - * \param[in] filename name of output directory + * \param[in] dir name of output directory * \param[in] particle_diags Each element of this vector handles output of 1 species. * \param[in] time the simulation time on the coarsest level * \param[in] isBTD whether this is a back-transformed diagnostic */ - void WriteParticles(const std::string& filename, + void WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, amrex::Real time, bool isBTD = false) const; - ~FlushFormatPlotfile() {} + FlushFormatPlotfile () = default; + ~FlushFormatPlotfile() override = default; + + FlushFormatPlotfile ( FlushFormatPlotfile const &) = default; + FlushFormatPlotfile& operator= ( FlushFormatPlotfile const & ) = default; + FlushFormatPlotfile ( FlushFormatPlotfile&& ) = default; + FlushFormatPlotfile& operator= ( FlushFormatPlotfile&& ) = default; }; #endif // WARPX_FLUSHFORMATPLOTFILE_H_ diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index db5dea10adc..df73ed34c94 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -55,7 +55,7 @@ namespace void FlushFormatPlotfile::WriteToFile ( - const amrex::Vector varnames, + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, @@ -86,7 +86,7 @@ FlushFormatPlotfile::WriteToFile ( Vector rfs; const VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); VisMF::SetHeaderVersion(amrex::VisMF::Header::Version_v1); - if (plot_raw_fields) rfs.emplace_back("raw_fields"); + if (plot_raw_fields) { rfs.emplace_back("raw_fields"); } amrex::WriteMultiLevelPlotfile(filename, nlev, amrex::GetVecOfConstPtrs(mf), varnames, geom, @@ -99,7 +99,7 @@ FlushFormatPlotfile::WriteToFile ( WriteAllRawFields(plot_raw_fields, nlev, filename, plot_raw_fields_guards); - WriteParticles(filename, particle_diags, time, isBTD); + WriteParticles(filename, particle_diags, static_cast(time), isBTD); WriteJobInfo(filename); @@ -245,8 +245,9 @@ FlushFormatPlotfile::WriteWarpXHeader( HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if( ! HeaderFile.good()) + if( ! HeaderFile.good()) { amrex::FileOpenFailed(HeaderFileName); + } HeaderFile.precision(17); @@ -342,7 +343,7 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, const amrex::Real time, bool isBTD) const { - for (auto& part_diag : particle_diags) { + for (const auto& part_diag : particle_diags) { WarpXParticleContainer* pc = part_diag.getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = part_diag.getPinnedParticleContainer(); auto tmp = isBTD ? @@ -484,7 +485,7 @@ WriteCoarseVector( const std::string field_name, const int lev, const bool plot_guards ) { IntVect ng(0); - if (plot_guards) ng = Fx_fp->nGrowVect(); + if (plot_guards) { ng = Fx_fp->nGrowVect(); } if (lev == 0) { // No coarse field for level 0: instead write a MultiFab @@ -521,7 +522,7 @@ WriteCoarseScalar( const std::string field_name, const int icomp ) { IntVect ng(0); - if (plot_guards) ng = F_fp->nGrowVect(); + if (plot_guards) { ng = F_fp->nGrowVect(); } if (lev == 0) { // No coarse field for level 0: instead write a MultiFab @@ -544,7 +545,7 @@ FlushFormatPlotfile::WriteAllRawFields( const bool plot_raw_fields, const int nlevels, const std::string& plotfilename, const bool plot_raw_fields_guards) const { - if (!plot_raw_fields) return; + if (!plot_raw_fields) { return; } auto & warpx = WarpX::GetInstance(); for (int lev = 0; lev < nlevels; ++lev) { diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H index 28785c1c905..54eb7099ba4 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H @@ -33,10 +33,12 @@ class FlushFormatSensei : public FlushFormat { public: FlushFormatSensei(); - ~FlushFormatSensei(); + ~FlushFormatSensei() override; - FlushFormatSensei(const FlushFormatSensei &) = delete; - void operator=(const FlushFormatSensei &) = delete; + FlushFormatSensei(const FlushFormatSensei &) = delete; + FlushFormatSensei& operator=(const FlushFormatSensei &) = delete; + FlushFormatSensei(FlushFormatSensei&&) = default; + FlushFormatSensei& operator=(FlushFormatSensei&& ) = default; /** Construct. * @@ -46,8 +48,8 @@ public: FlushFormatSensei (amrex::AmrMesh *amr_mesh, std::string diag_name); /** Do in-situ visualization for field and particle data */ - virtual void WriteToFile ( - amrex::Vector varnames, + void WriteToFile ( + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp index 060c0e4a5e9..e162b8b3121 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp @@ -7,14 +7,8 @@ # include #endif -FlushFormatSensei::FlushFormatSensei () : - m_insitu_pin_mesh(0), m_insitu_bridge(nullptr), - m_amr_mesh(nullptr) -{} - FlushFormatSensei::FlushFormatSensei (amrex::AmrMesh *amr_mesh, std::string diag_name) : - m_insitu_pin_mesh(0), m_insitu_bridge(nullptr), m_amr_mesh(amr_mesh) { #ifndef AMREX_USE_SENSEI_INSITU @@ -50,7 +44,7 @@ FlushFormatSensei::~FlushFormatSensei () = default; void FlushFormatSensei::WriteToFile ( - const amrex::Vector varnames, + const amrex::Vector& varnames, const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, diff --git a/Source/Diagnostics/FullDiagnostics.H b/Source/Diagnostics/FullDiagnostics.H index 4bb3e27895a..42b60c213f2 100644 --- a/Source/Diagnostics/FullDiagnostics.H +++ b/Source/Diagnostics/FullDiagnostics.H @@ -6,8 +6,7 @@ #include -class -FullDiagnostics final : public Diagnostics +class FullDiagnostics final : public Diagnostics { public: FullDiagnostics (int i, std::string name); @@ -27,7 +26,7 @@ private: */ bool m_solver_deposits_current = true; /** Flush m_mf_output and particles to file for the i^th buffer */ - void Flush (int i_buffer) override; + void Flush (int i_buffer, bool /* force_flush */) override; /** Flush raw data */ void FlushRaw (); /** whether to compute and pack cell-centered data in m_mf_output diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 7abe923b316..4f1e47a2a52 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -41,16 +41,14 @@ using namespace amrex::literals; -FullDiagnostics::FullDiagnostics (int i, std::string name) - : Diagnostics(i, name) +FullDiagnostics::FullDiagnostics (int i, std::string name): + Diagnostics{i, name}, + m_solver_deposits_current{ + !(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::None && + WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic)} { ReadParameters(); BackwardCompatibility(); - - m_solver_deposits_current = !( - WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::None && - WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic - ); } void @@ -128,7 +126,7 @@ FullDiagnostics::BackwardCompatibility () } void -FullDiagnostics::Flush ( int i_buffer ) +FullDiagnostics::Flush ( int i_buffer, bool /* force_flush */ ) { // This function should be moved to Diagnostics when plotfiles/openpmd format // is supported for BackTransformed Diagnostics, in BTDiagnostics class. @@ -149,7 +147,7 @@ FullDiagnostics::FlushRaw () {} bool FullDiagnostics::DoDump (int step, int /*i_buffer*/, bool force_flush) { - if (m_already_done) return false; + if (m_already_done) { return false; } if ( force_flush || (m_intervals.contains(step+1)) ){ m_already_done = true; return true; @@ -187,23 +185,30 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const int ncomp = ncomp_multimodefab; // This function is called multiple times, for different values of `lev` // but the `varnames` need only be updated once. + const bool update_varnames = (lev==0); if (update_varnames) { m_varnames.clear(); - const int n_rz = ncomp * m_varnames.size(); + const auto n_rz = ncomp * static_cast(m_varnames.size()); m_varnames.reserve(n_rz); } - // Reser field functors + // Add functors for average particle data for each species + const auto nvar = static_cast(m_varnames_fields.size()); + const auto nspec = static_cast(m_pfield_species.size()); + const auto ntot = static_cast(nvar + m_pfield_varnames.size() * nspec); + + // Reset field functors m_all_field_functors[lev].clear(); - m_all_field_functors[lev].resize(m_varnames_fields.size()); + m_all_field_functors[lev].resize(ntot); // Boolean flag for whether the current density should be deposited before // diagnostic output bool deposit_current = !m_solver_deposits_current; // Fill vector of functors for all components except individual cylindrical modes. - for (int comp=0, n=m_varnames_fields.size(); comp(m_varnames_fields.size()); + for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); @@ -322,6 +327,20 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) "Error: " + m_varnames_fields[comp] + " is not a known field output type in RZ geometry"); } } + + // Generate field functors for every particle field diagnostic for every species in m_pfield_species. + // The names of the diagnostics are output in the `[varname]_[species]` format. + for (int pcomp=0; pcomp(nullptr, + lev, m_crse_ratio, m_pfield_strings[pcomp], m_pfield_species_index[ispec], m_pfield_do_average[pcomp], + m_pfield_dofilter[pcomp], m_pfield_filter_strings[pcomp]); + if (update_varnames) { + AddRZModesToOutputNames(std::string(m_pfield_varnames[pcomp]) + "_" + std::string(m_pfield_species[ispec]), ncomp); + } + } + } + // Sum the number of components in input vector m_all_field_functors // and check that it corresponds to the number of components in m_varnames // and m_mf_output @@ -329,6 +348,7 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) for (int jj=0; jjnComp(); } + AMREX_ALWAYS_ASSERT( ncomp_from_src == m_varnames.size() ); #else amrex::ignore_unused(lev); @@ -340,7 +360,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) { #ifdef WARPX_DIM_RZ - if (!m_dump_rz_modes) return; + if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); @@ -371,7 +391,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) bool deposit_current = !m_solver_deposits_current; // First index of m_all_field_functors[lev] where RZ modes are stored - int icomp = m_all_field_functors[0].size(); + auto icomp =static_cast(m_all_field_functors[0].size()); const std::array coord {"r", "theta", "z"}; // Er, Etheta, Ez, Br, Btheta, Bz, jr, jtheta, jz @@ -487,11 +507,13 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); diag_dom.setHi(idim, std::min(m_hi[idim],warpx.Geom(lev).ProbHi(idim)) ); if ( std::fabs(warpx.Geom(lev).ProbLo(idim) - diag_dom.lo(idim)) - > warpx.Geom(lev).CellSize(idim) ) + > warpx.Geom(lev).CellSize(idim) ) { use_warpxba = false; + } if ( std::fabs(warpx.Geom(lev).ProbHi(idim) - diag_dom.hi(idim)) - > warpx.Geom(lev).CellSize(idim) ) + > warpx.Geom(lev).CellSize(idim) ) { use_warpxba = false; + } // User-defined value for coarsening should be an integer divisor of // blocking factor at level, lev. This assert is not relevant and thus @@ -553,7 +575,7 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { ba.coarsen(m_crse_ratio); // Generate a new distribution map if the physical m_lo and m_hi for the output // is different from the lo and hi physical co-ordinates of the simulation domain. - if (!use_warpxba) dmap = amrex::DistributionMapping{ba}; + if (!use_warpxba) { dmap = amrex::DistributionMapping{ba}; } // Allocate output MultiFab for diagnostics. The data will be stored at cell-centers. const int ngrow = (m_format == "sensei" || m_format == "ascent") ? 1 : 0; int const ncomp = static_cast(m_varnames.size()); @@ -607,7 +629,7 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev].resize(ntot); // Fill vector of functors for all components except individual cylindrical modes. for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bz" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); diff --git a/Source/Diagnostics/MultiDiagnostics.cpp b/Source/Diagnostics/MultiDiagnostics.cpp index 5a50dfb6565..ea14919c713 100644 --- a/Source/Diagnostics/MultiDiagnostics.cpp +++ b/Source/Diagnostics/MultiDiagnostics.cpp @@ -70,9 +70,9 @@ MultiDiagnostics::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diag_type_str == "Full" || diag_type_str == "BackTransformed" || diag_type_str == "BoundaryScraping", ".diag_type must be Full or BackTransformed or BoundaryScraping"); - if (diag_type_str == "Full") diags_types[i] = DiagTypes::Full; - if (diag_type_str == "BackTransformed") diags_types[i] = DiagTypes::BackTransformed; - if (diag_type_str == "BoundaryScraping") diags_types[i] = DiagTypes::BoundaryScraping; + if (diag_type_str == "Full") { diags_types[i] = DiagTypes::Full; } + if (diag_type_str == "BackTransformed") { diags_types[i] = DiagTypes::BackTransformed; } + if (diag_type_str == "BoundaryScraping") { diags_types[i] = DiagTypes::BoundaryScraping; } } } @@ -82,11 +82,13 @@ MultiDiagnostics::FilterComputePackFlush (int step, bool force_flush, bool BackT int i = 0; for (auto& diag : alldiags){ if (BackTransform) { - if (diags_types[i] == DiagTypes::BackTransformed) + if (diags_types[i] == DiagTypes::BackTransformed) { diag->FilterComputePackFlush (step, force_flush); + } } else { - if (diags_types[i] != DiagTypes::BackTransformed) + if (diags_types[i] != DiagTypes::BackTransformed) { diag->FilterComputePackFlush (step, force_flush); + } } ++i; } diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.H b/Source/Diagnostics/ParticleDiag/ParticleDiag.H index 03537ebfe43..7d8b5819f5f 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.H +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.H @@ -18,9 +18,9 @@ class ParticleDiag { public: ParticleDiag(std::string diag_name, std::string name, WarpXParticleContainer* pc, PinnedMemoryParticleContainer *pinned_pc = nullptr); - WarpXParticleContainer* getParticleContainer() const { return m_pc; } - PinnedMemoryParticleContainer* getPinnedParticleContainer() const { return m_pinned_pc; } - std::string getSpeciesName() const { return m_name; } + [[nodiscard]] WarpXParticleContainer* getParticleContainer() const { return m_pc; } + [[nodiscard]] PinnedMemoryParticleContainer* getPinnedParticleContainer() const { return m_pinned_pc; } + [[nodiscard]] std::string getSpeciesName() const { return m_name; } amrex::Vector m_plot_flags; bool m_do_random_filter = false; diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index c10e3468a71..7ca5e6541d7 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -95,7 +95,7 @@ RigidInjectedParticleContainer::WriteHeader (std::ostream& os) const PhysicalParticleContainer::WriteHeader( os ); // Write quantities that are specific to the rigid-injected species - const int nlevs = zinject_plane_levels.size(); + const auto nlevs = static_cast(zinject_plane_levels.size()); os << nlevs << "\n"; for (int i = 0; i < nlevs; ++i) { diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.H b/Source/Diagnostics/ReducedDiags/BeamRelevant.H index 4e0d0077dad..018ddef5463 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.H +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.H @@ -29,11 +29,11 @@ public: std::string m_beam_name; /** - * This function computes beam relevant quantites. + * This function computes beam relevant quantities. * * @param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; }; diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index ac5f3fd6632..fbe67ba6fc5 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -214,14 +214,21 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_pos0 = p.pos(0); const ParticleReal p_w = p.rdata(PIdx::w); -#if (defined WARPX_DIM_RZ) +#if defined(WARPX_DIM_3D) + const ParticleReal p_pos1 = p.pos(1); + const ParticleReal p_x_mean = p_pos0*p_w; + const ParticleReal p_y_mean = p_pos1*p_w; +#elif defined(WARPX_DIM_RZ) const ParticleReal p_theta = p.rdata(PIdx::theta); const ParticleReal p_x_mean = p_pos0*std::cos(p_theta)*p_w; const ParticleReal p_y_mean = p_pos0*std::sin(p_theta)*p_w; -#else - const ParticleReal p_pos1 = p.pos(1); +#elif defined(WARPX_DIM_XZ) const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = p_pos1*p_w; + const ParticleReal p_y_mean = 0; +#elif defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(p_pos0); + const ParticleReal p_x_mean = 0; + const ParticleReal p_y_mean = 0; #endif const ParticleReal p_z_mean = p.pos(index_z)*p_w; @@ -250,7 +257,7 @@ void BeamRelevant::ComputeDiags (int step) // reduced sum over mpi ranks (allreduce) amrex::ParallelAllReduce::Sum - ( values_per_rank_1st.data(), values_per_rank_1st.size(), ParallelDescriptor::Communicator()); + ( values_per_rank_1st.data(), static_cast(values_per_rank_1st.size()), ParallelDescriptor::Communicator()); const ParticleReal w_sum = values_per_rank_1st.at(0); const ParticleReal x_mean = values_per_rank_1st.at(1) /= w_sum; @@ -263,7 +270,7 @@ void BeamRelevant::ComputeDiags (int step) if (w_sum < std::numeric_limits::min() ) { - for (auto& item: m_data) item = 0.0_rt; + for (auto& item: m_data) { item = 0.0_rt; } return; } @@ -345,7 +352,7 @@ void BeamRelevant::ComputeDiags (int step) // reduced sum over mpi ranks (reduce to IO rank) ParallelDescriptor::ReduceRealSum - ( values_per_rank_2nd.data(), values_per_rank_2nd.size(), ParallelDescriptor::IOProcessorNumber()); + ( values_per_rank_2nd.data(), static_cast(values_per_rank_2nd.size()), ParallelDescriptor::IOProcessorNumber()); const ParticleReal x_ms = values_per_rank_2nd.at(0) /= w_sum; const ParticleReal y_ms = values_per_rank_2nd.at(1) /= w_sum; diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.H b/Source/Diagnostics/ReducedDiags/ChargeOnEB.H index c7aa58f06e2..ed69ce01767 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.H +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.H @@ -42,7 +42,7 @@ public: * * @param[in] step current time step */ - virtual void ComputeDiags (int step) override final; + void ComputeDiags (int step) final; private: /// Optional parser to add weight inside the integral diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.H b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H index 1998c56812d..fe00dd2e7e3 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.H +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H @@ -44,7 +44,7 @@ public: * [14]thetay_min, [15]thetay_ave, [16]thetay_max, [17]thetay_std * same for second species follows. */ - void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; private: /// auxiliary structure to store headers and indices of the reduced diagnostics diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 1e2c16ac737..a1a7b6cd61b 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -222,7 +222,7 @@ void ColliderRelevant::ComputeDiags (int step) using PType = typename WarpXParticleContainer::SuperParticleType; num_dens[i_s] = myspc.GetChargeDensity(0); - num_dens[i_s]->mult(1./q); + num_dens[i_s]->mult(1._prt/q); #if defined(WARPX_DIM_1D_Z) // w_tot @@ -456,7 +456,7 @@ void ColliderRelevant::ComputeDiags (int step) // Loop over boxes for (WarpXParIter pti(myspc, lev); pti.isValid(); ++pti) { - const auto GetPosition = GetParticlePosition(pti); + const auto GetPosition = GetParticlePosition(pti); // get particle arrays amrex::ParticleReal* const AMREX_RESTRICT ux = pti.GetAttribs()[PIdx::ux].dataPtr(); amrex::ParticleReal* const AMREX_RESTRICT uy = pti.GetAttribs()[PIdx::uy].dataPtr(); @@ -465,6 +465,13 @@ void ColliderRelevant::ComputeDiags (int step) // declare external fields const int offset = 0; const auto getExternalEB = GetExternalEBField(pti, offset); + const amrex::ParticleReal Ex_external_particle = myspc.m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = myspc.m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = myspc.m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = myspc.m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = myspc.m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = myspc.m_B_external_particle[2]; + // define variables in preparation for field gathering amrex::Box box = pti.tilebox(); box.grow(ngEB); @@ -491,8 +498,13 @@ void ColliderRelevant::ComputeDiags (int step) // get external fields amrex::ParticleReal xp, yp, zp; GetPosition(i, xp, yp, zp); - amrex::ParticleReal ex = 0._rt, ey = 0._rt, ez = 0._rt; - amrex::ParticleReal bx = 0._rt, by = 0._rt, bz = 0._rt; + amrex::ParticleReal ex = Ex_external_particle; + amrex::ParticleReal ey = Ey_external_particle; + amrex::ParticleReal ez = Ez_external_particle; + amrex::ParticleReal bx = Bx_external_particle; + amrex::ParticleReal by = By_external_particle; + amrex::ParticleReal bz = Bz_external_particle; + getExternalEB(i, ex, ey, ez, bx, by, bz); // gather E and B @@ -542,7 +554,7 @@ void ColliderRelevant::ComputeDiags (int step) // compute luminosity amrex::Real const n1_dot_n2 = amrex::MultiFab::Dot(mf_dst1, 0, mf_dst2, 0, 1, 0); - amrex::Real const lumi = 2. * PhysConst::c * n1_dot_n2 * dV; + amrex::Real const lumi = 2._rt * PhysConst::c * n1_dot_n2 * dV; m_data[get_idx("dL_dt")] = lumi; #endif // not RZ } diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.H b/Source/Diagnostics/ReducedDiags/FieldEnergy.H index 73ebad34ba0..4f5d1656841 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.H +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.H @@ -37,7 +37,7 @@ public: * * @param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; /** * \brief Calculate the integral of the field squared in RZ diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.H b/Source/Diagnostics/ReducedDiags/FieldMaximum.H index ea40c0e128c..95ae6ff16da 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.H +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.H @@ -31,7 +31,7 @@ public: * * @param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; }; diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.H b/Source/Diagnostics/ReducedDiags/FieldMomentum.H index 2dadb6a8e86..0f04b6413d1 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.H +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.H @@ -32,7 +32,7 @@ public: * * \param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; }; #endif diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index b9698d5ffd2..fe2040e50d2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -94,7 +94,7 @@ FieldMomentum::FieldMomentum (std::string rd_name) void FieldMomentum::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.H b/Source/Diagnostics/ReducedDiags/FieldProbe.H index 2b232264329..9f318c4f8a0 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.H +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.H @@ -46,18 +46,18 @@ public: /** * This function assins test/data particles to constructed environemnt */ - void InitData () override final; + void InitData () final; /** Redistribute parallel data structures during load balance */ - void LoadBalance () override final; + void LoadBalance () final; /** * This function computes the value of Ex, Ey, Ez, Bx, By, Bz and at a given point * * @param[in] step current time step */ - void ComputeDiags (int step) override final; + void ComputeDiags (int step) final; /* * Define constants used throughout FieldProbe @@ -113,7 +113,7 @@ private: /** * Built-in function in ReducedDiags to write out test data */ - virtual void WriteToFile (int step) const override; + void WriteToFile (int step) const override; /** Check if the probe is in the simulation domain boundary */ diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 7749d5ecc8d..9f45392bb0a 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -340,8 +340,8 @@ bool FieldProbe::ProbeInDomain () const auto & warpx = WarpX::GetInstance(); int const lev = 0; const amrex::Geometry& gm = warpx.Geom(lev); - const auto prob_lo = gm.ProbLo(); - const auto prob_hi = gm.ProbHi(); + const auto *const prob_lo = gm.ProbLo(); + const auto *const prob_hi = gm.ProbHi(); /* * Determine if probe exists within simulation boundaries. During 2D simulations, @@ -429,8 +429,8 @@ void FieldProbe::ComputeDiags (int step) for (MyParIter pti(m_probe, lev); pti.isValid(); ++pti) { - const auto getPosition = GetParticlePosition(pti); - auto setPosition = SetParticlePosition(pti); + const auto getPosition = GetParticlePosition(pti); + auto setPosition = SetParticlePosition(pti); const auto& aos = pti.GetArrayOfStructs(); const auto* AMREX_RESTRICT m_structs = aos().dataPtr(); @@ -591,7 +591,7 @@ void FieldProbe::ComputeDiags (int step) if (amrex::ParallelDescriptor::IOProcessor()) { length_vector.resize(mpisize, 0); } - localsize.resize(1, m_data.size()); + localsize.resize(1, static_cast(m_data.size())); // gather size of m_data from each processor amrex::ParallelDescriptor::Gather(localsize.data(), 1, @@ -600,7 +600,7 @@ void FieldProbe::ComputeDiags (int step) // IO processor sums values from length_array to get size of total output array. /* displs records the size of each m_data as well as previous displs. This array - * tells Gatherv where in the m_data_out array allocation to write incomming data. */ + * tells Gatherv where in the m_data_out array allocation to write incoming data. */ long total_data_size = 0; amrex::Vector displs_vector; if (amrex::ParallelDescriptor::IOProcessor()) { @@ -628,14 +628,15 @@ void FieldProbe::ComputeDiags (int step) void FieldProbe::WriteToFile (int step) const { - if (!(ProbeInDomain() && amrex::ParallelDescriptor::IOProcessor())) return; + if (!(ProbeInDomain() && amrex::ParallelDescriptor::IOProcessor())) { return; } // loop over num valid particles to find the lowest particle ID for later sorting - long int first_id = static_cast(m_data_out[0]); + auto first_id = static_cast(m_data_out[0]); for (long int i = 0; i < m_valid_particles; i++) { - if (m_data_out[i*noutputs] < first_id) + if (m_data_out[i*noutputs] < first_id) { first_id = static_cast(m_data_out[i*noutputs]); + } } // Create a new array to store probe data ordered by id, which will be printed to file. diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H index d658f209c8f..c85bf8fd541 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H @@ -19,21 +19,17 @@ * This enumerated struct is used to index the field probe particle * values that are being stored as SoA data. Nattribs * is enumerated to give the number of attributes stored. - * The strange insertion of `theta` below is due to the use of - * GetParticlePosition for the field probe locations, which reads the probe - * theta value from PIdx::theta = 4. */ struct FieldProbePIdx { enum { Ex = 0, Ey, Ez, - Bx, + Bx, By, Bz, + S, //!< the Poynting vector #ifdef WARPX_DIM_RZ theta, ///< RZ needs all three position components #endif - By, Bz, - S, //!< the Poynting vector nattribs }; }; @@ -48,7 +44,12 @@ class FieldProbeParticleContainer { public: FieldProbeParticleContainer (amrex::AmrCore* amr_core); - virtual ~FieldProbeParticleContainer() {} + ~FieldProbeParticleContainer() override = default; + + FieldProbeParticleContainer ( FieldProbeParticleContainer const &) = delete; + FieldProbeParticleContainer& operator= ( FieldProbeParticleContainer const & ) = delete; + FieldProbeParticleContainer ( FieldProbeParticleContainer&& ) = default; + FieldProbeParticleContainer& operator= ( FieldProbeParticleContainer&& ) = default; //! amrex iterator for our number of attributes using iterator = amrex::ParIter<0, 0, FieldProbePIdx::nattribs, 0>; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp index d928bd33fb9..1fd741ddc47 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp @@ -50,7 +50,6 @@ #include #include #include -#include #include @@ -76,7 +75,7 @@ FieldProbeParticleContainer::AddNParticles (int lev, AMREX_ALWAYS_ASSERT(x.size() == z.size()); // number of particles to add - int const np = x.size(); + auto const np = static_cast(x.size()); // have to resize here, not in the constructor because grids have not // been built when constructor was called. @@ -143,7 +142,7 @@ FieldProbeParticleContainer::AddNParticles (int lev, /* * Redistributes particles to their appropriate tiles if the box - * structure of the simulation changes to accomodate data more + * structure of the simulation changes to accommodate data more * efficiently. */ Redistribute(); diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index c4c81c0161a..e99c87ad14f 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -58,7 +58,7 @@ public: * * @param[in] step the timestep */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; /** * This function queries deprecated input parameters and aborts diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H index f2bfc12d7e1..0ecb74d4bf5 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H @@ -62,7 +62,7 @@ public: * * @param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; /** * write to file function for costs; this differs from the base class @@ -71,7 +71,7 @@ public: * * @param[in] step current time step */ - virtual void WriteToFile(int step) const override final; + void WriteToFile(int step) const final; }; diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 1ce88d043ff..893b00a5f00 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -87,7 +87,7 @@ void LoadBalanceCosts::ComputeDiags (int step) int nBoxes = 0; for (int lev = 0; lev < nLevels; ++lev) { - const auto cost = WarpX::getCosts(lev); + auto *const cost = WarpX::getCosts(lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( cost, "ERROR: costs are not initialized on level " + std::to_string(lev) + " !"); nBoxes += cost->size(); @@ -144,7 +144,7 @@ void LoadBalanceCosts::ComputeDiags (int step) #else m_data[shift_m_data + mfi.index()*m_nDataFields + 5] = 0.; #endif - m_data[shift_m_data + mfi.index()*m_nDataFields + 6] = tbx.d_numPts(); // note: difference to volume + m_data[shift_m_data + mfi.index()*m_nDataFields + 6] = static_cast(tbx.d_numPts()); // note: difference to volume m_data[shift_m_data + mfi.index()*m_nDataFields + 7] = countBoxMacroParticles(mfi, lev); #ifdef AMREX_USE_GPU m_data[shift_m_data + mfi.index()*m_nDataFields + 8] = amrex::Gpu::Device::deviceId(); @@ -158,7 +158,7 @@ void LoadBalanceCosts::ComputeDiags (int step) // parallel reduce to IO proc and get data over all procs ParallelDescriptor::ReduceRealSum(m_data.data(), - m_data.size(), + static_cast(m_data.size()), ParallelDescriptor::IOProcessorNumber()); #ifdef AMREX_USE_MPI @@ -282,7 +282,7 @@ void LoadBalanceCosts::WriteToFile (int step) const // get a reference to WarpX instance auto& warpx = WarpX::GetInstance(); - if (!ParallelDescriptor::IOProcessor()) return; + if (!ParallelDescriptor::IOProcessor()) { return; } // final step is a special case, fill jagged array with NaN if (m_intervals.nextContains(step+1) > warpx.maxStep()) @@ -347,7 +347,7 @@ void LoadBalanceCosts::WriteToFile (int step) const while (std::getline(ss, token, m_sep[0])) { cnt += 1; - if (ss.peek() == m_sep[0]) ss.ignore(); + if (ss.peek() == m_sep[0]) { ss.ignore(); } } // 2 columns for step, time; then nBoxes*nDatafields columns for data; @@ -367,7 +367,11 @@ void LoadBalanceCosts::WriteToFile (int step) const ofstmp.close(); // remove the original, rename tmp file - std::remove(fileDataName.c_str()); - std::rename(fileTmpName.c_str(), fileDataName.c_str()); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + std::remove(fileDataName.c_str()) == EXIT_SUCCESS, + "Failed to remove " + fileDataName); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + std::rename(fileTmpName.c_str(), fileDataName.c_str()) == EXIT_SUCCESS, + "Failed to rename " + fileTmpName + " into " + fileDataName); } } diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H index d0c823fb949..a43868ccc5f 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H @@ -31,7 +31,7 @@ public: * * @param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; }; #endif diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.H b/Source/Diagnostics/ReducedDiags/ParticleEnergy.H index b143cc93f04..b0df66709f3 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.H +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.H @@ -35,7 +35,7 @@ public: * * @param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; }; diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp index a132c135471..157b4bfb368 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp @@ -88,7 +88,7 @@ ParticleEnergy::ParticleEnergy (std::string rd_name) void ParticleEnergy::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get MultiParticleContainer class object const auto & mypc = WarpX::GetInstance().GetPartContainer(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H index b6bfb7c5e22..ed5cce1d7e0 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H @@ -31,11 +31,11 @@ public: std::string m_species_name; /** - * This funciton computes the particle extrema + * This function computes the particle extrema * * @param[in] step current time step */ - void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; private: /// auxiliary structure to store headers and indices of the reduced diagnostics diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index e125f44e08f..ba8e7e373de 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -388,7 +388,7 @@ void ParticleExtrema::ComputeDiags (int step) // Loop over boxes for (WarpXParIter pti(myspc, lev); pti.isValid(); ++pti) { - const auto GetPosition = GetParticlePosition(pti); + const auto GetPosition = GetParticlePosition(pti); // get particle arrays amrex::ParticleReal* const AMREX_RESTRICT ux = pti.GetAttribs()[PIdx::ux].dataPtr(); amrex::ParticleReal* const AMREX_RESTRICT uy = pti.GetAttribs()[PIdx::uy].dataPtr(); @@ -396,6 +396,13 @@ void ParticleExtrema::ComputeDiags (int step) // declare external fields const int offset = 0; const auto getExternalEB = GetExternalEBField(pti, offset); + const amrex::ParticleReal Ex_external_particle = myspc.m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = myspc.m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = myspc.m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = myspc.m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = myspc.m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = myspc.m_B_external_particle[2]; + // define variables in preparation for field gathering amrex::Box box = pti.tilebox(); box.grow(ngEB); @@ -422,8 +429,13 @@ void ParticleExtrema::ComputeDiags (int step) // get external fields ParticleReal xp, yp, zp; GetPosition(i, xp, yp, zp); - ParticleReal ex = 0._rt, ey = 0._rt, ez = 0._rt; - ParticleReal bx = 0._rt, by = 0._rt, bz = 0._rt; + amrex::ParticleReal ex = Ex_external_particle; + amrex::ParticleReal ey = Ey_external_particle; + amrex::ParticleReal ez = Ez_external_particle; + amrex::ParticleReal bx = Bx_external_particle; + amrex::ParticleReal by = By_external_particle; + amrex::ParticleReal bz = Bz_external_particle; + getExternalEB(i, ex, ey, ez, bx, by, bz); // gather E and B diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram.H index 07a2b9599fb..758f4a399a9 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.H @@ -62,7 +62,7 @@ public: * * @param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; }; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp index 242659236d6..512e6f1394a 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp @@ -50,8 +50,8 @@ struct NormalizationType { }; // constructor -ParticleHistogram::ParticleHistogram (std::string rd_name) -: ReducedDiags{rd_name} +ParticleHistogram::ParticleHistogram (std::string rd_name): + ReducedDiags{rd_name} { const ParmParse pp_rd_name(rd_name); @@ -60,10 +60,15 @@ ParticleHistogram::ParticleHistogram (std::string rd_name) pp_rd_name.get("species",selected_species_name); // read bin parameters - utils::parser::getWithParser(pp_rd_name, "bin_number",m_bin_num); - utils::parser::getWithParser(pp_rd_name, "bin_max", m_bin_max); - utils::parser::getWithParser(pp_rd_name, "bin_min", m_bin_min); - m_bin_size = (m_bin_max - m_bin_min) / m_bin_num; + int bin_num = 0; + amrex::Real bin_max = 0.0_rt, bin_min = 0.0_rt; + utils::parser::getWithParser(pp_rd_name, "bin_number", bin_num); + utils::parser::getWithParser(pp_rd_name, "bin_max", bin_max); + utils::parser::getWithParser(pp_rd_name, "bin_min", bin_min); + m_bin_num = bin_num; + m_bin_max = bin_max; + m_bin_min = bin_min; + m_bin_size = (bin_max - bin_min) / bin_num; // read histogram function std::string function_string; @@ -153,7 +158,7 @@ ParticleHistogram::ParticleHistogram (std::string rd_name) void ParticleHistogram::ComputeDiags (int step) { // Judge if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); @@ -196,7 +201,7 @@ void ParticleHistogram::ComputeDiags (int step) { for (WarpXParIter pti(myspc, lev); pti.isValid(); ++pti) { - auto const GetPosition = GetParticlePosition(pti); + auto const GetPosition = GetParticlePosition(pti); auto & attribs = pti.GetAttribs(); ParticleReal* const AMREX_RESTRICT d_w = attribs[PIdx::w].dataPtr(); @@ -218,14 +223,16 @@ void ParticleHistogram::ComputeDiags (int step) auto const uz = d_uz[i] / PhysConst::c; // don't count a particle if it is filtered out - if (do_parser_filter) - if (!fun_filterparser(t, x, y, z, ux, uy, uz)) + if (do_parser_filter) { + if (fun_filterparser(t, x, y, z, ux, uy, uz) == 0._rt) { return; + } + } // continue function if particle is not filtered out auto const f = fun_partparser(t, x, y, z, ux, uy, uz); // determine particle bin int const bin = int(Math::floor((f-bin_min)/bin_size)); - if ( bin<0 || bin>=num_bins ) return; // discard if out-of-range + if ( bin<0 || bin>=num_bins ) { return; } // discard if out-of-range // add particle to histogram bin //! @todo performance: on CPU, we are probably faster by @@ -247,7 +254,7 @@ void ParticleHistogram::ComputeDiags (int step) // reduced sum over mpi ranks ParallelDescriptor::ReduceRealSum - (m_data.data(), m_data.size(), ParallelDescriptor::IOProcessorNumber()); + (m_data.data(), static_cast(m_data.size()), ParallelDescriptor::IOProcessorNumber()); // normalize the maximum value to be one if ( m_norm == NormalizationType::max_to_unity ) @@ -255,11 +262,11 @@ void ParticleHistogram::ComputeDiags (int step) Real f_max = 0.0_rt; for ( int i = 0; i < m_bin_num; ++i ) { - if ( m_data[i] > f_max ) f_max = m_data[i]; + if ( m_data[i] > f_max ) { f_max = m_data[i]; } } for ( int i = 0; i < m_bin_num; ++i ) { - if ( f_max > std::numeric_limits::min() ) m_data[i] /= f_max; + if ( f_max > std::numeric_limits::min() ) { m_data[i] /= f_max; } } return; } @@ -274,7 +281,7 @@ void ParticleHistogram::ComputeDiags (int step) } for ( int i = 0; i < m_bin_num; ++i ) { - if ( f_area > std::numeric_limits::min() ) m_data[i] /= f_area; + if ( f_area > std::numeric_limits::min() ) { m_data[i] /= f_area; } } return; } diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H index 6b82514567d..98697380a4a 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H @@ -84,14 +84,14 @@ public: * * @param[in] step current time step */ - void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; /** * write to file function * * @param[in] step current time step */ - void WriteToFile (int step) const override final; + void WriteToFile (int step) const final; }; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index 0567f1cfc29..7cac712b4fc 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -61,8 +61,9 @@ ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) pp_rd_name.query("openpmd_backend", m_openpmd_backend); // pick first available backend if default is chosen - if( m_openpmd_backend == "default" ) + if( m_openpmd_backend == "default" ) { m_openpmd_backend = WarpXOpenPMDFileType(); + } pp_rd_name.add("openpmd_backend", m_openpmd_backend); // read species @@ -130,7 +131,7 @@ ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) void ParticleHistogram2D::ComputeDiags (int step) { // Judge if the diags should be done at this step - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // resize data array Array tlo{0,0}; // lower bounds @@ -193,7 +194,7 @@ void ParticleHistogram2D::ComputeDiags (int step) { for (WarpXParIter pti(myspc, lev); pti.isValid(); ++pti) { - auto const GetPosition = GetParticlePosition(pti); + auto const GetPosition = GetParticlePosition(pti); auto & attribs = pti.GetAttribs(); ParticleReal* const AMREX_RESTRICT d_w = attribs[PIdx::w].dataPtr(); @@ -215,9 +216,11 @@ void ParticleHistogram2D::ComputeDiags (int step) auto const uz = d_uz[i] / PhysConst::c; // don't count a particle if it is filtered out - if (do_parser_filter) - if(!static_cast(fun_filterparser(t, x, y, z, ux, uy, uz, w))) + if (do_parser_filter) { + if(!static_cast(fun_filterparser(t, x, y, z, ux, uy, uz, w))) { return; + } + } // continue function if particle is not filtered out auto const f_abs = fun_partparser_abs(t, x, y, z, ux, uy, uz, w); @@ -226,10 +229,10 @@ void ParticleHistogram2D::ComputeDiags (int step) // determine particle bin int const bin_abs = int(Math::floor((f_abs-bin_min_abs)/bin_size_abs)); - if ( bin_abs<0 || bin_abs>=num_bins_abs ) return; // discard if out-of-range + if ( bin_abs<0 || bin_abs>=num_bins_abs ) { return; } // discard if out-of-range int const bin_ord = int(Math::floor((f_ord-bin_min_ord)/bin_size_ord)); - if ( bin_ord<0 || bin_ord>=num_bins_ord ) return; // discard if out-of-range + if ( bin_ord<0 || bin_ord>=num_bins_ord ) { return; } // discard if out-of-range amrex::Real &data = d_table(bin_abs, bin_ord); amrex::HostDevice::Atomic::Add(&data, weight); diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.H b/Source/Diagnostics/ReducedDiags/ParticleMomentum.H index ec16d8184c5..feb8957e516 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.H +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.H @@ -34,7 +34,7 @@ public: * * \param [in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; }; #endif diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp index 5b370d29979..5f278d48f14 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp @@ -116,7 +116,7 @@ ParticleMomentum::ParticleMomentum (std::string rd_name) void ParticleMomentum::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get MultiParticleContainer class object const auto & mypc = WarpX::GetInstance().GetPartContainer(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.H b/Source/Diagnostics/ReducedDiags/ParticleNumber.H index ad6c886e269..3143a387a74 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.H +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.H @@ -32,7 +32,7 @@ public: * * @param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; }; diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.H b/Source/Diagnostics/ReducedDiags/ReducedDiags.H index c4fc82fbddf..01847a1ff12 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.H +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.H @@ -59,6 +59,13 @@ public: */ virtual ~ReducedDiags () = default; + // Default move and copy operations + ReducedDiags(const ReducedDiags&) = default; + ReducedDiags& operator=(const ReducedDiags&) = default; + ReducedDiags(ReducedDiags&&) = default; + ReducedDiags& operator=(ReducedDiags&&) = default; + + /** * function to initialize data after amr * levels are initialized. diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index 758b48b7262..b4f617a8520 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -116,7 +116,7 @@ void ReducedDiags::WriteToFile (int step) const ofs << WarpX::GetInstance().gett_new(0); // loop over data size and write - for (const auto& item : m_data) ofs << m_sep << item; + for (const auto& item : m_data) { ofs << m_sep << item; } // end loop over data size diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.H b/Source/Diagnostics/ReducedDiags/RhoMaximum.H index e7386f73383..6f306a5e7ad 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.H +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.H @@ -36,7 +36,7 @@ public: * * @param[in] step current time step */ - virtual void ComputeDiags(int step) override final; + void ComputeDiags(int step) final; private: /** Vector of (pointers to) functors to compute rho, per level, per species. We reuse here the diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp index 79698682b97..787a95ce412 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp @@ -133,7 +133,7 @@ void RhoMaximum::ComputeDiags (int step) // get number of levels const auto nLevel = warpx.finestLevel() + 1; - const int n_charged_species = m_rho_functors[0].size() - 1; + const auto n_charged_species = static_cast(m_rho_functors[0].size() - 1); // Min and max of total rho + max of |rho| for each species const int noutputs_per_level = 2+n_charged_species; diff --git a/Source/Diagnostics/SliceDiagnostic.H b/Source/Diagnostics/SliceDiagnostic.H index 977b147e4e8..000b01d470f 100644 --- a/Source/Diagnostics/SliceDiagnostic.H +++ b/Source/Diagnostics/SliceDiagnostic.H @@ -19,20 +19,20 @@ std::unique_ptr CreateSlice( const amrex::MultiFab& mf, amrex::IntVect &slice_cr_ratio ); void CheckSliceInput( amrex::RealBox real_box, - amrex::RealBox &slice_cc_nd_box, amrex::RealBox &slice_realbox, - amrex::IntVect &slice_cr_ratio, amrex::Vector dom_geom, - amrex::IntVect SliceType, amrex::IntVect &slice_lo, - amrex::IntVect &slice_hi, amrex::IntVect &interp_lo); + amrex::RealBox &slice_cc_nd_box, amrex::RealBox &slice_realbox, + amrex::IntVect &slice_cr_ratio, amrex::Vector dom_geom, + amrex::IntVect SliceType, amrex::IntVect &slice_lo, + amrex::IntVect &slice_hi, amrex::IntVect &interp_lo); void InterpolateSliceValues( amrex::MultiFab& smf, - amrex::IntVect interp_lo, amrex::RealBox slice_realbox, - amrex::Vector geom, int ncomp, int nghost, - amrex::IntVect slice_lo, amrex::IntVect slice_hi, - amrex::IntVect SliceType, amrex::RealBox real_box); + amrex::IntVect interp_lo, amrex::RealBox slice_realbox, + amrex::Vector geom, int ncomp, int nghost, + amrex::IntVect slice_lo, amrex::IntVect slice_hi, + amrex::IntVect SliceType, amrex::RealBox real_box); void InterpolateLo( const amrex::Box& bx, amrex::FArrayBox &fabox, - amrex::IntVect slice_lo, amrex::Vector geom, - int idir, amrex::IntVect IndType, amrex::RealBox slice_realbox, - int srccomp, int ncomp, int nghost, amrex::RealBox real_box); + amrex::IntVect slice_lo, amrex::Vector geom, + int idir, amrex::IntVect IndType, amrex::RealBox slice_realbox, + int srccomp, int ncomp, int nghost, amrex::RealBox real_box); #endif diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index aede8b303b8..71a585aea35 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -44,7 +44,7 @@ using namespace amrex; /* \brief * The functions creates the slice for diagnostics based on the user-input. - * The slice can be 1D, 2D, or 3D and it inherts the index type of the underlying data. + * The slice can be 1D, 2D, or 3D and it inherits the index type of the underlying data. * The implementation assumes that the slice is aligned with the coordinate axes. * The input parameters are modified if the user-input does not comply with requirements of coarsenability or if the slice extent is not contained within the simulation domain. * First a slice multifab (smf) with cell size equal to that of the simulation grid is created such that it extends from slice.dim_lo to slice.dim_hi and shares the same index space as the source multifab (mf) @@ -101,31 +101,31 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, // Determine if interpolation is required and number of cells in slice // for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - // Flag for interpolation if required // - if ( interp_lo[idim] == 1) { - interpolate = 1; - } - - // For the case when a dimension is reduced // - if ( ( slice_hi[idim] - slice_lo[idim]) == 1) { - slice_ncells[idim] = 1; - } - else { - slice_ncells[idim] = ( slice_hi[idim] - slice_lo[idim] + 1 ) - / slice_cr_ratio[idim]; - - const int refined_ncells = slice_hi[idim] - slice_lo[idim] + 1 ; - if ( slice_cr_ratio[idim] > 1) { - coarsen = true; - - // modify slice_grid_size if >= refines_cells // - if ( slice_grid_size >= refined_ncells ) { - slice_grid_size = refined_ncells - 1; - } - - } - configuration_dim += 1; - } + // Flag for interpolation if required // + if ( interp_lo[idim] == 1) { + interpolate = true; + } + + // For the case when a dimension is reduced // + if ( ( slice_hi[idim] - slice_lo[idim]) == 1) { + slice_ncells[idim] = 1; + } + else { + slice_ncells[idim] = ( slice_hi[idim] - slice_lo[idim] + 1 ) + / slice_cr_ratio[idim]; + + const int refined_ncells = slice_hi[idim] - slice_lo[idim] + 1 ; + if ( slice_cr_ratio[idim] > 1) { + coarsen = true; + + // modify slice_grid_size if >= refines_cells // + if ( slice_grid_size >= refined_ncells ) { + slice_grid_size = refined_ncells - 1; + } + + } + configuration_dim += 1; + } } if (configuration_dim==1) { ablastr::warn_manager::WMRecordWarning("Diagnostics", @@ -151,7 +151,7 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, const amrex::IntVect nghost_vect(AMREX_D_DECL(nghost, nghost, nghost)); ablastr::utils::communication::ParallelCopy(*smf, mf, 0, 0, ncomp, nghost_vect, nghost_vect, WarpX::do_single_precision_comms); - // inteprolate if required on refined slice // + // interpolate if required on refined slice // if (interpolate == 1 ) { InterpolateSliceValues( *smf, interp_lo, slice_cc_nd_box, dom_geom, ncomp, nghost, slice_lo, slice_hi, SliceType, real_box); @@ -162,70 +162,70 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, return smf; } else { - Vector crse_ba(1); - crse_ba[0] = sba[0]; - crse_ba[0].coarsen(slice_cr_ratio); + Vector crse_ba(1); + crse_ba[0] = sba[0]; + crse_ba[0].coarsen(slice_cr_ratio); - AMREX_ALWAYS_ASSERT(crse_ba[0].size() == sba[0].size()); + AMREX_ALWAYS_ASSERT(crse_ba[0].size() == sba[0].size()); - cs_mf = std::make_unique(amrex::convert(crse_ba[0],SliceType), - sdmap[0], ncomp,nghost); + cs_mf = std::make_unique(amrex::convert(crse_ba[0],SliceType), + sdmap[0], ncomp,nghost); - const MultiFab& mfSrc = *smf; - MultiFab& mfDst = *cs_mf; + const MultiFab& mfSrc = *smf; + MultiFab& mfDst = *cs_mf; - MFIter mfi_dst(mfDst); - for (MFIter mfi(mfSrc); mfi.isValid(); ++mfi) { + MFIter mfi_dst(mfDst); + for (MFIter mfi(mfSrc); mfi.isValid(); ++mfi) { - Array4 const& Src_fabox = mfSrc.const_array(mfi); + Array4 const& Src_fabox = mfSrc.const_array(mfi); - const Box& Dst_bx = mfi_dst.validbox(); - Array4 const& Dst_fabox = mfDst.array(mfi_dst); + const Box& Dst_bx = mfi_dst.validbox(); + Array4 const& Dst_fabox = mfDst.array(mfi_dst); - const int scomp = 0; - const int dcomp = 0; + const int scomp = 0; + const int dcomp = 0; - const IntVect cctype(AMREX_D_DECL(0,0,0)); - if( SliceType==cctype ) { - amrex::amrex_avgdown(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, - ncomp, slice_cr_ratio); - } - const IntVect ndtype(AMREX_D_DECL(1,1,1)); - if( SliceType == ndtype ) { - amrex::amrex_avgdown_nodes(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio); - } - if( SliceType == WarpX::GetInstance().getEfield(0,0).ixType().toIntVect() ) { - amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 0); - } - if( SliceType == WarpX::GetInstance().getEfield(0,1).ixType().toIntVect() ) { - amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 1); - } - if( SliceType == WarpX::GetInstance().getEfield(0,2).ixType().toIntVect() ) { - amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 2); - } - if( SliceType == WarpX::GetInstance().getBfield(0,0).ixType().toIntVect() ) { - amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 0); - } - if( SliceType == WarpX::GetInstance().getBfield(0,1).ixType().toIntVect() ) { - amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 1); - } - if( SliceType == WarpX::GetInstance().getBfield(0,2).ixType().toIntVect() ) { - amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, - scomp, ncomp, slice_cr_ratio, 2); - } + const IntVect cctype(AMREX_D_DECL(0,0,0)); + if( SliceType==cctype ) { + amrex::amrex_avgdown(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, + ncomp, slice_cr_ratio); + } + const IntVect ndtype(AMREX_D_DECL(1,1,1)); + if( SliceType == ndtype ) { + amrex::amrex_avgdown_nodes(Dst_bx, Dst_fabox, Src_fabox, dcomp, + scomp, ncomp, slice_cr_ratio); + } + if( SliceType == WarpX::GetInstance().getEfield(0,0).ixType().toIntVect() ) { + amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, + scomp, ncomp, slice_cr_ratio, 0); + } + if( SliceType == WarpX::GetInstance().getEfield(0,1).ixType().toIntVect() ) { + amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, + scomp, ncomp, slice_cr_ratio, 1); + } + if( SliceType == WarpX::GetInstance().getEfield(0,2).ixType().toIntVect() ) { + amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, + scomp, ncomp, slice_cr_ratio, 2); + } + if( SliceType == WarpX::GetInstance().getBfield(0,0).ixType().toIntVect() ) { + amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, + scomp, ncomp, slice_cr_ratio, 0); + } + if( SliceType == WarpX::GetInstance().getBfield(0,1).ixType().toIntVect() ) { + amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, + scomp, ncomp, slice_cr_ratio, 1); + } + if( SliceType == WarpX::GetInstance().getBfield(0,2).ixType().toIntVect() ) { + amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, + scomp, ncomp, slice_cr_ratio, 2); + } - if ( mfi_dst.isValid() ) { - ++mfi_dst; - } + if ( mfi_dst.isValid() ) { + ++mfi_dst; + } - } - return cs_mf; + } + return cs_mf; } } @@ -299,7 +299,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, warnMsg.str(), ablastr::warn_manager::WarnPriority::low); } - const auto very_small_number = 1E-10; + const auto very_small_number = 1E-10; // Factor to ensure index values computation depending on index type // const double fac = ( 1.0 - SliceType[idim] )*dom_geom[0].CellSize(idim)*0.5; @@ -309,7 +309,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, slice_cc_nd_box.setLo( idim, slice_realbox.lo(idim) ); slice_cc_nd_box.setHi( idim, slice_realbox.hi(idim) ); - if ( slice_cr_ratio[idim] > 1) slice_cr_ratio[idim] = 1; + if ( slice_cr_ratio[idim] > 1) { slice_cr_ratio[idim] = 1; } // check for interpolation -- compute index lo with floor and ceil if ( slice_cc_nd_box.lo(idim) - real_box.lo(idim) >= fac ) { @@ -349,7 +349,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, } else { - // moving realbox.lo and reabox.hi to nearest coarsenable grid point // + // moving realbox.lo and realbox.hi to nearest coarsenable grid point // auto index_lo = static_cast(floor(((slice_realbox.lo(idim) + very_small_number - (real_box.lo(idim))) / dom_geom[0].CellSize(idim))) ); auto index_hi = static_cast(ceil(((slice_realbox.hi(idim) - very_small_number @@ -425,15 +425,15 @@ InterpolateSliceValues(MultiFab& smf, IntVect interp_lo, RealBox slice_realbox, { for (MFIter mfi(smf); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); - FArrayBox& fabox = smf[mfi]; + const Box& bx = mfi.tilebox(); + FArrayBox& fabox = smf[mfi]; - for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( interp_lo[idim] == 1 ) { + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if ( interp_lo[idim] == 1 ) { InterpolateLo( bx, fabox, slice_lo, geom, idim, SliceType, slice_realbox, 0, ncomp, nghost, real_box); - } - } + } + } } } diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 2ae990851e2..91409bc294d 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -66,7 +66,7 @@ WarpX::GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, in ParallelDescriptor::ReadAndBcastFile(DMFileName, fileCharPtr); const std::string fileCharPtrString(fileCharPtr.dataPtr()); std::istringstream DMFile(fileCharPtrString, std::istringstream::in); - if ( ! DMFile.good()) amrex::FileOpenFailed(DMFileName); + if ( ! DMFile.good()) { amrex::FileOpenFailed(DMFileName); } DMFile.exceptions(std::ios_base::failbit | std::ios_base::badbit); int nprocs_in_checkpoint; @@ -382,11 +382,13 @@ WarpX::InitFromCheckpoint () if (do_pml) { for (int lev = 0; lev < nlevs; ++lev) { - if (pml[lev]) + if (pml[lev]) { pml[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); + } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) - if (pml_rz[lev]) + if (pml_rz[lev]) { pml_rz[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); + } #endif } } diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 62a1fa3755f..bf83c1ebb0b 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -88,6 +88,7 @@ public: * @param engine_type ADIOS engine for output * @param engine_parameters map of parameters for the engine * @param fieldPMLdirections PML field solver, @see WarpX::getPMLdirections() + * @param authors a string specifying the authors of the simulation (can be empty) */ WarpXOpenPMDPlot (openPMD::IterationEncoding ie, std::string filetype, @@ -95,10 +96,16 @@ public: std::map< std::string, std::string > operator_parameters, std::string engine_type, std::map< std::string, std::string > engine_parameters, - std::vector fieldPMLdirections); + std::vector fieldPMLdirections, + const std::string& authors); ~WarpXOpenPMDPlot (); + WarpXOpenPMDPlot ( WarpXOpenPMDPlot const &) = delete; + WarpXOpenPMDPlot& operator= ( WarpXOpenPMDPlot const & ) = delete; + WarpXOpenPMDPlot ( WarpXOpenPMDPlot&& ) = default; + WarpXOpenPMDPlot& operator= ( WarpXOpenPMDPlot&& ) = default; + /** Set Iteration Step for the series * * @note If an iteration has been written, then it will give a warning @@ -160,7 +167,7 @@ private: * @param[in] isBTD is this a backtransformed diagnostics write? * @return the iteration object */ - inline openPMD::Iteration GetIteration (int const iteration, bool const isBTD) const + [[nodiscard]] inline openPMD::Iteration GetIteration (int const iteration, bool const isBTD) const { if (isBTD) { @@ -198,15 +205,15 @@ private: * @param[in] varname name from WarpX * @param[out] field_name field name for openPMD-api output * @param[in] comp_name comp name for openPMD-api output - * @param[in] is_theta_mode indicate if this field will be output with theta - * modes (instead of a reconstructed 2D slice) + * @param[in] var_in_theta_mode indicate if this field will be output with theta + * modes (instead of a reconstructed 2D slice) */ void GetMeshCompNames ( int meshLevel, const std::string& varname, std::string& field_name, std::string& comp_name, - bool is_theta_mode + bool var_in_theta_mode ) const; /** This function sets up the entries for storing the particle positions and global IDs @@ -334,6 +341,9 @@ private: // meta data std::vector< bool > m_fieldPMLdirections; //! @see WarpX::getPMLdirections() + + // The authors' string + std::string m_authors; }; #endif // WARPX_USE_OPENPMD diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 69163b9c529..71d96a47927 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -69,7 +69,7 @@ namespace detail snakeToCamel (const std::string& snake_string) { std::string camelString = snake_string; - const int n = camelString.length(); + const auto n = static_cast(camelString.length()); for (int x = 0; x < n; x++) { if (x == 0) @@ -100,8 +100,9 @@ namespace detail std::string const & engine_type, std::map< std::string, std::string > const & engine_parameters) { - if (operator_type.empty() && engine_type.empty()) + if (operator_type.empty() && engine_type.empty()) { return "{}"; + } std::string options; std::string top_block; @@ -111,7 +112,7 @@ namespace detail std::string op_parameters; for (const auto& kv : operator_parameters) { - if (!op_parameters.empty()) op_parameters.append(",\n"); + if (!op_parameters.empty()) { op_parameters.append(",\n"); } op_parameters.append(std::string(12, ' ')) /* just pretty alignment */ .append("\"").append(kv.first).append("\": ") /* key */ .append("\"").append(kv.second).append("\""); /* value (as string) */ @@ -119,7 +120,7 @@ namespace detail std::string en_parameters; for (const auto& kv : engine_parameters) { - if (!en_parameters.empty()) en_parameters.append(",\n"); + if (!en_parameters.empty()) { en_parameters.append(",\n"); } en_parameters.append(std::string(12, ' ')) /* just pretty alignment */ .append("\"").append(kv.first).append("\": ") /* key */ .append("\"").append(kv.second).append("\""); /* value (as string) */ @@ -154,8 +155,9 @@ namespace detail } ] })END"; - if (!engine_type.empty() || !en_parameters.empty()) + if (!engine_type.empty() || !en_parameters.empty()) { op_block += ","; + } } // end operator string block // add the engine string block @@ -170,8 +172,9 @@ namespace detail "type": ")END"; en_block += engine_type + "\""; - if(!en_parameters.empty()) + if(!en_parameters.empty()) { en_block += ","; + } } // non-default engine parameters @@ -217,10 +220,8 @@ namespace detail getParticlePositionComponentLabels (bool ignore_dims=false) { using vs = std::vector< std::string >; - vs positionComponents; - if (ignore_dims) { - positionComponents = vs{"x", "y", "z"}; - } else { + auto positionComponents = vs{"x", "y", "z"}; + if (!ignore_dims) { #if defined(WARPX_DIM_1D_Z) positionComponents = vs{"z"}; #elif defined(WARPX_DIM_XZ) @@ -306,33 +307,29 @@ namespace detail getUnitDimension ( std::string const & record_name ) { - if( (record_name == "position") || (record_name == "positionOffset") ) return { - {openPMD::UnitDimension::L, 1.} - }; - else if( record_name == "momentum" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -1.} - }; - else if( record_name == "charge" ) return { - {openPMD::UnitDimension::T, 1.}, - {openPMD::UnitDimension::I, 1.} - }; - else if( record_name == "mass" ) return { - {openPMD::UnitDimension::M, 1.} - }; - else if( record_name == "E" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -3.}, - {openPMD::UnitDimension::I, -1.}, - }; - else if( record_name == "B" ) return { - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::I, -1.}, - {openPMD::UnitDimension::T, -2.} - }; - else return {}; + if( (record_name == "position") || (record_name == "positionOffset") ) { + return {{openPMD::UnitDimension::L, 1.}}; + } else if( record_name == "momentum" ) { + return {{openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -1.}}; + } else if( record_name == "charge" ) { + return {{openPMD::UnitDimension::T, 1.}, + {openPMD::UnitDimension::I, 1.}}; + } else if( record_name == "mass" ) { + return {{openPMD::UnitDimension::M, 1.}}; + } else if( record_name == "E" ) { + return {{openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -3.}, + {openPMD::UnitDimension::I, -1.}}; + } else if( record_name == "B" ) { + return {{openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::I, -1.}, + {openPMD::UnitDimension::T, -2.}}; + } else { + return {}; + } } /** \brief For a given field that is to be written to an openPMD file, @@ -378,11 +375,15 @@ WarpXOpenPMDPlot::WarpXOpenPMDPlot ( std::map< std::string, std::string > operator_parameters, std::string engine_type, std::map< std::string, std::string > engine_parameters, - std::vector fieldPMLdirections) - :m_Series(nullptr), - m_Encoding(ie), - m_OpenPMDFileType(std::move(openPMDFileType)), - m_fieldPMLdirections(std::move(fieldPMLdirections)) + std::vector fieldPMLdirections, + const std::string& authors) + : m_Series(nullptr), + m_MPIRank{amrex::ParallelDescriptor::MyProc()}, + m_MPISize{amrex::ParallelDescriptor::NProcs()}, + m_Encoding(ie), + m_OpenPMDFileType(std::move(openPMDFileType)), + m_fieldPMLdirections(std::move(fieldPMLdirections)), + m_authors{authors} { m_OpenPMDoptions = detail::getSeriesOptions(operator_type, operator_parameters, engine_type, engine_parameters); @@ -400,23 +401,23 @@ WarpXOpenPMDPlot::~WarpXOpenPMDPlot () std::string WarpXOpenPMDPlot::GetFileName (std::string& filepath) { - filepath.append("/"); - // transform paths for Windows -#ifdef _WIN32 - filepath = openPMD::auxiliary::replace_all(filepath, "/", "\\"); -#endif + filepath.append("/"); + // transform paths for Windows + #ifdef _WIN32 + filepath = openPMD::auxiliary::replace_all(filepath, "/", "\\"); + #endif - std::string filename = "openpmd"; - // - // OpenPMD supports timestepped names - // - if (m_Encoding == openPMD::IterationEncoding::fileBased) { - const std::string fileSuffix = std::string("_%0") + std::to_string(m_file_min_digits) + std::string("T"); - filename = filename.append(fileSuffix); - } - filename.append(".").append(m_OpenPMDFileType); - filepath.append(filename); - return filename; + std::string filename = "openpmd"; + // + // OpenPMD supports timestepped names + // + if (m_Encoding == openPMD::IterationEncoding::fileBased) { + const std::string fileSuffix = std::string("_%0") + std::to_string(m_file_min_digits) + std::string("T"); + filename = filename.append(fileSuffix); + } + filename.append(".").append(m_OpenPMDFileType); + filepath.append(filename); + return filename; } void WarpXOpenPMDPlot::SetStep (int ts, const std::string& dirPrefix, int file_min_digits, @@ -447,7 +448,7 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) // default close is true bool callClose = true; // close BTD file only when isLastBTDFlush is true - if (isBTD and !isLastBTDFlush) callClose = false; + if (isBTD and !isLastBTDFlush) { callClose = false; } if (callClose) { if (m_Series) { GetIteration(m_CurrentStep, isBTD).close(); @@ -470,8 +471,9 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) void WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) { - if( isBTD && m_Series != nullptr ) + if( isBTD && m_Series != nullptr ) { return; // already open for this snapshot (aka timestep in lab frame) + } // either for the next ts file, // or init a single file for all ts @@ -480,10 +482,11 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) // close a previously open series before creating a new one // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686 - if ( m_Encoding == openPMD::IterationEncoding::fileBased ) + if ( m_Encoding == openPMD::IterationEncoding::fileBased ) { m_Series = nullptr; - else if ( m_Series != nullptr ) + } else if ( m_Series != nullptr ) { return; + } if (amrex::ParallelDescriptor::NProcs() > 1) { #if defined(AMREX_USE_MPI) @@ -492,22 +495,19 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) amrex::ParallelDescriptor::Communicator(), m_OpenPMDoptions ); - m_MPISize = amrex::ParallelDescriptor::NProcs(); - m_MPIRank = amrex::ParallelDescriptor::MyProc(); #else WARPX_ABORT_WITH_MESSAGE("openPMD-api not built with MPI support!"); #endif } else { m_Series = std::make_unique(filepath, access, m_OpenPMDoptions); - m_MPISize = 1; - m_MPIRank = 1; } m_Series->setIterationEncoding( m_Encoding ); // input file / simulation setup author - if( !WarpX::authors.empty()) - m_Series->setAuthor( WarpX::authors ); + if( !m_authors.empty()) { + m_Series->setAuthor( m_authors ); + } // more natural naming for PIC m_Series->setMeshesPath( "fields" ); // conform to ED-PIC extension of openPMD @@ -523,15 +523,17 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector& part const bool isBTD, const bool isLastBTDFlush, const amrex::Vector& totalParticlesFlushedAlready) { - WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDParticles()"); +WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDParticles()"); - for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { +for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { WarpXParticleContainer* pc = particle_diags[i].getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = particle_diags[i].getPinnedParticleContainer(); - if (isBTD || use_pinned_pc) - if (!pinned_pc->isDefined()) + if (isBTD || use_pinned_pc) { + if (!pinned_pc->isDefined()) { continue; // Skip to the next particle container + } + } PinnedMemoryParticleContainer tmp = (isBTD || use_pinned_pc) ? pinned_pc->make_alike() : @@ -574,65 +576,74 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector& part int_flags.resize(tmp.NumIntComps(), 1); const auto mass = pc->AmIA() ? PhysConst::m_e : pc->getMass(); - RandomFilter const random_filter(particle_diags[i].m_do_random_filter, - particle_diags[i].m_random_fraction); - UniformFilter const uniform_filter(particle_diags[i].m_do_uniform_filter, - particle_diags[i].m_uniform_stride); - ParserFilter parser_filter(particle_diags[i].m_do_parser_filter, - utils::parser::compileParser + RandomFilter const random_filter(particle_diags[i].m_do_random_filter, + particle_diags[i].m_random_fraction); + UniformFilter const uniform_filter(particle_diags[i].m_do_uniform_filter, + particle_diags[i].m_uniform_stride); + ParserFilter parser_filter(particle_diags[i].m_do_parser_filter, + utils::parser::compileParser (particle_diags[i].m_particle_filter_parser.get()), pc->getMass(), time); - parser_filter.m_units = InputUnits::SI; - GeometryFilter const geometry_filter(particle_diags[i].m_do_geom_filter, + parser_filter.m_units = InputUnits::SI; + GeometryFilter const geometry_filter(particle_diags[i].m_do_geom_filter, particle_diags[i].m_diag_domain); - if (isBTD || use_pinned_pc) { - tmp.copyParticles(*pinned_pc, true); - particlesConvertUnits(ConvertDirection::WarpX_to_SI, &tmp, mass); - } else { - particlesConvertUnits(ConvertDirection::WarpX_to_SI, pc, mass); - using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; - tmp.copyParticles(*pc, - [random_filter,uniform_filter,parser_filter,geometry_filter] - AMREX_GPU_HOST_DEVICE - (const SrcData& src, int ip, const amrex::RandomEngine& engine) - { - const SuperParticleType& p = src.getSuperParticle(ip); - return random_filter(p, engine) * uniform_filter(p, engine) - * parser_filter(p, engine) * geometry_filter(p, engine); - }, true); - particlesConvertUnits(ConvertDirection::SI_to_WarpX, pc, mass); - } + if (isBTD || use_pinned_pc) { + particlesConvertUnits(ConvertDirection::WarpX_to_SI, pinned_pc, mass); + using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; + tmp.copyParticles(*pinned_pc, + [random_filter,uniform_filter,parser_filter,geometry_filter] + AMREX_GPU_HOST_DEVICE + (const SrcData& src, int ip, const amrex::RandomEngine& engine) + { + const SuperParticleType& p = src.getSuperParticle(ip); + return random_filter(p, engine) * uniform_filter(p, engine) + * parser_filter(p, engine) * geometry_filter(p, engine); + }, true); + particlesConvertUnits(ConvertDirection::SI_to_WarpX, pinned_pc, mass); + } else { + particlesConvertUnits(ConvertDirection::WarpX_to_SI, pc, mass); + using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; + tmp.copyParticles(*pc, + [random_filter,uniform_filter,parser_filter,geometry_filter] + AMREX_GPU_HOST_DEVICE + (const SrcData& src, int ip, const amrex::RandomEngine& engine) + { + const SuperParticleType& p = src.getSuperParticle(ip); + return random_filter(p, engine) * uniform_filter(p, engine) + * parser_filter(p, engine) * geometry_filter(p, engine); + }, true); + particlesConvertUnits(ConvertDirection::SI_to_WarpX, pc, mass); + } // real_names contains a list of all real particle attributes. // real_flags is 1 or 0, whether quantity is dumped or not. - { - if (isBTD) { - DumpToFile(&tmp, - particle_diags[i].getSpeciesName(), - m_CurrentStep, - real_flags, - int_flags, - real_names, int_names, - pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush, - totalParticlesFlushedAlready[i] - ); - } else { - DumpToFile(&tmp, - particle_diags[i].getSpeciesName(), - m_CurrentStep, - real_flags, - int_flags, - real_names, int_names, - pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush, - 0 - ); - } + if (isBTD) { + DumpToFile(&tmp, + particle_diags[i].getSpeciesName(), + m_CurrentStep, + real_flags, + int_flags, + real_names, int_names, + pc->getCharge(), pc->getMass(), + isBTD, isLastBTDFlush, + totalParticlesFlushedAlready[i] + ); + } else { + DumpToFile(&tmp, + particle_diags[i].getSpeciesName(), + m_CurrentStep, + real_flags, + int_flags, + real_names, int_names, + pc->getCharge(), pc->getMass(), + isBTD, isLastBTDFlush, + 0 + ); + } } - } +} } void @@ -685,8 +696,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // we will set up empty particles unless it's BTD, where we might add some in a following buffer dump // during this setup, we mark some particle properties as constant and potentially zero-sized bool doParticleSetup = true; - if (isBTD) + if (isBTD) { doParticleSetup = is_first_flush_with_particles || is_last_flush_and_never_particles; + } // this setup stage also implicitly calls "makeEmpty" if needed (i.e., is_last_flush_and_never_particles) // for BTD, we call this multiple times as we may resize in subsequent dumps if number of particles in the buffer > 0 @@ -706,17 +718,17 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // dump individual particles bool contributed_particles = false; // did the local MPI rank contribute particles? for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) { - uint64_t offset = static_cast( counter.m_ParticleOffsetAtRank[currentLevel] ); + auto offset = static_cast( counter.m_ParticleOffsetAtRank[currentLevel] ); // For BTD, the offset include the number of particles already flushed - if (isBTD) offset += ParticleFlushOffset; + if (isBTD) { offset += ParticleFlushOffset; } for (ParticleIter pti(*pc, currentLevel); pti.isValid(); ++pti) { auto const numParticleOnTile = pti.numParticles(); - uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); + auto const numParticleOnTile64 = static_cast( numParticleOnTile ); // Do not call storeChunk() with zero-sized particle tiles: // https://github.com/openPMD/openPMD-api/issues/1147 // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 - if (numParticleOnTile == 0) continue; + if (numParticleOnTile == 0) { continue; } contributed_particles = true; @@ -732,8 +744,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, new amrex::ParticleReal[numParticleOnTile], [](amrex::ParticleReal const *p) { delete[] p; } ); - for (auto i = 0; i < numParticleOnTile; i++) + for (auto i = 0; i < numParticleOnTile; i++) { z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} + } std::string const positionComponent = "z"; currSpecies["position"]["z"].storeChunk(z, {offset}, {numParticleOnTile64}); } @@ -783,9 +796,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, [](uint64_t const *p) { delete[] p; } ); for (auto i = 0; i < numParticleOnTile; i++) { - ids.get()[i] = ablastr::particles::localIDtoGlobal(aos[i].id(), aos[i].cpu()); + ids.get()[i] = ablastr::particles::localIDtoGlobal(static_cast(aos[i].id()), static_cast(aos[i].cpu())); } - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64}); } @@ -813,7 +826,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, if (is_resizing_flush && !contributed_particles && isBTD && m_Series->backend() == "ADIOS2") { for( auto & [record_name, record] : currSpecies ) { for( auto & [comp_name, comp] : record ) { - if (comp.constant()) continue; + if (comp.constant()) { continue; } auto dtype = comp.getDatatype(); switch (dtype) { @@ -863,7 +876,7 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, const unsigned long long np, bool const isBTD) const { std::string options = "{}"; - if (isBTD) options = "{ \"resizable\": true }"; + if (isBTD) { options = "{ \"resizable\": true }"; } auto dtype_real = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); auto dtype_int = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); // @@ -900,14 +913,16 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, [[maybe_unused]] const auto [_, newRecord] = addedRecords.insert(record_name); if( newRecord ) { currRecord.setUnitDimension( detail::getUnitDimension(record_name) ); - if( record_name == "weighting" ) + if( record_name == "weighting" ) { currRecord.setAttribute( "macroWeighted", 1u ); - else + } else { currRecord.setAttribute( "macroWeighted", 0u ); - if( record_name == "momentum" || record_name == "weighting" ) + } + if( record_name == "momentum" || record_name == "weighting" ) { currRecord.setAttribute( "weightingPower", 1.0 ); - else + } else { currRecord.setAttribute( "weightingPower", 0.0 ); + } } } } @@ -923,10 +938,11 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, if( newRecord ) { currRecord.setUnitDimension( detail::getUnitDimension(record_name) ); currRecord.setAttribute( "macroWeighted", 0u ); - if( record_name == "momentum" || record_name == "weighting" ) + if( record_name == "momentum" || record_name == "weighting" ) { currRecord.setAttribute( "weightingPower", 1.0 ); - else + } else { currRecord.setAttribute( "weightingPower", 0.0 ); + } } } } @@ -943,30 +959,31 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, { auto const numParticleOnTile = pti.numParticles(); - uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); + auto const numParticleOnTile64 = static_cast( numParticleOnTile ); auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile auto const& soa = pti.GetStructOfArrays(); - // first we concatinate the AoS into contiguous arrays + // first we concatenate the AoS into contiguous arrays { // note: WarpX does not yet use extra AoS Real attributes for( auto idx=0; idx d( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - - for( auto kk=0; kk d( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p){ delete[] p; } + ); + + for( auto kk=0; kk(), {np}, options); - auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, options); - - auto const positionComponents = detail::getParticlePositionComponentLabels(); - for( auto const& comp : positionComponents ) { - currSpecies["position"][comp].resetDataset( realType ); - } + std::string options = "{}"; + if (isBTD) { options = "{ \"resizable\": true }"; } + auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); + auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, options); + + auto const positionComponents = detail::getParticlePositionComponentLabels(); + for( auto const& comp : positionComponents ) { + currSpecies["position"][comp].resetDataset( realType ); + } - auto const scalar = openPMD::RecordComponent::SCALAR; - currSpecies["id"][scalar].resetDataset( idType ); + const auto *const scalar = openPMD::RecordComponent::SCALAR; + currSpecies["id"][scalar].resetDataset( idType ); } void @@ -1029,7 +1046,7 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( amrex::ParticleReal const mass) { auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}); - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; // define record shape to be number of particles auto const positionComponents = detail::getParticlePositionComponentLabels(true); @@ -1141,67 +1158,74 @@ void WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, amrex::Geometry& full_geom ) const { - // meta data for ED-PIC extension - auto const period = full_geom.periodicity(); // TODO double-check: is this the proper global bound or of some level? - std::vector fieldBoundary(6, "reflecting"); - std::vector particleBoundary(6, "absorbing"); - fieldBoundary.resize(AMREX_SPACEDIM * 2); - particleBoundary.resize(AMREX_SPACEDIM * 2); - - for (auto i = 0u; i < fieldBoundary.size() / 2u; ++i) - if (m_fieldPMLdirections.at(i)) - fieldBoundary.at(i) = "open"; - - for (auto i = 0u; i < fieldBoundary.size() / 2u; ++i) - if (period.isPeriodic(i)) { - fieldBoundary.at(2u * i) = "periodic"; - fieldBoundary.at(2u * i + 1u) = "periodic"; - particleBoundary.at(2u * i) = "periodic"; - particleBoundary.at(2u * i + 1u) = "periodic"; - } - - meshes.setAttribute("fieldSolver", []() { - switch (WarpX::electromagnetic_solver_id) { - case ElectromagneticSolverAlgo::Yee : - return "Yee"; - case ElectromagneticSolverAlgo::CKC : - return "CK"; - case ElectromagneticSolverAlgo::PSATD : - return "PSATD"; - default: - return "other"; - } - }()); - meshes.setAttribute("fieldBoundary", fieldBoundary); - meshes.setAttribute("particleBoundary", particleBoundary); - meshes.setAttribute("currentSmoothing", []() { - if (WarpX::use_filter) return "Binomial"; - else return "none"; - }()); - if (WarpX::use_filter) - meshes.setAttribute("currentSmoothingParameters", []() { - std::stringstream ss; - ss << "period=1;compensator=false"; + // meta data for ED-PIC extension + auto const period = full_geom.periodicity(); // TODO double-check: is this the proper global bound or of some level? + std::vector fieldBoundary(6, "reflecting"); + std::vector particleBoundary(6, "absorbing"); + fieldBoundary.resize(AMREX_SPACEDIM * 2); + particleBoundary.resize(AMREX_SPACEDIM * 2); + + const auto HalfFieldBoundarySize = static_cast(fieldBoundary.size() / 2u); + + for (auto i = 0; i < HalfFieldBoundarySize; ++i) { + if (m_fieldPMLdirections.at(i)) { + fieldBoundary.at(i) = "open"; + } + } + + for (int i = 0; i < HalfFieldBoundarySize; ++i) { + if (period.isPeriodic(i)) { + fieldBoundary.at(2u * i) = "periodic"; + fieldBoundary.at(2u * i + 1u) = "periodic"; + particleBoundary.at(2u * i) = "periodic"; + particleBoundary.at(2u * i + 1u) = "periodic"; + } + } + + meshes.setAttribute("fieldSolver", []() { + switch (WarpX::electromagnetic_solver_id) { + case ElectromagneticSolverAlgo::Yee : + return "Yee"; + case ElectromagneticSolverAlgo::CKC : + return "CK"; + case ElectromagneticSolverAlgo::PSATD : + return "PSATD"; + default: + return "other"; + } + }()); + meshes.setAttribute("fieldBoundary", fieldBoundary); + meshes.setAttribute("particleBoundary", particleBoundary); + meshes.setAttribute("currentSmoothing", []() { + if (WarpX::use_filter) { return "Binomial"; } + else { return "none"; } + }()); + if (WarpX::use_filter) { + meshes.setAttribute("currentSmoothingParameters", []() { + std::stringstream ss; + ss << "period=1;compensator=false"; #if (AMREX_SPACEDIM >= 2) - ss << ";numPasses_x=" << WarpX::filter_npass_each_dir[0]; + ss << ";numPasses_x=" << WarpX::filter_npass_each_dir[0]; #endif #if defined(WARPX_DIM_3D) - ss << ";numPasses_y=" << WarpX::filter_npass_each_dir[1]; - ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[2]; + ss << ";numPasses_y=" << WarpX::filter_npass_each_dir[1]; + ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[2]; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[1]; + ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[1]; #elif defined(WARPX_DIM_1D_Z) - ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[0]; + ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[0]; #endif - std::string currentSmoothingParameters = ss.str(); - return currentSmoothingParameters; - }()); - meshes.setAttribute("chargeCorrection", []() { - if (WarpX::do_dive_cleaning) return "hyperbolic"; // TODO or "spectral" or something? double-check - else return "none"; - }()); - if (WarpX::do_dive_cleaning) - meshes.setAttribute("chargeCorrectionParameters", "period=1"); + std::string currentSmoothingParameters = ss.str(); + return currentSmoothingParameters; + }()); + } + meshes.setAttribute("chargeCorrection", []() { + if (WarpX::do_dive_cleaning) { return "hyperbolic"; // TODO or "spectral" or something? double-check + } else { return "none"; } + }()); + if (WarpX::do_dive_cleaning) { + meshes.setAttribute("chargeCorrectionParameters", "period=1"); + } } @@ -1280,8 +1304,9 @@ WarpXOpenPMDPlot::GetMeshCompNames (int meshLevel, } } - if ( 0 == meshLevel ) + if ( 0 == meshLevel ) { return; + } field_name += std::string("_lvl").append(std::to_string(meshLevel)); } @@ -1364,19 +1389,21 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename, } // If there are no fields to be written, interrupt the function here - if ( varnames.empty() ) return; + if ( varnames.empty() ) { return; } // loop over levels up to output_levels // note: this is usually the finestLevel, not the maxLevel for (int lev=0; lev < output_levels; lev++) { amrex::Geometry full_geom = geom[lev]; - if( isBTD ) + if( isBTD ) { full_geom = full_BTD_snapshot; + } // setup is called once. So it uses property "period" from first // geometry for field levels. - if ( (0 == lev) && first_write_to_iteration ) + if ( (0 == lev) && first_write_to_iteration ) { SetupFields(meshes, full_geom); + } amrex::Box const & global_box = full_geom.Domain(); @@ -1481,38 +1508,38 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename, // // // -WarpXParticleCounter::WarpXParticleCounter (ParticleContainer* pc) +WarpXParticleCounter::WarpXParticleCounter (ParticleContainer* pc): + m_MPIRank{amrex::ParallelDescriptor::MyProc()}, + m_MPISize{amrex::ParallelDescriptor::NProcs()} { - m_MPISize = amrex::ParallelDescriptor::NProcs(); - m_MPIRank = amrex::ParallelDescriptor::MyProc(); - - m_ParticleCounterByLevel.resize(pc->finestLevel()+1); - m_ParticleOffsetAtRank.resize(pc->finestLevel()+1); - m_ParticleSizeAtRank.resize(pc->finestLevel()+1); + m_ParticleCounterByLevel.resize(pc->finestLevel()+1); + m_ParticleOffsetAtRank.resize(pc->finestLevel()+1); + m_ParticleSizeAtRank.resize(pc->finestLevel()+1); - for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) + for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) { - long numParticles = 0; // numParticles in this processor + long numParticles = 0; // numParticles in this processor - for (ParticleIter pti(*pc, currentLevel); pti.isValid(); ++pti) { - auto numParticleOnTile = pti.numParticles(); - numParticles += numParticleOnTile; - } + for (ParticleIter pti(*pc, currentLevel); pti.isValid(); ++pti) { + auto numParticleOnTile = pti.numParticles(); + numParticles += numParticleOnTile; + } - unsigned long long offset=0; // offset of this level - unsigned long long sum=0; // numParticles in this level (sum from all processors) + unsigned long long offset=0; // offset of this level + unsigned long long sum=0; // numParticles in this level (sum from all processors) - GetParticleOffsetOfProcessor(numParticles, offset, sum); + GetParticleOffsetOfProcessor(numParticles, offset, sum); - m_ParticleCounterByLevel[currentLevel] = sum; - m_ParticleOffsetAtRank[currentLevel] = offset; - m_ParticleSizeAtRank[currentLevel] = numParticles; + m_ParticleCounterByLevel[currentLevel] = sum; + m_ParticleOffsetAtRank[currentLevel] = offset; + m_ParticleSizeAtRank[currentLevel] = numParticles; - // adjust offset, it should be numbered after particles from previous levels - for (auto lv=0; lv(result.size()); for (int i=0; i& distance_t { const auto plo = pc.Geom(lev).ProbLoArray(); const auto dxi = pc.Geom(lev).InvCellSizeArray(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif for(WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { - const auto getPosition = GetParticlePosition(pti); + const auto getPosition = GetParticlePosition(pti); auto& tile = pti.GetParticleTile(); auto ptd = tile.getParticleTileData(); const auto np = tile.numParticles(); diff --git a/Source/Evolve/CMakeLists.txt b/Source/Evolve/CMakeLists.txt index 3ef0dae5e8e..a84d6e1e42b 100644 --- a/Source/Evolve/CMakeLists.txt +++ b/Source/Evolve/CMakeLists.txt @@ -4,5 +4,6 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE WarpXEvolve.cpp WarpXComputeDt.cpp + WarpXOneStepImplicitPicard.cpp ) endforeach() diff --git a/Source/Evolve/Make.package b/Source/Evolve/Make.package index 275b8bfde5a..a9a877475b9 100644 --- a/Source/Evolve/Make.package +++ b/Source/Evolve/Make.package @@ -1,4 +1,5 @@ CEXE_sources += WarpXEvolve.cpp CEXE_sources += WarpXComputeDt.cpp +CEXE_sources += WarpXOneStepImplicitPicard.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Evolve diff --git a/Source/Evolve/WarpXComputeDt.cpp b/Source/Evolve/WarpXComputeDt.cpp index e52f92c4acb..c1a87166920 100644 --- a/Source/Evolve/WarpXComputeDt.cpp +++ b/Source/Evolve/WarpXComputeDt.cpp @@ -37,12 +37,13 @@ WarpX::ComputeDt () electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { std::stringstream errorMsg; - if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) + if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) { errorMsg << "warpx.const_dt must be specified with the electrostatic solver."; - else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) + } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { errorMsg << "warpx.const_dt must be specified with the hybrid-PIC solver."; - else + } else { errorMsg << "warpx.const_dt must be specified when not using a field solver."; + } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_const_dt.has_value(), errorMsg.str()); for (int lev=0; lev<=max_level; lev++) { diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 067cd9a11b6..b062501ed1b 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -24,8 +24,10 @@ #endif #include "Parallelization/GuardCellManager.H" #include "Particles/MultiParticleContainer.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Particles/ParticleBoundaryBuffer.H" -#include "Python/WarpX_py.H" +#include "Python/callbacks.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXUtil.H" @@ -77,7 +79,7 @@ WarpX::Evolve (int numsteps) for (int step = istep[0]; step < numsteps_max && cur_time < stop_time; ++step) { WARPX_PROFILE("WarpX::Evolve::step"); - const Real evolve_time_beg_step = amrex::second(); + const auto evolve_time_beg_step = static_cast(amrex::second()); //Check and clear signal flags and asynchronously broadcast them from process 0 SignalHandling::CheckSignals(); @@ -116,54 +118,57 @@ WarpX::Evolve (int numsteps) } } - // At the beginning, we have B^{n} and E^{n}. - // Particles have p^{n} and x^{n}. - // is_synchronized is true. - if (is_synchronized) { - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - // Not called at each iteration, so exchange all guard cells - FillBoundaryE(guard_cells.ng_alloc_EB); - FillBoundaryB(guard_cells.ng_alloc_EB); - } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - // on first step, push p by -0.5*dt - for (int lev = 0; lev <= finest_level; ++lev) - { - mypc->PushP(lev, -0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); - } - is_synchronized = false; - - } else { - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - // Beyond one step, we have E^{n} and B^{n}. - // Particles have p^{n-1/2} and x^{n}. - - // E and B are up-to-date inside the domain only - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - // E and B: enough guard cells to update Aux or call Field Gather in fp and cp - // Need to update Aux on lower levels, to interpolate to higher levels. - if (fft_do_time_averaging) + if (evolve_scheme == EvolveScheme::Explicit) { + // At the beginning, we have B^{n} and E^{n}. + // Particles have p^{n} and x^{n}. + // is_synchronized is true. + if (is_synchronized) { + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + // Not called at each iteration, so exchange all guard cells + FillBoundaryE(guard_cells.ng_alloc_EB); + FillBoundaryB(guard_cells.ng_alloc_EB); + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + // on first step, push p by -0.5*dt + for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); + mypc->PushP(lev, -0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + } + is_synchronized = false; + + } else { + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + // Beyond one step, we have E^{n} and B^{n}. + // Particles have p^{n-1/2} and x^{n}. + + // E and B are up-to-date inside the domain only + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + // E and B: enough guard cells to update Aux or call Field Gather in fp and cp + // Need to update Aux on lower levels, to interpolate to higher levels. + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + FillBoundaryAux(guard_cells.ng_UpdateAux); + } } - // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) - FillBoundaryAux(guard_cells.ng_UpdateAux); + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); } // If needed, deposit the initial ion charge and current densities that // will be used to update the E-field in Ohm's law. if (step == step_begin && electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC - ) HybridPICDepositInitialRhoAndJ(); + ) { HybridPICDepositInitialRhoAndJ(); } // Run multi-physics modules: // ionization, Coulomb collisions, QED @@ -180,13 +185,18 @@ WarpX::Evolve (int numsteps) // gather fields, push particles, deposit sources, update fields ExecutePythonCallback("particleinjection"); - // Electrostatic or hybrid-PIC case: only gather fields and push - // particles, deposition and calculation of fields done further below - if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || + + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + OneStep_ImplicitPicard(cur_time); + } + else if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) { + // Electrostatic or hybrid-PIC case: only gather fields and push + // particles, deposition and calculation of fields done further below const bool skip_deposition = true; - PushParticlesandDepose(cur_time, skip_deposition); + PushParticlesandDeposit(cur_time, skip_deposition); } // Electromagnetic case: multi-J algorithm else if (do_multi_J) @@ -218,31 +228,33 @@ WarpX::Evolve (int numsteps) // value of step in code (first step is 0) mypc->doResampling(istep[0]+1, verbose); - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date - // B : guard cells are NOT up-to-date - } - - if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { - // At the end of last step, push p by 0.5*dt to synchronize - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); + if (evolve_scheme == EvolveScheme::Explicit) { + if (num_mirrors>0){ + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date + // B : guard cells are NOT up-to-date } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - for (int lev = 0; lev <= finest_level; ++lev) { - mypc->PushP(lev, 0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1], - *Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1], - *Bfield_aux[lev][2]); + + if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { + // At the end of last step, push p by 0.5*dt to synchronize + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + for (int lev = 0; lev <= finest_level; ++lev) { + mypc->PushP(lev, 0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1], + *Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1], + *Bfield_aux[lev][2]); + } + is_synchronized = true; } - is_synchronized = true; } for (int lev = 0; lev <= max_level; ++lev) { @@ -376,7 +388,7 @@ WarpX::Evolve (int numsteps) } // create ending time stamp for calculating elapsed time each iteration - const Real evolve_time_end_step = amrex::second(); + const auto evolve_time_end_step = static_cast(amrex::second()); evolve_time += evolve_time_end_step - evolve_time_beg_step; HandleSignals(); @@ -402,7 +414,7 @@ WarpX::Evolve (int numsteps) if (istep[0] == max_step || (stop_time - 1.e-3*dt[0] <= cur_time && cur_time < stop_time + dt[0]) || exit_loop_due_to_interrupt_signal) { multi_diags->FilterComputePackFlushLastTimestep( istep[0] ); - if (exit_loop_due_to_interrupt_signal) ExecutePythonCallback("onbreaksignal"); + if (exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } } @@ -423,7 +435,7 @@ WarpX::OneStep_nosub (Real cur_time) ExecutePythonCallback("particlescraper"); ExecutePythonCallback("beforedeposition"); - PushParticlesandDepose(cur_time); + PushParticlesandDeposit(cur_time); ExecutePythonCallback("afterdeposition"); @@ -437,8 +449,8 @@ WarpX::OneStep_nosub (Real cur_time) // solve. // For extended PML: copy J from regular grid to PML, and damp J in PML - if (do_pml && pml_has_particles) CopyJPML(); - if (do_pml && do_pml_j_damping) DampJPML(); + if (do_pml && pml_has_particles) { CopyJPML(); } + if (do_pml && do_pml_j_damping) { DampJPML(); } ExecutePythonCallback("beforeEsolve"); @@ -465,14 +477,12 @@ WarpX::OneStep_nosub (Real cur_time) else { FillBoundaryE(guard_cells.ng_afterPushPSATD, WarpX::sync_nodal_points); FillBoundaryB(guard_cells.ng_afterPushPSATD, WarpX::sync_nodal_points); - if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) + if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) { FillBoundaryF(guard_cells.ng_alloc_F, WarpX::sync_nodal_points); - if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) + } + if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) { FillBoundaryG(guard_cells.ng_alloc_G, WarpX::sync_nodal_points); - } - - if (do_pml) { - NodalSyncPML(); + } } } else { EvolveF(0.5_rt * dt[0], DtType::FirstHalf); @@ -500,17 +510,17 @@ WarpX::OneStep_nosub (Real cur_time) if (do_pml) { DampPML(); - NodalSyncPML(); - FillBoundaryE(guard_cells.ng_MovingWindow); - FillBoundaryB(guard_cells.ng_MovingWindow); - FillBoundaryF(guard_cells.ng_MovingWindow); - FillBoundaryG(guard_cells.ng_MovingWindow); + FillBoundaryE(guard_cells.ng_MovingWindow, WarpX::sync_nodal_points); + FillBoundaryB(guard_cells.ng_MovingWindow, WarpX::sync_nodal_points); + FillBoundaryF(guard_cells.ng_MovingWindow, WarpX::sync_nodal_points); + FillBoundaryG(guard_cells.ng_MovingWindow, WarpX::sync_nodal_points); } // E and B are up-to-date in the domain, but all guard cells are // outdated. - if (safe_guard_cells) + if (safe_guard_cells) { FillBoundaryB(guard_cells.ng_alloc_EB); + } } // !PSATD ExecutePythonCallback("afterEsolve"); @@ -552,7 +562,7 @@ void WarpX::SyncCurrentAndRho () { // TODO This works only without mesh refinement const int lev = 0; - if (use_filter) ApplyFilterJ(current_fp_vay, lev); + if (use_filter) { ApplyFilterJ(current_fp_vay, lev); } } } } @@ -600,17 +610,17 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Push particle from x^{n} to x^{n+1} // from p^{n-1/2} to p^{n+1/2} const bool skip_deposition = true; - PushParticlesandDepose(cur_time, skip_deposition); + PushParticlesandDeposit(cur_time, skip_deposition); // Initialize multi-J loop: // 1) Prepare E,B,F,G fields in spectral space PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::do_dive_cleaning) PSATDForwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDForwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } // 2) Set the averaged fields to zero - if (WarpX::fft_do_time_averaging) PSATDEraseAverageFields(); + if (WarpX::fft_do_time_averaging) { PSATDEraseAverageFields(); } // 3) Deposit rho (in rho_new, since it will be moved during the loop) // (after checking that pointer to rho_fp on MR level 0 is not null) @@ -642,29 +652,29 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) } // Number of depositions for multi-J scheme - const int n_depose = WarpX::do_multi_J_n_depositions; + const int n_deposit = WarpX::do_multi_J_n_depositions; // Time sub-step for each multi-J deposition - const amrex::Real sub_dt = dt[0] / static_cast(n_depose); + const amrex::Real sub_dt = dt[0] / static_cast(n_deposit); // Whether to perform multi-J depositions on a time interval that spans // one or two full time steps (from n*dt to (n+1)*dt, or from n*dt to (n+2)*dt) - const int n_loop = (WarpX::fft_do_time_averaging) ? 2*n_depose : n_depose; + const int n_loop = (WarpX::fft_do_time_averaging) ? 2*n_deposit : n_deposit; // Loop over multi-J depositions - for (int i_depose = 0; i_depose < n_loop; i_depose++) + for (int i_deposit = 0; i_deposit < n_loop; i_deposit++) { // Move J from new to old if J is linear in time - if (J_in_time == JInTime::Linear) PSATDMoveJNewToJOld(); + if (J_in_time == JInTime::Linear) { PSATDMoveJNewToJOld(); } - const amrex::Real t_depose_current = (J_in_time == JInTime::Linear) ? - (i_depose-n_depose+1)*sub_dt : (i_depose-n_depose+0.5_rt)*sub_dt; + const amrex::Real t_deposit_current = (J_in_time == JInTime::Linear) ? + (i_deposit-n_deposit+1)*sub_dt : (i_deposit-n_deposit+0.5_rt)*sub_dt; - const amrex::Real t_depose_charge = (rho_in_time == RhoInTime::Linear) ? - (i_depose-n_depose+1)*sub_dt : (i_depose-n_depose+0.5_rt)*sub_dt; + const amrex::Real t_deposit_charge = (rho_in_time == RhoInTime::Linear) ? + (i_deposit-n_deposit+1)*sub_dt : (i_deposit-n_deposit+0.5_rt)*sub_dt; - // Deposit new J at relative time t_depose_current with time step dt + // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) auto& current = (WarpX::do_current_centering) ? current_fp_nodal : current_fp; - mypc->DepositCurrent(current, dt[0], t_depose_current); + mypc->DepositCurrent(current, dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering @@ -679,10 +689,10 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) if (rho_fp[0]) { // Move rho from new to old if rho is linear in time - if (rho_in_time == RhoInTime::Linear) PSATDMoveRhoNewToRhoOld(); + if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } - // Deposit rho at relative time t_depose_charge - mypc->DepositCharge(rho_fp, t_depose_charge); + // Deposit rho at relative time t_deposit_charge + mypc->DepositCharge(rho_fp, t_deposit_charge); // Filter, exchange boundary, and interpolate across levels SyncRho(rho_fp, rho_cp, charge_buf); // Forward FFT of rho @@ -699,13 +709,13 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Advance E,B,F,G fields in time and update the average fields PSATDPushSpectralFields(); - // Transform non-average fields E,B,F,G after n_depose pushes + // Transform non-average fields E,B,F,G after n_deposit pushes // (the relative time reached here coincides with an integer full time step) - if (i_depose == n_depose-1) + if (i_deposit == n_deposit-1) { PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::do_dive_cleaning) PSATDBackwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDBackwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } } @@ -725,9 +735,9 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) pml[lev]->PushPSATD(lev); } ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) ApplyEfieldBoundary(lev, PatchType::coarse); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } } // Damp fields in PML before exchanging guard cells @@ -739,16 +749,13 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Exchange guard cells and synchronize nodal points FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) + if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) { FillBoundaryF(guard_cells.ng_alloc_F, WarpX::sync_nodal_points); - if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) + } + if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) { FillBoundaryG(guard_cells.ng_alloc_G, WarpX::sync_nodal_points); - - // Synchronize fields on nodal points in PML - if (do_pml) - { - NodalSyncPML(); } + #else amrex::ignore_unused(cur_time); WARPX_ABORT_WITH_MESSAGE( @@ -772,7 +779,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) * */ void -WarpX::OneStep_sub1 (Real curtime) +WarpX::OneStep_sub1 (Real cur_time) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electrostatic_solver_id == ElectrostaticSolverAlgo::None, @@ -786,10 +793,10 @@ WarpX::OneStep_sub1 (Real curtime) const int coarse_lev = 0; // i) Push particles and fields on the fine patch (first fine step) - PushParticlesandDepose(fine_lev, curtime, DtType::FirstHalf); + PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) ApplyFilterJ(current_fp, fine_lev); + if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, 2*ncomps); @@ -817,7 +824,7 @@ WarpX::OneStep_sub1 (Real curtime) // ii) Push particles on the coarse patch and mother grid. // Push the fields on the coarse patch and mother grid // by only half a coarse step (first half) - PushParticlesandDepose(coarse_lev, curtime, DtType::Full); + PushParticlesandDeposit(coarse_lev, cur_time, DtType::Full); StoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); AddRhoFromFineLevelandSumBoundary(rho_fp, rho_cp, charge_buf, coarse_lev, 0, ncomps); @@ -847,10 +854,10 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryAux(guard_cells.ng_UpdateAux); // iv) Push particles and fields on the fine patch (second fine step) - PushParticlesandDepose(fine_lev, curtime+dt[fine_lev], DtType::SecondHalf); + PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) ApplyFilterJ(current_fp, fine_lev); + if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, ncomps); @@ -871,8 +878,9 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); } - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); + } FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); // v) Push the fields on the coarse patch and mother grid @@ -918,16 +926,15 @@ WarpX::OneStep_sub1 (Real curtime) WarpX::sync_nodal_points); } DampPML(coarse_lev, PatchType::fine); - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); + } } - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - - // Synchronize nodal points at the end of the time step - if (do_pml) NodalSyncPML(); + } } void @@ -965,17 +972,18 @@ WarpX::doQEDEvents (int lev) #endif void -WarpX::PushParticlesandDepose (amrex::Real cur_time, bool skip_deposition) +WarpX::PushParticlesandDeposit (amrex::Real cur_time, bool skip_current, PushType push_type) { // Evolve particles to p^{n+1/2} and x^{n+1} - // Depose current, j^{n+1/2} + // Deposit current, j^{n+1/2} for (int lev = 0; lev <= finest_level; ++lev) { - PushParticlesandDepose(lev, cur_time, DtType::Full, skip_deposition); + PushParticlesandDeposit(lev, cur_time, DtType::Full, skip_current, push_type); } } void -WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_deposition) +WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_current, + PushType push_type) { amrex::MultiFab* current_x = nullptr; amrex::MultiFab* current_y = nullptr; @@ -1002,15 +1010,15 @@ WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, } mypc->Evolve(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2], + *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], + *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], *current_x, *current_y, *current_z, current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), rho_fp[lev].get(), charge_buf[lev].get(), Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), - cur_time, dt[lev], a_dt_type, skip_deposition); - if (! skip_deposition) { + cur_time, dt[lev], a_dt_type, skip_current, push_type); + if (! skip_current) { #ifdef WARPX_DIM_RZ // This is called after all particles have deposited their current and charge. ApplyInverseVolumeScalingToCurrentDensity(current_fp[lev][0].get(), current_fp[lev][1].get(), current_fp[lev][2].get(), lev); @@ -1031,6 +1039,12 @@ WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, // of the filter to avoid incorrect results (moved to `SyncCurrentAndRho()`). // Might this be related to issue #1943? #endif + if (do_fluid_species) { + myfl->Evolve(lev, + *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], + *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], + rho_fp[lev].get(), *current_x, *current_y, *current_z, cur_time, skip_current); + } } } @@ -1080,8 +1094,8 @@ WarpX::applyMirrors(Real time) NullifyMF(Bz, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_fp[lev]) NullifyMF(*F_fp[lev], lev, z_min, z_max); - if (G_fp[lev]) NullifyMF(*G_fp[lev], lev, z_min, z_max); + if (F_fp[lev]) { NullifyMF(*F_fp[lev], lev, z_min, z_max); } + if (G_fp[lev]) { NullifyMF(*G_fp[lev], lev, z_min, z_max); } if (lev>0) { @@ -1102,8 +1116,8 @@ WarpX::applyMirrors(Real time) NullifyMF(cBz, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_cp[lev]) NullifyMF(*F_cp[lev], lev, z_min, z_max); - if (G_cp[lev]) NullifyMF(*G_cp[lev], lev, z_min, z_max); + if (F_cp[lev]) { NullifyMF(*F_cp[lev], lev, z_min, z_max); } + if (G_cp[lev]) { NullifyMF(*G_cp[lev], lev, z_min, z_max); } } } } diff --git a/Source/Evolve/WarpXOneStepImplicitPicard.cpp b/Source/Evolve/WarpXOneStepImplicitPicard.cpp new file mode 100644 index 00000000000..56ed26db1e2 --- /dev/null +++ b/Source/Evolve/WarpXOneStepImplicitPicard.cpp @@ -0,0 +1,423 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "WarpX.H" + +#include "BoundaryConditions/PML.H" +#include "Diagnostics/MultiDiagnostics.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" +#ifdef WARPX_USE_PSATD +# ifdef WARPX_DIM_RZ +# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" +# else +# include "FieldSolver/SpectralSolver/SpectralSolver.H" +# endif +#endif +#include "Parallelization/GuardCellManager.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleBoundaryBuffer.H" +#include "Python/callbacks.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" +#include "Utils/WarpXUtil.H" +#include "Utils/WarpXConst.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +void +WarpX::EvolveImplicitPicardInit (int lev) +{ + + if (lev == 0) { + // Add space to save the positions and velocities at the start of the time steps + for (auto const& pc : *mypc) { +#if (AMREX_SPACEDIM >= 2) + pc->AddRealComp("x_n"); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + pc->AddRealComp("y_n"); +#endif + pc->AddRealComp("z_n"); + pc->AddRealComp("ux_n"); + pc->AddRealComp("uy_n"); + pc->AddRealComp("uz_n"); + } + } + + // Initialize MultiFabs to hold the E and B fields at the start of the time steps + // Only one refinement level is supported + const int nlevs_max = maxLevel() + 1; + Efield_n.resize(nlevs_max); + Efield_save.resize(nlevs_max); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + Bfield_n.resize(nlevs_max); + Bfield_save.resize(nlevs_max); + } + + // The Efield_n and Bfield_n will hold the fields at the start of the time step. + // This is needed since in each iteration the fields are advanced from the values + // at the start of the step. + // The Efield_save and Bfield_save will hold the fields from the previous iteration, + // to check the change in the fields after the iterations to check for convergence. + // The Efiel_fp and Bfield_fp will hole the n+theta during the iterations and then + // advance to the n+1 time level after the iterations complete. + AllocInitMultiFabFromModel(Efield_n[lev][0], *Efield_fp[0][0], lev, "Efield_n[0]"); + AllocInitMultiFabFromModel(Efield_n[lev][1], *Efield_fp[0][1], lev, "Efield_n[1]"); + AllocInitMultiFabFromModel(Efield_n[lev][2], *Efield_fp[0][2], lev, "Efield_n[2]"); + AllocInitMultiFabFromModel(Efield_save[lev][0], *Efield_fp[0][0], lev, "Efield_save[0]"); + AllocInitMultiFabFromModel(Efield_save[lev][1], *Efield_fp[0][1], lev, "Efield_save[1]"); + AllocInitMultiFabFromModel(Efield_save[lev][2], *Efield_fp[0][2], lev, "Efield_save[2]"); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + AllocInitMultiFabFromModel(Bfield_n[lev][0], *Bfield_fp[0][0], lev, "Bfield_n[0]"); + AllocInitMultiFabFromModel(Bfield_n[lev][1], *Bfield_fp[0][1], lev, "Bfield_n[1]"); + AllocInitMultiFabFromModel(Bfield_n[lev][2], *Bfield_fp[0][2], lev, "Bfield_n[2]"); + AllocInitMultiFabFromModel(Bfield_save[lev][0], *Bfield_fp[0][0], lev, "Bfield_save[0]"); + AllocInitMultiFabFromModel(Bfield_save[lev][1], *Bfield_fp[0][1], lev, "Bfield_save[1]"); + AllocInitMultiFabFromModel(Bfield_save[lev][2], *Bfield_fp[0][2], lev, "Bfield_save[2]"); + } + +} + +void +WarpX::OneStep_ImplicitPicard(amrex::Real cur_time) +{ + using namespace amrex::literals; + + // We have E^{n}. + // Particles have p^{n} and x^{n}. + // With full implicit, B^{n} + // With semi-implicit, B^{n-1/2} + + // Save the values at the start of the time step, + // copying particle data to x_n etc. + for (auto const& pc : *mypc) { + SaveParticlesAtImplicitStepStart (*pc, 0); + } + + // Save the fields at the start of the step + amrex::MultiFab::Copy(*Efield_n[0][0], *Efield_fp[0][0], 0, 0, ncomps, Efield_fp[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_n[0][1], *Efield_fp[0][1], 0, 0, ncomps, Efield_fp[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_n[0][2], *Efield_fp[0][2], 0, 0, ncomps, Efield_fp[0][2]->nGrowVect()); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + amrex::MultiFab::Copy(*Bfield_n[0][0], *Bfield_fp[0][0], 0, 0, ncomps, Bfield_fp[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_n[0][1], *Bfield_fp[0][1], 0, 0, ncomps, Bfield_fp[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_n[0][2], *Bfield_fp[0][2], 0, 0, ncomps, Bfield_fp[0][2]->nGrowVect()); + } else if (evolve_scheme == EvolveScheme::SemiImplicitPicard) { + // This updates Bfield_fp so it holds the new B at n+1/2 + EvolveB(dt[0], DtType::Full); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + } + + // Start the iterations + amrex::Real deltaE = 1._rt; + amrex::Real deltaB = 1._rt; + int iteration_count = 0; + while (iteration_count < max_picard_iterations && + (deltaE > picard_iteration_tolerance || deltaB > picard_iteration_tolerance)) { + iteration_count++; + + // Advance the particle positions by 1/2 dt, + // particle velocities by dt, then take average of old and new v, + // deposit currents, giving J at n+1/2 + // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. + bool skip_current = false; + PushType push_type = PushType::Implicit; + PushParticlesandDeposit(cur_time, skip_current, push_type); + + SyncCurrentAndRho(); + + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Save the E at n+1/2 from the previous iteration so that the change + // in this iteration can be calculated + amrex::MultiFab::Copy(*Efield_save[0][0], *Efield_fp[0][0], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Efield_save[0][1], *Efield_fp[0][1], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Efield_save[0][2], *Efield_fp[0][2], 0, 0, ncomps, 0); + } + + // Copy Efield_n into Efield_fp since EvolveE updates Efield_fp in place + amrex::MultiFab::Copy(*Efield_fp[0][0], *Efield_n[0][0], 0, 0, ncomps, Efield_n[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][1], *Efield_n[0][1], 0, 0, ncomps, Efield_n[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][2], *Efield_n[0][2], 0, 0, ncomps, Efield_n[0][2]->nGrowVect()); + + // Updates Efield_fp so it holds the new E at n+1/2 + EvolveE(0.5_rt*dt[0]); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyEfieldBoundary(0, PatchType::fine); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Save the B at n+1/2 from the previous iteration so that the change + // in this iteration can be calculated + amrex::MultiFab::Copy(*Bfield_save[0][0], *Bfield_fp[0][0], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Bfield_save[0][1], *Bfield_fp[0][1], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Bfield_save[0][2], *Bfield_fp[0][2], 0, 0, ncomps, 0); + } + + // Copy Bfield_n into Bfield_fp since EvolveB updates Bfield_fp in place + amrex::MultiFab::Copy(*Bfield_fp[0][0], *Bfield_n[0][0], 0, 0, ncomps, Bfield_n[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][1], *Bfield_n[0][1], 0, 0, ncomps, Bfield_n[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][2], *Bfield_n[0][2], 0, 0, ncomps, Bfield_n[0][2]->nGrowVect()); + + // This updates Bfield_fp so it holds the new B at n+1/2 + EvolveB(0.5_rt*dt[0], DtType::Full); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + } + + // The B field update needs + if (num_mirrors>0){ + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date from the mirrors + // B : guard cells are NOT up-to-date from the mirrors + } + + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Calculate the change in E and B from this iteration + // deltaE = abs(Enew - Eold)/max(abs(Enew)) + Efield_save[0][0]->minus(*Efield_fp[0][0], 0, ncomps, 0); + Efield_save[0][1]->minus(*Efield_fp[0][1], 0, ncomps, 0); + Efield_save[0][2]->minus(*Efield_fp[0][2], 0, ncomps, 0); + amrex::Real maxE0 = std::max(1._rt, Efield_fp[0][0]->norm0(0, 0)); + amrex::Real maxE1 = std::max(1._rt, Efield_fp[0][1]->norm0(0, 0)); + amrex::Real maxE2 = std::max(1._rt, Efield_fp[0][2]->norm0(0, 0)); + amrex::Real deltaE0 = Efield_save[0][0]->norm0(0, 0)/maxE0; + amrex::Real deltaE1 = Efield_save[0][1]->norm0(0, 0)/maxE1; + amrex::Real deltaE2 = Efield_save[0][2]->norm0(0, 0)/maxE2; + deltaE = std::max(std::max(deltaE0, deltaE1), deltaE2); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + Bfield_save[0][0]->minus(*Bfield_fp[0][0], 0, ncomps, 0); + Bfield_save[0][1]->minus(*Bfield_fp[0][1], 0, ncomps, 0); + Bfield_save[0][2]->minus(*Bfield_fp[0][2], 0, ncomps, 0); + amrex::Real maxB0 = std::max(1._rt, Bfield_fp[0][0]->norm0(0, 0)); + amrex::Real maxB1 = std::max(1._rt, Bfield_fp[0][1]->norm0(0, 0)); + amrex::Real maxB2 = std::max(1._rt, Bfield_fp[0][2]->norm0(0, 0)); + amrex::Real deltaB0 = Bfield_save[0][0]->norm0(0, 0)/maxB0; + amrex::Real deltaB1 = Bfield_save[0][1]->norm0(0, 0)/maxB1; + amrex::Real deltaB2 = Bfield_save[0][2]->norm0(0, 0)/maxB2; + deltaB = std::max(std::max(deltaB0, deltaB1), deltaB2); + } else { + deltaB = 0.; + } + amrex::Print() << "Max delta " << iteration_count << " " << deltaE << " " << deltaB << "\n"; + } + + // Now, the particle positions and velocities and the Efield_fp and Bfield_fp hold + // the new values at n+1/2 + } + + amrex::Print() << "Picard iterations = " << iteration_count << ", Eerror = " << deltaE << ", Berror = " << deltaB << "\n"; + if (picard_iteration_tolerance > 0. && iteration_count == max_picard_iterations) { + std::stringstream convergenceMsg; + convergenceMsg << "The Picard implicit solver failed to converge after " << iteration_count << " iterations, with Eerror = " << deltaE << ", Berror = " << deltaB << " with a tolerance of " << picard_iteration_tolerance; + if (require_picard_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); + } + } + + // Advance particles to step n+1 + for (auto const& pc : *mypc) { + FinishImplicitParticleUpdate(*pc, 0); + } + + // Advance fields to step n+1 + // WarpX::sync_nodal_points is used to avoid instability + FinishImplicitFieldUpdate(Efield_fp, Efield_n); + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + FinishImplicitFieldUpdate(Bfield_fp, Bfield_n); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + } + +} + +void +WarpX::SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev) +{ + +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc.getParticleComps(); + + for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + x_n[ip] = xp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + y_n[ip] = yp; +#endif + z_n[ip] = zp; + + ux_n[ip] = ux[ip]; + uy_n[ip] = uy[ip]; + uz_n[ip] = uz[ip]; + + }); + + } + } +} + +void +WarpX::FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev) +{ + using namespace amrex::literals; + +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc.getParticleComps(); + + for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + const auto setPosition = SetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + xp = 2._rt*xp - x_n[ip]; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + yp = 2._rt*yp - y_n[ip]; +#endif + zp = 2._rt*zp - z_n[ip]; + + ux[ip] = 2._rt*ux[ip] - ux_n[ip]; + uy[ip] = 2._rt*uy[ip] - uy_n[ip]; + uz[ip] = 2._rt*uz[ip] - uz_n[ip]; + + setPosition(ip, xp, yp, zp); + }); + + } + } +} + +void +WarpX::FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Field_fp, + amrex::Vector, 3 > >& Field_n) +{ + using namespace amrex::literals; + + for (int lev = 0; lev <= finest_level; ++lev) { + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); + amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); + amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); + + amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); + amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); + amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); + + amrex::Box tbx = mfi.tilebox(Field_fp[lev][0]->ixType().toIntVect()); + amrex::Box tby = mfi.tilebox(Field_fp[lev][1]->ixType().toIntVect()); + amrex::Box tbz = mfi.tilebox(Field_fp[lev][2]->ixType().toIntVect()); + + amrex::ParallelFor( + tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fx(i,j,k,n) = 2._rt*Fx(i,j,k,n) - Fx_n(i,j,k,n); + }, + tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fy(i,j,k,n) = 2._rt*Fy(i,j,k,n) - Fy_n(i,j,k,n); + }, + tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fz(i,j,k,n) = 2._rt*Fz(i,j,k,n) - Fz_n(i,j,k,n); + }); + } + } +} diff --git a/Source/Evolve/WarpXPushType.H b/Source/Evolve/WarpXPushType.H new file mode 100644 index 00000000000..dbca64a398c --- /dev/null +++ b/Source/Evolve/WarpXPushType.H @@ -0,0 +1,18 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_PUSHTYPE_H_ +#define WARPX_PUSHTYPE_H_ + +// Specify which scheme to use for the particle advance +enum struct PushType : int +{ + Explicit = 0, // Use the standard leap-frog scheme + Implicit // Use the Crank-Nicolson scheme. + // See for example Eqs. 15-18 in Chen, JCP 407 (2020) 109228 +}; + +#endif // WARPX_PUSHTYPE_H_ diff --git a/Source/FieldSolver/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolver.H index 1e0f180b0fe..7fc6c9bf6da 100644 --- a/Source/FieldSolver/ElectrostaticSolver.H +++ b/Source/FieldSolver/ElectrostaticSolver.H @@ -52,7 +52,7 @@ namespace ElectrostaticSolver { void buildParsers (); void buildParsersEB (); - /* \brief Sets the EB potential string and updates the parsers + /* \brief Sets the EB potential string and updates the function parser * * \param [in] potential The string value of the potential */ @@ -61,7 +61,9 @@ namespace ElectrostaticSolver { buildParsersEB(); } - PhiCalculatorEB getPhiEB(amrex::Real t) const noexcept { + [[nodiscard]] PhiCalculatorEB + getPhiEB(amrex::Real t) const noexcept + { return PhiCalculatorEB{t, potential_eb}; } diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index a6932f223e2..bd3086c0438 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -7,10 +7,12 @@ #include "WarpX.H" #include "FieldSolver/ElectrostaticSolver.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Parallelization/GuardCellManager.H" #include "Particles/MultiParticleContainer.H" #include "Particles/WarpXParticleContainer.H" -#include "Python/WarpX_py.H" +#include "Python/callbacks.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" @@ -103,8 +105,9 @@ WarpX::AddBoundaryField () // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } // Allocate fields for charge and potential const int num_levels = max_level + 1; @@ -144,8 +147,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -179,7 +183,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) bool const local_average = false; // Average across all MPI ranks std::array beta_pr = pc.meanParticleVelocity(local_average); std::array beta; - for (int i=0 ; i < static_cast(beta.size()) ; i++) beta[i] = beta_pr[i]/PhysConst::c; // Normalize + for (int i=0 ; i < static_cast(beta.size()) ; i++) { + beta[i] = beta_pr[i]/PhysConst::c; // Normalize + } // Compute the potential phi, by solving the Poisson equation computePhi( rho, phi, beta, pc.self_fields_required_precision, @@ -199,8 +205,9 @@ WarpX::AddSpaceChargeFieldLabFrame () // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -209,6 +216,10 @@ WarpX::AddSpaceChargeFieldLabFrame () // Deposit particle charge density (source of Poisson solver) mypc->DepositCharge(rho_fp, 0.0_rt); + if (do_fluid_species) { + int const lev = 0; + myfl->DepositCharge( lev, *rho_fp[lev] ); + } SyncRho(rho_fp, rho_cp, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels #ifndef WARPX_DIM_RZ @@ -375,7 +386,7 @@ void WarpX::setPhiBC ( amrex::Vector>& phi ) const { // check if any dimension has non-periodic boundary conditions - if (!m_poisson_boundary_handler.has_non_periodic) return; + if (!m_poisson_boundary_handler.has_non_periodic) { return; } // get the boundary potentials at the current time amrex::Array phi_bc_values_lo; @@ -411,7 +422,7 @@ WarpX::setPhiBC ( amrex::Vector>& phi ) const // loop over dimensions for (int idim=0; idim, 3> > std::array const beta ) const { // return early if beta is 0 since there will be no B-field - if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) return; + if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) { return; } for (int lev = 0; lev <= max_level; lev++) { @@ -967,7 +978,7 @@ void ElectrostaticSolver::PoissonBoundaryHandler::definePhiBCs (const amrex::Geo lobc[0] = LinOpBCType::Neumann; dirichlet_flag[0] = false; - // handle the r_max boundary explicity + // handle the r_max boundary explicitly if (WarpX::field_boundary_hi[0] == FieldBoundaryType::PEC) { hibc[0] = LinOpBCType::Dirichlet; dirichlet_flag[1] = true; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index 4c6748cfbce..36be09696fd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -64,7 +64,7 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_h_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_h_stencil_coefs_z.size()); // Extract cylindrical specific parameters Real const dr = m_dr; @@ -106,13 +106,15 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // At the +z boundary (innermost guard cell) if ( apply_hi_z && (j==domain_box.bigEnd(1)+1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Br(i,j,0,m) = coef1_z*Br(i,j,0,m) - coef2_z*Et(i,j,0,m); + } } // At the -z boundary (innermost guard cell) if ( apply_lo_z && (j==domain_box.smallEnd(1)-1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Br(i,j,0,m) = coef1_z*Br(i,j,0,m) + coef2_z*Et(i,j+1,0,m); + } } }, @@ -120,13 +122,15 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // At the +z boundary (innermost guard cell) if ( apply_hi_z && (j==domain_box.bigEnd(1)+1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Bt(i,j,0,m) = coef1_z*Bt(i,j,0,m) + coef2_z*Er(i,j,0,m); + } } // At the -z boundary (innermost guard cell) if ( apply_lo_z && (j==domain_box.smallEnd(1)-1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Bt(i,j,0,m) = coef1_z*Bt(i,j,0,m) - coef2_z*Er(i,j+1,0,m); + } } // At the +r boundary (innermost guard cell) if ( apply_hi_r && (i==domain_box.bigEnd(0)+1) ){ @@ -233,31 +237,39 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #ifdef WARPX_DIM_3D // At the +y boundary (innermost guard cell) - if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) { Bx(i,j,k) = coef1_y * Bx(i,j,k) + coef2_y * Ez(i,j,k); + } // At the -y boundary (innermost guard cell) - if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) { Bx(i,j,k) = coef1_y * Bx(i,j,k) - coef2_y * Ez(i,j+1,k); + } // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) + if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) + if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i,j,k+1); + } #elif WARPX_DIM_XZ // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i,j+1,k); + } #elif WARPX_DIM_1D_Z // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i+1,j,k); + } #endif }, @@ -266,33 +278,41 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #if (defined WARPX_DIM_3D || WARPX_DIM_XZ) // At the +x boundary (innermost guard cell) - if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) { By(i,j,k) = coef1_x * By(i,j,k) - coef2_x * Ez(i,j,k); + } // At the -x boundary (innermost guard cell) - if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) { By(i,j,k) = coef1_x * By(i,j,k) + coef2_x * Ez(i+1,j,k); + } #endif #ifdef WARPX_DIM_3D // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) + if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) + if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i,j,k+1); + } #elif WARPX_DIM_XZ // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i,j+1,k); + } #elif WARPX_DIM_1D_Z // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i+1,j,k); + } #endif }, @@ -301,19 +321,23 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #if (defined WARPX_DIM_3D || WARPX_DIM_XZ) // At the +x boundary (innermost guard cell) - if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) { Bz(i,j,k) = coef1_x * Bz(i,j,k) + coef2_x * Ey(i,j,k); + } // At the -x boundary (innermost guard cell) - if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) { Bz(i,j,k) = coef1_x * Bz(i,j,k) - coef2_x * Ey(i+1,j,k); + } #endif #ifdef WARPX_DIM_3D // At the +y boundary (innermost guard cell) - if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) { Bz(i,j,k) = coef1_y * Bz(i,j,k) - coef2_y * Ex(i,j,k); + } // At the -y boundary (innermost guard cell) - if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) { Bz(i,j,k) = coef1_y * Bz(i,j,k) + coef2_y * Ex(i,j+1,k); + } #elif WARPX_DIM_1D_Z ignore_unused(i,j,k); #endif diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp index 163792a2737..3268c81ac81 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp @@ -43,8 +43,8 @@ void FiniteDifferenceSolver::ComputeDivE ( const std::array,3>& Efield, amrex::MultiFab& divEfield ) { - // Select algorithm (The choice of algorithm is a runtime option, - // but we compile code for each algorithm, using templates) + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ if (m_fdtd_algo == ElectromagneticSolverAlgo::Yee || m_fdtd_algo == ElectromagneticSolverAlgo::HybridPIC){ @@ -94,11 +94,11 @@ void FiniteDifferenceSolver::ComputeDivECartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract tileboxes for which to loop Box const& tdive = mfi.tilebox(divEfield.ixType().toIntVect()); @@ -140,9 +140,9 @@ void FiniteDifferenceSolver::ComputeDivECylindrical ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); - int const n_coefs_r = m_stencil_coefs_r.size(); + auto const n_coefs_r = static_cast(m_stencil_coefs_r.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract cylindrical specific parameters Real const dr = m_dr; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index 0cbad684c68..4425e655917 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -63,8 +63,8 @@ void FiniteDifferenceSolver::EvolveB ( amrex::ignore_unused(area_mod, ECTRhofield, Venl, flag_info_cell, borrowing); #endif - // Select algorithm (The choice of algorithm is a runtime option, - // but we compile code for each algorithm, using templates) + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ if ((m_fdtd_algo == ElectromagneticSolverAlgo::Yee)|| (m_fdtd_algo == ElectromagneticSolverAlgo::HybridPIC)){ @@ -120,7 +120,7 @@ void FiniteDifferenceSolver::EvolveBCartesian ( { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Extract field data for this grid/tile Array4 const& Bx = Bfield[0]->array(mfi); @@ -132,11 +132,11 @@ void FiniteDifferenceSolver::EvolveBCartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract tileboxes for which to loop Box const& tbx = mfi.tilebox(Bfield[0]->ixType().toIntVect()); @@ -195,7 +195,7 @@ void FiniteDifferenceSolver::EvolveBCartesian ( if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -233,7 +233,7 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { // Extract field data for this grid/tile @@ -356,7 +356,7 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -385,7 +385,7 @@ void FiniteDifferenceSolver::EvolveBCylindrical ( { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Extract field data for this grid/tile Array4 const& Br = Bfield[0]->array(mfi); @@ -397,9 +397,9 @@ void FiniteDifferenceSolver::EvolveBCylindrical ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); - int const n_coefs_r = m_stencil_coefs_r.size(); + auto const n_coefs_r = static_cast(m_stencil_coefs_r.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract cylindrical specific parameters Real const dr = m_dr; @@ -462,7 +462,7 @@ void FiniteDifferenceSolver::EvolveBCylindrical ( }, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ - Real const r = rmin + (i + 0.5)*dr; // r on a cell-centered grid (Bz is cell-centered in r) + Real const r = rmin + (i + 0.5_rt)*dr; // r on a cell-centered grid (Bz is cell-centered in r) Bz(i, j, 0, 0) += dt*( - T_Algo::UpwardDrr_over_r(Et, r, dr, coefs_r, n_coefs_r, i, j, 0, 0)); for (int m=1 ; m(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 9ae67d4f960..1fb2dd85e60 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -46,11 +46,11 @@ void FiniteDifferenceSolver::EvolveBPML ( amrex::Real const dt, const bool dive_cleaning) { - // Select algorithm (The choice of algorithm is a runtime option, - // but we compile code for each algorithm, using templates) + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ amrex::ignore_unused(Bfield, Efield, dt, dive_cleaning); - WARPX_ABORT_WITH_MESSAGE( + WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else if (m_grid_type == GridType::Collocated) { @@ -98,11 +98,11 @@ void FiniteDifferenceSolver::EvolveBPMLCartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract tileboxes for which to loop Box const& tbx = mfi.tilebox(Bfield[0]->ixType().ixType()); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index b044986383e..74922650a42 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -117,7 +117,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Extract field data for this grid/tile Array4 const& Ex = Efield[0]->array(mfi); @@ -138,11 +138,11 @@ void FiniteDifferenceSolver::EvolveECartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract tileboxes for which to loop Box const& tex = mfi.tilebox(Efield[0]->ixType().toIntVect()); @@ -221,7 +221,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -249,7 +249,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Extract field data for this grid/tile Array4 const& Er = Efield[0]->array(mfi); @@ -264,9 +264,9 @@ void FiniteDifferenceSolver::EvolveECylindrical ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); - int const n_coefs_r = m_stencil_coefs_r.size(); + auto const n_coefs_r = static_cast(m_stencil_coefs_r.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract cylindrical specific parameters Real const dr = m_dr; @@ -284,7 +284,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( amrex::ParallelFor(ter, tet, tez, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ - Real const r = rmin + (i + 0.5)*dr; // r on cell-centered point (Er is cell-centered in r) + Real const r = rmin + (i + 0.5_rt)*dr; // r on cell-centered point (Er is cell-centered in r) Er(i, j, 0, 0) += c2 * dt*( - T_Algo::DownwardDz(Bt, coefs_z, n_coefs_z, i, j, 0, 0) - PhysConst::mu0 * jr(i, j, 0, 0) ); // Mode m=0 @@ -423,7 +423,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } // end of loop over grid/tiles diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp index b7a42ad28b7..95f899c98e1 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp @@ -88,7 +88,7 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Extract field data for this grid/tile amrex::Array4 const &Ex = Efield[0]->array(mfi); @@ -149,7 +149,7 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } #ifdef WARPX_DIM_XZ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp index 2ecc9ee0e04..93352ce9896 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp @@ -52,8 +52,8 @@ void FiniteDifferenceSolver::EvolveEPML ( MultiSigmaBox const& sigba, amrex::Real const dt, bool pml_has_particles ) { - // Select algorithm (The choice of algorithm is a runtime option, - // but we compile code for each algorithm, using templates) + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ amrex::ignore_unused(Efield, Bfield, Jfield, Ffield, sigba, dt, pml_has_particles, edge_lengths); WARPX_ABORT_WITH_MESSAGE( @@ -117,11 +117,11 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract tileboxes for which to loop Box const& tex = mfi.tilebox(Efield[0]->ixType().ixType()); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp index c43d965b3a4..83014a08026 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveF.cpp @@ -50,8 +50,8 @@ void FiniteDifferenceSolver::EvolveF ( int const rhocomp, amrex::Real const dt ) { - // Select algorithm (The choice of algorithm is a runtime option, - // but we compile code for each algorithm, using templates) + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ if (m_fdtd_algo == ElectromagneticSolverAlgo::Yee){ @@ -103,11 +103,11 @@ void FiniteDifferenceSolver::EvolveFCartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z =static_cast(m_stencil_coefs_z.size()); // Extract tileboxes for which to loop Box const& tf = mfi.tilebox(Ffield->ixType().toIntVect()); @@ -156,9 +156,9 @@ void FiniteDifferenceSolver::EvolveFCylindrical ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); - int const n_coefs_r = m_stencil_coefs_r.size(); + auto const n_coefs_r = static_cast(m_stencil_coefs_r.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract cylindrical specific parameters Real const dr = m_dr; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp index 5b69737c79c..4ef056c937a 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveFPML.cpp @@ -44,8 +44,8 @@ void FiniteDifferenceSolver::EvolveFPML ( std::array< amrex::MultiFab*, 3 > const Efield, amrex::Real const dt ) { - // Select algorithm (The choice of algorithm is a runtime option, - // but we compile code for each algorithm, using templates) + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ amrex::ignore_unused(Ffield, Efield, dt); WARPX_ABORT_WITH_MESSAGE( @@ -92,11 +92,11 @@ void FiniteDifferenceSolver::EvolveFPMLCartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract tileboxes for which to loop Box const& tf = mfi.tilebox(Ffield->ixType().ixType()); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp index 30d19efe008..b6bc8fdca7f 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveG.cpp @@ -95,9 +95,9 @@ void FiniteDifferenceSolver::EvolveGCartesian ( amrex::Real const* const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); amrex::Real const* const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - const int n_coefs_x = m_stencil_coefs_x.size(); - const int n_coefs_y = m_stencil_coefs_y.size(); - const int n_coefs_z = m_stencil_coefs_z.size(); + const auto n_coefs_x = static_cast(m_stencil_coefs_x.size()); + const auto n_coefs_y = static_cast(m_stencil_coefs_y.size()); + const auto n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract tilebox to loop over amrex::Box const& tf = mfi.tilebox(Gfield->ixType().toIntVect()); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H index b6c5c0a8ce8..e49995557a0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H @@ -21,7 +21,7 @@ /** * This struct contains only static functions to initialize the stencil coefficients - * and to compute finite-difference derivatives for the Cartesian Yee algorithm. + * and to compute finite-difference derivatives for the Cylindrical Yee algorithm. */ struct CylindricalYeeAlgorithm { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index d877db5253d..861b2648c1e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -139,22 +139,24 @@ class FiniteDifferenceSolver * \param[out] Efield vector of electric field MultiFabs updated at a given level * \param[in] Jfield vector of total current MultiFabs at a given level * \param[in] Jifield vector of ion current density MultiFabs at a given level + * \param[in] Jextfield vector of external current density MultiFabs at a given level * \param[in] Bfield vector of magnetic field MultiFabs at a given level * \param[in] rhofield scalar ion charge density Multifab at a given level * \param[in] Pefield scalar electron pressure MultiFab at a given level * \param[in] edge_lengths length of edges along embedded boundaries * \param[in] lev level number for the calculation - * \param[in] hybrid_pic_model instance of the hybrid-PIC model + * \param[in] hybrid_model instance of the hybrid-PIC model * \param[in] include_resistivity_term boolean flag for whether to include resistivity */ void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3>& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); /** @@ -233,11 +235,12 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); template @@ -337,11 +340,12 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); template diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp index 1bb557d009d..1bbc6c9b337 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp @@ -30,15 +30,15 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( int const fdtd_algo, std::array cell_size, - short grid_type) { - + short grid_type): // Register the type of finite-difference algorithm - m_fdtd_algo = fdtd_algo; - m_grid_type = grid_type; - + m_fdtd_algo{fdtd_algo}, + m_grid_type{grid_type} +{ // return if not FDTD - if (fdtd_algo == ElectromagneticSolverAlgo::None || fdtd_algo == ElectromagneticSolverAlgo::PSATD) + if (fdtd_algo == ElectromagneticSolverAlgo::None || fdtd_algo == ElectromagneticSolverAlgo::PSATD) { return; + } // Calculate coefficients of finite-difference stencil #ifdef WARPX_DIM_RZ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 51e11aa46f3..c7ea9b46458 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -46,6 +46,20 @@ public: void InitData (); + /** + * \brief + * Function to evaluate the external current expressions and populate the + * external current multifab. Note the external current can be a function + * of time and therefore this should be re-evaluated at every step. + */ + void GetCurrentExternal ( + amrex::Vector, 3>> const& edge_lengths + ); + void GetCurrentExternal ( + std::array< std::unique_ptr, 3> const& edge_lengths, + int lev + ); + /** * \brief * Function to calculate the total current based on Ampere's law while @@ -107,12 +121,12 @@ public: * charge density (and assumption of quasi-neutrality) using the user * specified electron equation of state. * - * \param[out] Pe scalar electron pressure MultiFab at a given level - * \param[in] rhofield scalar ion chrge density Multifab at a given level + * \param[out] Pe_filed scalar electron pressure MultiFab at a given level + * \param[in] rho_field scalar ion charge density Multifab at a given level */ void FillElectronPressureMF ( - std::unique_ptr const& Pe, - amrex::MultiFab* const& rhofield ); + std::unique_ptr const& Pe_field, + amrex::MultiFab* const& rho_field ); // Declare variables to hold hybrid-PIC model parameters /** Number of substeps to take when evolving B */ @@ -133,15 +147,33 @@ public: std::unique_ptr m_resistivity_parser; amrex::ParserExecutor<1> m_eta; + /** External current */ + std::string m_Jx_ext_grid_function = "0.0"; + std::string m_Jy_ext_grid_function = "0.0"; + std::string m_Jz_ext_grid_function = "0.0"; + std::array< std::unique_ptr, 3> m_J_external_parser; + std::array< amrex::ParserExecutor<4>, 3> m_J_external; + bool m_external_field_has_time_dependence = false; + // Declare multifabs specifically needed for the hybrid-PIC model amrex::Vector< std::unique_ptr > rho_fp_temp; amrex::Vector, 3 > > current_fp_temp; amrex::Vector, 3 > > current_fp_ampere; + amrex::Vector, 3 > > current_fp_external; amrex::Vector< std::unique_ptr > electron_pressure_fp; // Helper functions to retrieve hybrid-PIC multifabs - amrex::MultiFab * get_pointer_current_fp_ampere (int lev, int direction) const { return current_fp_ampere[lev][direction].get(); } - amrex::MultiFab * get_pointer_electron_pressure_fp (int lev) const { return electron_pressure_fp[lev].get(); } + [[nodiscard]] amrex::MultiFab* + get_pointer_current_fp_ampere (int lev, int direction) const + { + return current_fp_ampere[lev][direction].get(); + } + + [[nodiscard]] amrex::MultiFab* + get_pointer_electron_pressure_fp (int lev) const + { + return electron_pressure_fp[lev].get(); + } /** Gpu Vector with index type of the Jx multifab */ amrex::GpuArray Jx_IndexType; @@ -176,7 +208,7 @@ struct ElectronPressure { amrex::Real const T0, amrex::Real const gamma, amrex::Real const rho) { - return n0 * T0 * pow((rho/PhysConst::q_e)/n0, gamma); + return n0 * T0 * std::pow((rho/PhysConst::q_e)/n0, gamma); } }; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 7213e599b67..5a6f0018b2a 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -42,6 +42,11 @@ void HybridPICModel::ReadParameters () // convert electron temperature from eV to J m_elec_temp *= PhysConst::q_e; + + // external currents + pp_hybrid.query("Jx_external_grid_function(x,y,z,t)", m_Jx_ext_grid_function); + pp_hybrid.query("Jy_external_grid_function(x,y,z,t)", m_Jy_ext_grid_function); + pp_hybrid.query("Jz_external_grid_function(x,y,z,t)", m_Jz_ext_grid_function); } void HybridPICModel::AllocateMFs (int nlevs_max) @@ -50,6 +55,7 @@ void HybridPICModel::AllocateMFs (int nlevs_max) rho_fp_temp.resize(nlevs_max); current_fp_temp.resize(nlevs_max); current_fp_ampere.resize(nlevs_max); + current_fp_external.resize(nlevs_max); } void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm, @@ -86,6 +92,21 @@ void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const Distri dm, ncomps, ngJ, lev, "current_fp_ampere[y]", 0.0_rt); WarpX::AllocInitMultiFab(current_fp_ampere[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp_ampere[z]", 0.0_rt); + + // the external current density multifab is made nodal to avoid needing to interpolate + // to a nodal grid as has to be done for the ion and total current density multifabs + WarpX::AllocInitMultiFab(current_fp_external[lev][0], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, ngJ, lev, "current_fp_external[x]", 0.0_rt); + WarpX::AllocInitMultiFab(current_fp_external[lev][1], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, ngJ, lev, "current_fp_external[y]", 0.0_rt); + WarpX::AllocInitMultiFab(current_fp_external[lev][2], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, ngJ, lev, "current_fp_external[z]", 0.0_rt); + +#ifdef WARPX_DIM_RZ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (ncomps == 1), + "Ohm's law solver only support m = 0 azimuthal mode at present."); +#endif } void HybridPICModel::ClearLevel (int lev) @@ -95,6 +116,7 @@ void HybridPICModel::ClearLevel (int lev) for (int i = 0; i < 3; ++i) { current_fp_temp[lev][i].reset(); current_fp_ampere[lev][i].reset(); + current_fp_external[lev][i].reset(); } } @@ -104,6 +126,22 @@ void HybridPICModel::InitData () utils::parser::makeParser(m_eta_expression, {"rho"})); m_eta = m_resistivity_parser->compile<1>(); + m_J_external_parser[0] = std::make_unique( + utils::parser::makeParser(m_Jx_ext_grid_function,{"x","y","z","t"})); + m_J_external_parser[1] = std::make_unique( + utils::parser::makeParser(m_Jy_ext_grid_function,{"x","y","z","t"})); + m_J_external_parser[2] = std::make_unique( + utils::parser::makeParser(m_Jz_ext_grid_function,{"x","y","z","t"})); + m_J_external[0] = m_J_external_parser[0]->compile<4>(); + m_J_external[1] = m_J_external_parser[1]->compile<4>(); + m_J_external[2] = m_J_external_parser[2]->compile<4>(); + + // check if the external current parsers depend on time + for (int i=0; i<3; i++) { + const std::set J_ext_symbols = m_J_external_parser[i]->symbols(); + m_external_field_has_time_dependence += J_ext_symbols.count("t"); + } + auto & warpx = WarpX::GetInstance(); // Get the grid staggering of the fields involved in calculating E @@ -175,6 +213,178 @@ void HybridPICModel::InitData () Ey_IndexType[1] = 1; Ez_IndexType[1] = 1; #endif + + // Initialize external current - note that this approach skips the check + // if the current is time dependent which is what needs to be done to + // write time independent fields on the first step. + std::array< std::unique_ptr, 3 > edge_lengths; + + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) + { +#ifdef AMREX_USE_EB + auto& edge_lengths_x = warpx.getedgelengths(lev, 0); + edge_lengths[0] = std::make_unique( + edge_lengths_x, amrex::make_alias, 0, edge_lengths_x.nComp() + ); + auto& edge_lengths_y = warpx.getedgelengths(lev, 1); + edge_lengths[1] = std::make_unique( + edge_lengths_y, amrex::make_alias, 0, edge_lengths_y.nComp() + ); + auto& edge_lengths_z = warpx.getedgelengths(lev, 2); + edge_lengths[2] = std::make_unique( + edge_lengths_z, amrex::make_alias, 0, edge_lengths_z.nComp() + ); +#endif + GetCurrentExternal(edge_lengths, lev); + } +} + +void HybridPICModel::GetCurrentExternal ( + amrex::Vector, 3>> const& edge_lengths) +{ + if (!m_external_field_has_time_dependence) { return; } + + auto& warpx = WarpX::GetInstance(); + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) + { + GetCurrentExternal(edge_lengths[lev], lev); + } +} + + +void HybridPICModel::GetCurrentExternal ( + std::array< std::unique_ptr, 3> const& edge_lengths, + int lev) +{ + // This logic matches closely to WarpX::InitializeExternalFieldsOnGridUsingParser + // except that the parsers include time dependence. + auto & warpx = WarpX::GetInstance(); + + auto t = warpx.gett_new(lev); + + auto dx_lev = warpx.Geom(lev).CellSizeArray(); + const RealBox& real_box = warpx.Geom(lev).ProbDomain(); + + auto& mfx = current_fp_external[lev][0]; + auto& mfy = current_fp_external[lev][1]; + auto& mfz = current_fp_external[lev][2]; + + const amrex::IntVect x_nodal_flag = mfx->ixType().toIntVect(); + const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); + const amrex::IntVect z_nodal_flag = mfz->ixType().toIntVect(); + + // avoid implicit lambda capture + auto Jx_external = m_J_external[0]; + auto Jy_external = m_J_external[1]; + auto Jz_external = m_J_external[2]; + + for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); + + auto const& mfxfab = mfx->array(mfi); + auto const& mfyfab = mfy->array(mfi); + auto const& mfzfab = mfz->array(mfi); + +#ifdef AMREX_USE_EB + amrex::Array4 const& lx = edge_lengths[0]->array(mfi); + amrex::Array4 const& ly = edge_lengths[1]->array(mfi); + amrex::Array4 const& lz = edge_lengths[2]->array(mfi); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ignore_unused(ly); +#endif +#else + amrex::ignore_unused(edge_lengths); +#endif + + amrex::ParallelFor (tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // skip if node is covered by an embedded boundary +#ifdef AMREX_USE_EB + if (lx(i, j, k) <= 0) return; +#endif + // Shift required in the x-, y-, or z- position + // depending on the index type of the multifab +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0._rt; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - x_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; +#else + const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1._rt - x_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1._rt - x_nodal_flag[2]) * dx_lev[2] * 0.5_rt; + const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; +#endif + // Initialize the x-component of the field. + mfxfab(i,j,k) = Jx_external(x,y,z,t); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // skip if node is covered by an embedded boundary +#ifdef AMREX_USE_EB + if (ly(i, j, k) <= 0) return; +#endif +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0._rt; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - y_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; +#elif defined(WARPX_DIM_3D) + const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1._rt - y_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1._rt - y_nodal_flag[2]) * dx_lev[2] * 0.5_rt; + const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; +#endif + // Initialize the y-component of the field. + mfyfab(i,j,k) = Jy_external(x,y,z,t); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // skip if node is covered by an embedded boundary +#ifdef AMREX_USE_EB + if (lz(i, j, k) <= 0) return; +#endif +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0._rt; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - z_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; +#elif defined(WARPX_DIM_3D) + const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1._rt - z_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1._rt - z_nodal_flag[2]) * dx_lev[2] * 0.5_rt; + const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; +#endif + // Initialize the z-component of the field. + mfzfab(i,j,k) = Jz_external(x,y,z,t); + } + ); + } } void HybridPICModel::CalculateCurrentAmpere ( @@ -204,7 +414,7 @@ void HybridPICModel::CalculateCurrentAmpere ( // the boundary correction was already applied to J_i and the B-field // boundary ensures that J itself complies with the boundary conditions, right? // ApplyJfieldBoundary(lev, Jfield[0].get(), Jfield[1].get(), Jfield[2].get()); - for (int i=0; i<3; i++) current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); + for (int i=0; i<3; i++) { current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); } } void HybridPICModel::HybridPICSolveE ( @@ -259,7 +469,8 @@ void HybridPICModel::HybridPICSolveE ( // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( - Efield, current_fp_ampere[lev], Jfield, Bfield, rhofield, + Efield, current_fp_ampere[lev], Jfield, current_fp_external[lev], + Bfield, rhofield, electron_pressure_fp[lev], edge_lengths, lev, this, include_resistivity_term ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index f4f236e61d1..1a72fee53c2 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -28,8 +28,8 @@ void FiniteDifferenceSolver::CalculateCurrentAmpere ( std::array< std::unique_ptr, 3 > const& edge_lengths, int lev ) { - // Select algorithm (The choice of algorithm is a runtime option, - // but we compile code for each algorithm, using templates) + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) if (m_fdtd_algo == ElectromagneticSolverAlgo::HybridPIC) { #ifdef WARPX_DIM_RZ CalculateCurrentAmpereCylindrical ( @@ -49,10 +49,10 @@ void FiniteDifferenceSolver::CalculateCurrentAmpere ( } // /** -// * \brief Calculate electron current from Ampere's law without displacement -// * current and the kinetically tracked ion currents i.e. J_e = curl B. +// * \brief Calculate total current from Ampere's law without displacement +// * current i.e. J = 1/mu_0 curl x B. // * -// * \param[out] Jfield vector of electron current MultiFabs at a given level +// * \param[out] Jfield vector of total current MultiFabs at a given level // * \param[in] Bfield vector of magnetic field MultiFabs at a given level // */ #ifdef WARPX_DIM_RZ @@ -64,11 +64,189 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( int lev ) { - amrex::ignore_unused(Jfield, Bfield, edge_lengths, lev); - amrex::Abort(Utils::TextMsg::Err( - "currently hybrid E-solve does not work for RZ")); + // for the profiler + amrex::LayoutData* cost = WarpX::getCosts(lev); + +#ifndef AMREX_USE_EB + amrex::ignore_unused(edge_lengths); +#endif + + // reset Jfield + Jfield[0]->setVal(0); + Jfield[1]->setVal(0); + Jfield[2]->setVal(0); + + // Loop through the grids, and over the tiles within each grid +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*Jfield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + } + Real wt = static_cast(amrex::second()); + + // Extract field data for this grid/tile + Array4 const& Jr = Jfield[0]->array(mfi); + Array4 const& Jt = Jfield[1]->array(mfi); + Array4 const& Jz = Jfield[2]->array(mfi); + Array4 const& Br = Bfield[0]->array(mfi); + Array4 const& Bt = Bfield[1]->array(mfi); + Array4 const& Bz = Bfield[2]->array(mfi); + +#ifdef AMREX_USE_EB + amrex::Array4 const& lr = edge_lengths[0]->array(mfi); + amrex::Array4 const& lt = edge_lengths[1]->array(mfi); + amrex::Array4 const& lz = edge_lengths[2]->array(mfi); +#endif + + // Extract stencil coefficients + Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); + int const n_coefs_r = static_cast(m_stencil_coefs_r.size()); + Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); + int const n_coefs_z = static_cast(m_stencil_coefs_z.size()); + + // Extract cylindrical specific parameters + Real const dr = m_dr; + int const nmodes = m_nmodes; + Real const rmin = m_rmin; + + // Extract tileboxes for which to loop + Box const& tjr = mfi.tilebox(Jfield[0]->ixType().toIntVect()); + Box const& tjt = mfi.tilebox(Jfield[1]->ixType().toIntVect()); + Box const& tjz = mfi.tilebox(Jfield[2]->ixType().toIntVect()); + + Real const one_over_mu0 = 1._rt / PhysConst::mu0; + + // Calculate the total current, using Ampere's law, on the same grid + // as the E-field + amrex::ParallelFor(tjr, tjt, tjz, + + // Jr calculation + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ +#ifdef AMREX_USE_EB + // Skip if this cell is fully covered by embedded boundaries + if (lr(i, j, 0) <= 0) return; +#endif + // Mode m=0 + Jr(i, j, 0, 0) = one_over_mu0 * ( + - T_Algo::DownwardDz(Bt, coefs_z, n_coefs_z, i, j, 0, 0) + ); + + // Higher-order modes + // r on cell-centered point (Jr is cell-centered in r) + Real const r = rmin + (i + 0.5_rt)*dr; + for (int m=1; m 0.5_rt*dr) { + // Mode m=0 + Jt(i, j, 0, 0) = one_over_mu0 * ( + - T_Algo::DownwardDr(Bz, coefs_r, n_coefs_r, i, j, 0, 0) + + T_Algo::DownwardDz(Br, coefs_z, n_coefs_z, i, j, 0, 0) + ); + + // Higher-order modes + for (int m=1 ; m 0.5_rt*dr) { + // Mode m=0 + Jz(i, j, 0, 0) = one_over_mu0 * ( + T_Algo::DownwardDrr_over_r(Bt, r, dr, coefs_r, n_coefs_r, i, j, 0, 0) + ); + // Higher-order modes + for (int m=1 ; m(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } } + #else + template void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( std::array< std::unique_ptr, 3 >& Jfield, @@ -98,7 +276,7 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Extract field data for this grid/tile Array4 const& Jx = Jfield[0]->array(mfi); @@ -116,11 +294,11 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // Extract tileboxes for which to loop Box const& tjx = mfi.tilebox(Jfield[0]->ixType().toIntVect()); @@ -129,8 +307,8 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( Real const one_over_mu0 = 1._rt / PhysConst::mu0; - // First calculate the total current using Ampere's law on the - // same grid as the E-field + // Calculate the total current, using Ampere's law, on the same grid + // as the E-field amrex::ParallelFor(tjx, tjy, tjz, // Jx calculation @@ -179,7 +357,7 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -191,6 +369,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 >& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -198,20 +377,20 @@ void FiniteDifferenceSolver::HybridPICSolveE ( int lev, HybridPICModel const* hybrid_model, const bool include_resistivity_term ) { - // Select algorithm (The choice of algorithm is a runtime option, - // but we compile code for each algorithm, using templates) + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) if (m_fdtd_algo == ElectromagneticSolverAlgo::HybridPIC) { #ifdef WARPX_DIM_RZ HybridPICSolveECylindrical ( - Efield, Jfield, Jifield, Bfield, rhofield, Pefield, + Efield, Jfield, Jifield, Jextfield, Bfield, rhofield, Pefield, edge_lengths, lev, hybrid_model, include_resistivity_term ); #else HybridPICSolveECartesian ( - Efield, Jfield, Jifield, Bfield, rhofield, Pefield, + Efield, Jfield, Jifield, Jextfield, Bfield, rhofield, Pefield, edge_lengths, lev, hybrid_model, include_resistivity_term ); @@ -228,6 +407,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -238,12 +418,258 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( #ifndef AMREX_USE_EB amrex::ignore_unused(edge_lengths); #endif - amrex::ignore_unused( - Efield, Jfield, Jifield, Bfield, rhofield, Pefield, edge_lengths, - lev, hybrid_model, include_resistivity_term - ); - amrex::Abort(Utils::TextMsg::Err( - "currently hybrid E-solve does not work for RZ")); + + // Both steps below do not currently support m > 0 and should be + // modified if such support wants to be added + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (m_nmodes == 1), + "Ohm's law solver only support m = 0 azimuthal mode at present."); + + // for the profiler + amrex::LayoutData* cost = WarpX::getCosts(lev); + + using namespace ablastr::coarsen::sample; + + // get hybrid model parameters + const auto eta = hybrid_model->m_eta; + const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; + + // Index type required for interpolating fields from their respective + // staggering to the Ex, Ey, Ez locations + amrex::GpuArray const& Er_stag = hybrid_model->Ex_IndexType; + amrex::GpuArray const& Et_stag = hybrid_model->Ey_IndexType; + amrex::GpuArray const& Ez_stag = hybrid_model->Ez_IndexType; + amrex::GpuArray const& Jr_stag = hybrid_model->Jx_IndexType; + amrex::GpuArray const& Jt_stag = hybrid_model->Jy_IndexType; + amrex::GpuArray const& Jz_stag = hybrid_model->Jz_IndexType; + amrex::GpuArray const& Br_stag = hybrid_model->Bx_IndexType; + amrex::GpuArray const& Bt_stag = hybrid_model->By_IndexType; + amrex::GpuArray const& Bz_stag = hybrid_model->Bz_IndexType; + + // Parameters for `interp` that maps from Yee to nodal mesh and back + amrex::GpuArray const& nodal = {1, 1, 1}; + // The "coarsening is just 1 i.e. no coarsening" + amrex::GpuArray const& coarsen = {1, 1, 1}; + + // The E-field calculation is done in 2 steps: + // 1) The J x B term is calculated on a nodal mesh in order to ensure + // energy conservation. + // 2) The nodal E-field values are averaged onto the Yee grid and the + // electron pressure & resistivity terms are added (these terms are + // naturally located on the Yee grid). + + // Create a temporary multifab to hold the nodal E-field values + // Note the multifab has 3 values for Ex, Ey and Ez which we can do here + // since all three components will be calculated on the same grid. + // Also note that enE_nodal_mf does not need to have any guard cells since + // these values will be interpolated to the Yee mesh which is contained + // by the nodal mesh. + auto const& ba = convert(rhofield->boxArray(), IntVect::TheNodeVector()); + MultiFab enE_nodal_mf(ba, rhofield->DistributionMap(), 3, IntVect::TheZeroVector()); + + // Loop through the grids, and over the tiles within each grid for the + // initial, nodal calculation of E +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(enE_nodal_mf, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + } + Real wt = static_cast(amrex::second()); + + Array4 const& enE_nodal = enE_nodal_mf.array(mfi); + Array4 const& Jr = Jfield[0]->const_array(mfi); + Array4 const& Jt = Jfield[1]->const_array(mfi); + Array4 const& Jz = Jfield[2]->const_array(mfi); + Array4 const& Jir = Jifield[0]->const_array(mfi); + Array4 const& Jit = Jifield[1]->const_array(mfi); + Array4 const& Jiz = Jifield[2]->const_array(mfi); + Array4 const& Jextr = Jextfield[0]->const_array(mfi); + Array4 const& Jextt = Jextfield[1]->const_array(mfi); + Array4 const& Jextz = Jextfield[2]->const_array(mfi); + Array4 const& Br = Bfield[0]->const_array(mfi); + Array4 const& Bt = Bfield[1]->const_array(mfi); + Array4 const& Bz = Bfield[2]->const_array(mfi); + + // Loop over the cells and update the nodal E field + amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ + + // interpolate the total current to a nodal grid + auto const jr_interp = Interp(Jr, Jr_stag, nodal, coarsen, i, j, 0, 0); + auto const jt_interp = Interp(Jt, Jt_stag, nodal, coarsen, i, j, 0, 0); + auto const jz_interp = Interp(Jz, Jz_stag, nodal, coarsen, i, j, 0, 0); + + // interpolate the ion current to a nodal grid + auto const jir_interp = Interp(Jir, Jr_stag, nodal, coarsen, i, j, 0, 0); + auto const jit_interp = Interp(Jit, Jt_stag, nodal, coarsen, i, j, 0, 0); + auto const jiz_interp = Interp(Jiz, Jz_stag, nodal, coarsen, i, j, 0, 0); + + // interpolate the B field to a nodal grid + auto const Br_interp = Interp(Br, Br_stag, nodal, coarsen, i, j, 0, 0); + auto const Bt_interp = Interp(Bt, Bt_stag, nodal, coarsen, i, j, 0, 0); + auto const Bz_interp = Interp(Bz, Bz_stag, nodal, coarsen, i, j, 0, 0); + + // calculate enE = (J - Ji) x B + enE_nodal(i, j, 0, 0) = ( + (jt_interp - jit_interp - Jextt(i, j, 0)) * Bz_interp + - (jz_interp - jiz_interp - Jextz(i, j, 0)) * Bt_interp + ); + enE_nodal(i, j, 0, 1) = ( + (jz_interp - jiz_interp - Jextz(i, j, 0)) * Br_interp + - (jr_interp - jir_interp - Jextr(i, j, 0)) * Bz_interp + ); + enE_nodal(i, j, 0, 2) = ( + (jr_interp - jir_interp - Jextr(i, j, 0)) * Bt_interp + - (jt_interp - jit_interp - Jextt(i, j, 0)) * Br_interp + ); + }); + + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + wt = static_cast(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } + + // Loop through the grids, and over the tiles within each grid again + // for the Yee grid calculation of the E field +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*Efield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + } + Real wt = static_cast(amrex::second()); + + // Extract field data for this grid/tile + Array4 const& Er = Efield[0]->array(mfi); + Array4 const& Et = Efield[1]->array(mfi); + Array4 const& Ez = Efield[2]->array(mfi); + Array4 const& Jr = Jfield[0]->const_array(mfi); + Array4 const& Jt = Jfield[1]->const_array(mfi); + Array4 const& Jz = Jfield[2]->const_array(mfi); + Array4 const& enE = enE_nodal_mf.const_array(mfi); + Array4 const& rho = rhofield->const_array(mfi); + Array4 const& Pe = Pefield->array(mfi); + +#ifdef AMREX_USE_EB + amrex::Array4 const& lr = edge_lengths[0]->array(mfi); + amrex::Array4 const& lt = edge_lengths[1]->array(mfi); + amrex::Array4 const& lz = edge_lengths[2]->array(mfi); +#endif + + // Extract stencil coefficients + Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); + int const n_coefs_r = static_cast(m_stencil_coefs_r.size()); + Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); + int const n_coefs_z = static_cast(m_stencil_coefs_z.size()); + + // Extract cylindrical specific parameters + Real const dr = m_dr; + Real const rmin = m_rmin; + + Box const& ter = mfi.tilebox(Efield[0]->ixType().toIntVect()); + Box const& tet = mfi.tilebox(Efield[1]->ixType().toIntVect()); + Box const& tez = mfi.tilebox(Efield[2]->ixType().toIntVect()); + + // Loop over the cells and update the E field + amrex::ParallelFor(ter, tet, tez, + + // Er calculation + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ +#ifdef AMREX_USE_EB + // Skip if this cell is fully covered by embedded boundaries + if (lr(i, j, 0) <= 0) return; +#endif + // Interpolate to get the appropriate charge density in space + Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); + + // safety condition since we divide by rho_val later + if (rho_val < rho_floor) { rho_val = rho_floor; } + + // Get the gradient of the electron pressure + auto grad_Pe = T_Algo::UpwardDr(Pe, coefs_r, n_coefs_r, i, j, 0, 0); + + // interpolate the nodal neE values to the Yee grid + auto enE_r = Interp(enE, nodal, Er_stag, coarsen, i, j, 0, 0); + + Er(i, j, 0) = (enE_r - grad_Pe) / rho_val; + + // Add resistivity only if E field value is used to update B + if (include_resistivity_term) { Er(i, j, 0) += eta(rho_val) * Jr(i, j, 0); } + }, + + // Et calculation + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ +#ifdef AMREX_USE_EB + // In RZ Et is associated with a mesh node, so we need to check if the mesh node is covered + amrex::ignore_unused(lt); + if (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0) return; +#endif + // r on a nodal grid (Et is nodal in r) + Real const r = rmin + i*dr; + // Mode m=0: // Ensure that Et remains 0 on axis + if (r < 0.5_rt*dr) { + Et(i, j, 0, 0) = 0.; + return; + } + + // Interpolate to get the appropriate charge density in space + Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); + + // safety condition since we divide by rho_val later + if (rho_val < rho_floor) { rho_val = rho_floor; } + + // Get the gradient of the electron pressure + // -> d/dt = 0 for m = 0 + auto grad_Pe = 0.0_rt; + + // interpolate the nodal neE values to the Yee grid + auto enE_t = Interp(enE, nodal, Et_stag, coarsen, i, j, 0, 1); + + Et(i, j, 0) = (enE_t - grad_Pe) / rho_val; + + // Add resistivity only if E field value is used to update B + if (include_resistivity_term) { Et(i, j, 0) += eta(rho_val) * Jt(i, j, 0); } + }, + + // Ez calculation + [=] AMREX_GPU_DEVICE (int i, int j, int k){ +#ifdef AMREX_USE_EB + // Skip field solve if this cell is fully covered by embedded boundaries + if (lz(i,j,0) <= 0) { return; } +#endif + // Interpolate to get the appropriate charge density in space + Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); + + // safety condition since we divide by rho_val later + if (rho_val < rho_floor) { rho_val = rho_floor; } + + // Get the gradient of the electron pressure + auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k, 0); + + // interpolate the nodal neE values to the Yee grid + auto enE_z = Interp(enE, nodal, Ez_stag, coarsen, i, j, k, 2); + + Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; + + // Add resistivity only if E field value is used to update B + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); } + } + ); + + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + wt = static_cast(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } } #else @@ -253,6 +679,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -316,7 +743,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); Array4 const& enE_nodal = enE_nodal_mf.array(mfi); Array4 const& Jx = Jfield[0]->const_array(mfi); @@ -325,6 +752,9 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Array4 const& Jix = Jifield[0]->const_array(mfi); Array4 const& Jiy = Jifield[1]->const_array(mfi); Array4 const& Jiz = Jifield[2]->const_array(mfi); + Array4 const& Jextx = Jextfield[0]->const_array(mfi); + Array4 const& Jexty = Jextfield[1]->const_array(mfi); + Array4 const& Jextz = Jextfield[2]->const_array(mfi); Array4 const& Bx = Bfield[0]->const_array(mfi); Array4 const& By = Bfield[1]->const_array(mfi); Array4 const& Bz = Bfield[2]->const_array(mfi); @@ -349,23 +779,23 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // calculate enE = (J - Ji) x B enE_nodal(i, j, k, 0) = ( - (jy_interp - jiy_interp) * Bz_interp - - (jz_interp - jiz_interp) * By_interp + (jy_interp - jiy_interp - Jexty(i, j, k)) * Bz_interp + - (jz_interp - jiz_interp - Jextz(i, j, k)) * By_interp ); enE_nodal(i, j, k, 1) = ( - (jz_interp - jiz_interp) * Bx_interp - - (jx_interp - jix_interp) * Bz_interp + (jz_interp - jiz_interp - Jextz(i, j, k)) * Bx_interp + - (jx_interp - jix_interp - Jextx(i, j, k)) * Bz_interp ); enE_nodal(i, j, k, 2) = ( - (jx_interp - jix_interp) * By_interp - - (jy_interp - jiy_interp) * Bx_interp + (jx_interp - jix_interp - Jextx(i, j, k)) * By_interp + - (jy_interp - jiy_interp - Jexty(i, j, k)) * Bx_interp ); }); if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -380,7 +810,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Extract field data for this grid/tile Array4 const& Ex = Efield[0]->array(mfi); @@ -401,11 +831,11 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); Box const& tex = mfi.tilebox(Efield[0]->ixType().toIntVect()); Box const& tey = mfi.tilebox(Efield[1]->ixType().toIntVect()); @@ -424,7 +854,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Real rho_val = Interp(rho, nodal, Ex_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDx(Pe, coefs_x, n_coefs_x, i, j, k); @@ -435,7 +865,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ex(i, j, k) = (enE_x - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ex(i, j, k) += eta(rho_val) * Jx(i, j, k); + if (include_resistivity_term) { Ex(i, j, k) += eta(rho_val) * Jx(i, j, k); } }, // Ey calculation @@ -443,18 +873,18 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; + if (ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; + if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) { return; } #endif #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ey_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDy(Pe, coefs_y, n_coefs_y, i, j, k); @@ -465,21 +895,20 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ey(i, j, k) = (enE_y - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ey(i, j, k) += eta(rho_val) * Jy(i, j, k); + if (include_resistivity_term) { Ey(i, j, k) += eta(rho_val) * Jy(i, j, k); } }, // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ - #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; + if (lz(i,j,k) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k); @@ -490,14 +919,14 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); } } ); if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index fe9aa979a7e..3aee7697073 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -43,8 +43,8 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( std::unique_ptr const& macroscopic_properties) { - // Select algorithm (The choice of algorithm is a runtime option, - // but we compile code for each algorithm, using templates) + // Select algorithm (The choice of algorithm is a runtime option, + // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ amrex::ignore_unused(Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); @@ -152,11 +152,11 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); - int const n_coefs_x = m_stencil_coefs_x.size(); + auto const n_coefs_x = static_cast(m_stencil_coefs_x.size()); Real const * const AMREX_RESTRICT coefs_y = m_stencil_coefs_y.dataPtr(); - int const n_coefs_y = m_stencil_coefs_y.size(); + auto const n_coefs_y = static_cast(m_stencil_coefs_y.size()); Real const * const AMREX_RESTRICT coefs_z = m_stencil_coefs_z.dataPtr(); - int const n_coefs_z = m_stencil_coefs_z.size(); + auto const n_coefs_z = static_cast(m_stencil_coefs_z.size()); // This functor computes Hx = Bx/mu // Note that mu is cell-centered here and will be interpolated/averaged diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index d67978f5f3e..205b009b2eb 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -27,81 +27,81 @@ * \brief This class contains the macroscopic properties of the medium needed to * evaluate macroscopic Maxwell equation. */ -class -MacroscopicProperties +class MacroscopicProperties { public: - MacroscopicProperties (); // constructor - /** Read user-defined macroscopic properties. Called in constructor. */ - void ReadParameters (); - /** Initialize multifabs storing macroscopic multifabs */ - void InitData (); - - /** return MultiFab, sigma (conductivity) of the medium. */ - amrex::MultiFab& getsigma_mf () {return (*m_sigma_mf);} - /** return MultiFab, epsilon (permittivity) of the medium. */ - amrex::MultiFab& getepsilon_mf () {return (*m_eps_mf);} - /** return MultiFab, mu (permeability) of the medium. */ - amrex::MultiFab& getmu_mf () {return (*m_mu_mf);} - - /** Initializes the Multifabs storing macroscopic properties - * with user-defined functions(x,y,z). - */ - void InitializeMacroMultiFabUsingParser (amrex::MultiFab *macro_mf, + MacroscopicProperties (); // constructor + /** Read user-defined macroscopic properties. Called in constructor. */ + void ReadParameters (); + /** Initialize multifabs storing macroscopic multifabs */ + void InitData (); + + /** return MultiFab, sigma (conductivity) of the medium. */ + amrex::MultiFab& getsigma_mf () {return (*m_sigma_mf);} + /** return MultiFab, epsilon (permittivity) of the medium. */ + amrex::MultiFab& getepsilon_mf () {return (*m_eps_mf);} + /** return MultiFab, mu (permeability) of the medium. */ + amrex::MultiFab& getmu_mf () {return (*m_mu_mf);} + + /** Initializes the Multifabs storing macroscopic properties + * with user-defined functions(x,y,z). + */ + void InitializeMacroMultiFabUsingParser (amrex::MultiFab *macro_mf, amrex::ParserExecutor<3> const& macro_parser, - int lev); - - /** Gpu Vector with index type of the conductivity multifab */ - amrex::GpuArray sigma_IndexType; - /** Gpu Vector with index type of the permittivity multifab */ - amrex::GpuArray epsilon_IndexType; - /** Gpu Vector with index type of the permeability multifab */ - amrex::GpuArray mu_IndexType; - /** Gpu Vector with index type of the Ex multifab */ - amrex::GpuArray Ex_IndexType; - /** Gpu Vector with index type of the Ey multifab */ - amrex::GpuArray Ey_IndexType; - /** Gpu Vector with index type of the Ez multifab */ - amrex::GpuArray Ez_IndexType; - /** Gpu Vector with index type of coarsening ratio with default value (1,1,1) */ - amrex::GpuArray macro_cr_ratio; + const amrex::GpuArray& dx_lev, + const amrex::RealBox& prob_domain_lev); + + /** Gpu Vector with index type of the conductivity multifab */ + amrex::GpuArray sigma_IndexType; + /** Gpu Vector with index type of the permittivity multifab */ + amrex::GpuArray epsilon_IndexType; + /** Gpu Vector with index type of the permeability multifab */ + amrex::GpuArray mu_IndexType; + /** Gpu Vector with index type of the Ex multifab */ + amrex::GpuArray Ex_IndexType; + /** Gpu Vector with index type of the Ey multifab */ + amrex::GpuArray Ey_IndexType; + /** Gpu Vector with index type of the Ez multifab */ + amrex::GpuArray Ez_IndexType; + /** Gpu Vector with index type of coarsening ratio with default value (1,1,1) */ + amrex::GpuArray macro_cr_ratio; private: - /** Conductivity, sigma, of the medium */ - amrex::Real m_sigma = 0.0; - /** Permittivity, epsilon, of the medium */ - amrex::Real m_epsilon = PhysConst::ep0; - /** Permeability, mu, of the medium */ - amrex::Real m_mu = PhysConst::mu0; - /** Multifab for m_sigma */ - std::unique_ptr m_sigma_mf; - /** Multifab for m_epsilon */ - std::unique_ptr m_eps_mf; - /** Multifab for m_mu */ - std::unique_ptr m_mu_mf; - - /** Stores initialization type for conductivity : constant or parser */ - std::string m_sigma_s = "constant"; - /** Stores initialization type for permittivity : constant or parser */ - std::string m_epsilon_s = "constant"; - /** Stores initialization type for permeability : constant or parser */ - std::string m_mu_s = "constant"; - - /** string for storing parser function */ - std::string m_str_sigma_function; - std::string m_str_epsilon_function; - std::string m_str_mu_function; - /** Parser Wrappers */ - std::unique_ptr m_sigma_parser; - std::unique_ptr m_epsilon_parser; - std::unique_ptr m_mu_parser; + /** Conductivity, sigma, of the medium */ + amrex::Real m_sigma = 0.0; + /** Permittivity, epsilon, of the medium */ + amrex::Real m_epsilon = PhysConst::ep0; + /** Permeability, mu, of the medium */ + amrex::Real m_mu = PhysConst::mu0; + /** Multifab for m_sigma */ + std::unique_ptr m_sigma_mf; + /** Multifab for m_epsilon */ + std::unique_ptr m_eps_mf; + /** Multifab for m_mu */ + std::unique_ptr m_mu_mf; + + /** Stores initialization type for conductivity : constant or parser */ + std::string m_sigma_s = "constant"; + /** Stores initialization type for permittivity : constant or parser */ + std::string m_epsilon_s = "constant"; + /** Stores initialization type for permeability : constant or parser */ + std::string m_mu_s = "constant"; + + /** string for storing parser function */ + std::string m_str_sigma_function; + std::string m_str_epsilon_function; + std::string m_str_mu_function; + /** Parser Wrappers */ + std::unique_ptr m_sigma_parser; + std::unique_ptr m_epsilon_parser; + std::unique_ptr m_mu_parser; }; /** * \brief - * This struct contains only static functions to compute the co-efficients for the + * This struct contains only static functions to compute the coefficients for the * Lax-Wendroff scheme of macroscopic Maxwells equations using * macroscopic properties, namely, conductivity (sigma), permittivity (epsilon). * Permeability of the material, mu, is used as (beta/mu) for the E-update @@ -109,31 +109,31 @@ private: */ struct LaxWendroffAlgo { - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - static amrex::Real alpha (amrex::Real const sigma, - amrex::Real const epsilon, - amrex::Real dt) { - using namespace amrex; - const amrex::Real fac1 = 0.5_rt * sigma * dt / epsilon; - const amrex::Real alpha = (1._rt - fac1)/(1._rt + fac1); - return alpha; - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - static amrex::Real beta (amrex::Real const sigma, + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real alpha (amrex::Real const sigma, amrex::Real const epsilon, amrex::Real dt) { - using namespace amrex; - const amrex::Real fac1 = 0.5_rt * sigma * dt / epsilon; - const amrex::Real beta = dt / ( epsilon * (1._rt + fac1) ); - return beta; - } + using namespace amrex; + const amrex::Real fac1 = 0.5_rt * sigma * dt / epsilon; + const amrex::Real alpha = (1._rt - fac1)/(1._rt + fac1); + return alpha; + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real beta (amrex::Real const sigma, + amrex::Real const epsilon, + amrex::Real dt) { + using namespace amrex; + const amrex::Real fac1 = 0.5_rt * sigma * dt / epsilon; + const amrex::Real beta = dt / ( epsilon * (1._rt + fac1) ); + return beta; + } }; /** * \brief - * This struct contains only static functions to compute the co-efficients for the + * This struct contains only static functions to compute the coefficients for the * BackwardEuler scheme of macroscopic Maxwells equations using * macroscopic properties, namely, conductivity (sigma) and permittivity (epsilon). * Permeability of the material, mu, is used as (beta/mu) for the E-update @@ -141,25 +141,25 @@ struct LaxWendroffAlgo { */ struct BackwardEulerAlgo { - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - static amrex::Real alpha (amrex::Real const sigma, - amrex::Real const epsilon, - amrex::Real dt) { - using namespace amrex; - const amrex::Real fac1 = sigma * dt / epsilon; - const amrex::Real alpha = (1._rt)/(1._rt + fac1); - return alpha; - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - static amrex::Real beta (amrex::Real const sigma, + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real alpha (amrex::Real const sigma, amrex::Real const epsilon, amrex::Real dt) { - using namespace amrex; - const amrex::Real fac1 = sigma * dt / epsilon; - const amrex::Real beta = dt / ( epsilon * (1._rt + fac1) ); - return beta; - } + using namespace amrex; + const amrex::Real fac1 = sigma * dt / epsilon; + const amrex::Real alpha = (1._rt)/(1._rt + fac1); + return alpha; + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real beta (amrex::Real const sigma, + amrex::Real const epsilon, + amrex::Real dt) { + using namespace amrex; + const amrex::Real fac1 = sigma * dt / epsilon; + const amrex::Real beta = dt / ( epsilon * (1._rt + fac1) ); + return beta; + } }; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 858972f2eb4..965db68a558 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -66,6 +66,7 @@ MacroscopicProperties::ReadParameters () utils::parser::makeParser(m_str_sigma_function,{"x","y","z"})); } + // Query input for material permittivity, epsilon. bool epsilon_specified = false; if (utils::parser::queryWithParser(pp_macroscopic, "epsilon", m_epsilon)) { m_epsilon_s = "constant"; @@ -91,7 +92,7 @@ MacroscopicProperties::ReadParameters () utils::parser::makeParser(m_str_epsilon_function,{"x","y","z"})); } - // Query input for material permittivity, epsilon. + // Query input for material permeability, mu. bool mu_specified = false; if (utils::parser::queryWithParser(pp_macroscopic, "mu", m_mu)) { m_mu_s = "constant"; @@ -144,7 +145,8 @@ MacroscopicProperties::InitData () } else if (m_sigma_s == "parse_sigma_function") { - InitializeMacroMultiFabUsingParser(m_sigma_mf.get(), m_sigma_parser->compile<3>(), lev); + InitializeMacroMultiFabUsingParser(m_sigma_mf.get(), m_sigma_parser->compile<3>(), + warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); } // Initialize epsilon if (m_epsilon_s == "constant") { @@ -153,7 +155,8 @@ MacroscopicProperties::InitData () } else if (m_epsilon_s == "parse_epsilon_function") { - InitializeMacroMultiFabUsingParser(m_eps_mf.get(), m_epsilon_parser->compile<3>(), lev); + InitializeMacroMultiFabUsingParser(m_eps_mf.get(), m_epsilon_parser->compile<3>(), + warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); } // In the Maxwell solver, `epsilon` is used in the denominator. @@ -169,7 +172,8 @@ MacroscopicProperties::InitData () } else if (m_mu_s == "parse_mu_function") { - InitializeMacroMultiFabUsingParser(m_mu_mf.get(), m_mu_parser->compile<3>(), lev); + InitializeMacroMultiFabUsingParser(m_mu_mf.get(), m_mu_parser->compile<3>(), + warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); } @@ -204,11 +208,9 @@ void MacroscopicProperties::InitializeMacroMultiFabUsingParser ( amrex::MultiFab *macro_mf, amrex::ParserExecutor<3> const& macro_parser, - const int lev) + const amrex::GpuArray& dx_lev, + const amrex::RealBox& prob_domain_lev) { - WarpX& warpx = WarpX::GetInstance(); - const amrex::GpuArray dx_lev = warpx.Geom(lev).CellSizeArray(); - const amrex::RealBox& real_box = warpx.Geom(lev).ProbDomain(); const amrex::IntVect iv = macro_mf->ixType().toIntVect(); for ( amrex::MFIter mfi(*macro_mf, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { // Initialize ghost cells in addition to valid cells @@ -223,20 +225,20 @@ MacroscopicProperties::InitializeMacroMultiFabUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - iv[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j * dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = j * dx_lev[0] + prob_domain_lev.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - iv[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real x = i * dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real x = i * dx_lev[0] + prob_domain_lev.lo(0) + fac_x; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - iv[1]) * dx_lev[1] * 0.5_rt; - const amrex::Real z = j * dx_lev[1] + real_box.lo(1) + fac_z; + const amrex::Real z = j * dx_lev[1] + prob_domain_lev.lo(1) + fac_z; #else const amrex::Real fac_x = (1._rt - iv[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real x = i * dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real x = i * dx_lev[0] + prob_domain_lev.lo(0) + fac_x; const amrex::Real fac_y = (1._rt - iv[1]) * dx_lev[1] * 0.5_rt; - const amrex::Real y = j * dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real y = j * dx_lev[1] + prob_domain_lev.lo(1) + fac_y; const amrex::Real fac_z = (1._rt - iv[2]) * dx_lev[2] * 0.5_rt; - const amrex::Real z = k * dx_lev[2] + real_box.lo(2) + fac_z; + const amrex::Real z = k * dx_lev[2] + prob_domain_lev.lo(2) + fac_z; #endif // initialize the macroparameter macro_fab(i,j,k) = macro_parser(x,y,z); diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 7e9c3e83969..192017656ce 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -10,7 +10,7 @@ #include "Parallelization/GuardCellManager.H" #include "Particles/MultiParticleContainer.H" #include "Particles/WarpXParticleContainer.H" -#include "Python/WarpX_py.H" +#include "Python/callbacks.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/TextMsg.H" @@ -206,7 +206,7 @@ void WarpX::setVectorPotentialBC ( amrex::Vector,3>>& A ) const { // check if any dimension has non-periodic boundary conditions - if (!m_vector_poisson_boundary_handler.has_non_periodic) return; + if (!m_vector_poisson_boundary_handler.has_non_periodic) { return; } auto dirichlet_flag = m_vector_poisson_boundary_handler.dirichlet_flag; @@ -228,7 +228,7 @@ WarpX::setVectorPotentialBC ( amrex::Vector& v_comoving, const amrex::Real dt, const bool update_with_rho) - // Members initialization + // Members initialization : SpectralBaseAlgorithm{spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type}, // Initialize the infinite-order k vectors (the argument n_order = -1 selects // the infinite order option, the argument grid_type=GridType::Staggered is then irrelevant) diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H index dea72bec396..8fde50d8172 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H @@ -62,7 +62,7 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm * * \param[in,out] f all the fields in spectral space */ - virtual void pushSpectralFields (SpectralFieldData& f) const override final; + void pushSpectralFields (SpectralFieldData& f) const final; /** * \brief Virtual function for current correction in Fourier space @@ -73,7 +73,7 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm * * \param[in,out] field_data All fields in Fourier space */ - virtual void CurrentCorrection (SpectralFieldData& field_data) override final; + void CurrentCorrection (SpectralFieldData& field_data) final; /** * \brief Virtual function for Vay current deposition in Fourier space @@ -84,7 +84,7 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm * * \param[in,out] field_data All fields in Fourier space */ - virtual void VayDeposition (SpectralFieldData& field_data) override final; + void VayDeposition (SpectralFieldData& field_data) final; private: diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp index e396efe12fb..6d4d8613940 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp @@ -184,7 +184,7 @@ PsatdAlgorithmFirstOrder::pushSpectralFields (SpectralFieldData& f) const C10 = (div_cleaning) ? 0._rt : mu0*c*kx*ky*(knorm*S-dt*c*knorm2)*inv_knorm4; C11 = (div_cleaning) ? 0._rt : mu0*c*kx*kz*(knorm*S-dt*c*knorm2)*inv_knorm4; C12 = 0._rt; // This is not redundant, do not remove this - if (J_linear) C12 = (div_cleaning) ? mu0*(C-1._rt)*inv_knorm2 : mu0*(2._rt*ky2*(C-1._rt)+2._rt*kz2*(C-1._rt)-dt2*c2*kx2*knorm2)*inv_knorm4*0.5_rt; + if (J_linear) { C12 = (div_cleaning) ? mu0*(C-1._rt)*inv_knorm2 : mu0*(2._rt*ky2*(C-1._rt)+2._rt*kz2*(C-1._rt)-dt2*c2*kx2*knorm2)*inv_knorm4*0.5_rt; } C13 = (J_linear && !div_cleaning) ? mu0*kx*ky*(2._rt*(1._rt-C)-dt2*c2*knorm2)*inv_knorm4*0.5_rt : 0._rt; C14 = (J_linear && !div_cleaning) ? mu0*kx*kz*(2._rt*(1._rt-C)-dt2*c2*knorm2)*inv_knorm4*0.5_rt : 0._rt; C15 = (div_cleaning) ? I*mu0*c2*kx*(C-1._rt)*inv_knorm2 : 0._rt; @@ -211,7 +211,7 @@ PsatdAlgorithmFirstOrder::pushSpectralFields (SpectralFieldData& f) const C11 = (div_cleaning) ? 0._rt : mu0*c*ky*kz*(knorm*S-dt*c*knorm2)*inv_knorm4; C12 = (J_linear && !div_cleaning) ? mu0*kx*ky*(2._rt*(1._rt-C)-dt2*c2*knorm2)*inv_knorm4*0.5_rt : 0._rt; C13 = 0._rt; // This is not redundant, do not remove this - if (J_linear) C13 = (div_cleaning) ? mu0*(C-1._rt)*inv_knorm2 : mu0*(2._rt*kx2*(C-1._rt)+2._rt*kz2*(C-1._rt)-dt2*c2*ky2*knorm2)*inv_knorm4*0.5_rt; + if (J_linear) { C13 = (div_cleaning) ? mu0*(C-1._rt)*inv_knorm2 : mu0*(2._rt*kx2*(C-1._rt)+2._rt*kz2*(C-1._rt)-dt2*c2*ky2*knorm2)*inv_knorm4*0.5_rt; } C14 = (J_linear && !div_cleaning) ? mu0*ky*kz*(2._rt*(1._rt-C)-dt2*c2*knorm2)*inv_knorm4*0.5_rt : 0._rt; C15 = (div_cleaning) ? I*mu0*c2*ky*(C-1._rt)*inv_knorm2 : 0._rt; C16 = (div_cleaning && rho_linear) ? I*mu0*c*ky*(knorm*S-dt*c*knorm2)*inv_knorm4 : 0._rt; @@ -238,7 +238,7 @@ PsatdAlgorithmFirstOrder::pushSpectralFields (SpectralFieldData& f) const C12 = (J_linear && !div_cleaning) ? mu0*kx*kz*(2._rt*(1._rt-C)-dt2*c2*knorm2)*inv_knorm4*0.5_rt : 0._rt; C13 = (J_linear && !div_cleaning) ? mu0*ky*kz*(2._rt*(1._rt-C)-dt2*c2*knorm2)*inv_knorm4*0.5_rt : 0._rt; C14 = 0._rt; // This is not redundant, do not remove this - if (J_linear) C14 = (div_cleaning) ? mu0*(C-1._rt)*inv_knorm2 : mu0*(2._rt*kx2*(C-1._rt)+2._rt*ky2*(C-1._rt)-dt2*c2*kz2*knorm2)*inv_knorm4*0.5_rt; + if (J_linear) { C14 = (div_cleaning) ? mu0*(C-1._rt)*inv_knorm2 : mu0*(2._rt*kx2*(C-1._rt)+2._rt*ky2*(C-1._rt)-dt2*c2*kz2*knorm2)*inv_knorm4*0.5_rt; } C15 = (div_cleaning) ? I*mu0*c2*kz*(C-1._rt)*inv_knorm2 : 0._rt; C16 = (div_cleaning && rho_linear) ? I*mu0*c*kz*(knorm*S-dt*c*knorm2)*inv_knorm4 : 0._rt; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H index 37d76373990..ad0aca1ecf5 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H @@ -25,19 +25,19 @@ class PsatdAlgorithmGalileanRZ : public SpectralBaseAlgorithmRZ amrex::Real dt_step, bool update_with_rho); // Redefine functions from base class - virtual void pushSpectralFields (SpectralFieldDataRZ & f) override final; + void pushSpectralFields (SpectralFieldDataRZ & f) final; void InitializeSpectralCoefficients (SpectralFieldDataRZ const & f); /** * \brief Virtual function for current correction in Fourier space * This function overrides the virtual function \c CurrentCorrection in the - * base class \c SpectralBaseAlgorithmRZ (qualifier \c override) and cannot be + * base class \c SpectralBaseAlgorithmRZ and cannot be * overridden by further derived classes (qualifier \c final). * * \param[in,out] field_data all fields in Fourier space */ - virtual void CurrentCorrection (SpectralFieldDataRZ& field_data) override final; + void CurrentCorrection (SpectralFieldDataRZ& field_data) final; /** * \brief Virtual function for Vay current deposition in Fourier space @@ -47,11 +47,11 @@ class PsatdAlgorithmGalileanRZ : public SpectralBaseAlgorithmRZ * * \param[in,out] field_data All fields in Fourier space */ - virtual void VayDeposition (SpectralFieldDataRZ& field_data) override final; + void VayDeposition (SpectralFieldDataRZ& field_data) final; private: - bool coefficients_initialized; + bool coefficients_initialized = false; // Note that dt and v_galilean are saved to use in InitializeSpectralCoefficients amrex::Real m_dt; amrex::Vector m_v_galilean; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp index 3188712d61a..71cee95524e 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp @@ -23,14 +23,13 @@ PsatdAlgorithmGalileanRZ::PsatdAlgorithmGalileanRZ (SpectralKSpaceRZ const & spe short const grid_type, const amrex::Vector& v_galilean, amrex::Real const dt, - bool const update_with_rho) - // Initialize members of base class - : SpectralBaseAlgorithmRZ(spectral_kspace, dm, spectral_index, norder_z, grid_type), - m_dt(dt), - m_v_galilean(v_galilean), - m_update_with_rho(update_with_rho) + bool const update_with_rho): + // Initialize members of base class + SpectralBaseAlgorithmRZ{spectral_kspace, dm, spectral_index, norder_z, grid_type}, + m_dt{dt}, + m_v_galilean{v_galilean}, + m_update_with_rho{update_with_rho} { - // Allocate the arrays of coefficients amrex::BoxArray const & ba = spectral_kspace.spectralspace_ba; C_coef = SpectralRealCoefficients(ba, dm, n_rz_azimuthal_modes, 0); @@ -41,8 +40,6 @@ PsatdAlgorithmGalileanRZ::PsatdAlgorithmGalileanRZ (SpectralKSpaceRZ const & spe X4_coef = SpectralComplexCoefficients(ba, dm, n_rz_azimuthal_modes, 0); Theta2_coef = SpectralComplexCoefficients(ba, dm, n_rz_azimuthal_modes, 0); T_rho_coef = SpectralComplexCoefficients(ba, dm, n_rz_azimuthal_modes, 0); - - coefficients_initialized = false; } /* Advance the E and B field in spectral space (stored in `f`) diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H index ed293fd1efa..94f00d6d4eb 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H @@ -65,7 +65,7 @@ class PsatdAlgorithmJConstantInTime : public SpectralBaseAlgorithm * * \param[in,out] f all the fields in spectral space */ - virtual void pushSpectralFields (SpectralFieldData& f) const override final; + void pushSpectralFields (SpectralFieldData& f) const final; /** * \brief Initializes the coefficients used in \c pushSpectralFields to update the E and B fields @@ -101,7 +101,7 @@ class PsatdAlgorithmJConstantInTime : public SpectralBaseAlgorithm * * \param[in,out] field_data All fields in Fourier space */ - virtual void CurrentCorrection (SpectralFieldData& field_data) override final; + void CurrentCorrection (SpectralFieldData& field_data) final; /** * \brief Virtual function for Vay current deposition in Fourier space @@ -112,7 +112,7 @@ class PsatdAlgorithmJConstantInTime : public SpectralBaseAlgorithm * * \param[in,out] field_data All fields in Fourier space */ - virtual void VayDeposition (SpectralFieldData& field_data) override final; + void VayDeposition (SpectralFieldData& field_data) final; private: diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp index 11d9bbd94d3..167d7565bc0 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp @@ -60,12 +60,12 @@ PsatdAlgorithmJConstantInTime::PsatdAlgorithmJConstantInTime( m_update_with_rho(update_with_rho), m_time_averaging(time_averaging), m_dive_cleaning(dive_cleaning), - m_divb_cleaning(divb_cleaning) + m_divb_cleaning(divb_cleaning), + m_is_galilean{ + (v_galilean[0] != 0.) || (v_galilean[1] != 0.) || (v_galilean[2] != 0.)} { const amrex::BoxArray& ba = spectral_kspace.spectralspace_ba; - m_is_galilean = (v_galilean[0] != 0.) || (v_galilean[1] != 0.) || (v_galilean[2] != 0.); - // Always allocate these coefficients C_coef = SpectralRealCoefficients(ba, dm, 1, 0); S_ck_coef = SpectralRealCoefficients(ba, dm, 1, 0); @@ -845,18 +845,18 @@ PsatdAlgorithmJConstantInTime::VayDeposition (SpectralFieldData& field_data) #endif // Compute Jx - if (kx_mod != 0._rt) fields(i,j,k,Idx.Jx_mid) = I * Dx / kx_mod; - else fields(i,j,k,Idx.Jx_mid) = 0._rt; + if (kx_mod != 0._rt) { fields(i,j,k,Idx.Jx_mid) = I * Dx / kx_mod; } + else { fields(i,j,k,Idx.Jx_mid) = 0._rt; } #if defined(WARPX_DIM_3D) // Compute Jy - if (ky_mod != 0._rt) fields(i,j,k,Idx.Jy_mid) = I * Dy / ky_mod; - else fields(i,j,k,Idx.Jy_mid) = 0._rt; + if (ky_mod != 0._rt) { fields(i,j,k,Idx.Jy_mid) = I * Dy / ky_mod; } + else { fields(i,j,k,Idx.Jy_mid) = 0._rt; } #endif // Compute Jz - if (kz_mod != 0._rt) fields(i,j,k,Idx.Jz_mid) = I * Dz / kz_mod; - else fields(i,j,k,Idx.Jz_mid) = 0._rt; + if (kz_mod != 0._rt) { fields(i,j,k,Idx.Jz_mid) = I * Dz / kz_mod; } + else { fields(i,j,k,Idx.Jz_mid) = 0._rt; } }); } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H index 94eda8a2f72..aa3bcc08e77 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H @@ -64,7 +64,7 @@ class PsatdAlgorithmJLinearInTime : public SpectralBaseAlgorithm * * \param[in,out] f all the fields in spectral space */ - virtual void pushSpectralFields (SpectralFieldData& f) const override final; + void pushSpectralFields (SpectralFieldData& f) const final; /** * \brief Initializes the coefficients used in \c pushSpectralFields to update the E and B fields @@ -100,7 +100,7 @@ class PsatdAlgorithmJLinearInTime : public SpectralBaseAlgorithm * * \param[in,out] field_data All fields in Fourier space */ - virtual void CurrentCorrection (SpectralFieldData& field_data) override final; + void CurrentCorrection (SpectralFieldData& field_data) final; /** * \brief Virtual function for Vay current deposition in Fourier space @@ -111,7 +111,7 @@ class PsatdAlgorithmJLinearInTime : public SpectralBaseAlgorithm * * \param[in,out] field_data All fields in Fourier space */ - virtual void VayDeposition (SpectralFieldData& field_data) override final; + void VayDeposition (SpectralFieldData& field_data) final; private: diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp index 30243e63058..92e35c8c03b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp @@ -130,8 +130,8 @@ PsatdAlgorithmJLinearInTime::pushSpectralFields (SpectralFieldData& f) const const Complex rho_new = fields(i,j,k,Idx.rho_new); Complex F_old, G_old; - if (dive_cleaning) F_old = fields(i,j,k,Idx.F); - if (divb_cleaning) G_old = fields(i,j,k,Idx.G); + if (dive_cleaning) { F_old = fields(i,j,k,Idx.F); } + if (divb_cleaning) { G_old = fields(i,j,k,Idx.G); } // k vector values const amrex::Real kx = modified_kx_arr[i]; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H index 7f50d1cd8ea..1c25ca91903 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H @@ -21,28 +21,60 @@ #if WARPX_USE_PSATD -/* \brief Class that updates the field in spectral space +/* + * \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ class PsatdAlgorithmPml : public SpectralBaseAlgorithm { public: - PsatdAlgorithmPml(const SpectralKSpace& spectral_kspace, - const amrex::DistributionMapping& dm, - const SpectralFieldIndex& spectral_index, - int norder_x, int norder_y, - int norder_z, short grid_type, - amrex::Real dt, - bool dive_cleaning, - bool divb_cleaning); - void InitializeSpectralCoefficients( + /** + * \brief Constructor of the class PsatdAlgorithmPml + * + * \param[in] spectral_kspace Spectral space + * \param[in] dm Distribution mapping + * \param[in] spectral_index Object containing indices to access data in spectral space + * \param[in] norder_x Order of the spectral solver along x + * \param[in] norder_y Order of the spectral solver along y + * \param[in] norder_z Order of the spectral solver along z + * \param[in] grid_type Type of grid (collocated or not) + * \param[in] v_galilean Galilean velocity + * \param[in] dt Time step of the simulation + * \param[in] dive_cleaning Whether to use divergence correction for E (F term) + * \param[in] divb_cleaning Whether to use divergence correction for B (G term) + */ + PsatdAlgorithmPml( const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, - amrex::Real dt); + const SpectralFieldIndex& spectral_index, + int norder_x, + int norder_y, + int norder_z, + short grid_type, + const amrex::Vector& v_galilean, + amrex::Real dt, + bool dive_cleaning, + bool divb_cleaning); + + /** + * \brief Initializes the coefficients used in \c pushSpectralFields + * to update the E and B fields + * + * \param[in] spectral_kspace Spectral space + * \param[in] dm Distribution mapping + */ + void InitializeSpectralCoefficients( + const SpectralKSpace& spectral_kspace, + const amrex::DistributionMapping& dm); - // Redefine functions from base class - virtual void pushSpectralFields(SpectralFieldData& f) const override final; + /** + * \brief Updates the E and B fields in spectral space, + * according to the relevant PSATD equations + * + * \param[in,out] f All fields in spectral space + */ + void pushSpectralFields(SpectralFieldData& f) const final; /** * \brief Virtual function for current correction in Fourier space @@ -53,7 +85,7 @@ class PsatdAlgorithmPml : public SpectralBaseAlgorithm * * \param[in,out] field_data All fields in Fourier space */ - virtual void CurrentCorrection (SpectralFieldData& field_data) override final; + void CurrentCorrection (SpectralFieldData& field_data) final; /** * \brief Virtual function for Vay current deposition in Fourier space @@ -64,13 +96,23 @@ class PsatdAlgorithmPml : public SpectralBaseAlgorithm * * \param[in,out] field_data All fields in Fourier space */ - virtual void VayDeposition (SpectralFieldData& field_data) override final; + void VayDeposition (SpectralFieldData& field_data) final; private: + SpectralRealCoefficients C_coef, S_ck_coef, inv_k2_coef; + SpectralComplexCoefficients T2_coef; + // Centered modified finite-order k vectors + KVectorComponent modified_kx_vec_centered; +#if defined(WARPX_DIM_3D) + KVectorComponent modified_ky_vec_centered; +#endif + KVectorComponent modified_kz_vec_centered; + amrex::Vector m_v_galilean; amrex::Real m_dt; bool m_dive_cleaning; bool m_divb_cleaning; + bool m_is_galilean; }; #endif // WARPX_USE_PSATD diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp index 1a4039915e0..92b8b3e900d 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp @@ -9,6 +9,7 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData.H" #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpX_Complex.H" @@ -29,59 +30,84 @@ using namespace amrex; -/* \brief Initialize coefficients for the update equation */ -PsatdAlgorithmPml::PsatdAlgorithmPml(const SpectralKSpace& spectral_kspace, - const DistributionMapping& dm, - const SpectralFieldIndex& spectral_index, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type, - const Real dt, - const bool dive_cleaning, const bool divb_cleaning) - // Initialize members of base class - : SpectralBaseAlgorithm(spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type), - m_dt(dt), - m_dive_cleaning(dive_cleaning), - m_divb_cleaning(divb_cleaning) +PsatdAlgorithmPml::PsatdAlgorithmPml( + const SpectralKSpace& spectral_kspace, + const DistributionMapping& dm, + const SpectralFieldIndex& spectral_index, + int norder_x, + int norder_y, + int norder_z, + short grid_type, + const amrex::Vector& v_galilean, + Real dt, + bool dive_cleaning, + bool divb_cleaning) : + SpectralBaseAlgorithm(spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type), + // Initialize the centered finite-order modified k vectors: + // these are computed always with the assumption of centered grids + // (argument grid_type=GridType::Collocated), for both collocated and staggered grids + modified_kx_vec_centered(spectral_kspace.getModifiedKComponent(dm, 0, norder_x, GridType::Collocated)), +#if defined(WARPX_DIM_3D) + modified_ky_vec_centered(spectral_kspace.getModifiedKComponent(dm, 1, norder_y, GridType::Collocated)), + modified_kz_vec_centered(spectral_kspace.getModifiedKComponent(dm, 2, norder_z, GridType::Collocated)), +#else + modified_kz_vec_centered(spectral_kspace.getModifiedKComponent(dm, 1, norder_z, GridType::Collocated)), +#endif + m_v_galilean(v_galilean), + m_dt(dt), + m_dive_cleaning(dive_cleaning), + m_divb_cleaning(divb_cleaning), + m_is_galilean{(v_galilean[0] != 0.) || (v_galilean[1] != 0.) || (v_galilean[2] != 0.)} { const BoxArray& ba = spectral_kspace.spectralspace_ba; - // Allocate the arrays of coefficients - C_coef = SpectralRealCoefficients(ba, dm, 1, 0); - S_ck_coef = SpectralRealCoefficients(ba, dm, 1, 0); + // Allocate arrays of coefficients + C_coef = SpectralRealCoefficients(ba, dm, 1, 0); + S_ck_coef = SpectralRealCoefficients(ba, dm, 1, 0); inv_k2_coef = SpectralRealCoefficients(ba, dm, 1, 0); - InitializeSpectralCoefficients(spectral_kspace, dm, dt); -} + // Allocate this coefficient only with Galilean PSATD + if (m_is_galilean) + { + T2_coef = SpectralComplexCoefficients(ba, dm, 1, 0); + } -/* Advance the E and B field in spectral space (stored in `f`) - * over one time step */ -void -PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { + InitializeSpectralCoefficients(spectral_kspace, dm); +} +void PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const +{ const bool dive_cleaning = m_dive_cleaning; const bool divb_cleaning = m_divb_cleaning; + const bool is_galilean = m_is_galilean; const SpectralFieldIndex& Idx = m_spectral_index; // Loop over boxes - for (MFIter mfi(f.fields); mfi.isValid(); ++mfi){ - - const Box& bx = f.fields[mfi].box(); + for (amrex::MFIter mfi(f.fields); mfi.isValid(); ++mfi) + { + const amrex::Box& bx = f.fields[mfi].box(); // Extract arrays for the fields to be updated - const Array4 fields = f.fields[mfi].array(); + const amrex::Array4 fields = f.fields[mfi].array(); // Extract arrays for the coefficients - const Array4 C_arr = C_coef[mfi].array(); - const Array4 S_ck_arr = S_ck_coef[mfi].array(); - const Array4 inv_k2_arr = inv_k2_coef[mfi].array(); + const amrex::Array4 C_arr = C_coef[mfi].array(); + const amrex::Array4 S_ck_arr = S_ck_coef[mfi].array(); + const amrex::Array4 inv_k2_arr = inv_k2_coef[mfi].array(); + + amrex::Array4 T2_arr; + if (is_galilean) + { + T2_arr = T2_coef[mfi].array(); + } // Extract pointers for the k vectors - const Real* modified_kx_arr = modified_kx_vec[mfi].dataPtr(); + const amrex::Real* modified_kx_arr = modified_kx_vec[mfi].dataPtr(); #if defined(WARPX_DIM_3D) - const Real* modified_ky_arr = modified_ky_vec[mfi].dataPtr(); + const amrex::Real* modified_ky_arr = modified_ky_vec[mfi].dataPtr(); #endif - const Real* modified_kz_arr = modified_kz_vec[mfi].dataPtr(); + const amrex::Real* modified_kz_arr = modified_kz_vec[mfi].dataPtr(); const amrex::Real dt = m_dt; @@ -154,28 +180,29 @@ PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { } // k vector values, and coefficients - const Real kx = modified_kx_arr[i]; + const amrex::Real kx = modified_kx_arr[i]; #if defined(WARPX_DIM_3D) - const Real ky = modified_ky_arr[j]; - const Real kz = modified_kz_arr[k]; + const amrex::Real ky = modified_ky_arr[j]; + const amrex::Real kz = modified_kz_arr[k]; #else - constexpr Real ky = 0._rt; - const Real kz = modified_kz_arr[j]; + constexpr amrex::Real ky = 0._rt; + const amrex::Real kz = modified_kz_arr[j]; #endif - constexpr Real c2 = PhysConst::c * PhysConst::c; + constexpr amrex::Real c2 = PhysConst::c*PhysConst::c; const Complex I = Complex{0._rt, 1._rt}; - const Real kx2 = kx*kx; - const Real ky2 = ky*ky; - const Real kz2 = kz*kz; - const Real k_norm = std::sqrt(kx2 + ky2 + kz2); - - if (k_norm != 0._rt) { + const amrex::Real kx2 = kx*kx; + const amrex::Real ky2 = ky*ky; + const amrex::Real kz2 = kz*kz; + const amrex::Real knorm = std::sqrt(kx2 + ky2 + kz2); + if (knorm != 0._rt) + { const amrex::Real C = C_arr(i,j,k); const amrex::Real S_ck = S_ck_arr(i,j,k); const amrex::Real inv_k2 = inv_k2_arr(i,j,k); + const Complex T2 = (is_galilean) ? T2_arr(i,j,k) : 1.0_rt; const amrex::Real C1 = (kx2 * C + ky2 + kz2) * inv_k2; const amrex::Real C2 = (kx2 + ky2 * C + kz2) * inv_k2; @@ -218,42 +245,42 @@ PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { const Complex C22_c2 = C22 / c2; // Update E - fields(i,j,k,Idx.Exy) = C2 * Exy + C5 * Exz + C9 * Ey - + C10 * Bx + C11 * By + C19 * Bz; + fields(i,j,k,Idx.Exy) = T2 * (C2 * Exy + C5 * Exz + C9 * Ey + + C10 * Bx + C11 * By + C19 * Bz); - fields(i,j,k,Idx.Exz) = C6 * Exy + C3 * Exz + C8 * Ez - - C10 * Bx - C22 * By - C12 * Bz; + fields(i,j,k,Idx.Exz) = T2 * (C6 * Exy + C3 * Exz + C8 * Ez + - C10 * Bx - C22 * By - C12 * Bz); - fields(i,j,k,Idx.Eyz) = C3 * Eyz + C6 * Eyx + C7 * Ez - + C21 * Bx + C10 * By + C13 * Bz; + fields(i,j,k,Idx.Eyz) = T2 * (C3 * Eyz + C6 * Eyx + C7 * Ez + + C21 * Bx + C10 * By + C13 * Bz); - fields(i,j,k,Idx.Eyx) = C9 * Ex + C4 * Eyz + C1 * Eyx - - C14 * Bx - C10 * By - C18 * Bz; + fields(i,j,k,Idx.Eyx) = T2 * (C9 * Ex + C4 * Eyz + C1 * Eyx + - C14 * Bx - C10 * By - C18 * Bz); - fields(i,j,k,Idx.Ezx) = C8 * Ex + C1 * Ezx + C4 * Ezy - + C15 * Bx + C17 * By + C10 * Bz; + fields(i,j,k,Idx.Ezx) = T2 * (C8 * Ex + C1 * Ezx + C4 * Ezy + + C15 * Bx + C17 * By + C10 * Bz); - fields(i,j,k,Idx.Ezy) = C7 * Ey + C5 * Ezx + C2 * Ezy - - C20 * Bx - C16 * By - C10 * Bz; + fields(i,j,k,Idx.Ezy) = T2 * (C7 * Ey + C5 * Ezx + C2 * Ezy + - C20 * Bx - C16 * By - C10 * Bz); // Update B - fields(i,j,k,Idx.Bxy) = C2 * Bxy + C5 * Bxz + C9 * By - - C10_c2 * Ex - C11_c2 * Ey - C19_c2 * Ez; + fields(i,j,k,Idx.Bxy) = T2 * (C2 * Bxy + C5 * Bxz + C9 * By + - C10_c2 * Ex - C11_c2 * Ey - C19_c2 * Ez); - fields(i,j,k,Idx.Bxz) = C6 * Bxy + C3 * Bxz + C8 * Bz - + C10_c2 * Ex + C22_c2 * Ey + C12_c2 * Ez; + fields(i,j,k,Idx.Bxz) = T2 * (C6 * Bxy + C3 * Bxz + C8 * Bz + + C10_c2 * Ex + C22_c2 * Ey + C12_c2 * Ez); - fields(i,j,k,Idx.Byz) = C3 * Byz + C6 * Byx + C7 * Bz - - C21_c2 * Ex - C10_c2 * Ey - C13_c2 * Ez; + fields(i,j,k,Idx.Byz) = T2 * (C3 * Byz + C6 * Byx + C7 * Bz + - C21_c2 * Ex - C10_c2 * Ey - C13_c2 * Ez); - fields(i,j,k,Idx.Byx) = C9 * Bx + C4 * Byz + C1 * Byx - + C14_c2 * Ex + C10_c2 * Ey + C18_c2 * Ez; + fields(i,j,k,Idx.Byx) = T2 * (C9 * Bx + C4 * Byz + C1 * Byx + + C14_c2 * Ex + C10_c2 * Ey + C18_c2 * Ez); - fields(i,j,k,Idx.Bzx) = C8 * Bx + C1 * Bzx + C4 * Bzy - - C15_c2 * Ex - C17_c2 * Ey - C10_c2 * Ez; + fields(i,j,k,Idx.Bzx) = T2 * (C8 * Bx + C1 * Bzx + C4 * Bzy + - C15_c2 * Ex - C17_c2 * Ey - C10_c2 * Ez); - fields(i,j,k,Idx.Bzy) = C7 * By + C5 * Bzx + C2 * Bzy - + C20_c2 * Ex + C16_c2 * Ey + C10_c2 * Ez; + fields(i,j,k,Idx.Bzy) = T2 * (C7 * By + C5 * Bzx + C2 * Bzy + + C20_c2 * Ex + C16_c2 * Ey + C10_c2 * Ez); } else if (dive_cleaning && divb_cleaning) { @@ -266,80 +293,80 @@ PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { const Complex C25_c2 = C25 / c2; // Update E - fields(i,j,k,Idx.Exx) = C1 * Exx + C4 * Exy + C4 * Exz - - C9 * Ey - C8 * Ez + C23 * F; + fields(i,j,k,Idx.Exx) = T2 * (C1 * Exx + C4 * Exy + C4 * Exz + - C9 * Ey - C8 * Ez + C23 * F); - fields(i,j,k,Idx.Exy) = C5 * Exx + C2 * Exy + C5 * Exz - + C9 * Ey + C24 * Bz - C7 * G; + fields(i,j,k,Idx.Exy) = T2 * (C5 * Exx + C2 * Exy + C5 * Exz + + C9 * Ey + C24 * Bz - C7 * G); - fields(i,j,k,Idx.Exz) = C6 * Exx + C6 * Exy + C3 * Exz - + C8 * Ez - C25 * By + C7 * G; + fields(i,j,k,Idx.Exz) = T2 * (C6 * Exx + C6 * Exy + C3 * Exz + + C8 * Ez - C25 * By + C7 * G); - fields(i,j,k,Idx.Eyx) = C9 * Ex + C1 * Eyx + C4 * Eyy - + C4 * Eyz - C23 * Bz + C8 * G; + fields(i,j,k,Idx.Eyx) = T2 * (C9 * Ex + C1 * Eyx + C4 * Eyy + + C4 * Eyz - C23 * Bz + C8 * G); - fields(i,j,k,Idx.Eyy) = - C9 * Ex + C5 * Eyx + C2 * Eyy - + C5 * Eyz - C7 * Ez + C24 * F; + fields(i,j,k,Idx.Eyy) = T2 * (- C9 * Ex + C5 * Eyx + C2 * Eyy + + C5 * Eyz - C7 * Ez + C24 * F); - fields(i,j,k,Idx.Eyz) = C6 * Eyx + C6 * Eyy + C3 * Eyz - + C7 * Ez + C25 * Bx - C8 * G; + fields(i,j,k,Idx.Eyz) = T2 * (C6 * Eyx + C6 * Eyy + C3 * Eyz + + C7 * Ez + C25 * Bx - C8 * G); - fields(i,j,k,Idx.Ezx) = C8 * Ex + C1 * Ezx + C4 * Ezy - + C4 * Ezz + C23 * By - C9 * G; + fields(i,j,k,Idx.Ezx) = T2 * (C8 * Ex + C1 * Ezx + C4 * Ezy + + C4 * Ezz + C23 * By - C9 * G); - fields(i,j,k,Idx.Ezy) = C7 * Ey + C5 * Ezx + C2 * Ezy - + C5 * Ezz - C24 * Bx + C9 * G; + fields(i,j,k,Idx.Ezy) = T2 * (C7 * Ey + C5 * Ezx + C2 * Ezy + + C5 * Ezz - C24 * Bx + C9 * G); - fields(i,j,k,Idx.Ezz) = - C8 * Ex - C7 * Ey + C6 * Ezx - + C6 * Ezy + C3 * Ezz + C25 * F; + fields(i,j,k,Idx.Ezz) = T2 * (- C8 * Ex - C7 * Ey + C6 * Ezx + + C6 * Ezy + C3 * Ezz + C25 * F); // Update B - fields(i,j,k,Idx.Bxx) = C1 * Bxx + C4 * Bxy + C4 * Bxz - - C9 * By - C8 * Bz + C23_c2 * G; + fields(i,j,k,Idx.Bxx) = T2 * (C1 * Bxx + C4 * Bxy + C4 * Bxz + - C9 * By - C8 * Bz + C23_c2 * G); - fields(i,j,k,Idx.Bxy) = - C24_c2 * Ez + C5 * Bxx + C2 * Bxy - + C5 * Bxz + C9 * By + C7 * F; + fields(i,j,k,Idx.Bxy) = T2 * (- C24_c2 * Ez + C5 * Bxx + C2 * Bxy + + C5 * Bxz + C9 * By + C7 * F); - fields(i,j,k,Idx.Bxz) = C25_c2 * Ey + C6 * Bxx + C6 * Bxy - + C3 * Bxz + C8 * Bz - C7 * F; + fields(i,j,k,Idx.Bxz) = T2 * (C25_c2 * Ey + C6 * Bxx + C6 * Bxy + + C3 * Bxz + C8 * Bz - C7 * F); - fields(i,j,k,Idx.Byx) = C23_c2 * Ez + C9 * Bx + C1 * Byx - + C4 * Byy + C4 * Byz - C8 * F; + fields(i,j,k,Idx.Byx) = T2 * (C23_c2 * Ez + C9 * Bx + C1 * Byx + + C4 * Byy + C4 * Byz - C8 * F); - fields(i,j,k,Idx.Byy) = - C9 * Bx + C5 * Byx + C2 * Byy - + C5 * Byz - C7 * Bz + C24_c2 * G; + fields(i,j,k,Idx.Byy) = T2 * (- C9 * Bx + C5 * Byx + C2 * Byy + + C5 * Byz - C7 * Bz + C24_c2 * G); - fields(i,j,k,Idx.Byz) = - C25_c2 * Ex + C6 * Byx + C6 * Byy - + C3 * Byz + C7 * Bz + C8 * F; + fields(i,j,k,Idx.Byz) = T2 * (- C25_c2 * Ex + C6 * Byx + C6 * Byy + + C3 * Byz + C7 * Bz + C8 * F); - fields(i,j,k,Idx.Bzx) = - C23_c2 * Ey + C8 * Bx + C1 * Bzx - + C4 * Bzy + C4 * Bzz + C9 * F; + fields(i,j,k,Idx.Bzx) = T2 * (- C23_c2 * Ey + C8 * Bx + C1 * Bzx + + C4 * Bzy + C4 * Bzz + C9 * F); - fields(i,j,k,Idx.Bzy) = C24_c2 * Ex + C7 * By + C5 * Bzx - + C2 * Bzy + C5 * Bzz - C9 * F; + fields(i,j,k,Idx.Bzy) = T2 * (C24_c2 * Ex + C7 * By + C5 * Bzx + + C2 * Bzy + C5 * Bzz - C9 * F); - fields(i,j,k,Idx.Bzz) = - C8 * Bx - C7 * By + C6 * Bzx - + C6 * Bzy + C3 * Bzz + C25_c2 * G; + fields(i,j,k,Idx.Bzz) = T2 * (- C8 * Bx - C7 * By + C6 * Bzx + + C6 * Bzy + C3 * Bzz + C25_c2 * G); // Update F - fields(i,j,k,Idx.Fx) = C23_c2 * Ex + C8 * By - C9 * Bz - + C1 * Fx + C4 * Fy + C4 * Fz; + fields(i,j,k,Idx.Fx) = T2 * (C23_c2 * Ex + C8 * By - C9 * Bz + + C1 * Fx + C4 * Fy + C4 * Fz); - fields(i,j,k,Idx.Fy) = C24_c2 * Ey - C7 * Bx + C9 * Bz - + C5 * Fx + C2 * Fy + C5 * Fz; + fields(i,j,k,Idx.Fy) = T2 * (C24_c2 * Ey - C7 * Bx + C9 * Bz + + C5 * Fx + C2 * Fy + C5 * Fz); - fields(i,j,k,Idx.Fz) = C25_c2 * Ez + C7 * Bx - C8 * By - + C6 * Fx + C6 * Fy + C3 * Fz; + fields(i,j,k,Idx.Fz) = T2 * (C25_c2 * Ez + C7 * Bx - C8 * By + + C6 * Fx + C6 * Fy + C3 * Fz); // Update G - fields(i,j,k,Idx.Gx) = - C8 * Ey + C9 * Ez + C23 * Bx - + C1 * Gx + C4 * Gy + C4 * Gz; + fields(i,j,k,Idx.Gx) = T2 * (- C8 * Ey + C9 * Ez + C23 * Bx + + C1 * Gx + C4 * Gy + C4 * Gz); - fields(i,j,k,Idx.Gy) = C7 * Ex - C9 * Ez + C24 * By - + C5 * Gx + C2 * Gy + C5 * Gz; + fields(i,j,k,Idx.Gy) = T2 * (C7 * Ex - C9 * Ez + C24 * By + + C5 * Gx + C2 * Gy + C5 * Gz); - fields(i,j,k,Idx.Gz) = - C7 * Ex + C8 * Ey + C25 * Bz - + C6 * Gx + C6 * Gy + C3 * Gz; + fields(i,j,k,Idx.Gz) = T2 * (- C7 * Ex + C8 * Ey + C25 * Bz + + C6 * Gx + C6 * Gy + C3 * Gz); } } }); @@ -348,71 +375,95 @@ PsatdAlgorithmPml::pushSpectralFields(SpectralFieldData& f) const { void PsatdAlgorithmPml::InitializeSpectralCoefficients ( const SpectralKSpace& spectral_kspace, - const amrex::DistributionMapping& dm, - const amrex::Real dt) + const amrex::DistributionMapping& dm) { - const BoxArray& ba = spectral_kspace.spectralspace_ba; + const amrex::Real dt = m_dt; + const bool is_galilean = m_is_galilean; - // Fill them with the right values: - // Loop over boxes and allocate the corresponding coefficients - // for each box owned by the local MPI proc - for (MFIter mfi(ba, dm); mfi.isValid(); ++mfi) { + const amrex::BoxArray& ba = spectral_kspace.spectralspace_ba; - const Box& bx = ba[mfi]; + // Loop over boxes and allocate the corresponding coefficients + // for each box owned by the local MPI process + for (amrex::MFIter mfi(ba, dm); mfi.isValid(); ++mfi) + { + const amrex::Box& bx = ba[mfi]; // Extract pointers for the k vectors - const Real* modified_kx = modified_kx_vec[mfi].dataPtr(); + const amrex::Real* kx = modified_kx_vec[mfi].dataPtr(); + const amrex::Real* kx_c = modified_kx_vec_centered[mfi].dataPtr(); #if defined(WARPX_DIM_3D) - const Real* modified_ky = modified_ky_vec[mfi].dataPtr(); + const amrex::Real* ky = modified_ky_vec[mfi].dataPtr(); + const amrex::Real* ky_c = modified_ky_vec_centered[mfi].dataPtr(); #endif - const Real* modified_kz = modified_kz_vec[mfi].dataPtr(); + const amrex::Real* kz = modified_kz_vec[mfi].dataPtr(); + const amrex::Real* kz_c = modified_kz_vec_centered[mfi].dataPtr(); // Extract arrays for the coefficients - const Array4 C = C_coef[mfi].array(); - const Array4 S_ck = S_ck_coef[mfi].array(); - const Array4 inv_k2 = inv_k2_coef[mfi].array(); + const amrex::Array4 C = C_coef[mfi].array(); + const amrex::Array4 S_ck = S_ck_coef[mfi].array(); + const amrex::Array4 inv_k2 = inv_k2_coef[mfi].array(); - // Loop over indices within one box - ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + amrex::Array4 T2; + if (is_galilean) { - const Real kx = modified_kx[i]; + T2 = T2_coef[mfi].array(); + } + + // Extract Galilean velocity + amrex::Real vg_x = m_v_galilean[0]; #if defined(WARPX_DIM_3D) - const Real ky = modified_ky[j]; - const Real kz = modified_kz[k]; -#else - constexpr Real ky = 0._rt; - const Real kz = modified_kz[j]; + amrex::Real vg_y = m_v_galilean[1]; #endif + amrex::Real vg_z = m_v_galilean[2]; + // Loop over indices within one box + ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { // Calculate norm of vector - const Real k_norm = std::sqrt(kx*kx + ky*ky + kz*kz); - const Real k2 = k_norm * k_norm; + const amrex::Real knorm = std::sqrt( + amrex::Math::powi<2>(kx[i]) + +#if (AMREX_SPACEDIM==3) + amrex::Math::powi<2>(ky[j]) + amrex::Math::powi<2>(kz[k])); +#else + amrex::Math::powi<2>(kz[j])); +#endif + // Calculate the dot product of the k vector with the Galilean velocity. + // This has to be computed always with the centered (collocated) finite-order + // modified k vectors, to work correctly for both collocated and staggered grids. + // w_c = 0 always with standard PSATD (zero Galilean velocity). + const amrex::Real w_c = kx_c[i]*vg_x + +#if (AMREX_SPACEDIM==3) + ky_c[j]*vg_y + kz_c[k]*vg_z; +#else + kz_c[j]*vg_z; +#endif + constexpr amrex::Real c = PhysConst::c; + constexpr Complex I{0._rt,1._rt}; - // Calculate coefficients - constexpr Real c = PhysConst::c; + // Coefficients for knorm = 0 do not need to be set + if (knorm != 0._rt) + { + C(i,j,k) = std::cos(c*knorm*dt); + S_ck(i,j,k) = std::sin(c*knorm*dt) / (c*knorm); + inv_k2(i,j,k) = 1._rt / (knorm*knorm); - // Coefficients for k_norm = 0 do not need to be set - if (k_norm != 0._rt) { - C(i,j,k) = std::cos(c * k_norm * dt); - S_ck(i,j,k) = std::sin(c * k_norm * dt) / (c * k_norm); - inv_k2(i,j,k) = 1._rt / k2; + if (is_galilean) + { + T2(i,j,k) = amrex::exp(I*w_c*dt); + } } }); } } -void -PsatdAlgorithmPml::CurrentCorrection (SpectralFieldData& /*field_data*/) +void PsatdAlgorithmPml::CurrentCorrection (SpectralFieldData& /*field_data*/) { - WARPX_ABORT_WITH_MESSAGE( - "Current correction not implemented for PML PSATD"); + WARPX_ABORT_WITH_MESSAGE("Current correction not implemented for PML PSATD"); } -void -PsatdAlgorithmPml::VayDeposition (SpectralFieldData& /*field_data*/) +void PsatdAlgorithmPml::VayDeposition (SpectralFieldData& /*field_data*/) { - WARPX_ABORT_WITH_MESSAGE( - "Vay deposition not implemented for PML PSATD"); + WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented for PML PSATD"); } #endif // WARPX_USE_PSATD diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H index 33dafd382ac..00264956900 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H @@ -23,7 +23,7 @@ class PsatdAlgorithmPmlRZ : public SpectralBaseAlgorithmRZ short grid_type, amrex::Real dt_step); // Redefine functions from base class - virtual void pushSpectralFields (SpectralFieldDataRZ & f) override final; + void pushSpectralFields (SpectralFieldDataRZ & f) final; void InitializeSpectralCoefficients (SpectralFieldDataRZ const & f); @@ -36,7 +36,7 @@ class PsatdAlgorithmPmlRZ : public SpectralBaseAlgorithmRZ * * \param[in,out] field_data All fields in Fourier space */ - virtual void CurrentCorrection (SpectralFieldDataRZ& field_data) override final; + void CurrentCorrection (SpectralFieldDataRZ& field_data) final; /** * \brief Virtual function for Vay current deposition in Fourier space @@ -47,11 +47,11 @@ class PsatdAlgorithmPmlRZ : public SpectralBaseAlgorithmRZ * * \param[in,out] field_data All fields in Fourier space */ - virtual void VayDeposition (SpectralFieldDataRZ& field_data) override final; + void VayDeposition (SpectralFieldDataRZ& field_data) final; private: - bool coefficients_initialized; + bool coefficients_initialized = false; // Note that dt is saved to use in InitializeSpectralCoefficients amrex::Real m_dt; SpectralRealCoefficients C_coef, S_ck_coef; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp index 45a21f2ffec..66aa463503f 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp @@ -20,17 +20,15 @@ PsatdAlgorithmPmlRZ::PsatdAlgorithmPmlRZ (SpectralKSpaceRZ const & spectral_kspa amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, amrex::Real const dt) - // Initialize members of base class - : SpectralBaseAlgorithmRZ(spectral_kspace, dm, spectral_index, norder_z, grid_type), - m_dt(dt) + short const grid_type, amrex::Real const dt): + // Initialize members of base class and member variables + SpectralBaseAlgorithmRZ{spectral_kspace, dm, spectral_index, norder_z, grid_type}, + m_dt{dt} { // Allocate the arrays of coefficients amrex::BoxArray const & ba = spectral_kspace.spectralspace_ba; C_coef = SpectralRealCoefficients(ba, dm, n_rz_azimuthal_modes, 0); S_ck_coef = SpectralRealCoefficients(ba, dm, n_rz_azimuthal_modes, 0); - - coefficients_initialized = false; } /* Advance the E and B field in spectral space (stored in `f`) diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H index 5f02498f7d0..10feafec38b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H @@ -28,7 +28,7 @@ class PsatdAlgorithmRZ : public SpectralBaseAlgorithmRZ bool dive_cleaning, bool divb_cleaning); // Redefine functions from base class - virtual void pushSpectralFields(SpectralFieldDataRZ & f) override final; + void pushSpectralFields(SpectralFieldDataRZ & f) final; void InitializeSpectralCoefficients(SpectralFieldDataRZ const & f); @@ -41,7 +41,7 @@ class PsatdAlgorithmRZ : public SpectralBaseAlgorithmRZ * * \param[in,out] field_data All fields in Fourier space */ - virtual void CurrentCorrection (SpectralFieldDataRZ& field_data) override final; + void CurrentCorrection (SpectralFieldDataRZ& field_data) final; /** * \brief Virtual function for Vay current deposition in Fourier space @@ -52,7 +52,7 @@ class PsatdAlgorithmRZ : public SpectralBaseAlgorithmRZ * * \param[in,out] field_data All fields in Fourier space */ - virtual void VayDeposition (SpectralFieldDataRZ& field_data) override final; + void VayDeposition (SpectralFieldDataRZ& field_data) final; private: diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp index d5a95bc90bf..fcb89c39109 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp @@ -26,15 +26,15 @@ PsatdAlgorithmRZ::PsatdAlgorithmRZ (SpectralKSpaceRZ const & spectral_kspace, const int J_in_time, const int rho_in_time, const bool dive_cleaning, - const bool divb_cleaning) - // Initialize members of base class - : SpectralBaseAlgorithmRZ(spectral_kspace, dm, spectral_index, norder_z, grid_type), - m_dt(dt), - m_update_with_rho(update_with_rho), - m_time_averaging(time_averaging), - m_J_in_time(J_in_time), - m_dive_cleaning(dive_cleaning), - m_divb_cleaning(divb_cleaning) + const bool divb_cleaning): + // Initialize members of base class and member variables + SpectralBaseAlgorithmRZ{spectral_kspace, dm, spectral_index, norder_z, grid_type}, + m_dt{dt}, + m_update_with_rho{update_with_rho}, + m_time_averaging{time_averaging}, + m_J_in_time{J_in_time}, + m_dive_cleaning{dive_cleaning}, + m_divb_cleaning{divb_cleaning} { amrex::ignore_unused(rho_in_time); @@ -46,8 +46,6 @@ PsatdAlgorithmRZ::PsatdAlgorithmRZ (SpectralKSpaceRZ const & spectral_kspace, X2_coef = SpectralRealCoefficients(ba, dm, n_rz_azimuthal_modes, 0); X3_coef = SpectralRealCoefficients(ba, dm, n_rz_azimuthal_modes, 0); - coefficients_initialized = false; - if (time_averaging && J_in_time == JInTime::Linear) { X5_coef = SpectralRealCoefficients(ba, dm, n_rz_azimuthal_modes, 0); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H index 22fc506cc5a..4af24cfa559 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H @@ -41,7 +41,13 @@ class SpectralBaseAlgorithm // The destructor should also be a virtual function, so that // a pointer to subclass of `SpectraBaseAlgorithm` actually // calls the subclass's destructor. - virtual ~SpectralBaseAlgorithm() {} + virtual ~SpectralBaseAlgorithm() = default; + + // Default move and copy operations + SpectralBaseAlgorithm(const SpectralBaseAlgorithm&) = default; + SpectralBaseAlgorithm& operator=(const SpectralBaseAlgorithm&) = default; + SpectralBaseAlgorithm(SpectralBaseAlgorithm&&) = default; + SpectralBaseAlgorithm& operator=(SpectralBaseAlgorithm&&) = default; /** * \brief Virtual function for current correction in Fourier space diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp index 172715b988a..d635a5debe3 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp @@ -70,7 +70,7 @@ SpectralBaseAlgorithm::ComputeSpectralDivE ( const Box& bx = field_data.fields[mfi].box(); // Extract arrays for the fields to be updated - const Array4 fields = field_data.fields[mfi].array(); + const Array4 fields = field_data.fields[mfi].array(); // Extract pointers for the k vectors const Real* modified_kx_arr = modified_kx_vec[mfi].dataPtr(); #if defined(WARPX_DIM_3D) diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H index 5758f81e304..95c68f6d61b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H @@ -26,7 +26,28 @@ class SpectralBaseAlgorithmRZ // The destructor should also be a virtual function, so that // a pointer to subclass of `SpectraBaseAlgorithm` actually // calls the subclass's destructor. - virtual ~SpectralBaseAlgorithmRZ() {} + virtual ~SpectralBaseAlgorithmRZ() = default; + + /** + * \brief Default Copy constructor + */ + SpectralBaseAlgorithmRZ ( SpectralBaseAlgorithmRZ const &) = default; + + /** + * \brief Default Copy operator + */ + SpectralBaseAlgorithmRZ& operator= ( SpectralBaseAlgorithmRZ const & ) = default; + + + /** + * \brief Default Move constructor + */ + SpectralBaseAlgorithmRZ ( SpectralBaseAlgorithmRZ&& ) = default; + + /** + * \brief Default Move operator + */ + SpectralBaseAlgorithmRZ& operator= ( SpectralBaseAlgorithmRZ&& ) = default; /** * \brief Virtual function for current correction in Fourier space @@ -67,7 +88,7 @@ class SpectralBaseAlgorithmRZ // Compute and assign the modified k vectors : m_spectral_index(spectral_index), modified_kz_vec(spectral_kspace.getModifiedKComponent(dm, 1, norder_z, grid_type)) - {} + {} SpectralFieldIndex m_spectral_index; diff --git a/Source/FieldSolver/SpectralSolver/SpectralBinomialFilter.H b/Source/FieldSolver/SpectralSolver/SpectralBinomialFilter.H index ca30fb907ae..9d3af0f40a7 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralBinomialFilter.H +++ b/Source/FieldSolver/SpectralSolver/SpectralBinomialFilter.H @@ -23,7 +23,7 @@ class SpectralBinomialFilter // implements the filter. using KFilterArray = amrex::Gpu::DeviceVector; - SpectralBinomialFilter () {} + SpectralBinomialFilter () = default; void InitFilterArray (RealKVector const & kvec, amrex::Real dels, int npasses, diff --git a/Source/FieldSolver/SpectralSolver/SpectralBinomialFilter.cpp b/Source/FieldSolver/SpectralSolver/SpectralBinomialFilter.cpp index 14246b5be13..2fbab2422ea 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralBinomialFilter.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralBinomialFilter.cpp @@ -21,7 +21,7 @@ SpectralBinomialFilter::InitFilterArray (HankelTransform::RealVector const & kve bool const compensation, KFilterArray & filter) { - const int N = kvec.size(); + const auto N = static_cast(kvec.size()); filter.resize(N); amrex::Real* p_filter = filter.data(); amrex::Real const* p_kvec = kvec.data(); @@ -30,7 +30,7 @@ SpectralBinomialFilter::InitFilterArray (HankelTransform::RealVector const & kve { amrex::Real const ss = std::sin(0.5_rt*p_kvec[i]*dels); amrex::Real const ss2 = ss*ss; - amrex::Real filt = std::pow(1._rt - ss2, npasses); + auto filt = static_cast(std::pow(1._rt - ss2, npasses)); if (compensation) { filt *= (1._rt + npasses*ss2); } diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.H b/Source/FieldSolver/SpectralSolver/SpectralFieldData.H index 8fd23fac747..8ced43ced94 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.H +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.H @@ -10,10 +10,11 @@ #include "SpectralFieldData_fwd.H" -#include "AnyFFT.H" #include "SpectralKSpace.H" #include "Utils/WarpX_Complex.H" +#include + #include #include #include @@ -75,6 +76,26 @@ class SpectralFieldIndex */ ~SpectralFieldIndex () = default; + /** + * \brief Default Copy constructor + */ + SpectralFieldIndex ( SpectralFieldIndex const &) = default; + + /** + * \brief Default Copy operator + */ + SpectralFieldIndex& operator= ( SpectralFieldIndex const & ) = default; + + /** + * \brief Default Move constructor + */ + SpectralFieldIndex ( SpectralFieldIndex&& ) = default; + + /** + * \brief Default Move operator + */ + SpectralFieldIndex& operator= ( SpectralFieldIndex&& ) = default; + // Total number of fields that are actually allocated int n_fields; @@ -129,9 +150,14 @@ class SpectralFieldData int n_field_required, bool periodic_single_box); SpectralFieldData() = default; // Default constructor - SpectralFieldData& operator=(SpectralFieldData&& field_data) = default; ~SpectralFieldData(); + // default move and copy operations + SpectralFieldData(const SpectralFieldData&) = delete; + SpectralFieldData& operator=(const SpectralFieldData&) = delete; + SpectralFieldData(SpectralFieldData&&) = default; + SpectralFieldData& operator=(SpectralFieldData&& field_data) = default; + void ForwardTransform (int lev, const amrex::MultiFab& mf, int field_index, int i_comp); @@ -147,7 +173,7 @@ class SpectralFieldData // right before/after the Fourier transform SpectralField tmpSpectralField; // contains Complexs amrex::MultiFab tmpRealField; // contains Reals - AnyFFT::FFTplans forward_plan, backward_plan; + ablastr::math::anyfft::FFTplans forward_plan, backward_plan; // Correcting "shift" factors when performing FFT from/to // a cell-centered grid in real space, instead of a nodal grid SpectralShiftFactor xshift_FFTfromCell, xshift_FFTtoCell, diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp index 0cd7a346352..35d37df6038 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp @@ -63,9 +63,9 @@ SpectralFieldIndex::SpectralFieldIndex (const bool update_with_rho, Bx_avg = c++; By_avg = c++; Bz_avg = c++; } - if (dive_cleaning) F = c++; + if (dive_cleaning) { F = c++; } - if (divb_cleaning) G = c++; + if (divb_cleaning) { G = c++; } if (J_in_time == JInTime::Constant) { @@ -122,13 +122,12 @@ SpectralFieldData::SpectralFieldData( const int lev, const SpectralKSpace& k_space, const amrex::DistributionMapping& dm, const int n_field_required, - const bool periodic_single_box) + const bool periodic_single_box): + m_periodic_single_box{periodic_single_box} { amrex::LayoutData* cost = WarpX::getCosts(lev); const bool do_costs = WarpXUtilLoadBalance::doCosts(cost, realspace_ba, dm); - m_periodic_single_box = periodic_single_box; - const BoxArray& spectralspace_ba = k_space.spectralspace_ba; // Allocate the arrays that contain the fields in spectral space @@ -141,7 +140,7 @@ SpectralFieldData::SpectralFieldData( const int lev, tmpSpectralField = SpectralField(spectralspace_ba, dm, 1, 0); // By default, we assume the FFT is done from/to a nodal grid in real space - // It the FFT is performed from/to a cell-centered grid in real space, + // If the FFT is performed from/to a cell-centered grid in real space, // a correcting "shift" factor must be applied in spectral space. xshift_FFTfromCell = k_space.getSpectralShiftFactor(dm, 0, ShiftType::TransformFromCellCentered); @@ -164,8 +163,8 @@ SpectralFieldData::SpectralFieldData( const int lev, #endif // Allocate and initialize the FFT plans - forward_plan = AnyFFT::FFTplans(spectralspace_ba, dm); - backward_plan = AnyFFT::FFTplans(spectralspace_ba, dm); + forward_plan = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm); + backward_plan = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm); // Loop over boxes and allocate the corresponding plan // for each box owned by the local MPI proc for ( MFIter mfi(spectralspace_ba, dm); mfi.isValid(); ++mfi ){ @@ -173,27 +172,27 @@ SpectralFieldData::SpectralFieldData( const int lev, { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Note: the size of the real-space box and spectral-space box // differ when using real-to-complex FFT. When initializing // the FFT plan, the valid dimensions are those of the real-space box. const IntVect fft_size = realspace_ba[mfi].length(); - forward_plan[mfi] = AnyFFT::CreatePlan( + forward_plan[mfi] = ablastr::math::anyfft::CreatePlan( fft_size, tmpRealField[mfi].dataPtr(), - reinterpret_cast( tmpSpectralField[mfi].dataPtr()), - AnyFFT::direction::R2C, AMREX_SPACEDIM); + reinterpret_cast( tmpSpectralField[mfi].dataPtr()), + ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); - backward_plan[mfi] = AnyFFT::CreatePlan( + backward_plan[mfi] = ablastr::math::anyfft::CreatePlan( fft_size, tmpRealField[mfi].dataPtr(), - reinterpret_cast( tmpSpectralField[mfi].dataPtr()), - AnyFFT::direction::C2R, AMREX_SPACEDIM); + reinterpret_cast( tmpSpectralField[mfi].dataPtr()), + ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM); if (do_costs) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -204,8 +203,8 @@ SpectralFieldData::~SpectralFieldData() { if (!tmpRealField.empty()){ for ( MFIter mfi(tmpRealField); mfi.isValid(); ++mfi ){ - AnyFFT::DestroyPlan(forward_plan[mfi]); - AnyFFT::DestroyPlan(backward_plan[mfi]); + ablastr::math::anyfft::DestroyPlan(forward_plan[mfi]); + ablastr::math::anyfft::DestroyPlan(backward_plan[mfi]); } } } @@ -242,7 +241,7 @@ SpectralFieldData::ForwardTransform (const int lev, { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Copy the real-space field `mf` to the temporary field `tmpRealField` // This ensures that all fields have the same number of points @@ -267,7 +266,7 @@ SpectralFieldData::ForwardTransform (const int lev, } // Perform Fourier transform from `tmpRealField` to `tmpSpectralField` - AnyFFT::Execute(forward_plan[mfi]); + ablastr::math::anyfft::Execute(forward_plan[mfi]); // Copy the spectral-space field `tmpSpectralField` to the appropriate // index of the FabArray `fields` (specified by `field_index`) @@ -291,15 +290,15 @@ SpectralFieldData::ForwardTransform (const int lev, Complex spectral_field_value = tmp_arr(i,j,k); // Apply proper shift in each dimension #if (AMREX_SPACEDIM >= 2) - if (!is_nodal_x) spectral_field_value *= xshift_arr[i]; + if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } #endif #if defined(WARPX_DIM_3D) - if (!is_nodal_y) spectral_field_value *= yshift_arr[j]; - if (!is_nodal_z) spectral_field_value *= zshift_arr[k]; + if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } + if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } #elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) spectral_field_value *= zshift_arr[i]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into the right index fields_arr(i,j,k,field_index) = spectral_field_value; @@ -309,7 +308,7 @@ SpectralFieldData::ForwardTransform (const int lev, if (do_costs) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -367,7 +366,7 @@ SpectralFieldData::BackwardTransform (const int lev, { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Copy the spectral-space field `tmpSpectralField` to the appropriate // field (specified by the input argument field_index) @@ -391,15 +390,15 @@ SpectralFieldData::BackwardTransform (const int lev, Complex spectral_field_value = field_arr(i,j,k,field_index); // Apply proper shift in each dimension #if (AMREX_SPACEDIM >= 2) - if (!is_nodal_x) spectral_field_value *= xshift_arr[i]; + if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } #endif #if defined(WARPX_DIM_3D) - if (!is_nodal_y) spectral_field_value *= yshift_arr[j]; - if (!is_nodal_z) spectral_field_value *= zshift_arr[k]; + if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } + if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } #elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) spectral_field_value *= zshift_arr[i]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into temporary array tmp_arr(i,j,k) = spectral_field_value; @@ -407,7 +406,7 @@ SpectralFieldData::BackwardTransform (const int lev, } // Perform Fourier transform from `tmpSpectralField` to `tmpRealField` - AnyFFT::Execute(backward_plan[mfi]); + ablastr::math::anyfft::Execute(backward_plan[mfi]); // Copy the temporary field tmpRealField to the real-space field mf and // normalize, dividing by N, since (FFT + inverse FFT) results in a factor N @@ -448,7 +447,7 @@ SpectralFieldData::BackwardTransform (const int lev, { for (int dir = 0; dir < AMREX_SPACEDIM; dir++) { - if ((fill_guards[dir]) == 0) mf_box.grow(dir, -mf_ng[dir]); + if ((fill_guards[dir]) == 0) { mf_box.grow(dir, -mf_ng[dir]); } } } @@ -470,7 +469,7 @@ SpectralFieldData::BackwardTransform (const int lev, if (do_costs) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.H b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.H index b966c7c3c03..9fa7a65be3b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.H @@ -11,7 +11,8 @@ #include "SpectralFieldData.H" #include "SpectralHankelTransform/SpectralHankelTransformer.H" #include "SpectralKSpaceRZ.H" -#include "FieldSolver/SpectralSolver/AnyFFT.H" + +#include #include @@ -26,7 +27,7 @@ class SpectralFieldDataRZ // Define the FFTplans type, which holds one fft plan per box // (plans are only initialized for the boxes that are owned by // the local MPI rank) - using FFTplans = amrex::LayoutData; + using FFTplans = amrex::LayoutData; // Similarly, define the Hankel transformers and filter for each box. using MultiSpectralHankelTransformer = amrex::LayoutData; @@ -40,9 +41,18 @@ class SpectralFieldDataRZ int n_field_required, int n_modes); SpectralFieldDataRZ () = default; // Default constructor - SpectralFieldDataRZ& operator=(SpectralFieldDataRZ&& field_data) = default; ~SpectralFieldDataRZ (); + /** Delete Copy constructor */ + SpectralFieldDataRZ ( SpectralFieldDataRZ const &) = delete; + /** Delete Copy operator */ + SpectralFieldDataRZ& operator= ( SpectralFieldDataRZ const & ) = delete; + /** Default Move constructor */ + SpectralFieldDataRZ ( SpectralFieldDataRZ&& ) = default; + /** Default Move operator */ + SpectralFieldDataRZ& operator=(SpectralFieldDataRZ&& field_data) = default; + + void ForwardTransform (int lev, const amrex::MultiFab& mf, int field_index, int i_comp=0); void ForwardTransform (int lev, const amrex::MultiFab& mf_r, int field_index_r, diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp index db189f4287a..c4a693bf25a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp @@ -7,9 +7,9 @@ #include "SpectralFieldDataRZ.H" #include "Utils/WarpXUtil.H" -#include "FieldSolver/SpectralSolver/AnyFFT.H" #include "WarpX.H" +#include #include #include @@ -52,7 +52,7 @@ SpectralFieldDataRZ::SpectralFieldDataRZ (const int lev, tmpSpectralField = SpectralField(spectralspace_ba, dm, n_rz_azimuthal_modes, 0); // By default, we assume the z FFT is done from/to a nodal grid in real space. - // It the FFT is performed from/to a cell-centered grid in real space, + // If the FFT is performed from/to a cell-centered grid in real space, // a correcting "shift" factor must be applied in spectral space. zshift_FFTfromCell = k_space.getSpectralShiftFactor(dm, 1, ShiftType::TransformFromCellCentered); @@ -174,8 +174,10 @@ SpectralFieldDataRZ::SpectralFieldDataRZ (const int lev, dims, 2, // int howmany_rank, howmany_dims, - reinterpret_cast(tempHTransformed[mfi].dataPtr()), // complex *in - reinterpret_cast(tmpSpectralField[mfi].dataPtr()), // complex *out + reinterpret_cast< + ablastr::math::anyfft::Complex*>(tempHTransformed[mfi].dataPtr()), // complex *in + reinterpret_cast< + ablastr::math::anyfft::Complex*>(tmpSpectralField[mfi].dataPtr()), // complex *out FFTW_FORWARD, // int sign FFTW_ESTIMATE); // unsigned flags backward_plan[mfi] = @@ -188,8 +190,10 @@ SpectralFieldDataRZ::SpectralFieldDataRZ (const int lev, dims, 2, // int howmany_rank, howmany_dims, - reinterpret_cast(tmpSpectralField[mfi].dataPtr()), // complex *in - reinterpret_cast(tempHTransformed[mfi].dataPtr()), // complex *out + reinterpret_cast< + ablastr::math::anyfft::Complex*>(tmpSpectralField[mfi].dataPtr()), // complex *in + reinterpret_cast< + ablastr::math::anyfft::Complex*>(tempHTransformed[mfi].dataPtr()), // complex *out FFTW_BACKWARD, // int sign FFTW_ESTIMATE); // unsigned flags #endif @@ -265,8 +269,10 @@ SpectralFieldDataRZ::FABZForwardTransform (amrex::MFIter const & mfi, amrex::Box # else result = cufftExecZ2Z(forward_plan[mfi], # endif - reinterpret_cast(tempHTransformed[mfi].dataPtr(mode)), // Complex *in - reinterpret_cast(tmpSpectralField[mfi].dataPtr(mode)), // Complex *out + reinterpret_cast< + ablastr::math::anyfft::Complex*>(tempHTransformed[mfi].dataPtr(mode)), // Complex *in + reinterpret_cast< + ablastr::math::anyfft::Complex*>(tmpSpectralField[mfi].dataPtr(mode)), // Complex *out CUFFT_FORWARD); if (result != CUFFT_SUCCESS) { ablastr::warn_manager::WMRecordWarning("Spectral solver", @@ -325,7 +331,7 @@ SpectralFieldDataRZ::FABZForwardTransform (amrex::MFIter const & mfi, amrex::Box [=] AMREX_GPU_DEVICE(int i, int j, int k, int mode) noexcept { Complex spectral_field_value = tmp_arr(i,j,k,mode); // Apply proper shift. - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } // Copy field into the correct index. int const ic = field_index + mode*n_fields; fields_arr(i,j,k,ic) = spectral_field_value*inv_nz; @@ -363,7 +369,7 @@ SpectralFieldDataRZ::FABZBackwardTransform (amrex::MFIter const & mfi, amrex::Bo int const ic = field_index + mode*n_fields; Complex spectral_field_value = fields_arr(i,j,k,ic); // Apply proper shift. - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } // Copy field into the right index. tmp_arr(i,j,k,mode) = spectral_field_value; }); @@ -382,8 +388,10 @@ SpectralFieldDataRZ::FABZBackwardTransform (amrex::MFIter const & mfi, amrex::Bo # else result = cufftExecZ2Z(forward_plan[mfi], # endif - reinterpret_cast(tmpSpectralField[mfi].dataPtr(mode)), // Complex *in - reinterpret_cast(tempHTransformed[mfi].dataPtr(mode)), // Complex *out + reinterpret_cast< + ablastr::math::anyfft::Complex*>(tmpSpectralField[mfi].dataPtr(mode)), // Complex *in + reinterpret_cast< + ablastr::math::anyfft::Complex*>(tempHTransformed[mfi].dataPtr(mode)), // Complex *out CUFFT_INVERSE); if (result != CUFFT_SUCCESS) { ablastr::warn_manager::WMRecordWarning("Spectral solver", @@ -470,7 +478,7 @@ SpectralFieldDataRZ::ForwardTransform (const int lev, { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // Perform the Hankel transform first. // tempHTransformedSplit includes the imaginary component of mode 0. @@ -490,7 +498,7 @@ SpectralFieldDataRZ::ForwardTransform (const int lev, if (do_costs) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -527,7 +535,7 @@ SpectralFieldDataRZ::ForwardTransform (const int lev, { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); amrex::Box const& realspace_bx = tempHTransformed[mfi].box(); @@ -556,7 +564,7 @@ SpectralFieldDataRZ::ForwardTransform (const int lev, if (do_costs) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -589,7 +597,7 @@ SpectralFieldDataRZ::BackwardTransform (const int lev, { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); amrex::Box realspace_bx = tempHTransformed[mfi].box(); @@ -629,18 +637,18 @@ SpectralFieldDataRZ::BackwardTransform (const int lev, sign = +1._rt; } else { // Odd modes are anti-symmetric - int imode = (icomp + 1)/2; - sign = std::pow(-1._rt, imode); + const auto imode = (icomp + 1)/2; + sign = static_cast(std::pow(-1._rt, imode)); } } - int ic = icomp + i_comp; + const auto ic = icomp + i_comp; field_mf_array(i,j,k,ic) = sign*field_mf_copy_array(ii,j,k,icomp); }); if (do_costs) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -674,7 +682,7 @@ SpectralFieldDataRZ::BackwardTransform (const int lev, { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); amrex::Box realspace_bx = tempHTransformed[mfi].box(); @@ -720,7 +728,7 @@ SpectralFieldDataRZ::BackwardTransform (const int lev, } else { // Even modes are anti-symmetric int imode = (icomp + 1)/2; - sign = std::pow(-1._rt, imode+1); + sign = static_cast(std::pow(-1._rt, imode+1)); } } if (icomp == 0) { @@ -736,7 +744,7 @@ SpectralFieldDataRZ::BackwardTransform (const int lev, if (do_costs) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -773,7 +781,7 @@ SpectralFieldDataRZ::ApplyFilter (const int lev, int const field_index) { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); auto const & filter_r = binomialfilter[mfi].getFilterArrayR(); auto const & filter_z = binomialfilter[mfi].getFilterArrayZ(); @@ -798,7 +806,7 @@ SpectralFieldDataRZ::ApplyFilter (const int lev, int const field_index) if (do_costs) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -818,7 +826,7 @@ SpectralFieldDataRZ::ApplyFilter (const int lev, int const field_index1, { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); auto const & filter_r = binomialfilter[mfi].getFilterArrayR(); auto const & filter_z = binomialfilter[mfi].getFilterArrayZ(); @@ -847,7 +855,7 @@ SpectralFieldDataRZ::ApplyFilter (const int lev, int const field_index1, if (do_costs) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.H b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.H index 64ffc80ac37..2285ab05679 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.H +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.H @@ -32,14 +32,11 @@ ! (www.jpmoreau.fr) ! ------------------------------------------------------------------------ */ -#include "Utils/WarpXConst.H" +#ifndef WARPX_BESSEL_ROOTS_H_ +#define WARPX_BESSEL_ROOTS_H_ #include - -#include - - -void SecantRootFinder(int n, int nitmx, amrex::Real tol, amrex::Real *zeroj, int *ier); +#include /*---------------------------------------------------------------------- * calculate the first nk zeroes of bessel function j(n, x) @@ -56,112 +53,6 @@ void SecantRootFinder(int n, int nitmx, amrex::Real tol, amrex::Real *zeroj, int * abramowitz m. & stegun irene a. * handbook of mathematical functions */ -void GetBesselRoots(int n, int nk, amrex::Vector& roots, amrex::Vector &ier) { - using namespace amrex::literals; - - amrex::Real zeroj; - int ierror, ik, k; - - const amrex::Real tol = 1e-14_rt; - const amrex::Real nitmx = 10; - - const amrex::Real c1 = 1.8557571_rt; - const amrex::Real c2 = 1.033150_rt; - const amrex::Real c3 = 0.00397_rt; - const amrex::Real c4 = 0.0908_rt; - const amrex::Real c5 = 0.043_rt; - - const amrex::Real t0 = 4.0_rt*n*n; - const amrex::Real t1 = t0 - 1.0_rt; - const amrex::Real t3 = 4.0_rt*t1*(7.0_rt*t0 - 31.0_rt); - const amrex::Real t5 = 32.0_rt*t1*((83.0_rt*t0 - 982.0_rt)*t0 + 3779.0_rt); - const amrex::Real t7 = 64.0_rt*t1*(((6949.0_rt*t0 - 153855.0_rt)*t0 + 1585743.0_rt)*t0 - 6277237.0_rt); - - roots.resize(nk); - ier.resize(nk); - - // first zero - if (n == 0) { - zeroj = c1 + c2 - c3 - c4 + c5; - SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); - ier[0] = ierror; - roots[0] = zeroj; - ik = 1; - } - else { - // Include the trivial root - ier[0] = 0; - roots[0] = 0.; - const amrex::Real f1 = std::pow(n, (1.0_rt/3.0_rt)); - const amrex::Real f2 = f1*f1*n; - const amrex::Real f3 = f1*n*n; - zeroj = n + c1*f1 + (c2/f1) - (c3/n) - (c4/f2) + (c5/f3); - SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); - ier[1] = ierror; - roots[1] = zeroj; - ik = 2; - } - - // other zeroes - // k counts the nontrivial roots - // ik counts all roots - k = 2; - while (ik < nk) { - - // mac mahon's series for k >> n - const amrex::Real b0 = (k + 0.5_rt*n - 0.25_rt)*MathConst::pi; - const amrex::Real b1 = 8.0_rt*b0; - const amrex::Real b2 = b1*b1; - const amrex::Real b3 = 3.0_rt*b1*b2; - const amrex::Real b5 = 5.0_rt*b3*b2; - const amrex::Real b7 = 7.0_rt*b5*b2; - - zeroj = b0 - (t1/b1) - (t3/b3) - (t5/b5) - (t7/b7); - - const amrex::Real errj = std::abs(jn(n, zeroj)); - - // improve solution using procedure SecantRootFinder - if (errj > tol) SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); - - roots[ik] = zeroj; - ier[ik] = ierror; - - k++; - ik++; - } -} - -void SecantRootFinder(int n, int nitmx, amrex::Real tol, amrex::Real *zeroj, int *ier) { - using namespace amrex::literals; - - amrex::Real p0, p1, q0, q1, dp, p; - amrex::Real c[2]; - - c[0] = 0.95_rt; - c[1] = 0.999_rt; - *ier = 0; - - p = *zeroj; - for (int ntry=0 ; ntry <= 1 ; ntry++) { - p0 = c[ntry]*(*zeroj); +void GetBesselRoots(int n, int nk, amrex::Vector& roots, amrex::Vector &ier); - p1 = *zeroj; - q0 = jn(n, p0); - q1 = jn(n, p1); - for (int it=1; it <= nitmx; it++) { - if (q1 == q0) break; - p = p1 - q1*(p1 - p0)/(q1 - q0); - dp = p - p1; - if (it > 1 && std::abs(dp) < tol) { - *zeroj = p; - return; - } - p0 = p1; - q0 = q1; - p1 = p; - q1 = jn(n, p1); - } - } - *ier = 3; - *zeroj = p; -} +#endif // WARPX_BESSEL_ROOTS_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp new file mode 100644 index 00000000000..39782408eb0 --- /dev/null +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp @@ -0,0 +1,153 @@ +/* Copyright 2019 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +/* ------------------------------------------------------------------------- +! program to calculate the first zeroes (root abscissas) of the first +! kind bessel function of integer order n using the subroutine rootj. +! -------------------------------------------------------------------------- +! sample run: +! +! (calculate the first 10 zeroes of 1st kind bessel function of order 2). +! +! zeroes of bessel function of order: 2 +! +! number of calculated zeroes: 10 +! +! table of root abcissas (5 items per line) +! 5.135622 8.417244 11.619841 14.795952 17.959819 + 21.116997 24.270112 27.420574 30.569204 33.716520 +! +! table of error codes (5 items per line) +! 0 0 0 0 0 +! 0 0 0 0 0 +! +! -------------------------------------------------------------------------- +! reference: from numath library by tuan dang trong in fortran 77 +! [bibli 18]. +! +! c++ release 1.0 by j-p moreau, paris +! (www.jpmoreau.fr) +! ------------------------------------------------------------------------ */ + +#include "BesselRoots.H" + +#include "Utils/WarpXConst.H" + +#include + +namespace{ + + void SecantRootFinder(int n, int nitmx, amrex::Real tol, amrex::Real *zeroj, int *ier) { + using namespace amrex::literals; + + amrex::Real p0, p1, q0, q1, dp, p; + amrex::Real c[2]; + + c[0] = 0.95_rt; + c[1] = 0.999_rt; + *ier = 0; + + p = *zeroj; + for (int ntry=0 ; ntry <= 1 ; ntry++) { + p0 = c[ntry]*(*zeroj); + + p1 = *zeroj; + q0 = static_cast(jn(n, p0)); + q1 = static_cast(jn(n, p1)); + for (int it=1; it <= nitmx; it++) { + if (q1 == q0) { break; } + p = p1 - q1*(p1 - p0)/(q1 - q0); + dp = p - p1; + if (it > 1 && std::abs(dp) < tol) { + *zeroj = p; + return; + } + p0 = p1; + q0 = q1; + p1 = p; + q1 = static_cast(jn(n, p1)); + } + } + *ier = 3; + *zeroj = p; + } + +} + +void GetBesselRoots(int n, int nk, amrex::Vector& roots, amrex::Vector &ier) { + using namespace amrex::literals; + + amrex::Real zeroj; + int ierror, ik, k; + + const amrex::Real tol = 1e-14_rt; + const int nitmx = 10; + + const amrex::Real c1 = 1.8557571_rt; + const amrex::Real c2 = 1.033150_rt; + const amrex::Real c3 = 0.00397_rt; + const amrex::Real c4 = 0.0908_rt; + const amrex::Real c5 = 0.043_rt; + + const amrex::Real t0 = 4.0_rt*n*n; + const amrex::Real t1 = t0 - 1.0_rt; + const amrex::Real t3 = 4.0_rt*t1*(7.0_rt*t0 - 31.0_rt); + const amrex::Real t5 = 32.0_rt*t1*((83.0_rt*t0 - 982.0_rt)*t0 + 3779.0_rt); + const amrex::Real t7 = 64.0_rt*t1*(((6949.0_rt*t0 - 153855.0_rt)*t0 + 1585743.0_rt)*t0 - 6277237.0_rt); + + roots.resize(nk); + ier.resize(nk); + + // first zero + if (n == 0) { + zeroj = c1 + c2 - c3 - c4 + c5; + ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); + ier[0] = ierror; + roots[0] = zeroj; + ik = 1; + } + else { + // Include the trivial root + ier[0] = 0; + roots[0] = 0.; + const auto f1 = static_cast(std::pow(n, (1.0_rt/3.0_rt))); + const auto f2 = f1*f1*n; + const auto f3 = f1*n*n; + zeroj = n + c1*f1 + (c2/f1) - (c3/n) - (c4/f2) + (c5/f3); + ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); + ier[1] = ierror; + roots[1] = zeroj; + ik = 2; + } + + // other zeroes + // k counts the nontrivial roots + // ik counts all roots + k = 2; + while (ik < nk) { + + // mac mahon's series for k >> n + const amrex::Real b0 = (k + 0.5_rt*n - 0.25_rt)*MathConst::pi; + const amrex::Real b1 = 8.0_rt*b0; + const amrex::Real b2 = b1*b1; + const amrex::Real b3 = 3.0_rt*b1*b2; + const amrex::Real b5 = 5.0_rt*b3*b2; + const amrex::Real b7 = 7.0_rt*b5*b2; + + zeroj = b0 - (t1/b1) - (t3/b3) - (t5/b5) - (t7/b7); + + const auto errj = static_cast(std::abs(jn(n, zeroj))); + + // improve solution using procedure SecantRootFinder + if (errj > tol) { ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); } + + roots[ik] = zeroj; + ier[ik] = ierror; + + k++; + ik++; + } +} diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt index a3d2b60bcc4..70c9740367f 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/CMakeLists.txt @@ -1,5 +1,6 @@ target_sources(lib_rz PRIVATE + BesselRoots.cpp SpectralHankelTransformer.cpp HankelTransform.cpp ) diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp index da62382dffa..a8930e1d48b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp @@ -68,7 +68,7 @@ HankelTransform::HankelTransform (int const hankel_order, amrex::Vector denom(m_nk); for (int ik=0 ; ik < m_nk ; ik++) { - const amrex::Real jna = jn(p_denom, alphas[ik]); + const auto jna = static_cast(jn(p_denom, alphas[ik])); denom[ik] = MathConst::pi*rmax*rmax*jna*jna; } @@ -76,7 +76,7 @@ HankelTransform::HankelTransform (int const hankel_order, for (int ir=0 ; ir < m_nr ; ir++) { for (int ik=0 ; ik < m_nk ; ik++) { int const ii = ik + ir*m_nk; - num[ii] = jn(hankel_order, rmesh[ir]*kr[ik]); + num[ii] = static_cast(jn(hankel_order, rmesh[ir]*kr[ik])); } } @@ -99,7 +99,8 @@ HankelTransform::HankelTransform (int const hankel_order, if (hankel_order == azimuthal_mode-1) { for (int ir=0 ; ir < m_nr ; ir++) { int const ii = ir*m_nk; - invM[ii] = std::pow(rmesh[ir], (azimuthal_mode-1))/(MathConst::pi*std::pow(rmax, (azimuthal_mode+1))); + invM[ii] = static_cast( + std::pow(rmesh[ir], (azimuthal_mode-1))/(MathConst::pi*std::pow(rmax, (azimuthal_mode+1)))); } } else { for (int ir=0 ; ir < m_nr ; ir++) { @@ -121,7 +122,7 @@ HankelTransform::HankelTransform (int const hankel_order, // Calculate the matrix M by inverting invM if (azimuthal_mode !=0 && hankel_order != azimuthal_mode-1) { // In this case, invM is singular, thus we calculate the pseudo-inverse. - // The Moore-Penrose psuedo-inverse is calculated using the SVD method. + // The Moore-Penrose pseudo-inverse is calculated using the SVD method. M.resize(m_nk*m_nr, 0.); amrex::Vector invMcopy(invM); @@ -131,7 +132,7 @@ HankelTransform::HankelTransform (int const hankel_order, amrex::Vector sp((m_nr)*(m_nk-1), 0.); amrex::Vector temp((m_nr)*(m_nk-1), 0.); - // Calculate the singlular-value-decomposition of invM (leaving out the first row). + // Calculate the singular-value-decomposition of invM (leaving out the first row). // invM = u*sdiag*vt // Note that invMcopy.dataPtr()+1 is passed in so that the first ik row is skipped // A copy is passed in since the matrix is destroyed diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package index 8bb1d7ef7b4..37ca9a93186 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/Make.package @@ -1,3 +1,4 @@ +CEXE_sources += BesselRoots.cpp CEXE_sources += SpectralHankelTransformer.cpp CEXE_sources += HankelTransform.cpp diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/SpectralHankelTransformer.H b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/SpectralHankelTransformer.H index d14bb16681b..881a219c279 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/SpectralHankelTransformer.H +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/SpectralHankelTransformer.H @@ -23,7 +23,7 @@ class SpectralHankelTransformer { public: - SpectralHankelTransformer () {} + SpectralHankelTransformer () = default; SpectralHankelTransformer (int nr, int n_rz_azimuthal_modes, @@ -33,7 +33,7 @@ class SpectralHankelTransformer ExtractKrArray (); // Returns an array that holds the kr for all of the modes - HankelTransform::RealVector const & getKrArray () const {return m_kr;} + [[nodiscard]] HankelTransform::RealVector const & getKrArray () const {return m_kr;} // Converts a scalar field from the physical to the spectral space void diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp index b8d81000cda..91fe2953668 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp @@ -137,7 +137,7 @@ SpectralKSpace::getKComponent( const DistributionMapping& dm, * specified by `i_dim`. * * (By default, we assume the FFT is done from/to a collocated grid in real space - * It the FFT is performed from/to a cell-centered grid in real space, + * If the FFT is performed from/to a cell-centered grid in real space, * a correcting "shift" factor must be applied in spectral space.) */ SpectralShiftFactor @@ -147,14 +147,14 @@ SpectralKSpace::getSpectralShiftFactor( const DistributionMapping& dm, { // Initialize an empty DeviceVector in each box SpectralShiftFactor shift_factor( spectralspace_ba, dm ); - // Loop over boxes and allocate the corresponding DeviceVector + // Loop over boxes and allocate the corresponding DeviceVector // for each box owned by the local MPI proc for ( MFIter mfi(spectralspace_ba, dm); mfi.isValid(); ++mfi ){ const Gpu::DeviceVector& k = k_vec[i_dim][mfi]; Gpu::DeviceVector& shift = shift_factor[mfi]; // Allocate shift coefficients - const int N = k.size(); + const auto N = static_cast(k.size()); shift.resize(N); Real const* pk = k.data(); Complex* pshift = shift.data(); @@ -202,7 +202,7 @@ SpectralKSpace::getModifiedKComponent( const DistributionMapping& dm, Gpu::DeviceVector& modified_k = modified_k_comp[mfi]; // Allocate modified_k to the same size as k - const int N = k.size(); + const auto N = static_cast(k.size());; modified_k.resize(N); // Fill the modified k vector @@ -216,7 +216,7 @@ SpectralKSpace::getModifiedKComponent( const DistributionMapping& dm, Gpu::copyAsync(Gpu::hostToDevice, h_stencil_coef.begin(), h_stencil_coef.end(), d_stencil_coef.begin()); Gpu::synchronize(); - const int nstencil = d_stencil_coef.size(); + const auto nstencil = static_cast(d_stencil_coef.size()); Real const* p_stencil_coef = d_stencil_coef.data(); // Loop over boxes and allocate the corresponding DeviceVector @@ -227,7 +227,7 @@ SpectralKSpace::getModifiedKComponent( const DistributionMapping& dm, Gpu::DeviceVector& modified_k = modified_k_comp[mfi]; // Allocate modified_k to the same size as k - const int N = k.size(); + const auto N = static_cast(k.size());; modified_k.resize(N); Real const* p_k = k.data(); Real * p_modified_k = modified_k.data(); @@ -242,7 +242,7 @@ SpectralKSpace::getModifiedKComponent( const DistributionMapping& dm, std::sin( p_k[i]*(n+1)*delta_x )/( (n+1)*delta_x ); } else { p_modified_k[i] += p_stencil_coef[n]* \ - std::sin( p_k[i]*(n+0.5)*delta_x )/( (n+0.5)*delta_x ); + std::sin( p_k[i]*(n+0.5_rt)*delta_x )/( (n+0.5_rt)*delta_x ); } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 362af06a354..80fb2c39545 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -52,11 +52,11 @@ SpectralSolver::SpectralSolver( // - Select the algorithm depending on the input parameters // Initialize the corresponding coefficients over k space - if (pml) // PSATD equations in the PML region + if (pml) // PSATD or Galilean PSATD equations in the PML region { algorithm = std::make_unique( k_space, dm, m_spectral_index, norder_x, norder_y, norder_z, grid_type, - dt, dive_cleaning, divb_cleaning); + v_galilean, dt, dive_cleaning, divb_cleaning); } else // PSATD equations in the regular domain { diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index d515462d730..bb9ec04f266 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -183,11 +183,11 @@ WarpX::PSATDForwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { - if (F_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); + if (F_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); } if (spectral_solver_cp[lev]) { - if (F_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); + if (F_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); } } } } @@ -200,17 +200,17 @@ WarpX::PSATDBackwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (F_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); + if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); } #else - if (F_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); + if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (F_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); + if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); } #else - if (F_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); + if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); } #endif } } @@ -229,11 +229,11 @@ WarpX::PSATDForwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (G_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); + if (G_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); } if (spectral_solver_cp[lev]) { - if (G_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); + if (G_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); } } } } @@ -246,17 +246,17 @@ WarpX::PSATDBackwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (G_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); + if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); } #else - if (G_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); + if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (G_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); + if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); } #else - if (G_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); + if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); } #endif } } @@ -370,15 +370,15 @@ void WarpX::PSATDForwardTransformRho ( const amrex::Vector>& charge_cp, const int icomp, const int dcomp, const bool apply_kspace_filter) { - if (charge_fp[0] == nullptr) return; + if (charge_fp[0] == nullptr) { return; } for (int lev = 0; lev <= finest_level; ++lev) { - if (charge_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); + if (charge_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); } if (spectral_solver_cp[lev]) { - if (charge_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); + if (charge_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); } } } @@ -759,22 +759,23 @@ WarpX::PushPSATD () PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); #ifdef WARPX_DIM_RZ - if (pml_rz[0]) pml_rz[0]->PushPSATD(0); + if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } #endif // FFT of F and G - if (WarpX::do_dive_cleaning) PSATDForwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDForwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } // Update E, B, F, and G in k-space PSATDPushSpectralFields(); // Inverse FFT of E, B, F, and G PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::fft_do_time_averaging) + if (WarpX::fft_do_time_averaging) { PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp); - if (WarpX::do_dive_cleaning) PSATDBackwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDBackwardTransformG(); + } + if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } // Evolve the fields in the PML boxes for (int lev = 0; lev <= finest_level; ++lev) @@ -784,9 +785,9 @@ WarpX::PushPSATD () pml[lev]->PushPSATD(lev); } ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) ApplyEfieldBoundary(lev, PatchType::coarse); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } } #endif } @@ -916,7 +917,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) void WarpX::EvolveF (amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } for (int lev = 0; lev <= finest_level; ++lev) { @@ -927,16 +928,16 @@ WarpX::EvolveF (amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveF (int lev, amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } EvolveF(lev, PatchType::fine, a_dt, a_dt_type); - if (lev > 0) EvolveF(lev, PatchType::coarse, a_dt, a_dt_type); + if (lev > 0) { EvolveF(lev, PatchType::coarse, a_dt, a_dt_type); } } void WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } WARPX_PROFILE("WarpX::EvolveF()"); @@ -966,7 +967,7 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ void WarpX::EvolveG (amrex::Real a_dt, DtType a_dt_type) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } for (int lev = 0; lev <= finest_level; ++lev) { @@ -977,7 +978,7 @@ WarpX::EvolveG (amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveG (int lev, amrex::Real a_dt, DtType a_dt_type) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } EvolveG(lev, PatchType::fine, a_dt, a_dt_type); @@ -990,7 +991,7 @@ WarpX::EvolveG (int lev, amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_type*/) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } WARPX_PROFILE("WarpX::EvolveG()"); @@ -1076,8 +1077,8 @@ WarpX::DampFieldsInGuards(const int lev, { // Only apply to damped boundaries - if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) continue; - if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) continue; + if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) { continue; } + if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) { continue; } for ( amrex::MFIter mfi(*Efield[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) { @@ -1170,8 +1171,8 @@ void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& for (int iside = 0; iside < 2; iside++) { // Only apply to damped boundaries - if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) continue; - if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) continue; + if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) { continue; } + if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) { continue; } for (amrex::MFIter mfi(*mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { @@ -1207,7 +1208,7 @@ void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& } #ifdef WARPX_DIM_RZ -// This scales the current by the inverse volume and wraps around the depostion at negative radius. +// This scales the current by the inverse volume and wraps around the deposition at negative radius. // It is faster to apply this on the grid than to do it particle by particle. // It is put here since there isn't another nice place for it. void @@ -1291,8 +1292,8 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // Wrap the current density deposited in the guard cells around // to the cells above the axis. if (rmin == 0._rt && 1-ishift_r <= i && i < ngJ[0]-ishift_r) { - Jr_arr(i,j,0,2*imode-1) += std::pow(-1, imode+1)*Jr_arr(-ishift_r-i,j,0,2*imode-1); - Jr_arr(i,j,0,2*imode) += std::pow(-1, imode+1)*Jr_arr(-ishift_r-i,j,0,2*imode); + Jr_arr(i,j,0,2*imode-1) += static_cast(std::pow(-1, imode+1)*Jr_arr(-ishift_r-i,j,0,2*imode-1)); + Jr_arr(i,j,0,2*imode) += static_cast(std::pow(-1, imode+1)*Jr_arr(-ishift_r-i,j,0,2*imode)); } // Apply the inverse volume scaling // Jr is forced to zero on axis. @@ -1328,8 +1329,8 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // Wrap the current density deposited in the guard cells around // to the cells above the axis. if (rmin == 0._rt && 1-ishift_t <= i && i <= ngJ[0]-ishift_t) { - Jt_arr(i,j,0,2*imode-1) += std::pow(-1, imode+1)*Jt_arr(-ishift_t-i,j,0,2*imode-1); - Jt_arr(i,j,0,2*imode) += std::pow(-1, imode+1)*Jt_arr(-ishift_t-i,j,0,2*imode); + Jt_arr(i,j,0,2*imode-1) += static_cast(std::pow(-1, imode+1)*Jt_arr(-ishift_t-i,j,0,2*imode-1)); + Jt_arr(i,j,0,2*imode) += static_cast(std::pow(-1, imode+1)*Jt_arr(-ishift_t-i,j,0,2*imode)); } // Apply the inverse volume scaling @@ -1365,8 +1366,8 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // Wrap the current density deposited in the guard cells around // to the cells above the axis. if (rmin == 0._rt && 1-ishift_z <= i && i <= ngJ[0]-ishift_z) { - Jz_arr(i,j,0,2*imode-1) -= std::pow(-1, imode+1)*Jz_arr(-ishift_z-i,j,0,2*imode-1); - Jz_arr(i,j,0,2*imode) -= std::pow(-1, imode+1)*Jz_arr(-ishift_z-i,j,0,2*imode); + Jz_arr(i,j,0,2*imode-1) -= static_cast(std::pow(-1, imode+1)*Jz_arr(-ishift_z-i,j,0,2*imode-1)); + Jz_arr(i,j,0,2*imode) -= static_cast(std::pow(-1, imode+1)*Jz_arr(-ishift_z-i,j,0,2*imode)); } // Apply the inverse volume scaling @@ -1411,7 +1412,7 @@ WarpX::ApplyInverseVolumeScalingToChargeDensity (MultiFab* Rho, int lev) const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); const Dim3 lo = lbound(tilebox); const Real rmin = xyzmin[0]; - const Real rminr = xyzmin[0] + (tb.type(0) == NODE ? 0. : 0.5*dx[0]); + const Real rminr = xyzmin[0] + (tb.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); const int irmin = lo.x; const int ishift = (rminr > rmin ? 1 : 0); @@ -1445,7 +1446,7 @@ WarpX::ApplyInverseVolumeScalingToChargeDensity (MultiFab* Rho, int lev) else { imode = (icomp - ncomp/2 + 1)/2; } - Rho_arr(i,j,0,icomp) -= std::pow(-1, imode+1)*Rho_arr(-ishift-i,j,0,icomp); + Rho_arr(i,j,0,icomp) -= static_cast(std::pow(-1, imode+1)*Rho_arr(-ishift-i,j,0,icomp)); } // Apply the inverse volume scaling @@ -1453,7 +1454,7 @@ WarpX::ApplyInverseVolumeScalingToChargeDensity (MultiFab* Rho, int lev) if (r == 0.) { Rho_arr(i,j,0,icomp) /= (MathConst::pi*dr*axis_volume_factor); } else { - Rho_arr(i,j,0,icomp) /= (2.*MathConst::pi*r); + Rho_arr(i,j,0,icomp) /= (2._rt*MathConst::pi*r); } }); } diff --git a/Source/FieldSolver/WarpXPushFieldsEM_K.H b/Source/FieldSolver/WarpXPushFieldsEM_K.H index e8e675cd276..d6e7e353a54 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM_K.H +++ b/Source/FieldSolver/WarpXPushFieldsEM_K.H @@ -94,7 +94,7 @@ void damp_field_in_guards( // Apply damping factor in guards cells below the lower end of the domain const int n_guard = -tb_smallEnd; - const amrex::Real cell = static_cast(idx + n_guard); + const auto cell = static_cast(idx + n_guard); const amrex::Real phase = MathConst::pi * cell / n_guard; const amrex::Real sin_phase = std::sin(phase); @@ -107,7 +107,7 @@ void damp_field_in_guards( // Apply damping factor in guards cells above the upper end of the domain const int n_guard = tb_bigEnd - n_domain; - const amrex::Real cell = static_cast(tb_bigEnd - idx); + const auto cell = static_cast(tb_bigEnd - idx); const amrex::Real phase = MathConst::pi * cell / n_guard; const amrex::Real sin_phase = std::sin(phase); diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index b502620641b..9252671d1ff 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -10,6 +10,8 @@ #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" @@ -30,6 +32,13 @@ void WarpX::HybridPICEvolveFields () // Perform current deposition at t_{n+1/2}. mypc->DepositCurrent(current_fp, dt[0], -0.5_rt * dt[0]); + // Deposit cold-relativistic fluid charge and current + if (do_fluid_species) { + int const lev = 0; + myfl->DepositCharge(lev, *rho_fp[lev]); + myfl->DepositCurrent(lev, *current_fp[lev][0], *current_fp[lev][1], *current_fp[lev][2]); + } + // Synchronize J and rho: // filter (if used), exchange guard cells, interpolate across MR levels // and apply boundary conditions @@ -38,13 +47,18 @@ void WarpX::HybridPICEvolveFields () // SyncCurrent does not include a call to FillBoundary, but it is needed // for the hybrid-PIC solver since current values are interpolated to // a nodal grid - for (int lev = 0; lev <= finest_level; ++lev) - for (int idim = 0; idim < 3; ++idim) + for (int lev = 0; lev <= finest_level; ++lev) { + for (int idim = 0; idim < 3; ++idim) { current_fp[lev][idim]->FillBoundary(Geom(lev).periodicity()); + } + } // Get requested number of substeps to use int sub_steps = m_hybrid_pic_model->m_substeps / 2; + // Get the external current + m_hybrid_pic_model->GetCurrentExternal(m_edge_lengths); + // Reference hybrid-PIC multifabs auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; @@ -92,7 +106,7 @@ void WarpX::HybridPICEvolveFields () true ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - EvolveB(0.5 / sub_steps * dt[0], DtType::FirstHalf); + EvolveB(0.5_rt / sub_steps * dt[0], DtType::FirstHalf); FillBoundaryB(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); } @@ -120,7 +134,7 @@ void WarpX::HybridPICEvolveFields () true ); FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - EvolveB(0.5 / sub_steps * dt[0], DtType::SecondHalf); + EvolveB(0.5_rt / sub_steps * dt[0], DtType::SecondHalf); FillBoundaryB(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); } diff --git a/Source/FieldSolver/WarpX_FDTD.H b/Source/FieldSolver/WarpX_FDTD.H index 6f1ebad7819..c6f3c16a2c4 100644 --- a/Source/FieldSolver/WarpX_FDTD.H +++ b/Source/FieldSolver/WarpX_FDTD.H @@ -24,6 +24,9 @@ void warpx_computedivb(int i, int j, int k, int dcomp, #endif ) { + + using namespace amrex; + #if defined WARPX_DIM_3D divB(i,j,k,dcomp) = (Bx(i+1,j ,k ) - Bx(i,j,k))*dxinv + (By(i ,j+1,k ) - By(i,j,k))*dyinv @@ -37,8 +40,8 @@ void warpx_computedivb(int i, int j, int k, int dcomp, amrex::ignore_unused(j, Bx, dxinv); amrex::ignore_unused(k, By, dyinv); #elif defined WARPX_DIM_RZ - const amrex::Real ru = 1. + 0.5/(rmin*dxinv + i + 0.5); - const amrex::Real rd = 1. - 0.5/(rmin*dxinv + i + 0.5); + const amrex::Real ru = 1._rt + 0.5_rt/(rmin*dxinv + i + 0.5_rt); + const amrex::Real rd = 1._rt - 0.5_rt/(rmin*dxinv + i + 0.5_rt); divB(i,j,0,dcomp) = (ru*Bx(i+1,j,0) - rd*Bx(i,j,0))*dxinv + (Bz(i,j+1,0) - Bz(i,j,0))*dzinv; amrex::ignore_unused(k, By, dyinv); diff --git a/Source/FieldSolver/WarpX_QED_K.H b/Source/FieldSolver/WarpX_QED_K.H index a20dde8b9a7..6c86bba67b6 100644 --- a/Source/FieldSolver/WarpX_QED_K.H +++ b/Source/FieldSolver/WarpX_QED_K.H @@ -18,12 +18,12 @@ /** * calc_M calculates the Magnetization field of the vacuum at a specific point and returns it as a three component vector * \param[in] arr This is teh empty array that will be filled with the components of the M-field - * \param[in] ex The x-component of the E-field at the point at whicht the M-field is to be calculated - * \param[in] ey The y-component of the E-field at the point at whicht the M-field is to be calculated - * \param[in] ez The z-component of the E-field at the point at whicht the M-field is to be calculated - * \param[in] bx The x-component of the B-field at the point at whicht the M-field is to be calculated - * \param[in] by The y-component of the B-field at the point at whicht the M-field is to be calculated - * \param[in] bz The z-component of the B-field at the point at whicht the M-field is to be calculated + * \param[in] ex The x-component of the E-field at the point at which the M-field is to be calculated + * \param[in] ey The y-component of the E-field at the point at which the M-field is to be calculated + * \param[in] ez The z-component of the E-field at the point at which the M-field is to be calculated + * \param[in] bx The x-component of the B-field at the point at which the M-field is to be calculated + * \param[in] by The y-component of the B-field at the point at which the M-field is to be calculated + * \param[in] bz The z-component of the B-field at the point at which the M-field is to be calculated * \param[in] xi_c2 The quantum parameter * c2 being used for the simulation * \param[in] c2 the speed of light squared */ @@ -72,7 +72,7 @@ void calc_M(amrex::Real arr [], amrex::Real ex, amrex::Real ey, amrex::Real ez, * \param[in] Jz the current field in z * \param[in] dx The x spatial step, used for calculating curls * \param[in] dy The y spatial step, used for calculating curls - * \param[in] dz The z spatial step, used for calulating curls + * \param[in] dz The z spatial step, used for calculating curls * \param[in] dt The temporal step, used for the half push/correction to the E-fields at the end of the function * \param[in] xi_c2 Quantum parameter * c**2 */ @@ -109,7 +109,7 @@ const amrex::Real dyi = 1._rt/dy; amrex::Real Mpz [3] = {0._rt,0._rt,0._rt}; amrex::Real Mnz [3] = {0._rt,0._rt,0._rt}; - // Calcualting the M-field at the chosen stencil points + // Calculating the M-field at the chosen stencil points calc_M(Mpx, tmpEx(j+1,k,l), tmpEy(j+1,k,l), tmpEz(j+1,k,l), Bx(j+1,k,l), By(j+1,k,l), Bz(j+1,k,l), xi_c2, c2); @@ -182,7 +182,7 @@ const amrex::Real dyi = 1._rt/dy; + 7._rt*c2*bz*(BVxB + Bmu0J) ) }; - // Calcualting matrix values for the QED correction algorithm + // Calculating matrix values for the QED correction algorithm const amrex::Real a00 = beta + xi_c2*( 8._rt*c2i*ex*ex + 14._rt*bx*bx ); @@ -198,7 +198,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real detA = a00*( a11*a22 - a12*a12 ) - a01*( a01*a22 - a02*a12 )+a02*( a01*a12 - a02*a11 ); - // Calcualting the rows of the inverse matrix using the general 3x3 inverse form + // Calculating the rows of the inverse matrix using the general 3x3 inverse form const amrex::Real invAx[3] = {a22*a11 - a12*a12, a12*a02 - a22*a01, a12*a01 - a11*a02}; @@ -206,7 +206,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real invAz[3] = {a12*a01 - a02*a11, a02*a01 - a12*a00, a11*a00 - a01*a01}; - // Calcualting the final QED corrections by mutliplying the Omega vector with the inverse matrix + // Calculating the final QED corrections by mutliplying the Omega vector with the inverse matrix const amrex::Real dEx = (-1._rt/detA)*(invAx[0]*Omega[0] + invAx[1]*Omega[1] + @@ -220,7 +220,7 @@ const amrex::Real dyi = 1._rt/dy; invAz[1]*Omega[1] + invAz[2]*Omega[2]); - // Adding the QED corrections to the origional fields + // Adding the QED corrections to the original fields Ex(j,k,l) = Ex(j,k,l) + 0.5_rt*dt*dEx; @@ -239,7 +239,7 @@ const amrex::Real dyi = 1._rt/dy; amrex::Real Mpz [3] = {0._rt,0._rt,0._rt}; amrex::Real Mnz [3] = {0._rt,0._rt,0._rt}; - // Calcualting the M-field at the chosen stencil points + // Calculating the M-field at the chosen stencil points calc_M(Mpx, tmpEx(j+1,k,0), tmpEy(j+1,k,0), tmpEz(j+1,k,0), Bx(j+1,k,0), By(j+1,k,0), Bz(j+1,k,0), xi_c2, c2); @@ -308,7 +308,7 @@ const amrex::Real dyi = 1._rt/dy; + 7._rt*c2*bz*(BVxB + Bmu0J) ) }; - // Calcualting matrix values for the QED correction algorithm + // Calculating matrix values for the QED correction algorithm const amrex::Real a00 = beta + xi_c2*( 8._rt*c2i*ex*ex + 14._rt*bx*bx ); @@ -324,7 +324,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real detA = a00*( a11*a22 - a12*a12 ) - a01*( a01*a22 - a02*a12 ) + a02*( a01*a12 - a02*a11 ); - // Calcualting inverse matrix values using the general 3x3 form + // Calculating inverse matrix values using the general 3x3 form const amrex::Real invAx[3] = {a22*a11 - a12*a12, a12*a02 - a22*a01, a12*a01 - a11*a02}; @@ -332,7 +332,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real invAz[3] = {a12*a01 - a02*a11, a02*a01 - a12*a00, a11*a00 - a01*a01}; - // Calcualting the final QED corrections by mutliplying the Omega vector with the inverse matrix + // Calculating the final QED corrections by mutliplying the Omega vector with the inverse matrix const amrex::Real dEx = (-1._rt/detA)*(invAx[0]*Omega[0] + invAx[1]*Omega[1] + @@ -346,7 +346,7 @@ const amrex::Real dyi = 1._rt/dy; invAz[1]*Omega[1] + invAz[2]*Omega[2]); - // Adding the QED corrections to the origional fields + // Adding the QED corrections to the original fields Ex(j,k,0) = Ex(j,k,0) + 0.5_rt*dt*dEx; diff --git a/Source/Filter/BilinearFilter.cpp b/Source/Filter/BilinearFilter.cpp index c8e921aad31..a095bce6ae3 100644 --- a/Source/Filter/BilinearFilter.cpp +++ b/Source/Filter/BilinearFilter.cpp @@ -35,7 +35,7 @@ namespace { for(int ipass=1; ipass< lastpass; ipass++){ // element 0 has to be treated in its own way new_s.at(0) = 0.5_rt * old_s.at(0); - if (1(el); + } stencil_length_each_dir += 1.; #if defined(WARPX_DIM_3D) // npass_each_dir = npass_x npass_y npass_z diff --git a/Source/Filter/Filter.cpp b/Source/Filter/Filter.cpp index b6d5c503953..5952ffde59c 100644 --- a/Source/Filter/Filter.cpp +++ b/Source/Filter/Filter.cpp @@ -47,7 +47,7 @@ Filter::ApplyStencil (MultiFab& dstmf, const MultiFab& srcmf, const int lev, int { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); const auto& src = srcmf.array(mfi); const auto& dst = dstmf.array(mfi); @@ -59,7 +59,7 @@ Filter::ApplyStencil (MultiFab& dstmf, const MultiFab& srcmf, const int lev, int if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -218,7 +218,7 @@ Filter::ApplyStencil (amrex::MultiFab& dstmf, const amrex::MultiFab& srcmf, cons { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); const auto& srcfab = srcmf[mfi]; auto& dstfab = dstmf[mfi]; @@ -236,7 +236,7 @@ Filter::ApplyStencil (amrex::MultiFab& dstmf, const amrex::MultiFab& srcmf, cons if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } diff --git a/Source/Filter/NCIGodfreyFilter.H b/Source/Filter/NCIGodfreyFilter.H index 18b93189d76..91479cecd0d 100644 --- a/Source/Filter/NCIGodfreyFilter.H +++ b/Source/Filter/NCIGodfreyFilter.H @@ -16,7 +16,7 @@ enum class godfrey_coeff_set { Ex_Ey_Bz=0, Bx_By_Ez=1 }; /** - * \brief Class for Godrey's filter to suppress Numerical Cherenkov Instability + * \brief Class for Godfrey's filter to suppress Numerical Cherenkov Instability * * It derives from the base class Filter. * The filter stencil is initialized in method ComputeStencils. Computing the @@ -39,6 +39,8 @@ public: private: +//NCIGodfreyFilter not implemented in 1D +#if (AMREX_SPACEDIM >= 2) // Set of coefficients (different fields require to read // different coefficients from the table) godfrey_coeff_set m_coeff_set; @@ -46,6 +48,8 @@ private: amrex::Real m_cdtodz; // Whether the gather is from nodal fields or staggered fields bool m_nodal_gather; +#endif + }; #endif // #ifndef WARPX_GODFREY_FILTER_H_ diff --git a/Source/Filter/NCIGodfreyFilter.cpp b/Source/Filter/NCIGodfreyFilter.cpp index ff866332521..9567bdf1bb2 100644 --- a/Source/Filter/NCIGodfreyFilter.cpp +++ b/Source/Filter/NCIGodfreyFilter.cpp @@ -23,39 +23,36 @@ using namespace amrex; -NCIGodfreyFilter::NCIGodfreyFilter(godfrey_coeff_set coeff_set, amrex::Real cdtodz, bool nodal_gather){ - // Store parameters into class data members - m_coeff_set = coeff_set; - m_cdtodz = cdtodz; - m_nodal_gather = nodal_gather; +//NCIGodfreyFilter not implemented in 1D +#if (AMREX_SPACEDIM >= 2) +NCIGodfreyFilter::NCIGodfreyFilter(godfrey_coeff_set coeff_set, amrex::Real cdtodz, bool nodal_gather): + m_coeff_set{coeff_set}, // Store parameters into class data members + m_cdtodz{cdtodz}, + m_nodal_gather{nodal_gather} +{ // NCI Godfrey filter has fixed size, and is applied along z only. -#if defined(WARPX_DIM_3D) +# if defined(WARPX_DIM_3D) stencil_length_each_dir = {1,1,5}; slen = {1,1,5}; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +# else stencil_length_each_dir = {1,5}; slen = {1,5,1}; -#else - amrex::ignore_unused(coeff_set, cdtodz, nodal_gather); - WARPX_ABORT_WITH_MESSAGE( - "NCIGodfreyFilter not implemented in 1D!"); -#endif +# endif } -void NCIGodfreyFilter::ComputeStencils(){ - -#if (AMREX_SPACEDIM >= 2) +void NCIGodfreyFilter::ComputeStencils() +{ using namespace warpx::nci_godfrey; - // Sanity checks: filter length shoulz be 5 in z -#if defined(WARPX_DIM_3D) + // Sanity checks: filter length should be 5 in z +# if defined(WARPX_DIM_3D) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( slen.z==5,"ERROR: NCI filter requires 5 points in z"); -#else +# else WARPX_ALWAYS_ASSERT_WITH_MESSAGE( slen.y==5,"ERROR: NCI filter requires 5 points in z"); -#endif +# endif // Interpolate coefficients from the table, and store into prestencil. auto index = static_cast(tab_length*m_cdtodz); index = min(index, tab_length-2); @@ -111,33 +108,46 @@ void NCIGodfreyFilter::ComputeStencils(){ // so only 1 coeff, equal to 1) Vector h_stencil_x(1); h_stencil_x[0] = 1._rt; -#if defined(WARPX_DIM_3D) +# if defined(WARPX_DIM_3D) Vector h_stencil_y(1); h_stencil_y[0] = 1._rt; -#endif +# endif // Due to the way Filter::DoFilter() is written, // coefficient 0 has to be /2 h_stencil_x[0] /= 2._rt; -#if defined(WARPX_DIM_3D) +# if defined(WARPX_DIM_3D) h_stencil_y[0] /= 2._rt; -#endif +# endif h_stencil_z[0] /= 2._rt; stencil_x.resize(h_stencil_x.size()); -#if defined(WARPX_DIM_3D) +# if defined(WARPX_DIM_3D) stencil_y.resize(h_stencil_y.size()); -#endif +# endif stencil_z.resize(h_stencil_z.size()); Gpu::copyAsync(Gpu::hostToDevice,h_stencil_x.begin(),h_stencil_x.end(),stencil_x.begin()); -#if defined(WARPX_DIM_3D) +# if defined(WARPX_DIM_3D) Gpu::copyAsync(Gpu::hostToDevice,h_stencil_y.begin(),h_stencil_y.end(),stencil_y.begin()); -#endif +# endif Gpu::copyAsync(Gpu::hostToDevice,h_stencil_z.begin(),h_stencil_z.end(),stencil_z.begin()); Gpu::synchronize(); +} + #else - WARPX_ABORT_WITH_MESSAGE("NCIGodfreyFilter not implemented in 1D!"); -#endif + +NCIGodfreyFilter::NCIGodfreyFilter(godfrey_coeff_set, amrex::Real, bool) +{ + WARPX_ABORT_WITH_MESSAGE( + "NCIGodfreyFilter not implemented in 1D!"); } + +void NCIGodfreyFilter::ComputeStencils() +{ + WARPX_ABORT_WITH_MESSAGE( + "NCIGodfreyFilter not implemented in 1D!"); +} + +#endif diff --git a/Source/Fluids/CMakeLists.txt b/Source/Fluids/CMakeLists.txt new file mode 100644 index 00000000000..a5f28debbbd --- /dev/null +++ b/Source/Fluids/CMakeLists.txt @@ -0,0 +1,8 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + MultiFluidContainer.cpp + WarpXFluidContainer.cpp + ) +endforeach() diff --git a/Source/Fluids/Make.package b/Source/Fluids/Make.package new file mode 100644 index 00000000000..96bce415c4a --- /dev/null +++ b/Source/Fluids/Make.package @@ -0,0 +1,4 @@ +CEXE_sources += MultiFluidContainer.cpp +CEXE_sources += WarpXFluidContainer.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/Fluids diff --git a/Source/Fluids/MultiFluidContainer.H b/Source/Fluids/MultiFluidContainer.H new file mode 100644 index 00000000000..23f0c46590b --- /dev/null +++ b/Source/Fluids/MultiFluidContainer.H @@ -0,0 +1,83 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_MultiFluidContainer_H_ +#define WARPX_MultiFluidContainer_H_ +#include "Evolve/WarpXDtType.H" + +#include "WarpXFluidContainer_fwd.H" + +#include +#include + +#include + +/** + * The class MultiFluidContainer holds multiple instances of the + * class WarpXFluidContainer, stored in its member variable "allcontainers". + * The class WarpX typically has a single (pointer to an) instance of + * MultiFluidContainer. + * + * MultiFluidContainer typically has two types of functions: + * - Functions that loop over all instances of WarpXFluidContainer in + * allcontainers and calls the corresponding function (for instance, + * MultiFluidContainer::Evolve loops over all fluid containers and + * calls the corresponding WarpXFluidContainer::Evolve function). + * - Functions that specifically handle multiple species (for instance + * ReadParameters). + */ +class MultiFluidContainer +{ + +public: + + MultiFluidContainer (int nlevs_max); + + ~MultiFluidContainer() = default; + + MultiFluidContainer (MultiFluidContainer const &) = delete; + MultiFluidContainer& operator= (MultiFluidContainer const & ) = delete; + MultiFluidContainer(MultiFluidContainer&& ) = default; + MultiFluidContainer& operator=(MultiFluidContainer&& ) = default; + + [[nodiscard]] WarpXFluidContainer& + GetFluidContainer (int ispecies) const {return *allcontainers[ispecies];} + +#ifdef WARPX_USE_OPENPMD + std::unique_ptr& GetUniqueContainer(int ispecies) { + return allcontainers[ispecies]; + } +#endif + + void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); + + void InitData (int lev, amrex::Box init_box, amrex::Real cur_time); + + /// + /// This evolves all the fluids by one PIC time step, including current deposition, the + /// field solve, and pushing the fluids, for all the species in the MultiFluidContainer. + /// + void Evolve (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + amrex::MultiFab* rho, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, + amrex::Real cur_time, bool skip_deposition=false); + + [[nodiscard]] int nSpecies() const {return static_cast(species_names.size());} + + void DepositCharge (int lev, amrex::MultiFab &rho); + void DepositCurrent (int lev, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz); + +private: + + std::vector species_names; + + // Vector of fluid species + amrex::Vector> allcontainers; + +}; +#endif /*WARPX_MultiFluidContainer_H_*/ diff --git a/Source/Fluids/MultiFluidContainer.cpp b/Source/Fluids/MultiFluidContainer.cpp new file mode 100644 index 00000000000..234cefb4f07 --- /dev/null +++ b/Source/Fluids/MultiFluidContainer.cpp @@ -0,0 +1,73 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" +#include "Utils/Parser/ParserUtils.H" + +#include + +using namespace amrex; + +MultiFluidContainer::MultiFluidContainer (int nlevs_max) +{ + const ParmParse pp_fluids("fluids"); + pp_fluids.queryarr("species_names", species_names); + + const int nspecies = static_cast(species_names.size()); + + allcontainers.resize(nspecies); + for (int i = 0; i < nspecies; ++i) { + allcontainers[i] = std::make_unique(nlevs_max, i, species_names[i]); + } +} + +void +MultiFluidContainer::AllocateLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm) +{ + for (auto& fl : allcontainers) { + fl->AllocateLevelMFs(lev, ba, dm); + } +} + +void +MultiFluidContainer::InitData (int lev, amrex::Box init_box, amrex::Real cur_time) +{ + for (auto& fl : allcontainers) { + fl->InitData(lev, init_box, cur_time); + } +} + + +void +MultiFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho) +{ + for (auto& fl : allcontainers) { + fl->DepositCharge(lev,rho); + } +} + +void +MultiFluidContainer::DepositCurrent (int lev, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz) +{ + for (auto& fl : allcontainers) { + fl->DepositCurrent(lev,jx,jy,jz); + } +} + +void +MultiFluidContainer::Evolve (int lev, + const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, + const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, + MultiFab* rho, MultiFab& jx, MultiFab& jy, MultiFab& jz, + amrex::Real cur_time, bool skip_deposition) +{ + for (auto& fl : allcontainers) { + fl->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, rho, jx, jy, jz, cur_time, skip_deposition); + } +} diff --git a/Source/Fluids/MultiFluidContainer_fwd.H b/Source/Fluids/MultiFluidContainer_fwd.H new file mode 100644 index 00000000000..a1a55c3b450 --- /dev/null +++ b/Source/Fluids/MultiFluidContainer_fwd.H @@ -0,0 +1,12 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_MultiFluidContainer_fwd_H_ + +class MultiFluidContainer; + +#endif /* WARPX_MultiFluidContainer_fwd_H_ */ diff --git a/Source/Fluids/MusclHancockUtils.H b/Source/Fluids/MusclHancockUtils.H new file mode 100644 index 00000000000..2765f28f3b3 --- /dev/null +++ b/Source/Fluids/MusclHancockUtils.H @@ -0,0 +1,519 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_MusclHancock_H_ +#define WARPX_MusclHancock_H_ + +#include +#include +#include +#include + + +// Euler push for momentum source (r-direction) +// Note: assumes U normalized by c +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real F_r (amrex::Real r, amrex::Real u_r, amrex::Real u_theta, amrex::Real u_z, amrex::Real dt) +{ + using namespace amrex::literals; + return dt*(-u_theta*u_theta/r)/std::sqrt(1.0_rt + u_r*u_r + u_theta*u_theta + u_z*u_z) + u_r; +} + +// Euler push for momentum source (theta-direction) +// Note: assumes U normalized by c +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real F_theta (amrex::Real r, amrex::Real u_r, amrex::Real u_theta, amrex::Real u_z, amrex::Real dt) +{ + using namespace amrex::literals; + return dt*(u_theta*u_r/r)/std::sqrt(1.0_rt + u_r*u_r + u_theta*u_theta + u_z*u_z) + u_theta; +} +// Velocity at the half step +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real V_calc (const amrex::Array4& U, int i, int j, int k, int comp, amrex::Real c) +{ + using namespace amrex::literals; + // comp -> x, y, z -> 0, 1, 2, return Vx, Vy, or Vz: + amrex::Real gamma = std::sqrt(1.0_rt + (U(i,j,k,1)*U(i,j,k,1) + U(i,j,k,2)*U(i,j,k,2) + U(i,j,k,3)*U(i,j,k,3))/(c*c)); + return U(i,j,k,comp+1)/gamma; +} +// mindmod +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real minmod (amrex::Real a, amrex::Real b) +{ + using namespace amrex::literals; + if (a > 0.0_rt && b > 0.0_rt) { + return std::min(a, b); + } else if (a < 0.0_rt && b < 0.0_rt) { + return std::max(a, b); + } else { + return 0.0_rt; + } +} +// Min of 3 inputs +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real min3 (amrex::Real a, amrex::Real b, amrex::Real c) +{ + return std::min(a, std::min(b, c) ); +} +// Max of 3 inputs +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real max3 (amrex::Real a, amrex::Real b, amrex::Real c) +{ + return std::max(a, std::max(b, c) ); +} +// mindmod +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real minmod3 (amrex::Real a, amrex::Real b , amrex::Real c) +{ + using namespace amrex::literals; + if (a > 0.0_rt && b > 0.0_rt && c > 0.0_rt) { + return min3(a,b,c); + } else if (a < 0.0_rt && b < 0.0_rt && c < 0.0_rt) { + return max3(a,b,c); + } else { + return 0.0; + } +} +//maxmod +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real maxmod (amrex::Real a, amrex::Real b) +{ + using namespace amrex::literals; + if (a > 0.0_rt && b > 0.0_rt) { + return std::max(a, b); + } else if (a < 0.0_rt && b < 0.0_rt) { + return std::min(a, b); + } else { + return 0.0_rt; + } +} +// Rusanov Flux (density) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real flux_N (const amrex::Array4& Um, const amrex::Array4& Up, +int i, int j, int k, amrex::Real Vm, amrex::Real Vp) +{ + using namespace amrex::literals; + amrex::Real c = std::max( std::abs(Vm) , std::abs(Vp) ); + return 0.5_rt*(Vm*Um(i,j,k,0) + Vp*Up(i,j,k,0)) - (0.5_rt*c)*(Up(i,j,k,0) - Um(i,j,k,0)); +} +// Rusanov Flux (Momentum density x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real flux_NUx (const amrex::Array4& Um, const amrex::Array4& Up, +int i, int j, int k, amrex::Real Vm, amrex::Real Vp) +{ + using namespace amrex::literals; + amrex::Real c = std::max( std::abs(Vm) , std::abs(Vp) ); + return 0.5_rt*(Vm*Um(i,j,k,0)*Um(i,j,k,1) + Vp*Up(i,j,k,0)*Up(i,j,k,1)) + - (0.5_rt*c)*(Up(i,j,k,0)*Up(i,j,k,1) - Um(i,j,k,0)*Um(i,j,k,1)); +} +// Rusanov Flux (Momentum density y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real flux_NUy (const amrex::Array4& Um, const amrex::Array4& Up, +int i, int j, int k, amrex::Real Vm, amrex::Real Vp) +{ + using namespace amrex::literals; + amrex::Real c = std::max( std::abs(Vm) , std::abs(Vp) ); + return 0.5_rt*(Vm*Um(i,j,k,0)*Um(i,j,k,2) + Vp*Up(i,j,k,0)*Up(i,j,k,2)) + - (0.5_rt*c)*(Up(i,j,k,0)*Up(i,j,k,2) - Um(i,j,k,0)*Um(i,j,k,2)); +} +// Rusanov Flux (Momentum density z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real flux_NUz (const amrex::Array4& Um, const amrex::Array4& Up, +int i, int j, int k, amrex::Real Vm, amrex::Real Vp) +{ + using namespace amrex::literals; + amrex::Real c = std::max( std::abs(Vm) , std::abs(Vp) ); + return 0.5_rt*(Vm*Um(i,j,k,0)*Um(i,j,k,3) + Vp*Up(i,j,k,0)*Up(i,j,k,3)) + - (0.5_rt*c)*(Up(i,j,k,0)*Up(i,j,k,3) - Um(i,j,k,0)*Um(i,j,k,3)); +} +// ave_minmod high diffusivity, sigma can be between [1,2] +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real ave_adjustable_diff (amrex::Real a, amrex::Real b) +{ + using namespace amrex::literals; + constexpr auto sigma = static_cast(2.0*0.732050807568877); + if (a*b > 0.0_rt) { + return minmod3( (a+b)/2.0_rt, sigma*a, sigma*b ); + } else { + return 0.0_rt; + } +} +// ave_minmod Low diffusivity +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real ave (amrex::Real a, amrex::Real b) +{ + using namespace amrex::literals; + if (a*b > 0.0_rt) { + return minmod3( (a+b)/2.0_rt, 2.0_rt*a, 2.0_rt*b ); + } else { + return 0.0_rt; + } +} +// ave_superbee +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real ave_superbee (amrex::Real a, amrex::Real b) +{ + using namespace amrex::literals; + if (a*b > 0.0_rt) { + return minmod( maxmod(a,b), minmod(2.0_rt*a,2.0_rt*b)); + } else { + return 0.0_rt; + } +} +// stage2 slope limiting +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real ave_stage2 (amrex::Real dQ, amrex::Real a, amrex::Real b, amrex::Real c) +{ + using namespace amrex::literals; + // sigma = sqrt(3) -1 + constexpr auto sigma = 0.732050807568877_rt; + amrex::Real dq_min = 2.0_rt*std::min( b - min3(a,b,c), max3(a,b,c) - b); + return ( std::abs(dQ)/dQ ) * std::min( std::abs(dQ) , sigma*std::abs(dq_min) ); +} +// Returns the offset indices for the "plus" grid +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void plus_index_offsets (int i, int j, int k, int& ip, int& jp, int& kp, int comp) +{ + using namespace amrex::literals; + // Find the correct offsets +#if defined(WARPX_DIM_3D) + if (comp == 0) { //x + ip = i - 1; jp = j; kp = k; + } else if (comp == 1){ //y + ip = i; jp = j - 1; kp = k; + } else if (comp == 2){ //z + ip = i; jp = j; kp = k - 1; + } + #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + if (comp == 0) { //x + ip = i - 1; jp = j; kp = k; + } else if (comp == 2){ //z + ip = i; jp = j - 1; kp = k; + } +#else + if (comp == 2) { //z + ip = i - 1; jp = j; kp = k; + } +#endif +} +// Compute the zero edges +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void compute_U_edges (const amrex::Array4& Um, const amrex::Array4& Up, int i, int j, int k, amrex::Box box, +amrex::Real U_tilde0, amrex::Real U_tilde1, amrex::Real U_tilde2, amrex::Real U_tilde3, +amrex::Real dU0x, amrex::Real dU1x, amrex::Real dU2x, amrex::Real dU3x, int comp) +{ + using namespace amrex::literals; + // comp -> x, y, z -> 0, 1, 2 + int ip, jp, kp; + plus_index_offsets(i, j, k, ip, jp, kp, comp); + + if ( box.contains(i,j,k) ) { + Um(i,j,k,0) = U_tilde0 + dU0x/2.0_rt; + Um(i,j,k,1) = U_tilde1 + dU1x/2.0_rt; + Um(i,j,k,2) = U_tilde2 + dU2x/2.0_rt; + Um(i,j,k,3) = U_tilde3 + dU3x/2.0_rt; + } + + if ( box.contains(ip,jp,kp) ) { + Up(ip,jp,kp,0) = U_tilde0 - dU0x/2.0_rt; + Up(ip,jp,kp,1) = U_tilde1 - dU1x/2.0_rt; + Up(ip,jp,kp,2) = U_tilde2 - dU2x/2.0_rt; + Up(ip,jp,kp,3) = U_tilde3 - dU3x/2.0_rt; + } +} +// Compute the zero edges +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void set_U_edges_to_zero (const amrex::Array4& Um, +const amrex::Array4& Up, int i, int j, int k, amrex::Box box, int comp) +{ + using namespace amrex::literals; + // comp -> x, y, z -> 0, 1, 2 + int ip, jp, kp; + plus_index_offsets(i, j, k, ip, jp, kp, comp); + + if ( box.contains(i,j,k) ) { + Um(i,j,k,0) = 0.0_rt; + Um(i,j,k,1) = 0.0_rt; + Um(i,j,k,2) = 0.0_rt; + Um(i,j,k,3) = 0.0_rt; + } + + if ( box.contains(ip,jp,kp) ) { + Up(ip,jp,kp,0) = 0.0_rt; + Up(ip,jp,kp,1) = 0.0_rt; + Up(ip,jp,kp,2) = 0.0_rt; + Up(ip,jp,kp,3) = 0.0_rt; + } +} +// Positivity Limiter +// if Q_minus or Q_plus is zero for the density (i.e. component 0 of Q_minus/Q_plus), set dQ to 0 and recompute Q_minus / Q_plus +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void positivity_limiter (const amrex::Array4& U_edge_plus, +const amrex::Array4& U_edge_minus, const amrex::Array4& N_arr, +int i, int j, int k, amrex::Box box, amrex::Real Ux, amrex::Real Uy, amrex::Real Uz, +int comp) +{ + + using namespace amrex::literals; + // comp -> x, y, z -> 0, 1, 2 + int ip, jp, kp; + plus_index_offsets(i, j, k, ip, jp, kp, comp); + + // Set the edges to zero. If one edge in a cell is zero, we must self-consistently + // set the slope to zero (hence why we have the three cases, the first is when + // both points exist, and the second two are are edge cases) + if (( box.contains(i,j,k) ) && ( box.contains(ip,jp,kp) )) { + if ((U_edge_minus(i,j,k,0) < 0.0_rt) || (U_edge_plus(ip,jp,kp,0) < 0.0_rt)) { + U_edge_minus(i,j,k,0) = N_arr(i,j,k); + U_edge_minus(i,j,k,1) = Ux; + U_edge_minus(i,j,k,2) = Uy; + U_edge_minus(i,j,k,3) = Uz; + U_edge_plus(ip,jp,kp,0) = N_arr(i,j,k); + U_edge_plus(ip,jp,kp,1) = Ux; + U_edge_plus(ip,jp,kp,2) = Uy; + U_edge_plus(ip,jp,kp,3) = Uz; + } + } else if (( box.contains(i,j,k) ) && ( box.contains(ip,jp,kp) != 1)) { + if (U_edge_minus(i,j,k,0) < 0.0_rt) { + U_edge_minus(i,j,k,0) = N_arr(i,j,k); + U_edge_minus(i,j,k,1) = Ux; + U_edge_minus(i,j,k,2) = Uy; + U_edge_minus(i,j,k,3) = Uz; + } + } else if (( box.contains(i,j,k) != 1 ) && ( box.contains(ip,jp,kp) )) { + if (U_edge_plus(ip,jp,kp,0) < 0.0_rt){ + U_edge_plus(ip,jp,kp,0) = N_arr(i,j,k); + U_edge_plus(ip,jp,kp,1) = Ux; + U_edge_plus(ip,jp,kp,2) = Uy; + U_edge_plus(ip,jp,kp,3) = Uz; + } + } +} + +// Compute the difference in N (down-x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDx_N (const amrex::Array4& N, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + return N(i,j,k) - N(i-1,j,k); +#else + amrex::ignore_unused(N, i, j, k); + return 0.0_rt; +#endif +} +// Compute the difference in N (up-x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDx_N (const amrex::Array4& N, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + return N(i+1,j,k) - N(i,j,k); +#else + amrex::ignore_unused(N, i, j, k); + return 0.0_rt; +#endif +} +// Compute the difference in N (down-y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDy_N (const amrex::Array4& N, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) + return N(i,j,k) - N(i,j-1,k); +#else + amrex::ignore_unused(N, i, j, k); + return 0.0_rt; +#endif +} +// Compute the difference in N (up-y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDy_N (const amrex::Array4& N, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) + return N(i,j+1,k) - N(i,j,k); +#else + amrex::ignore_unused(N, i, j, k); + return 0.0_rt; +#endif +} +// Compute the difference in N (down-z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDz_N (const amrex::Array4& N, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) + return N(i,j,k) - N(i,j,k-1); +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + return N(i,j,k) - N(i,j-1,k); +#else + return N(i,j,k) - N(i-1,j,k); +#endif +} +// Compute the difference in N (up-z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDz_N (const amrex::Array4& N, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) + return N(i,j,k+1) - N(i,j,k); +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + return N(i,j+1,k) - N(i,j,k); +#else + return N(i+1,j,k) - N(i,j,k); +#endif +} + + +// Compute the difference in U (down-x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDx_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + // U is zero if N is zero, Check positivity before dividing + amrex::Real U_m = 0; + if (N(i-1,j,k) > 0) { U_m = NU(i-1,j,k)/N(i-1,j,k); } + return U - U_m; +#else + amrex::ignore_unused(N, NU, U, i, j, k); + return 0.0_rt; +#endif +} +// Compute the difference in U (up-x) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDx_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + // U is zero if N is zero, Check positivity before dividing + amrex::Real U_p = 0; + if (N(i+1,j,k) > 0) { U_p = NU(i+1,j,k)/N(i+1,j,k); } + return U_p - U; +#else + amrex::ignore_unused(N, NU, U, i, j, k); + return 0.0_rt; +#endif +} + +// Compute the difference in U (down-y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDy_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) + // U is zero if N is zero, Check positivity before dividing + amrex::Real U_m = 0; + if (N(i,j-1,k) > 0) { U_m = NU(i,j-1,k)/N(i,j-1,k); } + return U - U_m; +#else + amrex::ignore_unused(N, NU, U, i, j, k); + return 0.0_rt; +#endif +} +// Compute the difference in U (up-y) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDy_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences +#if defined(WARPX_DIM_3D) + // U is zero if N is zero, Check positivity before dividing + amrex::Real U_p = 0; + if (N(i,j+1,k) > 0) { U_p = NU(i,j+1,k)/N(i,j+1,k); } + return U_p - U; +#else + amrex::ignore_unused(N, NU, U, i, j, k); + return 0.0_rt; +#endif +} + +// Compute the difference in U (down-z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real DownDz_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences + amrex::Real U_m = 0_rt; + + // U is zero if N is zero, Check positivity before dividing +#if defined(WARPX_DIM_3D) + if (N(i,j,k-1) > 0) { U_m = NU(i,j,k-1)/N(i,j,k-1); } +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + if (N(i,j-1,k) > 0) { U_m = NU(i,j-1,k)/N(i,j-1,k); } +#else + if (N(i-1,j,k) > 0) { U_m = NU(i-1,j,k)/N(i-1,j,k); } +#endif + + // Return the difference + return U - U_m; +} +// Compute the difference in U (up-z) +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real UpDz_U (const amrex::Array4& N, +const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) +{ + using namespace amrex::literals; + // Write the correct differences + amrex::Real U_p = 0; + + // U is zero if N is zero, Check positivity before dividing +#if defined(WARPX_DIM_3D) + if (N(i,j,k+1) > 0) { U_p = NU(i,j,k+1)/N(i,j,k+1); } +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + if (N(i,j+1,k) > 0) { U_p = NU(i,j+1,k)/N(i,j+1,k); } +#else + if (N(i+1,j,k) > 0) { U_p = NU(i+1,j,k)/N(i+1,j,k); } +#endif + + // Return the difference + return U_p - U; +} + + +// Flux difference calculation +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +amrex::Real dF (const amrex::Array4& U_minus, +const amrex::Array4& U_plus,int i,int j,int k,amrex::Real clight, int comp, int dir) +{ + using namespace amrex::literals; + // dir -> x, y, z -> 0, 1, 2 + int ip, jp, kp; + plus_index_offsets(i, j, k, ip, jp, kp, dir); + + amrex::Real V_L_minus = V_calc(U_minus,ip,jp,kp,dir,clight); + amrex::Real V_I_minus = V_calc(U_minus,i,j,k,dir,clight); + amrex::Real V_L_plus = V_calc(U_plus,ip,jp,kp,dir,clight); + amrex::Real V_I_plus = V_calc(U_plus,i,j,k,dir,clight); + + // Flux differences depending on the component to compute + if (comp == 0){ + return flux_N( U_minus, U_plus, i, j, k, V_I_minus, V_I_plus) - flux_N( U_minus, U_plus, ip, jp, kp, V_L_minus, V_L_plus); + } else if (comp == 1){ + return flux_NUx( U_minus, U_plus, i, j, k, V_I_minus, V_I_plus) - flux_NUx( U_minus, U_plus, ip, jp, kp, V_L_minus, V_L_plus); + } else if (comp == 2){ + return flux_NUy( U_minus, U_plus, i, j, k, V_I_minus, V_I_plus) - flux_NUy( U_minus, U_plus, ip, jp, kp, V_L_minus, V_L_plus); + } else { //if (comp == 3) + return flux_NUz( U_minus, U_plus, i, j, k, V_I_minus, V_I_plus) - flux_NUz( U_minus, U_plus, ip, jp, kp, V_L_minus, V_L_plus); + } +} + +#endif /*WARPX_MusclHancock_H_*/ diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H new file mode 100644 index 00000000000..04ec4d9e80d --- /dev/null +++ b/Source/Fluids/WarpXFluidContainer.H @@ -0,0 +1,194 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_WarpXFluidContainer_H_ +#define WARPX_WarpXFluidContainer_H_ + +#include "Evolve/WarpXDtType.H" +#include "Initialization/PlasmaInjector.H" +#include "MultiFluidContainer.H" + +#include +#include + +#include + + +/** + * WarpXFluidContainer is the base class from which all concrete + * fluid container classes derive. + * + * WarpXFluidContainer contains the main functions for initialization, + * interaction with the grid (field gather and current deposition), fluid + * source and push, advective update and updates for non-inertial terms. + */ +class WarpXFluidContainer +{ +public: + friend MultiFluidContainer; + + WarpXFluidContainer (int nlevs_max, int ispecies, const std::string& name); + ~WarpXFluidContainer() = default; + + WarpXFluidContainer (WarpXFluidContainer const &) = delete; + WarpXFluidContainer& operator= (WarpXFluidContainer const & ) = delete; + WarpXFluidContainer(WarpXFluidContainer&& ) = default; + WarpXFluidContainer& operator=(WarpXFluidContainer&& ) = default; + + void AllocateLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); + + void InitData (int lev, amrex::Box init_box, amrex::Real cur_time); + + void ReadParameters (); + + /** + * Evolve updates a single timestep (dt) of the cold relativistic fluid equations + */ + void Evolve (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + amrex::MultiFab* rho, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, + amrex::Real cur_time, bool skip_deposition=false); + + /** + * AdvectivePush_Muscl takes a single timestep (dt) of the cold relativistic fluid equations + * using a Muscl-Handcock scheme + * + * \brief Advective term, cold-rel. fluids + * + * \param[in] lev refinement level + */ + void AdvectivePush_Muscl (int lev); + + + /** + * Apply (non-periodic) BC on the fluids (needed for spatial derivative), + * and communicate N, NU at boundaries + * + * \brief Apply non-periodic BC to fluids and communicate boundaries + * + * \param[in] lev refinement level + */ + void ApplyBcFluidsAndComms (int lev); + +#if defined(WARPX_DIM_RZ) + /** + * centrifugal_source_rz adds contributions due to curvature acceleration for a + * single timestep using an SSP-RK3 timestep for RZ specifically + * + * \brief Centrifugal source term + * + * \param[in] lev refinement level + */ + void centrifugal_source_rz (int lev); +#endif + + /** + * GatherAndPush introduces the Lorentz term in the cold relativistic fluid + * equations for a single timestep (dt) using Higuera and Cary Push + * + * \brief Lorentz Momentum Source + * + * \param[in] lev refinement level + * \param[in] Ex Yee electric field (x) + * \param[in] Ey Yee electric field (y) + * \param[in] Ez Yee electric field (z) + * \param[in] Bx Yee magnetic field (x) + * \param[in] By Yee magnetic field (y) + * \param[in] Bz Yee magnetic field (z) + * \param[in] t Current time + */ + void GatherAndPush (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + amrex::Real t); + + /** + * DepositCurrent interpolates the fluid current density comps. onto the Yee grid and + * sums the contributions to the particle current density + * + * \brief Deposit fluid current density. + * + * \param[in] lev refinement level + * \param[in,out] jx current density MultiFab x comp. + * \param[in,out] jy current density MultiFab y comp. + * \param[in,out] jz current density MultiFab z comp. + */ + void DepositCurrent (int lev, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz); + + /** + * DepositCharge interpolates the fluid charge density onto the Yee grid and + * sums the contributions to the particle charge density + * + * \brief Deposit fluid charge density. + * + * \param[in] lev refinement level + * \param[in,out] rho charge density MultiFab. + */ + void DepositCharge (int lev, amrex::MultiFab &rho, int icomp = 0); + + [[nodiscard]] amrex::Real getCharge () const {return charge;} + [[nodiscard]] amrex::Real getMass () const {return mass;} + +protected: + int species_id; + std::string species_name; + amrex::Real charge; + amrex::Real mass; + + int do_not_push = 0; + int do_not_gather = 0; + int do_not_deposit = 0; + PhysicalSpecies physical_species; + + // Parser for external fields + std::string m_B_ext_s = "none"; + std::string m_E_ext_s = "none"; + + // Parser for B_external on the particle + std::unique_ptr m_Bx_parser; + std::unique_ptr m_By_parser; + std::unique_ptr m_Bz_parser; + amrex::ParserExecutor<4> m_Bxfield_parser; + amrex::ParserExecutor<4> m_Byfield_parser; + amrex::ParserExecutor<4> m_Bzfield_parser; + + // Parser for E_external on the particle + std::unique_ptr m_Ex_parser; + std::unique_ptr m_Ey_parser; + std::unique_ptr m_Ez_parser; + amrex::ParserExecutor<4> m_Exfield_parser; + amrex::ParserExecutor<4> m_Eyfield_parser; + amrex::ParserExecutor<4> m_Ezfield_parser; + + std::unique_ptr h_inj_rho; + InjectorDensity* d_inj_rho = nullptr; + std::unique_ptr density_parser; + + std::unique_ptr h_inj_mom; + InjectorMomentum* d_inj_mom = nullptr; + std::unique_ptr ux_parser; + std::unique_ptr uy_parser; + std::unique_ptr uz_parser; + std::unique_ptr ux_th_parser; + std::unique_ptr uy_th_parser; + std::unique_ptr uz_th_parser; + + // Keep a pointer to TemperatureProperties to ensure the lifetime of the + // contained Parser + std::unique_ptr h_mom_temp; + std::unique_ptr h_mom_vel; + +public: + + // MultiFabs that contain the density (N) and momentum density (NU) of this fluid species, for each refinement level + amrex::Vector< std::unique_ptr > N; + amrex::Vector, 3 > > NU; + +}; + +#endif diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp new file mode 100644 index 00000000000..1915e4bf772 --- /dev/null +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -0,0 +1,1354 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "ablastr/coarsen/sample.H" +#include "Particles/Pusher/UpdateMomentumHigueraCary.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include "MusclHancockUtils.H" +#include "Fluids/WarpXFluidContainer.H" +#include "WarpX.H" +#include +#include "Utils/Parser/ParserUtils.H" +#include "Utils/WarpXUtil.H" +#include "Utils/SpeciesUtils.H" + +using namespace ablastr::utils::communication; +using namespace amrex; + +WarpXFluidContainer::WarpXFluidContainer(int nlevs_max, int ispecies, const std::string &name): + species_id{ispecies}, + species_name{name} +{ + ReadParameters(); + + // Initialize injection objects + const ParmParse pp_species_name(species_name); + SpeciesUtils::parseDensity(species_name, "", h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, "", "none", h_inj_mom, + ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); + if (h_inj_rho) { +#ifdef AMREX_USE_GPU + d_inj_rho = static_cast + (amrex::The_Arena()->alloc(sizeof(InjectorDensity))); + amrex::Gpu::htod_memcpy_async(d_inj_rho, h_inj_rho.get(), sizeof(InjectorDensity)); +#else + d_inj_rho = h_inj_rho.get(); +#endif + } + if (h_inj_mom) { +#ifdef AMREX_USE_GPU + d_inj_mom = static_cast + (amrex::The_Arena()->alloc(sizeof(InjectorMomentum))); + amrex::Gpu::htod_memcpy_async(d_inj_mom, h_inj_mom.get(), sizeof(InjectorMomentum)); +#else + d_inj_mom = h_inj_mom.get(); +#endif + } + amrex::Gpu::synchronize(); + + // Resize the list of MultiFabs for the right number of levels + N.resize(nlevs_max); + NU.resize(nlevs_max); +} + +void WarpXFluidContainer::ReadParameters() +{ + + // Extract charge, mass, species type + std::string injection_style = "none"; + SpeciesUtils::extractSpeciesProperties(species_name, injection_style, charge, mass, physical_species); + + const ParmParse pp_species_name(species_name); + pp_species_name.query("do_not_deposit", do_not_deposit); + pp_species_name.query("do_not_gather", do_not_gather); + pp_species_name.query("do_not_push", do_not_push); + + // default values of E_external and B_external + // are used to set the E and B field when "constant" or "parser" + // is not explicitly used in the input + pp_species_name.query("B_ext_init_style", m_B_ext_s); + std::transform(m_B_ext_s.begin(), + m_B_ext_s.end(), + m_B_ext_s.begin(), + ::tolower); + pp_species_name.query("E_ext_init_style", m_E_ext_s); + std::transform(m_E_ext_s.begin(), + m_E_ext_s.end(), + m_E_ext_s.begin(), + ::tolower); + + // Parse external fields: + // if the input string for B_ext_s is + // "parse_b_ext_function" then the mathematical expression + // for the Bx_, By_, Bz_external_function(x,y,z) + // must be provided in the input file. + if (m_B_ext_s == "parse_b_ext_function") { + // store the mathematical expression as string + std::string str_Bx_ext_function; + std::string str_By_ext_function; + std::string str_Bz_ext_function; + utils::parser::Store_parserString( + pp_species_name, "Bx_external_function(x,y,z,t)", + str_Bx_ext_function); + utils::parser::Store_parserString( + pp_species_name, "By_external_function(x,y,z,t)", + str_By_ext_function); + utils::parser::Store_parserString( + pp_species_name, "Bz_external_function(x,y,z,t)", + str_Bz_ext_function); + + // Parser for B_external on the fluid + m_Bx_parser = std::make_unique( + utils::parser::makeParser(str_Bx_ext_function,{"x","y","z","t"})); + m_By_parser = std::make_unique( + utils::parser::makeParser(str_By_ext_function,{"x","y","z","t"})); + m_Bz_parser = std::make_unique( + utils::parser::makeParser(str_Bz_ext_function,{"x","y","z","t"})); + + } + + // if the input string for E_ext_s is + // "parse_e_ext_function" then the mathematical expression + // for the Ex_, Ey_, Ez_external_function(x,y,z) + // must be provided in the input file. + if (m_E_ext_s == "parse_e_ext_function") { + // store the mathematical expression as string + std::string str_Ex_ext_function; + std::string str_Ey_ext_function; + std::string str_Ez_ext_function; + utils::parser::Store_parserString( + pp_species_name, "Ex_external_function(x,y,z,t)", + str_Ex_ext_function); + utils::parser::Store_parserString( + pp_species_name, "Ey_external_function(x,y,z,t)", + str_Ey_ext_function); + utils::parser::Store_parserString( + pp_species_name, "Ez_external_function(x,y,z,t)", + str_Ez_ext_function); + // Parser for E_external on the fluid + m_Ex_parser = std::make_unique( + utils::parser::makeParser(str_Ex_ext_function,{"x","y","z","t"})); + m_Ey_parser = std::make_unique( + utils::parser::makeParser(str_Ey_ext_function,{"x","y","z","t"})); + m_Ez_parser = std::make_unique( + utils::parser::makeParser(str_Ez_ext_function,{"x","y","z","t"})); + } +} + +void WarpXFluidContainer::AllocateLevelMFs(int lev, const BoxArray &ba, const DistributionMapping &dm) +{ + int ncomps = 1; + const amrex::IntVect nguards(AMREX_D_DECL(2, 2, 2)); + + // set human-readable tag for each MultiFab + auto const tag = [lev](std::string tagname) + { + tagname.append("[l=").append(std::to_string(lev)).append("]"); + return tagname; + }; + + WarpX::AllocInitMultiFab(N[lev], amrex::convert(ba, amrex::IntVect::TheNodeVector()), + dm, ncomps, nguards, lev, tag("fluid density"), 0.0_rt); + + WarpX::AllocInitMultiFab(NU[lev][0], amrex::convert(ba, amrex::IntVect::TheNodeVector()), + dm, ncomps, nguards, lev, tag("fluid momentum density [x]"), 0.0_rt); + WarpX::AllocInitMultiFab(NU[lev][1], amrex::convert(ba, amrex::IntVect::TheNodeVector()), + dm, ncomps, nguards, lev, tag("fluid momentum density [y]"), 0.0_rt); + WarpX::AllocInitMultiFab(NU[lev][2], amrex::convert(ba, amrex::IntVect::TheNodeVector()), + dm, ncomps, nguards, lev, tag("fluid momentum density [z]"), 0.0_rt); +} + +void WarpXFluidContainer::InitData(int lev, amrex::Box init_box, amrex::Real cur_time) +{ + WARPX_PROFILE("WarpXFluidContainer::InitData"); + + // Convert initialization box to nodal box + init_box.surroundingNodes(); + + // Create local copies of pointers for GPU kernels + InjectorDensity* inj_rho = d_inj_rho; + InjectorMomentum* inj_mom = d_inj_mom; + + // Extract grid geometry properties + WarpX &warpx = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx.Geom(lev); + const auto dx = geom.CellSizeArray(); + const auto problo = geom.ProbLoArray(); + const amrex::Real clight = PhysConst::c; + const amrex::Real gamma_boost = WarpX::gamma_boost; + const amrex::Real beta_boost = WarpX::beta_boost; + + // Loop through cells and initialize their value +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + + // Return the intersection of all cells and the ones we wish to update + amrex::Box init_box_intersection = init_box & tile_box; + + amrex::ParallelFor(init_box_intersection, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { +#if defined(WARPX_DIM_3D) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = problo[1] + j * dx[1]; + amrex::Real z = problo[2] + k * dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[1] + j * dx[1]; +#else + amrex::Real x = 0.0_rt; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[0] + i * dx[0]; +#endif + + // Lorentz transform z (from boosted to lab frame) + if (gamma_boost > 1._rt){ + z = gamma_boost*(z + beta_boost*clight*cur_time); + } + + amrex::Real n = inj_rho->getDensity(x, y, z); + amrex::XDim3 u = inj_mom->getBulkMomentum(x, y, z); + + // Give u the right dimensions of m/s + u.x = u.x * clight; + u.y = u.y * clight; + u.z = u.z * clight; + + // Check if n > 0 and if not, don't compute the boost + // Lorentz transform n, u (from lab to boosted frame) + if (n > 0.0){ + if (gamma_boost > 1._rt){ + amrex::Real gamma = std::sqrt(1.0_rt + (u.x*u.x + u.y*u.y + u.z*u.z)/(clight*clight)); + amrex::Real n_boosted = gamma_boost*n*( 1.0_rt - beta_boost*u.z/(gamma*clight) ); + amrex::Real uz_boosted = gamma_boost*(u.z - beta_boost*clight*gamma); + u.z = uz_boosted; + n = n_boosted; + } + } + + // Multiply by clight so u is back in SI units + N_arr(i, j, k) = n; + NUx_arr(i, j, k) = n * u.x; + NUy_arr(i, j, k) = n * u.y; + NUz_arr(i, j, k) = n * u.z; + + } + ); + } +} + + +void WarpXFluidContainer::Evolve( + int lev, + const amrex::MultiFab &Ex, const amrex::MultiFab &Ey, const amrex::MultiFab &Ez, + const amrex::MultiFab &Bx, const amrex::MultiFab &By, const amrex::MultiFab &Bz, + amrex::MultiFab* rho, amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz, + amrex::Real cur_time, bool skip_deposition) +{ + + WARPX_PROFILE("WarpXFluidContainer::Evolve"); + + if (rho && ! skip_deposition && ! do_not_deposit) { + // Deposit charge before particle push, in component 0 of MultiFab rho. + DepositCharge(lev, *rho, 0); + } + + // Step the Lorentz Term + if(!do_not_gather){ + GatherAndPush(lev, Ex, Ey, Ez, Bx, By, Bz, cur_time); + } + + // Cylindrical centrifugal term + if(!do_not_push){ +#if defined(WARPX_DIM_RZ) + centrifugal_source_rz(lev); +#endif + + // Apply (non-periodic) BC on the fluids (needed for spatial derivative), + // and communicate N, NU at boundaries + ApplyBcFluidsAndComms(lev); + + // Step the Advective term + AdvectivePush_Muscl(lev); + } + + // Deposit rho to the simulation mesh + // Deposit charge (end of the step) + if (rho && ! skip_deposition && ! do_not_deposit) { + DepositCharge(lev, *rho, 1); + } + + // Deposit J to the simulation mesh + if (!skip_deposition && ! do_not_deposit) { + DepositCurrent(lev, jx, jy, jz); + } +} + +// Momentum source due to curvature +void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) +{ + WARPX_PROFILE("WarpXFluidContainer::ApplyBcFluidsAndComms"); + + WarpX &warpx = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx.Geom(lev); + const amrex::Periodicity &period = geom.periodicity(); + const Array periodic_directions = geom.isPeriodic(); + amrex::Box domain = geom.Domain(); + // Convert to nodal box + domain.surroundingNodes(); + + // H&C push the momentum +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + + amrex::Array4 N_arr = N[lev]->array(mfi); + amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + + //Grow the tilebox + tile_box.grow(1); + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // If the cell is is first guard cell & the dimension is non + // periodic, then copy Q_{i+1} = Q_{i-1}. + // Don't check r-dir in Z: +#if defined(WARPX_DIM_3D) + + // Upper end (index 2) + if ( (periodic_directions[2] != 1) && (k==domain.bigEnd(2)+1) ){ + N_arr(i,j,k) = N_arr(i,j,k-2); + NUx_arr(i,j,k) = NUx_arr(i,j,k-2); + NUy_arr(i,j,k) = NUy_arr(i,j,k-2); + NUz_arr(i,j,k) = NUz_arr(i,j,k-2); + + // Lower end (index 2) + } else if ( (periodic_directions[2] != 1) && (k==domain.smallEnd(2)-1) ) { + N_arr(i,j,k) = N_arr(i,j,k+2); + NUx_arr(i,j,k) = NUx_arr(i,j,k+2); + NUy_arr(i,j,k) = NUy_arr(i,j,k+2); + NUz_arr(i,j,k) = NUz_arr(i,j,k+2); + } + +#elif ( defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) ) + + // Upper end (index 1) + if ( (periodic_directions[1] != 1) && (j==domain.bigEnd(1)+1) ){ + N_arr(i,j,k) = N_arr(i,j-2,k); + NUx_arr(i,j,k) = NUx_arr(i,j-2,k); + NUy_arr(i,j,k) = NUy_arr(i,j-2,k); + NUz_arr(i,j,k) = NUz_arr(i,j-2,k); + + // Lower end (index 1`) + } else if ( (periodic_directions[1] != 1) && (j==domain.smallEnd(1)-1) ) { + N_arr(i,j,k) = N_arr(i,j+2,k); + NUx_arr(i,j,k) = NUx_arr(i,j+2,k); + NUy_arr(i,j,k) = NUy_arr(i,j+2,k); + NUz_arr(i,j,k) = NUz_arr(i,j+2,k); + + } + +#elif ( defined(WARPX_DIM_1D_Z) || defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) ) + + // Upper end (index 0) + if ( (periodic_directions[0] != 1) && (i==domain.bigEnd(0)+1) ){ + N_arr(i,j,k) = N_arr(i-2,j,k); + NUx_arr(i,j,k) = NUx_arr(i-2,j,k); + NUy_arr(i,j,k) = NUy_arr(i-2,j,k); + NUz_arr(i,j,k) = NUz_arr(i-2,j,k); + + // Lower end (index 0) + } else if ( (periodic_directions[0] != 1) && (i==domain.smallEnd(0)-1) ) { + N_arr(i,j,k) = N_arr(i+2,j,k); + NUx_arr(i,j,k) = NUx_arr(i+2,j,k); + NUy_arr(i,j,k) = NUy_arr(i+2,j,k); + NUz_arr(i,j,k) = NUz_arr(i+2,j,k); + } + +#else + +#endif + } + ); + } + + // Fill guard cells + FillBoundary(*N[lev], N[lev]->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*NU[lev][0], NU[lev][0]->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*NU[lev][1], NU[lev][1]->nGrowVect(), WarpX::do_single_precision_comms, period); + FillBoundary(*NU[lev][2], NU[lev][2]->nGrowVect(), WarpX::do_single_precision_comms, period); +} + +// Muscl Advection Update +void WarpXFluidContainer::AdvectivePush_Muscl (int lev) +{ + WARPX_PROFILE("WarpXFluidContainer::AdvectivePush_Muscl"); + + // Grab the grid spacing + WarpX &warpx = WarpX::GetInstance(); + const Real dt = warpx.getdt(lev); + const amrex::Geometry &geom = warpx.Geom(lev); + const auto dx = geom.CellSizeArray(); + const amrex::Real clight = PhysConst::c; +#if defined(WARPX_DIM_3D) + amrex::Real dt_over_dx = (dt/dx[0]); + amrex::Real dt_over_dy = (dt/dx[1]); + amrex::Real dt_over_dz = (dt/dx[2]); + amrex::Real dt_over_dx_half = 0.5_rt*(dt/dx[0]); + amrex::Real dt_over_dy_half = 0.5_rt*(dt/dx[1]); + amrex::Real dt_over_dz_half = 0.5_rt*(dt/dx[2]); +#elif defined(WARPX_DIM_XZ) + amrex::Real dt_over_dx_half = 0.5_rt*(dt/dx[0]); + amrex::Real dt_over_dz_half = 0.5_rt*(dt/dx[1]); + amrex::Real dt_over_dx = (dt/dx[0]); + amrex::Real dt_over_dz = (dt/dx[1]); +#elif defined(WARPX_DIM_RZ) + const auto problo = geom.ProbLoArray(); + amrex::Real dt_over_dx_half = 0.5_rt*(dt/dx[0]); + amrex::Real dt_over_dz_half = 0.5_rt*(dt/dx[1]); + amrex::Box const& domain = geom.Domain(); +#else + amrex::Real dt_over_dz = (dt/dx[0]); + amrex::Real dt_over_dz_half = 0.5_rt*(dt/dx[0]); +#endif + + amrex::BoxArray ba = N[lev]->boxArray(); + + // Temporary Half-step values +#if defined(WARPX_DIM_3D) + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_y( amrex::convert(ba, IntVect(1,0,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_y( amrex::convert(ba, IntVect(1,0,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,1,0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,1,0)), N[lev]->DistributionMap(), 4, 1); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::MultiFab tmp_U_minus_x( amrex::convert(ba, IntVect(0,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_x( amrex::convert(ba, IntVect(0,1)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(1,0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(1,0)), N[lev]->DistributionMap(), 4, 1); +#else + amrex::MultiFab tmp_U_minus_z( amrex::convert(ba, IntVect(0)), N[lev]->DistributionMap(), 4, 1); + amrex::MultiFab tmp_U_plus_z( amrex::convert(ba, IntVect(0)), N[lev]->DistributionMap(), 4, 1); +#endif + + // Fill edge values of N and U at the half timestep for MUSCL +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + // Loop over a box with one extra gridpoint in the ghost region to avoid + // an extra MPI communication between the edge value computation loop and + // the flux calculation loop + amrex::Box tile_box = mfi.growntilebox(1); + + // Limit the grown box for RZ at r = 0, r_max +#if defined (WARPX_DIM_RZ) + const int idir = 0; + const int n_cell = -1; + tile_box.growLo(idir, n_cell); + tile_box.growHi(idir, n_cell); +#endif + + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + + // Boxes are computed to avoid going out of bounds. + // Grow the entire domain + amrex::Box box = mfi.validbox(); + box.grow(1); +#if defined(WARPX_DIM_3D) + amrex::Box const box_x = amrex::convert( box, tmp_U_minus_x.ixType() ); + amrex::Box const box_y = amrex::convert( box, tmp_U_minus_y.ixType() ); + amrex::Box const box_z = amrex::convert( box, tmp_U_minus_z.ixType() ); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Box const box_x = amrex::convert( box, tmp_U_minus_x.ixType() ); + amrex::Box const box_z = amrex::convert( box, tmp_U_minus_z.ixType() ); +#else + amrex::Box const box_z = amrex::convert( box, tmp_U_minus_z.ixType() ); +#endif + + //N and NU are always defined at the nodes, the tmp_Q_* are defined + //in between the nodes (i.e. on the staggered Yee grid) and store the + //values of N and U at these points. + //(i.e. the 4 components correspond to N + the 3 components of U) + // Extract the temporary arrays for edge values +#if defined(WARPX_DIM_3D) + amrex::Array4 U_minus_x = tmp_U_minus_x.array(mfi); + amrex::Array4 U_plus_x = tmp_U_plus_x.array(mfi); + amrex::Array4 U_minus_y = tmp_U_minus_y.array(mfi); + amrex::Array4 U_plus_y = tmp_U_plus_y.array(mfi); + amrex::Array4 U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 U_plus_z = tmp_U_plus_z.array(mfi); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array4 U_minus_x = tmp_U_minus_x.array(mfi); + amrex::Array4 U_plus_x = tmp_U_plus_x.array(mfi); + amrex::Array4 U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 U_plus_z = tmp_U_plus_z.array(mfi); +#else + amrex::Array4 U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 U_plus_z = tmp_U_plus_z.array(mfi); +#endif + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // Density positivity check (Makes the algorithm safe from divide by zeros) + if( N_arr(i,j,k) > 0.0){ + + // - Grab local Uz Uy Ux gamma + // Isolate U from NU + amrex::Real Ux = (NUx_arr(i, j, k) / N_arr(i,j,k)); + amrex::Real Uy = (NUy_arr(i, j, k) / N_arr(i,j,k)); + amrex::Real Uz = (NUz_arr(i, j, k) / N_arr(i,j,k)); + + // Compute useful quantities for J + amrex::Real c_sq = clight*clight; + amrex::Real gamma = std::sqrt(1.0_rt + (Ux*Ux + Uy*Uy + Uz*Uz)/(c_sq) ); + amrex::Real inv_c2_gamma3 = 1._rt/(c_sq*gamma*gamma*gamma); + + // J represents are 4x4 matrices that show up in the advection + // equations written as a function of U = {N, Ux, Uy, Uz}: + // \partial_t U + Jx \partial_x U + Jy \partial_y U + Jz \partial_z U = 0 +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + amrex::Real Vx = Ux/gamma; + // Compute the non-zero element of Jx + amrex::Real J00x = Vx; + amrex::Real J01x = N_arr(i,j,k)*(1/gamma)*(1-Vx*Vx/c_sq); + amrex::Real J02x = -N_arr(i,j,k)*Uy*Ux*inv_c2_gamma3; + amrex::Real J03x = -N_arr(i,j,k)*Uz*Ux*inv_c2_gamma3; + amrex::Real J11x = Vx; + amrex::Real J22x = Vx; + amrex::Real J33x = Vx; + + // Compute the cell slopes x + amrex::Real dU0x = ave( DownDx_N(N_arr,i,j,k), UpDx_N(N_arr,i,j,k) ); + amrex::Real dU1x = ave( DownDx_U(N_arr,NUx_arr,Ux,i,j,k), UpDx_U(N_arr,NUx_arr,Ux,i,j,k) ); + amrex::Real dU2x = ave( DownDx_U(N_arr,NUy_arr,Uy,i,j,k), UpDx_U(N_arr,NUy_arr,Uy,i,j,k) ); + amrex::Real dU3x = ave( DownDx_U(N_arr,NUz_arr,Uz,i,j,k), UpDx_U(N_arr,NUz_arr,Uz,i,j,k) ); + +#endif + +#if defined(WARPX_DIM_3D) + amrex::Real Vy = Uy/gamma; + // Compute the non-zero element of Jy + amrex::Real J00y = Vy; + amrex::Real J01y = -N_arr(i,j,k)*Ux*Uy*inv_c2_gamma3; + amrex::Real J02y = N_arr(i,j,k)*(1/gamma)*(1-Vy*Vy/c_sq); + amrex::Real J03y = -N_arr(i,j,k)*Uz*Uy*inv_c2_gamma3; + amrex::Real J11y = Vy; + amrex::Real J22y = Vy; + amrex::Real J33y = Vy; + + // Compute the cell slopes y + amrex::Real dU0y = ave( DownDy_N(N_arr,i,j,k), UpDy_N(N_arr,i,j,k) ); + amrex::Real dU1y = ave( DownDy_U(N_arr,NUx_arr,Ux,i,j,k), UpDy_U(N_arr,NUx_arr,Ux,i,j,k) ); + amrex::Real dU2y = ave( DownDy_U(N_arr,NUy_arr,Uy,i,j,k), UpDy_U(N_arr,NUy_arr,Uy,i,j,k) ); + amrex::Real dU3y = ave( DownDy_U(N_arr,NUz_arr,Uz,i,j,k), UpDy_U(N_arr,NUz_arr,Uz,i,j,k) ); + +#endif + amrex::Real Vz = Uz/gamma; + // Compute the non-zero element of Jz + amrex::Real J00z = Vz; + amrex::Real J01z = -N_arr(i,j,k)*Ux*Uz*inv_c2_gamma3; + amrex::Real J02z = -N_arr(i,j,k)*Uy*Uz*inv_c2_gamma3; + amrex::Real J03z = N_arr(i,j,k)*(1/gamma)*(1-Vz*Vz/c_sq); + amrex::Real J11z = Vz; + amrex::Real J22z = Vz; + amrex::Real J33z = Vz; + + // Compute the cell slopes z + amrex::Real dU0z = ave( DownDz_N(N_arr,i,j,k), UpDz_N(N_arr,i,j,k) ); + amrex::Real dU1z = ave( DownDz_U(N_arr,NUx_arr,Ux,i,j,k), UpDz_U(N_arr,NUx_arr,Ux,i,j,k) ); + amrex::Real dU2z = ave( DownDz_U(N_arr,NUy_arr,Uy,i,j,k), UpDz_U(N_arr,NUy_arr,Uy,i,j,k) ); + amrex::Real dU3z = ave( DownDz_U(N_arr,NUz_arr,Uz,i,j,k), UpDz_U(N_arr,NUz_arr,Uz,i,j,k) ); + + + // Select the specific implementation depending on dimensionality +#if defined(WARPX_DIM_3D) + + // Compute U ([ N, U]) at the halfsteps (U_tilde) using the slopes (dU) + amrex::Real JdU0x = J00x*dU0x + J01x*dU1x + J02x*dU2x + J03x*dU3x; + amrex::Real JdU1x = J11x*dU1x ; + amrex::Real JdU2x = J22x*dU2x ; + amrex::Real JdU3x = J33x*dU3x; + amrex::Real JdU0y = J00y*dU0y + J01y*dU1y + J02y*dU2y + J03y*dU3y; + amrex::Real JdU1y = J11y*dU1y; + amrex::Real JdU2y = J22y*dU2y; + amrex::Real JdU3y = J33y*dU3y; + amrex::Real JdU0z = J00z*dU0z + J01z*dU1z + J02z*dU2z + J03z*dU3z; + amrex::Real JdU1z = J11z*dU1z; + amrex::Real JdU2z = J22z*dU2z; + amrex::Real JdU3z = J33z*dU3z; + amrex::Real U_tilde0 = N_arr(i,j,k) - dt_over_dx_half*JdU0x - dt_over_dy_half*JdU0y - dt_over_dz_half*JdU0z; + amrex::Real U_tilde1 = Ux - dt_over_dx_half*JdU1x - dt_over_dy_half*JdU1y - dt_over_dz_half*JdU1z; + amrex::Real U_tilde2 = Uy - dt_over_dx_half*JdU2x - dt_over_dy_half*JdU2y - dt_over_dz_half*JdU2z; + amrex::Real U_tilde3 = Uz - dt_over_dx_half*JdU3x - dt_over_dy_half*JdU3y - dt_over_dz_half*JdU3z; + + + // Predict U at the cell edges (x) + compute_U_edges(U_minus_x, U_plus_x, i, j, k, box_x, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0x, dU1x, dU2x, dU3x,0); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_x, U_minus_x, N_arr, i, j, k, box_x, Ux, Uy, Uz, 0); + + // Predict U at the cell edges (y) + compute_U_edges(U_minus_y, U_plus_y, i, j, k, box_y, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0y, dU1y, dU2y, dU3y,1); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_y, U_minus_y, N_arr, i, j, k, box_y, Ux, Uy, Uz, 1); + + // Predict U at the cell edges (z) + compute_U_edges(U_minus_z, U_plus_z, i, j, k, box_z, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0z, dU1z, dU2z, dU3z,2); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_z, U_minus_z, N_arr, i, j, k, box_z, Ux, Uy, Uz, 2); + +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + + // Have no RZ-inertial source for primitive vars if in XZ + amrex::Real N_source = 0.0; + +#if defined(WARPX_DIM_RZ) + amrex::Real dr = dx[0]; + amrex::Real r = problo[0] + i * dr; + // Impose "none" boundaries + // Condition: dUx = 0 at r = 0 + if (i == domain.smallEnd(0)) { + // R|_{0+} -> L|_{0-} + // N -> N (N_arr(i-1,j,k) -> N_arr(i+1,j,k)) + // NUr -> -NUr (NUx_arr(i-1,j,k) -> -NUx_arr(i+1,j,k)) + // NUt -> -NUt (NUy_arr(i-1,j,k) -> -NUy_arr(i+1,j,k)) + // NUz -> -NUz (NUz_arr(i-1,j,k) -> NUz_arr(i+1,j,k)) + dU0x = ave( -UpDx_N(N_arr,i,j,k) , UpDx_N(N_arr,i,j,k) ); + // First term in the ave is: U_{x,y} + U_{x,y}_p, + // which can be written as 2*U_{x,y} + UpDx_U(U_{x,y}) + dU1x = ave( 2.0_rt*Ux + UpDx_U(N_arr,NUx_arr,Ux,i,j,k) , UpDx_U(N_arr,NUx_arr,Ux,i,j,k) ); + dU2x = ave( 2.0_rt*Uy + UpDx_U(N_arr,NUy_arr,Uy,i,j,k) , UpDx_U(N_arr,NUy_arr,Uy,i,j,k) ); + dU3x = ave( -UpDx_U(N_arr,NUz_arr,Uz,i,j,k) , UpDx_U(N_arr,NUz_arr,Uz,i,j,k) ); + } else if (i == domain.bigEnd(0)+1) { + dU0x = ave( DownDx_N(N_arr,i,j,k) , 0.0_rt ); + dU1x = ave( DownDx_U(N_arr,NUx_arr,Ux,i,j,k) , 0.0_rt ); + dU2x = ave( DownDx_U(N_arr,NUy_arr,Uy,i,j,k) , 0.0_rt ); + dU3x = ave( DownDx_U(N_arr,NUz_arr,Uz,i,j,k) , 0.0_rt ); + } + + // RZ sources: + if (i != domain.smallEnd(0)) { + N_source = N_arr(i,j,k)*Vx/r; + } +#endif + + // Compute U ([ N, U]) at the halfsteps (U_tilde) using the slopes (dU) + amrex::Real JdU0x = J00x*dU0x + J01x*dU1x + J02x*dU2x + J03x*dU3x; + amrex::Real JdU1x = J11x*dU1x; + amrex::Real JdU2x = J22x*dU2x; + amrex::Real JdU3x = J33x*dU3x; + amrex::Real JdU0z = J00z*dU0z + J01z*dU1z + J02z*dU2z + J03z*dU3z; + amrex::Real JdU1z = J11z*dU1z; + amrex::Real JdU2z = J22z*dU2z; + amrex::Real JdU3z = J33z*dU3z; + amrex::Real U_tilde0 = N_arr(i,j,k) - dt_over_dx_half*JdU0x - dt_over_dz_half*JdU0z - (dt/2.0_rt)*N_source; + amrex::Real U_tilde1 = Ux - dt_over_dx_half*JdU1x - dt_over_dz_half*JdU1z; + amrex::Real U_tilde2 = Uy - dt_over_dx_half*JdU2x - dt_over_dz_half*JdU2z; + amrex::Real U_tilde3 = Uz - dt_over_dx_half*JdU3x - dt_over_dz_half*JdU3z; + + // Predict U at the cell edges (x) + compute_U_edges(U_minus_x, U_plus_x, i, j, k, box_x, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0x, dU1x, dU2x, dU3x,0); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_x, U_minus_x, N_arr, i, j, k, box_x, Ux, Uy, Uz, 0); + + // Predict U at the cell edges (z) + compute_U_edges(U_minus_z, U_plus_z, i, j, k, box_z, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0z, dU1z, dU2z, dU3z,2); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_z, U_minus_z, N_arr, i, j, k, box_z, Ux, Uy, Uz, 2); + +#else + + // Compute U ([ N, U]) at the halfsteps (U_tilde) using the slopes (dU) + amrex::Real JdU0z = J00z*dU0z + J01z*dU1z + J02z*dU2z + J03z*dU3z; + amrex::Real JdU1z = J11z*dU1z; + amrex::Real JdU2z = J22z*dU2z; + amrex::Real JdU3z = J33z*dU3z; + amrex::Real U_tilde0 = N_arr(i,j,k) - dt_over_dz_half*JdU0z; + amrex::Real U_tilde1 = Ux - dt_over_dz_half*JdU1z; + amrex::Real U_tilde2 = Uy - dt_over_dz_half*JdU2z; + amrex::Real U_tilde3 = Uz - dt_over_dz_half*JdU3z; + + // Predict U at the cell edges (z) + compute_U_edges(U_minus_z, U_plus_z, i, j, k, box_z, U_tilde0, U_tilde1, U_tilde2, U_tilde3, dU0z, dU1z, dU2z, dU3z,2); + + // Positivity Limiter for density N, if N_edge < 0, + // then set the slope (dU) to to zero in that cell/direction + positivity_limiter (U_plus_z, U_minus_z, N_arr, i, j, k, box_z, Ux, Uy, Uz, 2); + +#endif + // If N<= 0 then set the edge values (U_minus/U_plus) to zero + } else { +#if defined(WARPX_DIM_3D) + set_U_edges_to_zero(U_minus_x, U_plus_x, i, j, k, box_x, 0); + set_U_edges_to_zero(U_minus_y, U_plus_y, i, j, k, box_y, 1); + set_U_edges_to_zero(U_minus_z, U_plus_z, i, j, k, box_z, 2); +#elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + set_U_edges_to_zero(U_minus_x, U_plus_x, i, j, k, box_x, 0); + set_U_edges_to_zero(U_minus_z, U_plus_z, i, j, k, box_z, 2); +#else + set_U_edges_to_zero(U_minus_z, U_plus_z, i, j, k, box_z, 2); +#endif + } + } + ); + } + + // Given the values of `U_minus` and `U_plus`, compute fluxes in between nodes, and update N, NU accordingly +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + amrex::Box tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Array4 N_arr = N[lev]->array(mfi); + amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + +#if defined(WARPX_DIM_3D) + amrex::Array4 const &U_minus_x = tmp_U_minus_x.array(mfi); + amrex::Array4 const &U_plus_x = tmp_U_plus_x.array(mfi); + amrex::Array4 const &U_minus_y = tmp_U_minus_y.array(mfi); + amrex::Array4 const &U_plus_y = tmp_U_plus_y.array(mfi); + amrex::Array4 const &U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 const &U_plus_z = tmp_U_plus_z.array(mfi); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array4 const &U_minus_x = tmp_U_minus_x.array(mfi); + amrex::Array4 const &U_plus_x = tmp_U_plus_x.array(mfi); + amrex::Array4 const &U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 const &U_plus_z = tmp_U_plus_z.array(mfi); +#else + amrex::Array4 const &U_minus_z = tmp_U_minus_z.array(mfi); + amrex::Array4 const &U_plus_z = tmp_U_plus_z.array(mfi); +#endif + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // Select the specific implementation depending on dimensionality +#if defined(WARPX_DIM_3D) + + // Update the conserved variables Q = [N, NU] from tn -> tn + dt + N_arr(i,j,k) = N_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,0,0) + - dt_over_dy*dF(U_minus_y,U_plus_y,i,j,k,clight,0,1) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,0,2); + NUx_arr(i,j,k) = NUx_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,1,0) + - dt_over_dy*dF(U_minus_y,U_plus_y,i,j,k,clight,1,1) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,1,2); + NUy_arr(i,j,k) = NUy_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,2,0) + - dt_over_dy*dF(U_minus_y,U_plus_y,i,j,k,clight,2,1) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,2,2); + NUz_arr(i,j,k) = NUz_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,3,0) + - dt_over_dy*dF(U_minus_y,U_plus_y,i,j,k,clight,3,1) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,3,2); + +#elif defined(WARPX_DIM_XZ) + + // Update the conserved variables Q = [N, NU] from tn -> tn + dt + N_arr(i,j,k) = N_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,0,0) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,0,2); + NUx_arr(i,j,k) = NUx_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,1,0) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,1,2); + NUy_arr(i,j,k) = NUy_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,2,0) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,2,2); + NUz_arr(i,j,k) = NUz_arr(i,j,k) - dt_over_dx*dF(U_minus_x,U_plus_x,i,j,k,clight,3,0) + - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,3,2); + +#elif defined(WARPX_DIM_RZ) + + // Compute the flux areas for RZ + // Cell-centered radius + amrex::Real dr = dx[0]; + amrex::Real dz = dx[1]; + amrex::Real r = problo[0] + i * dr; + amrex::Real Vij = 0.0_rt; + amrex::Real S_Az = 0.0_rt; + + // Volume element and z-facing surfaces + if (i == domain.smallEnd(0)) { + Vij = 2.0_rt*MathConst::pi*(dr/2.0_rt)*(dr/4.0_rt)*dz; + S_Az = 2.0_rt*MathConst::pi*(dr/4.0_rt)*(dr/2.0_rt); + } else if (i == domain.bigEnd(0)+1) { + Vij = 2.0_rt*MathConst::pi*(r - dr/4.0_rt)*(dr/2.0_rt)*dz; + S_Az = 2.0_rt*MathConst::pi*(r - dr/4.0_rt)*(dr/2.0_rt); + } else { + Vij = 2.0_rt*MathConst::pi*r*dr*dz; + S_Az = 2.0_rt*MathConst::pi*(r)*dr; + } + + // Radial Surfaces + amrex::Real S_Ar_plus = 2.0_rt*MathConst::pi*(r + dr/2.0_rt)*dz; + amrex::Real S_Ar_minus = 2.0_rt*MathConst::pi*(r - dr/2.0_rt)*dz; + if (i == domain.smallEnd(0)) { + S_Ar_minus = 0.0_rt; + } + if (i == domain.bigEnd(0)+1) { + S_Ar_plus = 2.0_rt*MathConst::pi*(r)*dz; + } + + // Impose "none" boundaries + // Condition: Vx(r) = 0 at boundaries + amrex::Real Vx_I_minus = V_calc(U_minus_x,i,j,k,0,clight); + amrex::Real Vx_L_plus = V_calc(U_plus_x,i-1,j,k,0,clight); + + // compute the fluxes: + // (note that _plus is shifted due to grid location) + amrex::Real Vx_L_minus = 0.0_rt, Vx_I_plus = 0.0_rt; + amrex::Real F0_minusx = 0.0_rt, F1_minusx = 0.0_rt, F2_minusx = 0.0_rt, F3_minusx = 0.0_rt; + amrex::Real F0_plusx = 0.0_rt, F1_plusx = 0.0_rt, F2_plusx = 0.0_rt, F3_plusx = 0.0_rt; + if (i != domain.smallEnd(0)) { + Vx_L_minus = V_calc(U_minus_x,i-1,j,k,0,clight); + F0_minusx = flux_N( U_minus_x, U_plus_x, i-1, j, k, Vx_L_minus, Vx_L_plus)*S_Ar_minus; + F1_minusx = flux_NUx(U_minus_x, U_plus_x, i-1, j, k, Vx_L_minus, Vx_L_plus)*S_Ar_minus; + F2_minusx = flux_NUy(U_minus_x, U_plus_x, i-1, j, k, Vx_L_minus, Vx_L_plus)*S_Ar_minus; + F3_minusx = flux_NUz(U_minus_x, U_plus_x, i-1, j, k, Vx_L_minus, Vx_L_plus)*S_Ar_minus; + } + if (i < domain.bigEnd(0)) { + Vx_I_plus = V_calc(U_plus_x,i,j,k,0,clight); + F0_plusx = flux_N( U_minus_x, U_plus_x, i , j, k, Vx_I_minus, Vx_I_plus)*S_Ar_plus; + F1_plusx = flux_NUx(U_minus_x, U_plus_x, i , j, k, Vx_I_minus, Vx_I_plus)*S_Ar_plus; + F2_plusx = flux_NUy(U_minus_x, U_plus_x, i , j, k, Vx_I_minus, Vx_I_plus)*S_Ar_plus; + F3_plusx = flux_NUz(U_minus_x, U_plus_x, i , j, k, Vx_I_minus, Vx_I_plus)*S_Ar_plus; + } + + // Update the conserved variables from tn -> tn + dt + N_arr(i,j,k) = N_arr(i,j,k) - (dt/Vij)*(F0_plusx - F0_minusx + dF(U_minus_z,U_plus_z,i,j,k,clight,0,2)*S_Az); + NUx_arr(i,j,k) = NUx_arr(i,j,k) - (dt/Vij)*(F1_plusx - F1_minusx + dF(U_minus_z,U_plus_z,i,j,k,clight,1,2)*S_Az); + NUy_arr(i,j,k) = NUy_arr(i,j,k) - (dt/Vij)*(F2_plusx - F2_minusx + dF(U_minus_z,U_plus_z,i,j,k,clight,2,2)*S_Az); + NUz_arr(i,j,k) = NUz_arr(i,j,k) - (dt/Vij)*(F3_plusx - F3_minusx + dF(U_minus_z,U_plus_z,i,j,k,clight,3,2)*S_Az); + +#else + + // Update the conserved variables Q = [N, NU] from tn -> tn + dt + N_arr(i,j,k) = N_arr(i,j,k) - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,0,2); + NUx_arr(i,j,k) = NUx_arr(i,j,k) - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,1,2); + NUy_arr(i,j,k) = NUy_arr(i,j,k) - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,2,2); + NUz_arr(i,j,k) = NUz_arr(i,j,k) - dt_over_dz*dF(U_minus_z,U_plus_z,i,j,k,clight,3,2); +#endif + } + ); + } +} + + +// Momentum source due to curvature +#if defined(WARPX_DIM_RZ) +void WarpXFluidContainer::centrifugal_source_rz (int lev) +{ + WARPX_PROFILE("WarpXFluidContainer::centrifugal_source_rz"); + + WarpX &warpx = WarpX::GetInstance(); + const Real dt = warpx.getdt(lev); + const amrex::Geometry &geom = warpx.Geom(lev); + const auto dx = geom.CellSizeArray(); + const auto problo = geom.ProbLoArray(); + const amrex::Real clight = PhysConst::c; + amrex::Box const& domain = geom.Domain(); + + // H&C push the momentum +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // Verify density is non-zero + if (N_arr(i,j,k)>0.0_rt) { + + // Compute r + amrex::Real r = problo[0] + i * dx[0]; + + // Isolate U from NU + amrex::Real u_r = (NUx_arr(i, j, k) / (N_arr(i,j,k) * clight )); + amrex::Real u_theta = (NUy_arr(i, j, k) / (N_arr(i,j,k) * clight )); + amrex::Real u_z = (NUz_arr(i, j, k) / (N_arr(i,j,k) * clight )); + + // (SSP-RK3) Push the fluid momentum (R and Theta) + // F_r, F_theta are first order euler pushes of our rhs operator + if (i != domain.smallEnd(0)) { + amrex::Real u_r_1 = F_r(r,u_r,u_theta,u_z,dt); + amrex::Real u_theta_1 = F_theta(r,u_r,u_theta,u_z,dt); + amrex::Real u_r_2 = (0.75_rt)*(u_r) + (0.25_rt)*F_r(r,u_r_1,u_theta_1,u_z,dt); + amrex::Real u_theta_2 = (0.75_rt)*(u_theta) + (0.25_rt)*F_theta(r,u_r_1,u_theta_1,u_z,dt); + u_r = (1.0_rt/3.0_rt)*(u_r) + (2.0_rt/3.0_rt)*F_r(r,u_r_2,u_theta_2,u_z,dt); + u_theta = (1.0_rt/3.0_rt)*(u_theta) + (2.0_rt/3.0_rt)*F_theta(r,u_r_2,u_theta_2,u_z,dt); + + // Calculate NU, save NUr, NUtheta + NUx_arr(i,j,k) = N_arr(i,j,k)*u_r*clight; + NUy_arr(i,j,k) = N_arr(i,j,k)*u_theta*clight; + + // BC r = 0, u_theta = 0, and there is no extra source terms + } else { + NUx_arr(i,j,k) = 0.0_rt; + NUy_arr(i,j,k) = 0.0_rt; + } + } + } + ); + } +} +#endif + +// Momentum source from fields +void WarpXFluidContainer::GatherAndPush ( + int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + Real t) +{ + WARPX_PROFILE("WarpXFluidContainer::GatherAndPush"); + + WarpX &warpx = WarpX::GetInstance(); + const amrex::Real q = getCharge(); + const amrex::Real m = getMass(); + const Real dt = warpx.getdt(lev); + const amrex::Geometry &geom = warpx.Geom(lev); + const auto dx = geom.CellSizeArray(); + const auto problo = geom.ProbLoArray(); + const amrex::Real gamma_boost = WarpX::gamma_boost; + const amrex::Real beta_boost = WarpX::beta_boost; + //Check whether m_E_ext_s is "none" + bool external_e_fields; // Needs intializing + bool external_b_fields; // Needs intializing + + + // Prepare interpolation of current components to cell center + auto Nodal_type = amrex::GpuArray{0, 0, 0}; + auto Ex_type = amrex::GpuArray{0, 0, 0}; + auto Ey_type = amrex::GpuArray{0, 0, 0}; + auto Ez_type = amrex::GpuArray{0, 0, 0}; + auto Bx_type = amrex::GpuArray{0, 0, 0}; + auto By_type = amrex::GpuArray{0, 0, 0}; + auto Bz_type = amrex::GpuArray{0, 0, 0}; + for (int i = 0; i < AMREX_SPACEDIM; ++i) + { + Nodal_type[i] = N[lev]->ixType()[i]; + Ex_type[i] = Ex.ixType()[i]; + Ey_type[i] = Ey.ixType()[i]; + Ez_type[i] = Ez.ixType()[i]; + Bx_type[i] = Bx.ixType()[i]; + By_type[i] = By.ixType()[i]; + Bz_type[i] = Bz.ixType()[i]; + } + + // External field parsers + external_e_fields = (m_E_ext_s == "parse_e_ext_function"); + external_b_fields = (m_B_ext_s == "parse_b_ext_function"); + amrex::ParserExecutor<4> Exfield_parser; + amrex::ParserExecutor<4> Eyfield_parser; + amrex::ParserExecutor<4> Ezfield_parser; + amrex::ParserExecutor<4> Bxfield_parser; + amrex::ParserExecutor<4> Byfield_parser; + amrex::ParserExecutor<4> Bzfield_parser; + if (external_e_fields){ + constexpr int num_arguments = 4; //x,y,z,t + Exfield_parser = m_Ex_parser->compile(); + Eyfield_parser = m_Ey_parser->compile(); + Ezfield_parser = m_Ez_parser->compile(); + } + + if (external_b_fields){ + constexpr int num_arguments = 4; //x,y,z,t + Bxfield_parser = m_Bx_parser->compile(); + Byfield_parser = m_By_parser->compile(); + Bzfield_parser = m_Bz_parser->compile(); + } + + + // H&C push the momentum +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 NUz_arr = NU[lev][2]->array(mfi); + + amrex::Array4 const& Ex_arr = Ex.array(mfi); + amrex::Array4 const& Ey_arr = Ey.array(mfi); + amrex::Array4 const& Ez_arr = Ez.array(mfi); + amrex::Array4 const& Bx_arr = Bx.array(mfi); + amrex::Array4 const& By_arr = By.array(mfi); + amrex::Array4 const& Bz_arr = Bz.array(mfi); + + // Here, we do not perform any coarsening. + amrex::GpuArray coarsening_ratio = {1, 1, 1}; + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + + // Only run if density is positive + if (N_arr(i,j,k)>0.0) { + + // Interpolate fields from tmp to Nodal points + amrex::Real Ex_Nodal = ablastr::coarsen::sample::Interp(Ex_arr, + Ex_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real Ey_Nodal = ablastr::coarsen::sample::Interp(Ey_arr, + Ey_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real Ez_Nodal = ablastr::coarsen::sample::Interp(Ez_arr, + Ez_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real Bx_Nodal = ablastr::coarsen::sample::Interp(Bx_arr, + Bx_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real By_Nodal = ablastr::coarsen::sample::Interp(By_arr, + By_type, Nodal_type, coarsening_ratio, i, j, k, 0); + amrex::Real Bz_Nodal = ablastr::coarsen::sample::Interp(Bz_arr, + Bz_type, Nodal_type, coarsening_ratio, i, j, k, 0); + + if (gamma_boost > 1._rt) { // Lorentz transform fields due to moving frame + if ( ( external_b_fields ) || ( external_e_fields ) ){ + + // Lorentz transform z (from boosted to lab frame) + amrex::Real Ex_ext_boost, Ey_ext_boost, Ez_ext_boost; + amrex::Real Bx_ext_boost, By_ext_boost, Bz_ext_boost; + amrex::Real Ex_ext_lab, Ey_ext_lab, Ez_ext_lab; + amrex::Real Bx_ext_lab, By_ext_lab, Bz_ext_lab; + + // Grab the location +#if defined(WARPX_DIM_3D) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = problo[1] + j * dx[1]; + amrex::Real z = problo[2] + k * dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[1] + j * dx[1]; +#else + amrex::Real x = 0.0_rt; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[0] + i * dx[0]; +#endif + + // Get the lab frame E and B + // Transform (boosted to lab) + amrex::Real t_lab = gamma_boost*(t + beta_boost*z/PhysConst::c); + amrex::Real z_lab = gamma_boost*(z + beta_boost*PhysConst::c*t); + + // Grab the external fields in the lab frame: + if ( external_e_fields ) { + Ex_ext_lab = Exfield_parser(x, y, z_lab, t_lab); + Ey_ext_lab = Eyfield_parser(x, y, z_lab, t_lab); + Ez_ext_lab = Ezfield_parser(x, y, z_lab, t_lab); + }else{ + Ex_ext_lab = 0.0; + Ey_ext_lab = 0.0; + Ez_ext_lab = 0.0; + } + if ( external_b_fields ) { + Bx_ext_lab = Bxfield_parser(x, y, z_lab, t_lab); + By_ext_lab = Byfield_parser(x, y, z_lab, t_lab); + Bz_ext_lab = Bzfield_parser(x, y, z_lab, t_lab); + }else{ + Bx_ext_lab = 0.0; + By_ext_lab = 0.0; + Bz_ext_lab = 0.0; + } + + // Transform E & B (lab to boosted frame) + // (Require both to for the lorentz transform) + // RHS m_parser + Ez_ext_boost = Ez_ext_lab; + Bz_ext_boost = Bz_ext_lab; + Ex_ext_boost = gamma_boost*(Ex_ext_lab - beta_boost*PhysConst::c*By_ext_lab); + Ey_ext_boost = gamma_boost*(Ey_ext_lab + beta_boost*PhysConst::c*Bx_ext_lab); + Bx_ext_boost = gamma_boost*(Bx_ext_lab + beta_boost*Ey_ext_lab/PhysConst::c); + By_ext_boost = gamma_boost*(By_ext_lab - beta_boost*Ex_ext_lab/PhysConst::c); + + // Then add to Nodal quantities in the boosted frame: + Ex_Nodal += Ex_ext_boost; + Ey_Nodal += Ey_ext_boost; + Ez_Nodal += Ez_ext_boost; + Bx_Nodal += Bx_ext_boost; + By_Nodal += By_ext_boost; + Bz_Nodal += Bz_ext_boost; + } + } else { + + // Added external e fields: + if ( external_e_fields ){ +#if defined(WARPX_DIM_3D) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = problo[1] + j * dx[1]; + amrex::Real z = problo[2] + k * dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[1] + j * dx[1]; +#else + amrex::Real x = 0.0_rt; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[0] + i * dx[0]; +#endif + + Ex_Nodal += Exfield_parser(x, y, z, t); + Ey_Nodal += Eyfield_parser(x, y, z, t); + Ez_Nodal += Ezfield_parser(x, y, z, t); + } + + // Added external b fields: + if ( external_b_fields ){ +#if defined(WARPX_DIM_3D) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = problo[1] + j * dx[1]; + amrex::Real z = problo[2] + k * dx[2]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Real x = problo[0] + i * dx[0]; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[1] + j * dx[1]; +#else + amrex::Real x = 0.0_rt; + amrex::Real y = 0.0_rt; + amrex::Real z = problo[0] + i * dx[0]; +#endif + + Bx_Nodal += Bxfield_parser(x, y, z, t); + By_Nodal += Byfield_parser(x, y, z, t); + Bz_Nodal += Bzfield_parser(x, y, z, t); + } + } + + // Isolate U from NU + amrex::Real tmp_Ux = (NUx_arr(i, j, k) / N_arr(i,j,k)); + amrex::Real tmp_Uy = (NUy_arr(i, j, k) / N_arr(i,j,k)); + amrex::Real tmp_Uz = (NUz_arr(i, j, k) / N_arr(i,j,k)); + + // Enforce RZ boundary conditions +#if defined(WARPX_DIM_RZ) + if ( i == 0 ){ + Ex_Nodal = 0.0; + Ey_Nodal = 0.0; + By_Nodal = 0.0; + Bx_Nodal = 0.0; + } +#endif + + // Push the fluid momentum + UpdateMomentumHigueraCary(tmp_Ux, tmp_Uy, tmp_Uz, + Ex_Nodal, Ey_Nodal, Ez_Nodal, + Bx_Nodal, By_Nodal, Bz_Nodal, q, m, dt ); + + // Calculate NU + NUx_arr(i,j,k) = N_arr(i,j,k)*tmp_Ux; + NUy_arr(i,j,k) = N_arr(i,j,k)*tmp_Uy; + NUz_arr(i,j,k) = N_arr(i,j,k)*tmp_Uz; + } + } + ); + } +} + +void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icomp) +{ + WARPX_PROFILE("WarpXFluidContainer::DepositCharge"); + + WarpX &warpx = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx.Geom(lev); + const amrex::Periodicity &period = geom.periodicity(); + const amrex::Real q = getCharge(); + auto const &owner_mask_rho = amrex::OwnerMask(rho, period); + + // Assertion, make sure rho is at the same location as N + AMREX_ALWAYS_ASSERT(rho.ixType().nodeCentered()); + + // Loop over and deposit charge density +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + + amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 rho_arr = rho.array(mfi); + amrex::Array4 owner_mask_rho_arr = owner_mask_rho->array(mfi); + + // Deposit Rho + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + if ( owner_mask_rho_arr(i,j,k) ) { rho_arr(i,j,k,icomp) += q*N_arr(i,j,k); } + } + ); + } +} + + +void WarpXFluidContainer::DepositCurrent( + int lev, + amrex::MultiFab &jx, amrex::MultiFab &jy, amrex::MultiFab &jz) +{ + WARPX_PROFILE("WarpXFluidContainer::DepositCurrent"); + + // Temporary nodal currents + amrex::MultiFab tmp_jx_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jy_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); + amrex::MultiFab tmp_jz_fluid(N[lev]->boxArray(), N[lev]->DistributionMap(), 1, 0); + + const amrex::Real inv_clight_sq = 1.0_prt / PhysConst::c / PhysConst::c; + const amrex::Real q = getCharge(); + + // Prepare interpolation of current components to cell center + auto j_nodal_type = amrex::GpuArray{0, 0, 0}; + auto jx_type = amrex::GpuArray{0, 0, 0}; + auto jy_type = amrex::GpuArray{0, 0, 0}; + auto jz_type = amrex::GpuArray{0, 0, 0}; + for (int i = 0; i < AMREX_SPACEDIM; ++i) + { + j_nodal_type[i] = tmp_jx_fluid.ixType()[i]; + jx_type[i] = jx.ixType()[i]; + jy_type[i] = jy.ixType()[i]; + jz_type[i] = jz.ixType()[i]; + } + + // We now need to create a mask to fix the double counting. + WarpX &warpx = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx.Geom(lev); + const amrex::Periodicity &period = geom.periodicity(); + auto const &owner_mask_x = amrex::OwnerMask(jx, period); + auto const &owner_mask_y = amrex::OwnerMask(jy, period); + auto const &owner_mask_z = amrex::OwnerMask(jz, period); + + // Calculate j at the nodes +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + amrex::Box const &tile_box = mfi.tilebox(N[lev]->ixType().toIntVect()); + + amrex::Array4 const &N_arr = N[lev]->array(mfi); + amrex::Array4 const &NUx_arr = NU[lev][0]->array(mfi); + amrex::Array4 const &NUy_arr = NU[lev][1]->array(mfi); + amrex::Array4 const &NUz_arr = NU[lev][2]->array(mfi); + + amrex::Array4 tmp_jx_fluid_arr = tmp_jx_fluid.array(mfi); + amrex::Array4 tmp_jy_fluid_arr = tmp_jy_fluid.array(mfi); + amrex::Array4 tmp_jz_fluid_arr = tmp_jz_fluid.array(mfi); + + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + // Calculate J from fluid quantities + amrex::Real gamma = 1.0_rt, Ux = 0.0_rt, Uy = 0.0_rt, Uz = 0.0_rt; + if (N_arr(i, j, k)>0.0_rt){ + Ux = NUx_arr(i, j, k)/N_arr(i, j, k); + Uy = NUy_arr(i, j, k)/N_arr(i, j, k); + Uz = NUz_arr(i, j, k)/N_arr(i, j, k); + gamma = std::sqrt(1.0_rt + ( Ux*Ux + Uy*Uy + Uz*Uz) * inv_clight_sq ) ; + } + tmp_jx_fluid_arr(i, j, k) = q * (NUx_arr(i, j, k) / gamma); + tmp_jy_fluid_arr(i, j, k) = q * (NUy_arr(i, j, k) / gamma); + tmp_jz_fluid_arr(i, j, k) = q * (NUz_arr(i, j, k) / gamma); + } + ); + } + + // Interpolate j from the nodes to the simulation mesh (typically Yee mesh) +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*N[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + amrex::Box const &tile_box_x = mfi.tilebox(jx.ixType().toIntVect()); + amrex::Box const &tile_box_y = mfi.tilebox(jy.ixType().toIntVect()); + amrex::Box const &tile_box_z = mfi.tilebox(jz.ixType().toIntVect()); + + amrex::Array4 jx_arr = jx.array(mfi); + amrex::Array4 jy_arr = jy.array(mfi); + amrex::Array4 jz_arr = jz.array(mfi); + + amrex::Array4 tmp_jx_fluid_arr = tmp_jx_fluid.array(mfi); + amrex::Array4 tmp_jy_fluid_arr = tmp_jy_fluid.array(mfi); + amrex::Array4 tmp_jz_fluid_arr = tmp_jz_fluid.array(mfi); + + amrex::Array4 owner_mask_x_arr = owner_mask_x->array(mfi); + amrex::Array4 owner_mask_y_arr = owner_mask_y->array(mfi); + amrex::Array4 owner_mask_z_arr = owner_mask_z->array(mfi); + + // When using the `Interp` function, one needs to specify whether coarsening is desired. + // Here, we do not perform any coarsening. + amrex::GpuArray coarsening_ratio = {1, 1, 1}; + + + // Interpolate fluid current and deposit it + // ( mask double counting ) + amrex::ParallelFor( tile_box_x, tile_box_y, tile_box_z, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real jx_tmp = ablastr::coarsen::sample::Interp(tmp_jx_fluid_arr, + j_nodal_type, jx_type, coarsening_ratio, i, j, k, 0); + if ( owner_mask_x_arr(i,j,k) ) { jx_arr(i, j, k) += jx_tmp; } + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real jy_tmp = ablastr::coarsen::sample::Interp(tmp_jy_fluid_arr, + j_nodal_type, jy_type, coarsening_ratio, i, j, k, 0); + if ( owner_mask_y_arr(i,j,k) ) { jy_arr(i, j, k) += jy_tmp; } + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + amrex::Real jz_tmp = ablastr::coarsen::sample::Interp(tmp_jz_fluid_arr, + j_nodal_type, jz_type, coarsening_ratio, i, j, k, 0); + if ( owner_mask_z_arr(i,j,k) ) { jz_arr(i, j, k) += jz_tmp; } + } + ); + } +} diff --git a/Source/Fluids/WarpXFluidContainer_fwd.H b/Source/Fluids/WarpXFluidContainer_fwd.H new file mode 100644 index 00000000000..77a6bc0c991 --- /dev/null +++ b/Source/Fluids/WarpXFluidContainer_fwd.H @@ -0,0 +1,12 @@ +/* Copyright 2023 Grant Johnson, Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_WarpXFluidContainer_fwd_H_ + +class WarpXFluidContainer; + +#endif /* WARPX_WarpXFluidContainer_fwd_H_ */ diff --git a/Source/Initialization/CMakeLists.txt b/Source/Initialization/CMakeLists.txt index 35767e87c50..8931de740ad 100644 --- a/Source/Initialization/CMakeLists.txt +++ b/Source/Initialization/CMakeLists.txt @@ -2,14 +2,15 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE - WarpXAMReXInit.cpp + ExternalField.cpp + GetTemperature.cpp + GetVelocity.cpp InjectorDensity.cpp InjectorMomentum.cpp PlasmaInjector.cpp - WarpXInitData.cpp TemperatureProperties.cpp VelocityProperties.cpp - GetTemperature.cpp - GetVelocity.cpp + WarpXAMReXInit.cpp + WarpXInitData.cpp ) endforeach() diff --git a/Source/Initialization/ExternalField.H b/Source/Initialization/ExternalField.H new file mode 100644 index 00000000000..7b64339a042 --- /dev/null +++ b/Source/Initialization/ExternalField.H @@ -0,0 +1,69 @@ +/* Copyright 2023 Luca Fedeli + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef EXTERNAL_FIELD_H_ +#define EXTERNAL_FIELD_H_ + +#include "ExternalField_fwd.H" + +#include +#include +#include +#include + +#include +#include + +enum class ExternalFieldType +{ + default_zero, + constant, + parse_ext_grid_function, + read_from_file +}; + +/** + * \brief Struct to store data related to external electromagnetic fields + * (flags, field values, and field parsers) + */ +struct ExternalFieldParams +{ + + /** + * \brief The constructor reads and stores the parameters related to the external fields. + * "pp_warpx" must point at the "warpx" parameter group in the inputfile. + */ + ExternalFieldParams(const amrex::ParmParse& pp_warpx); + + //! Initial electric field on the grid + amrex::GpuArray E_external_grid = {0,0,0}; + //! Initial magnetic field on the grid + amrex::GpuArray B_external_grid = {0,0,0}; + + //! Initialization type for external magnetic field on the grid + ExternalFieldType B_ext_grid_type = ExternalFieldType::default_zero; + //! Initialization type for external electric field on the grid + ExternalFieldType E_ext_grid_type = ExternalFieldType::default_zero; + + //! User-defined parser to initialize x-component of the magnetic field on the grid + std::unique_ptr Bxfield_parser; + //! User-defined parser to initialize y-component of the magnetic field on the grid + std::unique_ptr Byfield_parser; + //! User-defined parser to initialize z-component of the magnetic field on the grid + std::unique_ptr Bzfield_parser; + //! User-defined parser to initialize x-component of the electric field on the grid + std::unique_ptr Exfield_parser; + //! User-defined parser to initialize y-component of the electric field on the grid + std::unique_ptr Eyfield_parser; + //! User-defined parser to initialize z-component of the electric field on the grid + std::unique_ptr Ezfield_parser; + + //! Path of the file where external fields are stored + std::string external_fields_path; +}; + +#endif //EXTERNAL_FIELD_H_ diff --git a/Source/Initialization/ExternalField.cpp b/Source/Initialization/ExternalField.cpp new file mode 100644 index 00000000000..909e5c01977 --- /dev/null +++ b/Source/Initialization/ExternalField.cpp @@ -0,0 +1,182 @@ +/* Copyright 2023 Luca Fedeli + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "ExternalField.H" + +#include "Utils/TextMsg.H" +#include "Utils/Parser/ParserUtils.H" + +#include + +#include +#include + +namespace +{ + + enum class EMFieldType{E, B}; + + template + ExternalFieldType string_to_external_field_type(std::string s) + { + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + + if constexpr (T == EMFieldType::E){ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(s != "parse_b_ext_grid_function", + "parse_B_ext_grid_function can be used only for B_ext_grid_init_style"); + } + else{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(s != "parse_e_ext_grid_function", + "parse_E_ext_grid_function can be used only for E_ext_grid_init_style"); + } + + if ( s.empty() || s == "default"){ + return ExternalFieldType::default_zero; + } + else if ( s == "constant"){ + return ExternalFieldType::constant; + } + else if ( s == "parse_b_ext_grid_function" || s == "parse_e_ext_grid_function"){ + return ExternalFieldType::parse_ext_grid_function; + } + else if ( s == "read_from_file"){ + return ExternalFieldType::read_from_file; + } + else{ + WARPX_ABORT_WITH_MESSAGE( + "'" + s + "' is an unknown external field type!"); + } + + return ExternalFieldType::default_zero; + } +} + +ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) +{ + // default values of E_external_grid and B_external_grid + // are used to set the E and B field when "constant" or + // "parser" is not explicitly used in the input. + std::string B_ext_grid_s; + pp_warpx.query("B_ext_grid_init_style", B_ext_grid_s); + B_ext_grid_type = string_to_external_field_type(B_ext_grid_s); + + std::string E_ext_grid_s; + pp_warpx.query("E_ext_grid_init_style", E_ext_grid_s); + E_ext_grid_type = string_to_external_field_type(E_ext_grid_s); + + // + // Constant external field + // + + // if the input string is "constant", the values for the + // external grid must be provided in the input. + auto v_B = std::vector(3); + if (B_ext_grid_type == ExternalFieldType::constant) { + utils::parser::getArrWithParser(pp_warpx, "B_external_grid", v_B); + } + std::copy(v_B.begin(), v_B.end(), B_external_grid.begin()); + + // if the input string is "constant", the values for the + // external grid must be provided in the input. + auto v_E = std::vector(3); + if (E_ext_grid_type == ExternalFieldType::constant) { + utils::parser::getArrWithParser(pp_warpx, "E_external_grid", v_E); + } + std::copy(v_E.begin(), v_E.end(), E_external_grid.begin()); + //___________________________________________________________________________ + + + // + // External E field with parser + // + + // if the input string for the B-field is "parse_b_ext_grid_function", + // then the analytical expression or function must be + // provided in the input file. + if (B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + + //! Strings storing parser function to initialize the components of the magnetic field on the grid + std::string str_Bx_ext_grid_function; + std::string str_By_ext_grid_function; + std::string str_Bz_ext_grid_function; + +#ifdef WARPX_DIM_RZ + std::stringstream warnMsg; + warnMsg << "Parser for external B (r and theta) fields does not work with RZ\n" + << "The initial Br and Bt fields are currently hardcoded to 0.\n" + << "The initial Bz field should only be a function of z.\n"; + ablastr::warn_manager::WMRecordWarning( + "Inputs", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); + str_Bx_ext_grid_function = "0"; + str_By_ext_grid_function = "0"; +#else + utils::parser::Store_parserString(pp_warpx, "Bx_external_grid_function(x,y,z)", + str_Bx_ext_grid_function); + utils::parser::Store_parserString(pp_warpx, "By_external_grid_function(x,y,z)", + str_By_ext_grid_function); +#endif + utils::parser::Store_parserString(pp_warpx, "Bz_external_grid_function(x,y,z)", + str_Bz_ext_grid_function); + + Bxfield_parser = std::make_unique( + utils::parser::makeParser(str_Bx_ext_grid_function,{"x","y","z"})); + Byfield_parser = std::make_unique( + utils::parser::makeParser(str_By_ext_grid_function,{"x","y","z"})); + Bzfield_parser = std::make_unique( + utils::parser::makeParser(str_Bz_ext_grid_function,{"x","y","z"})); + } + //___________________________________________________________________________ + + + // + // External B field with parser + // + + // if the input string for the E-field is "parse_e_ext_grid_function", + // then the analytical expression or function must be + // provided in the input file. + if (E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + +#ifdef WARPX_DIM_RZ + WARPX_ABORT_WITH_MESSAGE( + "E parser for external fields does not work with RZ -- TO DO"); +#endif + + //! Strings storing parser function to initialize the components of the electric field on the grid + std::string str_Ex_ext_grid_function; + std::string str_Ey_ext_grid_function; + std::string str_Ez_ext_grid_function; + + utils::parser::Store_parserString(pp_warpx, "Ex_external_grid_function(x,y,z)", + str_Ex_ext_grid_function); + utils::parser::Store_parserString(pp_warpx, "Ey_external_grid_function(x,y,z)", + str_Ey_ext_grid_function); + utils::parser::Store_parserString(pp_warpx, "Ez_external_grid_function(x,y,z)", + str_Ez_ext_grid_function); + + Exfield_parser = std::make_unique( + utils::parser::makeParser(str_Ex_ext_grid_function,{"x","y","z"})); + Eyfield_parser = std::make_unique( + utils::parser::makeParser(str_Ey_ext_grid_function,{"x","y","z"})); + Ezfield_parser = std::make_unique( + utils::parser::makeParser(str_Ez_ext_grid_function,{"x","y","z"})); + } + //___________________________________________________________________________ + + + // + // External fields from file + // + + if (E_ext_grid_type == ExternalFieldType::read_from_file || + B_ext_grid_type == ExternalFieldType::read_from_file){ + std::string read_fields_from_path="./"; + pp_warpx.query("read_fields_from_path", external_fields_path); + } + //___________________________________________________________________________ +} diff --git a/Source/Initialization/ExternalField_fwd.H b/Source/Initialization/ExternalField_fwd.H new file mode 100644 index 00000000000..f7a8547ecce --- /dev/null +++ b/Source/Initialization/ExternalField_fwd.H @@ -0,0 +1,13 @@ +/* Copyright 2023 Luca Fedeli + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef EXTERNAL_FIELD_FWD_H_ +#define EXTERNAL_FIELD_FWD_H_ + +struct ExternalFieldParams; + +#endif //EXTERNAL_FIELD_FWD_H_ diff --git a/Source/Initialization/GetTemperature.cpp b/Source/Initialization/GetTemperature.cpp index 4aa1f7fbd74..2e8670bbe6c 100644 --- a/Source/Initialization/GetTemperature.cpp +++ b/Source/Initialization/GetTemperature.cpp @@ -8,8 +8,9 @@ #include "GetTemperature.H" -GetTemperature::GetTemperature (TemperatureProperties const& temp) noexcept { - m_type = temp.m_type; +GetTemperature::GetTemperature (TemperatureProperties const& temp) noexcept : + m_type{temp.m_type} +{ if (m_type == TempConstantValue) { m_temperature = temp.m_temperature; } diff --git a/Source/Initialization/GetVelocity.H b/Source/Initialization/GetVelocity.H index 7b1d5f21350..3123708bf06 100644 --- a/Source/Initialization/GetVelocity.H +++ b/Source/Initialization/GetVelocity.H @@ -27,7 +27,7 @@ struct GetVelocity int m_sign_dir; //! Sign of the velocity direction positive=1, negative=-1 /** Constant velocity value, if m_type == VelConstantValue */ - amrex::Real m_velocity; + amrex::Real m_velocity{0}; /** Velocity parser function, if m_type == VelParserFunction */ amrex::ParserExecutor<3> m_velocity_parser; @@ -79,6 +79,7 @@ struct GetVelocity * 1: y * 2: z */ + [[nodiscard]] AMREX_GPU_HOST_DEVICE int direction () const noexcept { diff --git a/Source/Initialization/GetVelocity.cpp b/Source/Initialization/GetVelocity.cpp index 2d2a342a7ed..4dd77a74383 100644 --- a/Source/Initialization/GetVelocity.cpp +++ b/Source/Initialization/GetVelocity.cpp @@ -7,10 +7,9 @@ #include "GetVelocity.H" -GetVelocity::GetVelocity (VelocityProperties const& vel) noexcept { - m_type = vel.m_type; - m_dir = vel.m_dir; - m_sign_dir = vel.m_sign_dir; +GetVelocity::GetVelocity (VelocityProperties const& vel) noexcept: + m_type{vel.m_type}, m_dir{vel.m_dir}, m_sign_dir{vel.m_sign_dir} +{ if (m_type == VelConstantValue) { m_velocity = vel.m_velocity; } diff --git a/Source/Initialization/InjectorDensity.H b/Source/Initialization/InjectorDensity.H index df85e0625f7..3848a42f4ee 100644 --- a/Source/Initialization/InjectorDensity.H +++ b/Source/Initialization/InjectorDensity.H @@ -26,6 +26,7 @@ struct InjectorDensityConstant { InjectorDensityConstant (amrex::Real a_rho) noexcept : m_rho(a_rho) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::Real getDensity (amrex::Real, amrex::Real, amrex::Real) const noexcept @@ -43,6 +44,7 @@ struct InjectorDensityParser InjectorDensityParser (amrex::ParserExecutor<3> const& a_parser) noexcept : m_parser(a_parser) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::Real getDensity (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept @@ -60,6 +62,7 @@ struct InjectorDensityPredefined void clear (); + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::Real getDensity (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept @@ -111,7 +114,7 @@ struct InjectorDensityPredefined private: enum struct Profile { null, parabolic_channel }; - Profile profile; + Profile profile{Profile::null}; amrex::GpuArray p; }; @@ -150,10 +153,14 @@ struct InjectorDensity void operator= (InjectorDensity const&) = delete; void operator= (InjectorDensity &&) = delete; + // Default destructor + ~InjectorDensity () = default; + void clear (); // call getDensity from the object stored in the union // (the union is called Object, and the instance is called object). + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::Real getDensity (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept diff --git a/Source/Initialization/InjectorDensity.cpp b/Source/Initialization/InjectorDensity.cpp index 959f412c8b3..92a7b5193dd 100644 --- a/Source/Initialization/InjectorDensity.cpp +++ b/Source/Initialization/InjectorDensity.cpp @@ -39,7 +39,6 @@ void InjectorDensity::clear () InjectorDensityPredefined::InjectorDensityPredefined ( std::string const& a_species_name) noexcept - : profile(Profile::null) { const ParmParse pp_species_name(a_species_name); diff --git a/Source/Initialization/InjectorFlux.H b/Source/Initialization/InjectorFlux.H index adfe75b3ed2..e889c8b8966 100644 --- a/Source/Initialization/InjectorFlux.H +++ b/Source/Initialization/InjectorFlux.H @@ -24,6 +24,7 @@ struct InjectorFluxConstant { InjectorFluxConstant (amrex::Real a_flux) noexcept : m_flux(a_flux) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::Real getFlux (amrex::Real, amrex::Real, amrex::Real, amrex::Real) const noexcept @@ -41,6 +42,7 @@ struct InjectorFluxParser InjectorFluxParser (amrex::ParserExecutor<4> const& a_parser) noexcept : m_parser(a_parser) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::Real getFlux (amrex::Real x, amrex::Real y, amrex::Real z, amrex::Real t) const noexcept @@ -79,6 +81,9 @@ struct InjectorFlux void operator= (InjectorFlux const&) = delete; void operator= (InjectorFlux &&) = delete; + // Default destructor + ~InjectorFlux () = default; + void clear () { switch (type) @@ -93,6 +98,7 @@ struct InjectorFlux // call getFlux from the object stored in the union // (the union is called Object, and the instance is called object). + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::Real getFlux (amrex::Real x, amrex::Real y, amrex::Real z, amrex::Real t) const noexcept diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index 87b81381ce9..f7ee6cff138 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -34,6 +34,7 @@ struct InjectorMomentumConstant InjectorMomentumConstant (amrex::Real a_ux, amrex::Real a_uy, amrex::Real a_uz) noexcept : m_ux(a_ux), m_uy(a_uy), m_uz(a_uz) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real, amrex::Real, amrex::Real, @@ -42,6 +43,7 @@ struct InjectorMomentumConstant return amrex::XDim3{m_ux,m_uy,m_uz}; } + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getBulkMomentum (amrex::Real, amrex::Real, amrex::Real) const noexcept @@ -64,6 +66,7 @@ struct InjectorMomentumGaussian m_ux_th(a_ux_th), m_uy_th(a_uy_th), m_uz_th(a_uz_th) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real /*x*/, amrex::Real /*y*/, amrex::Real /*z*/, @@ -74,6 +77,7 @@ struct InjectorMomentumGaussian amrex::RandomNormal(m_uz_m, m_uz_th, engine)}; } + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getBulkMomentum (amrex::Real /*x*/, amrex::Real /*y*/, amrex::Real /*z*/) const noexcept @@ -94,6 +98,8 @@ namespace { * @param u_th Momentum spread * @param engine Object used to generate random numbers */ + [[nodiscard]] + AMREX_FORCE_INLINE AMREX_GPU_HOST_DEVICE amrex::Real generateGaussianFluxDist( amrex::Real u_m, amrex::Real u_th, amrex::RandomEngine const& engine ) { @@ -103,37 +109,40 @@ namespace { // Momentum to be returned at the end of this function amrex::Real u = 0._rt; + const amrex::Real abs_u_m = std::abs(u_m); + if (u_th == 0._rt) { u = u_m; // Trivial case ; avoids division by 0 in the rest of the code below - } else if (u_m < 0.6*u_th) { - // Mean velocity is lower than thermal velocity - // Use the distribution u*exp(-u**2*(1-u_m/u_th)/(2*u_th**2)) as an approximation + } else if (abs_u_m < 0.6*u_th) { + // Mean velocity magnitude is less than thermal velocity + // Use the distribution u*exp(-u**2*(1-abs(u_m)/u_th)/(2*u_th**2)) as an approximation // and then use the rejection method to correct it - // ( stop rejecting with probability exp(-u_m/(2*u_th**3)*(u-u_th)**2) ) + // ( stop rejecting with probability exp(-abs(u_m)/(2*u_th**3)*(u-sign(u_m)*u_th)**2) ) // Note that this is the method that is used in the common case u_m=0 - const amrex::Real approx_u_th = u_th/std::sqrt( 1._rt - u_m/u_th ); - const amrex::Real reject_prefactor = (u_m/u_th)/(2._rt*u_th*u_th); // To save computation + const amrex::Real umsign = std::copysign(1._rt, u_m); + const amrex::Real approx_u_th = u_th/std::sqrt( 1._rt - abs_u_m/u_th ); + const amrex::Real reject_prefactor = (abs_u_m/u_th)/(2._rt*u_th*u_th); // To save computation bool reject = true; while (reject) { - // Generates u according to u*exp(-u**2/(2*approx_u_th**2), + // Generates u according to u*exp(-u**2/(2*approx_u_th**2)), // using the method of the inverse cumulative function amrex::Real xrand = 1._rt - amrex::Random(engine); // ensures urand > 0 u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); // Rejection method xrand = amrex::Random(engine); - if (xrand < std::exp(-reject_prefactor*(u-u_th)*(u-u_th))) reject = false; + if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } } } else { - // Mean velocity is greater than thermal velocity - // Use the distribution exp(-(u-u_m-u_th**2/u_m)**2/(2*u_th**2)) as an approximation + // Mean velocity magnitude is greater than thermal velocity + // Use the distribution exp(-(u-u_m-u_th**2/abs(u_m))**2/(2*u_th**2)) as an approximation // and then use the rejection method to correct it - // ( stop rejecting with probability (u/u_m)*exp(1-(u/u_m)) ; note + // ( stop rejecting with probability (u/abs(u_m))*exp(1-(u/abs(u_m))) ; note // that this number is always between 0 and 1 ) // Note that in the common case `u_m = 0`, this rejection method // is not used, and the above rejection method is used instead. bool reject = true; - const amrex::Real approx_u_m = u_m + u_th*u_th/u_m; - const amrex::Real inv_um = 1._rt/u_m; // To save computation + const amrex::Real approx_u_m = u_m + u_th*u_th/abs_u_m; + const amrex::Real inv_um = 1._rt/abs_u_m; // To save computation while (reject) { // Approximate distribution: normal distribution, where we only retain positive u u = -1._rt; @@ -142,7 +151,7 @@ namespace { } // Rejection method const amrex::Real xrand = amrex::Random(engine); - if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) reject = false; + if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } } } @@ -165,15 +174,9 @@ struct InjectorMomentumGaussianFlux m_flux_normal_axis(a_flux_normal_axis), m_flux_direction(a_flux_direction) { - // For now, do not allow negative `u_m` along the flux axis - bool raise_error = false; - if ((m_flux_normal_axis == 0) && (m_ux_m < 0)) raise_error = true; - if ((m_flux_normal_axis == 1) && (m_uy_m < 0)) raise_error = true; - if ((m_flux_normal_axis == 2) && (m_uz_m < 0)) raise_error = true; - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( raise_error==false, - "When using the `gaussianflux` distribution, the central momentum along the flux axis must be positive or zero." ); } + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real /*x*/, amrex::Real /*y*/, amrex::Real /*z*/, @@ -194,7 +197,7 @@ struct InjectorMomentumGaussianFlux u_th = m_uz_th; } amrex::Real u = generateGaussianFluxDist(u_m, u_th, engine); - if (m_flux_direction < 0) u = -u; + if (m_flux_direction < 0) { u = -u; } // Note: Here, in RZ geometry, the variables `ux` and `uy` actually // correspond to the radial and azimuthal component of the momentum @@ -205,6 +208,7 @@ struct InjectorMomentumGaussianFlux return amrex::XDim3{ux, uy, uz}; } + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getBulkMomentum (amrex::Real /*x*/, amrex::Real /*y*/, amrex::Real /*z*/) const noexcept @@ -228,16 +232,16 @@ struct InjectorMomentumUniform amrex::Real a_uz_min, amrex::Real a_ux_max, amrex::Real a_uy_max, amrex::Real a_uz_max) noexcept : m_ux_min(a_ux_min), m_uy_min(a_uy_min), m_uz_min(a_uz_min), - m_ux_max(a_ux_max), m_uy_max(a_uy_max), m_uz_max(a_uz_max) - { - m_Dux = m_ux_max - m_ux_min; - m_Duy = m_uy_max - m_uy_min; - m_Duz = m_uz_max - m_uz_min; - m_ux_h = 0.5 * (m_ux_max + m_ux_min); - m_uy_h = 0.5 * (m_uy_max + m_uy_min); - m_uz_h = 0.5 * (m_uz_max + m_uz_min); - } + m_ux_max(a_ux_max), m_uy_max(a_uy_max), m_uz_max(a_uz_max), + m_Dux(m_ux_max - m_ux_min), + m_Duy(m_uy_max - m_uy_min), + m_Duz(m_uz_max - m_uz_min), + m_ux_h(amrex::Real(0.5) * (m_ux_max + m_ux_min)), + m_uy_h(amrex::Real(0.5) * (m_uy_max + m_uy_min)), + m_uz_h(amrex::Real(0.5) * (m_uz_max + m_uz_min)) + {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real /*x*/, amrex::Real /*y*/, amrex::Real /*z*/, @@ -248,6 +252,7 @@ struct InjectorMomentumUniform m_uz_min + amrex::Random(engine) * m_Duz}; } + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getBulkMomentum (amrex::Real /*x*/, amrex::Real /*y*/, amrex::Real /*z*/) const noexcept @@ -258,8 +263,8 @@ struct InjectorMomentumUniform private: amrex::Real m_ux_min, m_uy_min, m_uz_min; amrex::Real m_ux_max, m_uy_max, m_uz_max; - amrex::Real m_ux_h, m_uy_h, m_uz_h; amrex::Real m_Dux, m_Duy, m_Duz; + amrex::Real m_ux_h, m_uy_h, m_uz_h; }; // struct whose getMomentum returns momentum for 1 particle with relativistic @@ -273,6 +278,7 @@ struct InjectorMomentumBoltzmann : velocity(b), temperature(t) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real const x, amrex::Real const y, amrex::Real const z, @@ -303,7 +309,7 @@ struct InjectorMomentumBoltzmann // The following condition is equation 32 in Zenitani 2015 // (Phys. Plasmas 22, 042116) , called the flipping method. It - // transforms the intergral: d3x' -> d3x where d3x' is the volume + // transforms the integral: d3x' -> d3x where d3x' is the volume // element for positions in the boosted frame. The particle positions // and densities can be initialized in the simulation frame. // The flipping method can transform any symmetric distribution from one @@ -325,16 +331,17 @@ struct InjectorMomentumBoltzmann return amrex::XDim3 {u[0],u[1],u[2]}; } + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getBulkMomentum (amrex::Real const x, amrex::Real const y, amrex::Real const z) const noexcept { using namespace amrex::literals; amrex::Real u[3]; - for (auto& el : u) el = 0.0_rt; + for (auto& el : u) { el = 0.0_rt; } const amrex::Real beta = velocity(x,y,z); int const dir = velocity.direction(); - const amrex::Real gamma = static_cast(1._rt/sqrt(1._rt-beta*beta)); + const auto gamma = 1._rt/std::sqrt(1._rt-beta*beta); u[dir] = gamma*beta; return amrex::XDim3 {u[0],u[1],u[2]}; } @@ -344,7 +351,7 @@ private: GetTemperature temperature; }; -// struct whose getMomentum returns momentum for 1 particle with relativistc +// struct whose getMomentum returns momentum for 1 particle with relativistic // drift velocity beta, from the Maxwell-Juttner distribution. Method is from // Zenitani 2015 (Phys. Plasmas 22, 042116). struct InjectorMomentumJuttner @@ -356,6 +363,7 @@ struct InjectorMomentumJuttner : velocity(b), temperature(t) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real const x, amrex::Real const y, amrex::Real const z, @@ -401,8 +409,8 @@ struct InjectorMomentumJuttner // The value of dir is the boost direction to be transformed. u[dir] = u[dir]*(2._rt*x1-1._rt); x1 = amrex::Random(engine); - // The following condition is equtaion 32 in Zenitani, called - // The flipping method. It transforms the intergral: d3x' -> d3x + // The following condition is equation 32 in Zenitani, called + // The flipping method. It transforms the integral: d3x' -> d3x // where d3x' is the volume element for positions in the boosted frame. // The particle positions and densities can be initialized in the // simulation frame with this method. @@ -426,16 +434,17 @@ struct InjectorMomentumJuttner return amrex::XDim3 {u[0],u[1],u[2]}; } + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getBulkMomentum (amrex::Real const x, amrex::Real const y, amrex::Real const z) const noexcept { using namespace amrex::literals; amrex::Real u[3]; - for (auto& el : u) el = 0.0_rt; + for (auto& el : u) { el = 0.0_rt; } amrex::Real const beta = velocity(x,y,z); int const dir = velocity.direction(); - amrex::Real const gamma = static_cast(1._rt/sqrt(1._rt-beta*beta)); + auto const gamma = 1._rt/std::sqrt(1._rt-beta*beta); u[dir] = gamma*beta; return amrex::XDim3 {u[0],u[1],u[2]}; } @@ -458,6 +467,7 @@ struct InjectorMomentumRadialExpansion : u_over_r(a_u_over_r) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real x, amrex::Real y, amrex::Real z, @@ -466,6 +476,7 @@ struct InjectorMomentumRadialExpansion return {x*u_over_r, y*u_over_r, z*u_over_r}; } + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getBulkMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept @@ -477,7 +488,7 @@ private: amrex::Real u_over_r; }; -// struct whose getMomentumm returns local momentum computed from parser. +// struct whose getMomentum returns local momentum computed from parser. struct InjectorMomentumParser { InjectorMomentumParser (amrex::ParserExecutor<3> const& a_ux_parser, @@ -486,6 +497,7 @@ struct InjectorMomentumParser : m_ux_parser(a_ux_parser), m_uy_parser(a_uy_parser), m_uz_parser(a_uz_parser) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real x, amrex::Real y, amrex::Real z, @@ -494,6 +506,7 @@ struct InjectorMomentumParser return amrex::XDim3{m_ux_parser(x,y,z),m_uy_parser(x,y,z),m_uz_parser(x,y,z)}; } + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getBulkMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept @@ -504,6 +517,47 @@ struct InjectorMomentumParser amrex::ParserExecutor<3> m_ux_parser, m_uy_parser, m_uz_parser; }; +// struct whose getMomentum returns local momentum and thermal spread computed from parser. +struct InjectorMomentumGaussianParser +{ + InjectorMomentumGaussianParser (amrex::ParserExecutor<3> const& a_ux_m_parser, + amrex::ParserExecutor<3> const& a_uy_m_parser, + amrex::ParserExecutor<3> const& a_uz_m_parser, + amrex::ParserExecutor<3> const& a_ux_th_parser, + amrex::ParserExecutor<3> const& a_uy_th_parser, + amrex::ParserExecutor<3> const& a_uz_th_parser) noexcept + : m_ux_m_parser(a_ux_m_parser), m_uy_m_parser(a_uy_m_parser), m_uz_m_parser(a_uz_m_parser), + m_ux_th_parser(a_ux_th_parser), m_uy_th_parser(a_uy_th_parser), m_uz_th_parser(a_uz_th_parser) {} + + [[nodiscard]] + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getMomentum (amrex::Real x, amrex::Real y, amrex::Real z, + amrex::RandomEngine const& engine) const noexcept + { + amrex::Real const ux_m = m_ux_m_parser(x,y,z); + amrex::Real const uy_m = m_uy_m_parser(x,y,z); + amrex::Real const uz_m = m_uz_m_parser(x,y,z); + amrex::Real const ux_th = m_ux_th_parser(x,y,z); + amrex::Real const uy_th = m_uy_th_parser(x,y,z); + amrex::Real const uz_th = m_uz_th_parser(x,y,z); + return amrex::XDim3{amrex::RandomNormal(ux_m, ux_th, engine), + amrex::RandomNormal(uy_m, uy_th, engine), + amrex::RandomNormal(uz_m, uz_th, engine)}; + } + + [[nodiscard]] + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getBulkMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + return amrex::XDim3{m_ux_m_parser(x,y,z), m_uy_m_parser(x,y,z), m_uz_m_parser(x,y,z)}; + } + + amrex::ParserExecutor<3> m_ux_m_parser, m_uy_m_parser, m_uz_m_parser; + amrex::ParserExecutor<3> m_ux_th_parser, m_uy_th_parser, m_uz_th_parser; +}; + // Base struct for momentum injector. // InjectorMomentum contains a union (called Object) that holds any one // instance of: @@ -512,6 +566,7 @@ struct InjectorMomentumParser // - InjectorMomentumGaussianFlux : to generate v*gaussian distribution; // - InjectorMomentumRadialExpansion: to generate radial expansion; // - InjectorMomentumParser : to generate momentum from parser; +// - InjectorMomentumGaussianParser : to generate momentum and thermal spread from parser; // The choice is made at runtime, depending in the constructor called. // This mimics virtual functions. struct InjectorMomentum @@ -532,6 +587,19 @@ struct InjectorMomentum object(t, a_ux_parser, a_uy_parser, a_uz_parser) { } + // This constructor stores a InjectorMomentumGaussianParser in union object. + InjectorMomentum (InjectorMomentumGaussianParser* t, + amrex::ParserExecutor<3> const& a_ux_m_parser, + amrex::ParserExecutor<3> const& a_uy_m_parser, + amrex::ParserExecutor<3> const& a_uz_m_parser, + amrex::ParserExecutor<3> const& a_ux_th_parser, + amrex::ParserExecutor<3> const& a_uy_th_parser, + amrex::ParserExecutor<3> const& a_uz_th_parser) + : type(Type::gaussianparser), + object(t, a_ux_m_parser, a_uy_m_parser, a_uz_m_parser, + a_ux_th_parser, a_uy_th_parser, a_uz_th_parser) + { } + // This constructor stores a InjectorMomentumGaussian in union object. InjectorMomentum (InjectorMomentumGaussian* t, amrex::Real a_ux_m, amrex::Real a_uy_m, amrex::Real a_uz_m, @@ -563,9 +631,9 @@ struct InjectorMomentum object(t, temperature, velocity) { } - // This constructor stores a InjectorMomentumJuttner in union object. - InjectorMomentum (InjectorMomentumJuttner* t, - GetTemperature const& temperature, GetVelocity const& velocity) + // This constructor stores a InjectorMomentumJuttner in union object. + InjectorMomentum (InjectorMomentumJuttner* t, + GetTemperature const& temperature, GetVelocity const& velocity) : type(Type::juttner), object(t, temperature, velocity) { } @@ -584,10 +652,14 @@ struct InjectorMomentum void operator= (InjectorMomentum const&) = delete; void operator= (InjectorMomentum &&) = delete; + // Default destructor + ~InjectorMomentum() = default; + void clear (); // call getMomentum from the object stored in the union // (the union is called Object, and the instance is called object). + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real x, amrex::Real y, amrex::Real z, @@ -603,6 +675,10 @@ struct InjectorMomentum { return object.gaussian.getMomentum(x,y,z,engine); } + case Type::gaussianparser: + { + return object.gaussianparser.getMomentum(x,y,z,engine); + } case Type::gaussianflux: { return object.gaussianflux.getMomentum(x,y,z,engine); @@ -637,6 +713,7 @@ struct InjectorMomentum // call getBulkMomentum from the object stored in the union // (the union is called Object, and the instance is called object). + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getBulkMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept @@ -651,6 +728,10 @@ struct InjectorMomentum { return object.gaussian.getBulkMomentum(x,y,z); } + case Type::gaussianparser: + { + return object.gaussianparser.getBulkMomentum(x,y,z); + } case Type::gaussianflux: { return object.gaussianflux.getBulkMomentum(x,y,z); @@ -683,7 +764,7 @@ struct InjectorMomentum } } - enum struct Type { constant, gaussian, gaussianflux, uniform, boltzmann, juttner, radial_expansion, parser }; + enum struct Type { constant, gaussian, gaussianflux, uniform, boltzmann, juttner, radial_expansion, parser, gaussianparser }; Type type; private: @@ -725,6 +806,15 @@ private: amrex::ParserExecutor<3> const& a_uy_parser, amrex::ParserExecutor<3> const& a_uz_parser) noexcept : parser(a_ux_parser, a_uy_parser, a_uz_parser) {} + Object (InjectorMomentumGaussianParser*, + amrex::ParserExecutor<3> const& a_ux_m_parser, + amrex::ParserExecutor<3> const& a_uy_m_parser, + amrex::ParserExecutor<3> const& a_uz_m_parser, + amrex::ParserExecutor<3> const& a_ux_th_parser, + amrex::ParserExecutor<3> const& a_uy_th_parser, + amrex::ParserExecutor<3> const& a_uz_th_parser) noexcept + : gaussianparser(a_ux_m_parser, a_uy_m_parser, a_uz_m_parser, + a_ux_th_parser, a_uy_th_parser, a_uz_th_parser) {} InjectorMomentumConstant constant; InjectorMomentumGaussian gaussian; InjectorMomentumGaussianFlux gaussianflux; @@ -732,7 +822,8 @@ private: InjectorMomentumBoltzmann boltzmann; InjectorMomentumJuttner juttner; InjectorMomentumRadialExpansion radial_expansion; - InjectorMomentumParser parser; + InjectorMomentumParser parser; + InjectorMomentumGaussianParser gaussianparser; }; Object object; }; diff --git a/Source/Initialization/InjectorMomentum.cpp b/Source/Initialization/InjectorMomentum.cpp index 13f39ef95a5..35513260827 100644 --- a/Source/Initialization/InjectorMomentum.cpp +++ b/Source/Initialization/InjectorMomentum.cpp @@ -15,6 +15,7 @@ void InjectorMomentum::clear () { case Type::parser: case Type::gaussian: + case Type::gaussianparser: case Type::gaussianflux: case Type::uniform: case Type::boltzmann: diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 17ce8325543..c86ed432bd1 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -18,6 +18,7 @@ // random distribution inside a unit cell. struct InjectorPositionRandom { + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getPositionUnitBox (int /*i_part*/, amrex::IntVect const /*ref_fac*/, @@ -33,6 +34,7 @@ struct InjectorPositionRandomPlane { InjectorPositionRandomPlane (int const& a_dir) noexcept : dir(a_dir) {} + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getPositionUnitBox (int /*i_part*/, amrex::IntVect const /*ref_fac*/, @@ -41,19 +43,19 @@ struct InjectorPositionRandomPlane using namespace amrex::literals; #if ((defined WARPX_DIM_3D) || (defined WARPX_DIM_RZ)) // In RZ, the 3 components of the `XDim3` vector below correspond to r, theta, z respectively - if (dir == 0) return amrex::XDim3{0._rt, amrex::Random(engine), amrex::Random(engine)}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), 0._rt, amrex::Random(engine)}; - else return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; + if (dir == 0) { return amrex::XDim3{0._rt, amrex::Random(engine), amrex::Random(engine)}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), 0._rt, amrex::Random(engine)}; } + else { return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; } #elif (defined(WARPX_DIM_XZ)) // In 2D, the 2 first components of the `XDim3` vector below correspond to x and z - if (dir == 0) return amrex::XDim3{0._rt, amrex::Random(engine), 0._rt}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; - else return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; + if (dir == 0) { return amrex::XDim3{0._rt, amrex::Random(engine), 0._rt}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; } + else { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt }; } #elif (defined(WARPX_DIM_1D_Z)) // In 2D, the first components of the `XDim3` vector below correspond to z - if (dir == 0) return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; - else return amrex::XDim3{0._rt, 0._rt, 0._rt}; + if (dir == 0) { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; } + else { return amrex::XDim3{0._rt, 0._rt, 0._rt}; } #endif } private: @@ -70,6 +72,7 @@ struct InjectorPositionRegular // particles within the cell. // ref_fac: the number of particles evenly-spaced within a cell // is a_ppc*(ref_fac**AMREX_SPACEDIM). + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getPositionUnitBox (int const i_part, amrex::IntVect const ref_fac, @@ -154,15 +157,18 @@ struct InjectorPosition zmin(a_zmin), zmax(a_zmax) { } + ~InjectorPosition () = default; + // Explicitly prevent the compiler from generating copy constructors // and copy assignment operators. InjectorPosition (InjectorPosition const&) = delete; - InjectorPosition (InjectorPosition&&) = delete; - void operator= (InjectorPosition const&) = delete; - void operator= (InjectorPosition &&) = delete; + InjectorPosition (InjectorPosition&&) = delete; + void operator= (InjectorPosition const&) = delete; + void operator= (InjectorPosition &&) = delete; // call getPositionUnitBox from the object stored in the union // (the union is called Object, and the instance is called object). + [[nodiscard]] AMREX_GPU_HOST_DEVICE amrex::XDim3 getPositionUnitBox (int const i_part, amrex::IntVect const ref_fac, @@ -190,6 +196,7 @@ struct InjectorPosition * \param x, y, z the point to check * \returns bool flag */ + [[nodiscard]] AMREX_GPU_HOST_DEVICE bool insideBounds (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept @@ -204,6 +211,7 @@ struct InjectorPosition * \param x, y, z the point to check * \returns bool flag */ + [[nodiscard]] AMREX_GPU_HOST_DEVICE bool insideBoundsInclusive (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept @@ -213,7 +221,8 @@ struct InjectorPosition z <= zmax and z >= zmin); } - // bool: whether the region defined by lo and hi overaps with the plasma region + // bool: whether the region defined by lo and hi overlaps with the plasma region + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool overlapsWith (const amrex::XDim3& lo, const amrex::XDim3& hi) const noexcept diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index 13eed49639f..8b4a4c1d669 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -1,11 +1,12 @@ -CEXE_sources += WarpXAMReXInit.cpp -CEXE_sources += WarpXInitData.cpp -CEXE_sources += PlasmaInjector.cpp +CEXE_sources += ExternalField.cpp +CEXE_sources += GetTemperature.cpp +CEXE_sources += GetVelocity.cpp CEXE_sources += InjectorDensity.cpp CEXE_sources += InjectorMomentum.cpp +CEXE_sources += PlasmaInjector.cpp CEXE_sources += TemperatureProperties.cpp -CEXE_sources += GetTemperature.cpp CEXE_sources += VelocityProperties.cpp -CEXE_sources += GetVelocity.cpp +CEXE_sources += WarpXAMReXInit.cpp +CEXE_sources += WarpXInitData.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index 89ea4a00a78..616b1a74b17 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -42,17 +43,27 @@ class PlasmaInjector public: + /** Default constructor*/ PlasmaInjector () = default; - PlasmaInjector (int ispecies, const std::string& name, const amrex::Geometry& geom); + PlasmaInjector (int ispecies, const std::string& name, const amrex::Geometry& geom, + const std::string& src_name=""); + + // Default move and copy operations + PlasmaInjector(const PlasmaInjector&) = delete; + PlasmaInjector& operator=(const PlasmaInjector&) = delete; + PlasmaInjector(PlasmaInjector&&) = default; + PlasmaInjector& operator=(PlasmaInjector&&) = default; ~PlasmaInjector (); // bool: whether the point (x, y, z) is inside the plasma region - bool insideBounds (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept; + [[nodiscard]] bool insideBounds ( + amrex::Real x, amrex::Real y, amrex::Real z) const noexcept; // bool: whether the region defined by lo and hi overlaps with the plasma region - bool overlapsWith (const amrex::XDim3& lo, const amrex::XDim3& hi) const noexcept; + [[nodiscard]] bool overlapsWith ( + const amrex::XDim3& lo, const amrex::XDim3& hi) const noexcept; int num_particles_per_cell; amrex::Real num_particles_per_cell_real; @@ -60,18 +71,18 @@ public: amrex::Vector num_particles_per_cell_each_dim; // gamma * beta - amrex::XDim3 getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept; + [[nodiscard]] amrex::XDim3 getMomentum ( + amrex::Real x, amrex::Real y, amrex::Real z) const noexcept; - amrex::Real getCharge () {return charge;} - amrex::Real getMass () {return mass;} - PhysicalSpecies getPhysicalSpecies() const {return physical_species;} + bool queryCharge (amrex::ParticleReal& a_charge) const; + bool queryMass (amrex::ParticleReal& a_mass) const; // bool: whether the initial injection of particles should be done // This routine is called during initialization of the plasma. - bool doInjection () const noexcept { return h_inj_pos != nullptr;} + [[nodiscard]] bool doInjection () const noexcept { return h_inj_pos != nullptr;} // bool: whether the flux injection of particles should be done. - bool doFluxInjection () const noexcept { return h_flux_pos != nullptr;} + [[nodiscard]] bool doFluxInjection () const noexcept { return h_flux_pos != nullptr;} bool add_single_particle = false; amrex::Vector single_particle_pos; @@ -120,11 +131,7 @@ public: bool radially_weighted = true; - std::string str_density_function; std::string str_flux_function; - std::string str_momentum_function_ux; - std::string str_momentum_function_uy; - std::string str_momentum_function_uz; amrex::Real xmin, xmax; amrex::Real ymin, ymax; @@ -132,25 +139,27 @@ public: amrex::Real density_min = std::numeric_limits::epsilon(); amrex::Real density_max = std::numeric_limits::max(); - InjectorPosition* getInjectorPosition (); - InjectorPosition* getInjectorFluxPosition (); - InjectorDensity* getInjectorDensity (); + [[nodiscard]] InjectorPosition* getInjectorPosition () const; + [[nodiscard]] InjectorPosition* getInjectorFluxPosition () const; + [[nodiscard]] InjectorDensity* getInjectorDensity () const; - InjectorFlux* getInjectorFlux (); - InjectorMomentum* getInjectorMomentumDevice (); - InjectorMomentum* getInjectorMomentumHost (); + [[nodiscard]] InjectorFlux* getInjectorFlux () const; + [[nodiscard]] InjectorMomentum* getInjectorMomentumDevice () const; + [[nodiscard]] InjectorMomentum* getInjectorMomentumHost () const; protected: - amrex::Real mass, charge; + bool mass_from_source = false; + bool charge_from_source = false; + amrex::ParticleReal mass, charge; PhysicalSpecies physical_species = PhysicalSpecies::unspecified; - amrex::Real density; amrex::Real flux; int species_id; std::string species_name; + std::string source_name; std::unique_ptr h_inj_pos; InjectorPosition* d_inj_pos = nullptr; @@ -171,23 +180,24 @@ protected: std::unique_ptr ux_parser; std::unique_ptr uy_parser; std::unique_ptr uz_parser; + std::unique_ptr ux_th_parser; + std::unique_ptr uy_th_parser; + std::unique_ptr uz_th_parser; // Keep a pointer to TemperatureProperties to ensure the lifetime of the // contained Parser std::unique_ptr h_mom_temp; std::unique_ptr h_mom_vel; - void setupSingleParticle (const amrex::ParmParse& pp_species_name); - void setupMultipleParticles (const amrex::ParmParse& pp_species_name); - void setupGaussianBeam (const amrex::ParmParse& pp_species_name); - void setupNRandomPerCell (const amrex::ParmParse& pp_species_name); - void setupNFluxPerCell (const amrex::ParmParse& pp_species_name); - void setupNuniformPerCell (const amrex::ParmParse& pp_species_name); - void setupExternalFile (const amrex::ParmParse& pp_species_name); - - void parseDensity (const amrex::ParmParse& pp_species_name); - void parseFlux (const amrex::ParmParse& pp_species_name); - void parseMomentum (const amrex::ParmParse& pp_species_name, const std::string& style); + void setupSingleParticle (amrex::ParmParse const& pp_species); + void setupMultipleParticles (amrex::ParmParse const& pp_species); + void setupGaussianBeam (amrex::ParmParse const& pp_species); + void setupNRandomPerCell (amrex::ParmParse const& pp_species); + void setupNFluxPerCell (amrex::ParmParse const& pp_species); + void setupNuniformPerCell (amrex::ParmParse const& pp_species); + void setupExternalFile (amrex::ParmParse const& pp_species); + + void parseFlux (amrex::ParmParse const& pp_species); }; #endif diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index b880d4a46e7..3b2ad76513a 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -14,8 +14,8 @@ #include "Initialization/InjectorDensity.H" #include "Initialization/InjectorMomentum.H" #include "Initialization/InjectorPosition.H" -#include "Particles/SpeciesPhysicalProperties.H" #include "Utils/Parser/ParserUtils.H" +#include "Utils/SpeciesUtils.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "WarpX.H" @@ -45,22 +45,10 @@ using namespace amrex::literals; -namespace { - void StringParseAbortMessage(const std::string& var, - const std::string& name) { - std::stringstream stringstream; - std::string string; - stringstream << var << " string '" << name << "' not recognized."; - string = stringstream.str(); - WARPX_ABORT_WITH_MESSAGE(string); - } -} - PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, - const amrex::Geometry& geom): - species_id{ispecies}, species_name{name} + const amrex::Geometry& geom, const std::string& src_name): + species_id{ispecies}, species_name{name}, source_name{src_name} { - const amrex::ParmParse pp_species_name(species_name); #ifdef AMREX_USE_GPU static_assert(std::is_trivially_copyable::value, @@ -71,7 +59,9 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, "InjectorMomentum must be trivially copyable"); #endif - pp_species_name.query("radially_weighted", radially_weighted); + const amrex::ParmParse pp_species(species_name); + + utils::parser::queryWithParser(pp_species, source_name, "radially_weighted", radially_weighted); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(radially_weighted, "ERROR: Only radially_weighted=true is supported"); // Unlimited boundaries @@ -113,93 +103,63 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, } # endif - utils::parser::queryWithParser(pp_species_name, "xmin", xmin); - utils::parser::queryWithParser(pp_species_name, "ymin", ymin); - utils::parser::queryWithParser(pp_species_name, "zmin", zmin); - utils::parser::queryWithParser(pp_species_name, "xmax", xmax); - utils::parser::queryWithParser(pp_species_name, "ymax", ymax); - utils::parser::queryWithParser(pp_species_name, "zmax", zmax); - - utils::parser::queryWithParser(pp_species_name, "density_min", density_min); - utils::parser::queryWithParser(pp_species_name, "density_max", density_max); - - std::string physical_species_s; - const bool species_is_specified = pp_species_name.query("species_type", physical_species_s); - if (species_is_specified){ - const auto physical_species_from_string = species::from_string( physical_species_s ); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(physical_species_from_string, - physical_species_s + " does not exist!"); - physical_species = physical_species_from_string.value(); - charge = species::get_charge( physical_species ); - mass = species::get_mass( physical_species ); - } + utils::parser::queryWithParser(pp_species, source_name, "xmin", xmin); + utils::parser::queryWithParser(pp_species, source_name, "ymin", ymin); + utils::parser::queryWithParser(pp_species, source_name, "zmin", zmin); + utils::parser::queryWithParser(pp_species, source_name, "xmax", xmax); + utils::parser::queryWithParser(pp_species, source_name, "ymax", ymax); + utils::parser::queryWithParser(pp_species, source_name, "zmax", zmax); + + utils::parser::queryWithParser(pp_species, source_name, "density_min", density_min); + utils::parser::queryWithParser(pp_species, source_name, "density_max", density_max); - // Parse injection style std::string injection_style = "none"; - pp_species_name.query("injection_style", injection_style); + utils::parser::query(pp_species, source_name, "injection_style", injection_style); std::transform(injection_style.begin(), injection_style.end(), injection_style.begin(), ::tolower); - // parse charge and mass - const bool charge_is_specified = - utils::parser::queryWithParser(pp_species_name, "charge", charge); - const bool mass_is_specified = - utils::parser::queryWithParser(pp_species_name, "mass", mass); - - if ( charge_is_specified && species_is_specified ){ - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + species_name + ".charge' and " + - species_name + ".species_type' are specified.\n" + - species_name + ".charge' will take precedence.\n"); - - } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - charge_is_specified || - species_is_specified || - (injection_style == "external_file"), - "Need to specify at least one of species_type or charge for species '" + - species_name + "'." - ); - - if ( mass_is_specified && species_is_specified ){ - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + species_name + ".mass' and " + - species_name + ".species_type' are specified.\n" + - species_name + ".mass' will take precedence.\n"); - } - - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - mass_is_specified || - species_is_specified || - (injection_style == "external_file"), - "Need to specify at least one of species_type or mass for species '" + - species_name + "'." - ); - num_particles_per_cell_each_dim.assign(3, 0); if (injection_style == "singleparticle") { - setupSingleParticle(pp_species_name); + setupSingleParticle(pp_species); return; } else if (injection_style == "multipleparticles") { - setupMultipleParticles(pp_species_name); + setupMultipleParticles(pp_species); return; } else if (injection_style == "gaussian_beam") { - setupGaussianBeam(pp_species_name); + setupGaussianBeam(pp_species); } else if (injection_style == "nrandompercell") { - setupNRandomPerCell(pp_species_name); + setupNRandomPerCell(pp_species); } else if (injection_style == "nfluxpercell") { - setupNFluxPerCell(pp_species_name); + setupNFluxPerCell(pp_species); } else if (injection_style == "nuniformpercell") { - setupNuniformPerCell(pp_species_name); + setupNuniformPerCell(pp_species); } else if (injection_style == "external_file") { - setupExternalFile(pp_species_name); + setupExternalFile(pp_species); } else if (injection_style != "none") { - StringParseAbortMessage("Injection style", injection_style); + SpeciesUtils::StringParseAbortMessage("Injection style", injection_style); } + if (h_inj_rho) { +#ifdef AMREX_USE_GPU + d_inj_rho = static_cast + (amrex::The_Arena()->alloc(sizeof(InjectorDensity))); + amrex::Gpu::htod_memcpy_async(d_inj_rho, h_inj_rho.get(), sizeof(InjectorDensity)); +#else + d_inj_rho = h_inj_rho.get(); +#endif + } + if (h_inj_mom) { +#ifdef AMREX_USE_GPU + d_inj_mom = static_cast + (amrex::The_Arena()->alloc(sizeof(InjectorMomentum))); + amrex::Gpu::htod_memcpy_async(d_inj_mom, h_inj_mom.get(), sizeof(InjectorMomentum)); +#else + d_inj_mom = h_inj_mom.get(); +#endif + } amrex::Gpu::synchronize(); } @@ -223,36 +183,26 @@ PlasmaInjector::~PlasmaInjector () PlasmaInjector::~PlasmaInjector () = default; #endif -void PlasmaInjector::setupSingleParticle (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupSingleParticle (amrex::ParmParse const& pp_species) { - utils::parser::getArrWithParser( - pp_species_name, "single_particle_pos", single_particle_pos, 0, 3); - utils::parser::getArrWithParser( - pp_species_name, "single_particle_u", single_particle_u, 0, 3); + utils::parser::getArrWithParser(pp_species, source_name, "single_particle_pos", single_particle_pos, 0, 3); + utils::parser::getArrWithParser(pp_species, source_name, "single_particle_u", single_particle_u, 0, 3); for (auto& x : single_particle_u) { x *= PhysConst::c; } - utils::parser::getWithParser( - pp_species_name, "single_particle_weight", single_particle_weight); + utils::parser::getWithParser(pp_species, source_name, "single_particle_weight", single_particle_weight); add_single_particle = true; } -void PlasmaInjector::setupMultipleParticles (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupMultipleParticles (amrex::ParmParse const& pp_species) { - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_pos_x", multiple_particles_pos_x); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_pos_y", multiple_particles_pos_y); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_pos_z", multiple_particles_pos_z); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_ux", multiple_particles_ux); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_uy", multiple_particles_uy); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_uz", multiple_particles_uz); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_weight", multiple_particles_weight); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_pos_x", multiple_particles_pos_x); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_pos_y", multiple_particles_pos_y); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_pos_z", multiple_particles_pos_z); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_ux", multiple_particles_ux); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_uy", multiple_particles_uy); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_uz", multiple_particles_uz); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_weight", multiple_particles_weight); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( ((multiple_particles_pos_x.size() == multiple_particles_pos_y.size()) && (multiple_particles_pos_x.size() == multiple_particles_pos_z.size()) && @@ -267,26 +217,29 @@ void PlasmaInjector::setupMultipleParticles (const amrex::ParmParse& pp_species_ add_multiple_particles = true; } -void PlasmaInjector::setupGaussianBeam (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupGaussianBeam (amrex::ParmParse const& pp_species) { - utils::parser::getWithParser(pp_species_name, "x_m", x_m); - utils::parser::getWithParser(pp_species_name, "y_m", y_m); - utils::parser::getWithParser(pp_species_name, "z_m", z_m); - utils::parser::getWithParser(pp_species_name, "x_rms", x_rms); - utils::parser::getWithParser(pp_species_name, "y_rms", y_rms); - utils::parser::getWithParser(pp_species_name, "z_rms", z_rms); - utils::parser::queryWithParser(pp_species_name, "x_cut", x_cut); - utils::parser::queryWithParser(pp_species_name, "y_cut", y_cut); - utils::parser::queryWithParser(pp_species_name, "z_cut", z_cut); - utils::parser::getWithParser(pp_species_name, "q_tot", q_tot); - utils::parser::getWithParser(pp_species_name, "npart", npart); - pp_species_name.query("do_symmetrize", do_symmetrize); - pp_species_name.query("symmetrization_order", symmetrization_order); + utils::parser::getWithParser(pp_species, source_name, "x_m", x_m); + utils::parser::getWithParser(pp_species, source_name, "y_m", y_m); + utils::parser::getWithParser(pp_species, source_name, "z_m", z_m); + utils::parser::getWithParser(pp_species, source_name, "x_rms", x_rms); + utils::parser::getWithParser(pp_species, source_name, "y_rms", y_rms); + utils::parser::getWithParser(pp_species, source_name, "z_rms", z_rms); + utils::parser::queryWithParser(pp_species, source_name, "x_cut", x_cut); + utils::parser::queryWithParser(pp_species, source_name, "y_cut", y_cut); + utils::parser::queryWithParser(pp_species, source_name, "z_cut", z_cut); + utils::parser::getWithParser(pp_species, source_name, "q_tot", q_tot); + utils::parser::getWithParser(pp_species, source_name, "npart", npart); + utils::parser::queryWithParser(pp_species, source_name, "do_symmetrize", do_symmetrize); + utils::parser::queryWithParser(pp_species, source_name, "symmetrization_order", symmetrization_order); const std::set valid_symmetries = {4,8}; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( valid_symmetries.count(symmetrization_order), "Error: Symmetrization only supported to orders 4 or 8 "); gaussian_beam = true; - parseMomentum(pp_species_name, "gaussian_beam"); + SpeciesUtils::parseMomentum(species_name, source_name, "gaussian_beam", h_inj_mom, + ux_parser, uy_parser, uz_parser, + ux_th_parser, uy_th_parser, uz_th_parser, + h_mom_temp, h_mom_vel); #if defined(WARPX_DIM_XZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( y_rms > 0._rt, "Error: Gaussian beam y_rms must be strictly greater than 0 in 2D " @@ -301,10 +254,9 @@ void PlasmaInjector::setupGaussianBeam (const amrex::ParmParse& pp_species_name) #endif } -void PlasmaInjector::setupNRandomPerCell (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupNRandomPerCell (amrex::ParmParse const& pp_species) { - utils::parser::getWithParser( - pp_species_name, "num_particles_per_cell", num_particles_per_cell); + utils::parser::getWithParser(pp_species, source_name, "num_particles_per_cell", num_particles_per_cell); #if WARPX_DIM_RZ if (WarpX::n_rz_azimuthal_modes > 1) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -326,14 +278,16 @@ void PlasmaInjector::setupNRandomPerCell (const amrex::ParmParse& pp_species_nam d_inj_pos = h_inj_pos.get(); #endif - parseDensity(pp_species_name); - parseMomentum(pp_species_name, "nrandompercell"); + SpeciesUtils::parseDensity(species_name, source_name, h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, source_name, "nrandompercell", h_inj_mom, + ux_parser, uy_parser, uz_parser, + ux_th_parser, uy_th_parser, uz_th_parser, + h_mom_temp, h_mom_vel); } -void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupNFluxPerCell (amrex::ParmParse const& pp_species) { - utils::parser::getWithParser( - pp_species_name, "num_particles_per_cell", num_particles_per_cell_real); + utils::parser::getWithParser(pp_species, source_name, "num_particles_per_cell", num_particles_per_cell_real); #ifdef WARPX_DIM_RZ if (WarpX::n_rz_azimuthal_modes > 1) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -343,14 +297,11 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) "(Please visit PR#765 for more information.)"); } #endif - utils::parser::getWithParser( - pp_species_name, "surface_flux_pos", surface_flux_pos); - utils::parser::queryWithParser( - pp_species_name, "flux_tmin", flux_tmin); - utils::parser::queryWithParser( - pp_species_name, "flux_tmax", flux_tmax); + utils::parser::getWithParser(pp_species, source_name, "surface_flux_pos", surface_flux_pos); + utils::parser::queryWithParser(pp_species, source_name, "flux_tmin", flux_tmin); + utils::parser::queryWithParser(pp_species, source_name, "flux_tmax", flux_tmax); std::string flux_normal_axis_string; - pp_species_name.get("flux_normal_axis", flux_normal_axis_string); + utils::parser::get(pp_species, source_name, "flux_normal_axis", flux_normal_axis_string); flux_normal_axis = -1; #ifdef WARPX_DIM_RZ if (flux_normal_axis_string == "r" || flux_normal_axis_string == "R") { @@ -387,7 +338,7 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) #endif WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_normal_axis >= 0, "Error: Invalid value for flux_normal_axis. It must be " + flux_normal_axis_help); - pp_species_name.get("flux_direction", flux_direction); + utils::parser::getWithParser(pp_species, source_name, "flux_direction", flux_direction); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_direction == +1 || flux_direction == -1, "Error: flux_direction must be -1 or +1."); // Construct InjectorPosition with InjectorPositionRandom. @@ -403,11 +354,15 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) d_flux_pos = h_flux_pos.get(); #endif - parseFlux(pp_species_name); - parseMomentum(pp_species_name, "nfluxpercell"); + parseFlux(pp_species); + SpeciesUtils::parseMomentum(species_name, source_name, "nfluxpercell", h_inj_mom, + ux_parser, uy_parser, uz_parser, + ux_th_parser, uy_th_parser, uz_th_parser, + h_mom_temp, h_mom_vel, + flux_normal_axis, flux_direction); } -void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupNuniformPerCell (amrex::ParmParse const& pp_species) { // Note that for RZ, three numbers are expected, r, theta, and z. // For 2D, only two are expected. The third is overwritten with 1. @@ -419,9 +374,8 @@ void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_na #else constexpr int num_required_ppc_each_dim = 3; #endif - utils::parser::getArrWithParser( - pp_species_name, "num_particles_per_cell_each_dim", - num_particles_per_cell_each_dim, 0, num_required_ppc_each_dim); + utils::parser::getArrWithParser(pp_species, source_name, "num_particles_per_cell_each_dim", num_particles_per_cell_each_dim, + 0, num_required_ppc_each_dim); #if WARPX_DIM_XZ num_particles_per_cell_each_dim.push_back(1); #endif @@ -455,11 +409,14 @@ void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_na num_particles_per_cell = num_particles_per_cell_each_dim[0] * num_particles_per_cell_each_dim[1] * num_particles_per_cell_each_dim[2]; - parseDensity(pp_species_name); - parseMomentum(pp_species_name, "nuniformpercell"); + SpeciesUtils::parseDensity(species_name, source_name, h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, source_name, "nuniformpercell", h_inj_mom, + ux_parser, uy_parser, uz_parser, + ux_th_parser, uy_th_parser, uz_th_parser, + h_mom_temp, h_mom_vel); } -void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupExternalFile (amrex::ParmParse const& pp_species) { #ifndef WARPX_USE_OPENPMD WARPX_ABORT_WITH_MESSAGE( @@ -468,16 +425,16 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) #endif external_file = true; std::string str_injection_file; - pp_species_name.get("injection_file", str_injection_file); + utils::parser::get(pp_species, source_name, "injection_file", str_injection_file); // optional parameters - utils::parser::queryWithParser(pp_species_name, "q_tot", q_tot); - utils::parser::queryWithParser(pp_species_name, "z_shift",z_shift); - - const bool charge_is_specified = pp_species_name.contains("charge"); - const bool mass_is_specified = pp_species_name.contains("mass"); - const bool species_is_specified = pp_species_name.contains("species_type"); + utils::parser::queryWithParser(pp_species, source_name, "q_tot", q_tot); + utils::parser::queryWithParser(pp_species, source_name, "z_shift",z_shift); #ifdef WARPX_USE_OPENPMD + const bool charge_is_specified = pp_species.contains("charge"); + const bool mass_is_specified = pp_species.contains("mass"); + const bool species_is_specified = pp_species.contains("species_type"); + if (amrex::ParallelDescriptor::IOProcessor()) { m_openpmd_input_series = std::make_unique( str_injection_file, openPMD::Access::READ_ONLY); @@ -492,70 +449,71 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) std::string const ps_name = it.particles.begin()->first; openPMD::ParticleSpecies ps = it.particles.begin()->second; - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ps.contains("charge") || charge_is_specified || species_is_specified, - std::string("'") + ps_name + - ".injection_file' does not contain a 'charge' species record. " - "Please specify '" + ps_name + ".charge' or " - "'" + ps_name + ".species_type' in your input file!\n"); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ps.contains("mass") || mass_is_specified || species_is_specified, - std::string("'") + ps_name + - ".injection_file' does not contain a 'mass' species record. " - "Please specify '" + ps_name + ".mass' or " - "'" + ps_name + ".species_type' in your input file!\n"); - - if (charge_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".charge' and '" + - ps_name + ".injection_file' specify a charge.\n'" + - ps_name + ".charge' will take precedence.\n"); + charge_from_source = ps.contains("charge"); + mass_from_source = ps.contains("mass"); + + if (charge_from_source) { + if (charge_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".charge' and '" + + ps_name + ".injection_file' specify a charge.\n'" + + ps_name + ".charge' will take precedence.\n"); + } + else if (species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".species_type' and '" + + ps_name + ".injection_file' specify a charge.\n'" + + ps_name + ".species_type' will take precedence.\n"); + } + else { + // TODO: Add ASSERT_WITH_MESSAGE to test if charge is a constant record + auto p_q_ptr = + ps["charge"][openPMD::RecordComponent::SCALAR].loadChunk(); + m_openpmd_input_series->flush(); + amrex::ParticleReal const p_q = p_q_ptr.get()[0]; + auto const charge_unit = static_cast(ps["charge"][openPMD::RecordComponent::SCALAR].unitSI()); + charge = p_q * charge_unit; + } } - else if (species_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".species_type' and '" + - ps_name + ".injection_file' specify a charge.\n'" + - ps_name + ".species_type' will take precedence.\n"); - } - else { - // TODO: Add ASSERT_WITH_MESSAGE to test if charge is a constant record - auto p_q_ptr = - ps["charge"][openPMD::RecordComponent::SCALAR].loadChunk(); - m_openpmd_input_series->flush(); - amrex::ParticleReal const p_q = p_q_ptr.get()[0]; - double const charge_unit = ps["charge"][openPMD::RecordComponent::SCALAR].unitSI(); - charge = p_q * charge_unit; - } - if (mass_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".mass' and '" + - ps_name + ".injection_file' specify a charge.\n'" + - ps_name + ".mass' will take precedence.\n"); - } - else if (species_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".species_type' and '" + - ps_name + ".injection_file' specify a mass.\n'" + - ps_name + ".species_type' will take precedence.\n"); - } - else { - // TODO: Add ASSERT_WITH_MESSAGE to test if mass is a constant record - auto p_m_ptr = - ps["mass"][openPMD::RecordComponent::SCALAR].loadChunk(); - m_openpmd_input_series->flush(); - amrex::ParticleReal const p_m = p_m_ptr.get()[0]; - double const mass_unit = ps["mass"][openPMD::RecordComponent::SCALAR].unitSI(); - mass = p_m * mass_unit; + if (mass_from_source) { + if (mass_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".mass' and '" + + ps_name + ".injection_file' specify a charge.\n'" + + ps_name + ".mass' will take precedence.\n"); + } + else if (species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".species_type' and '" + + ps_name + ".injection_file' specify a mass.\n'" + + ps_name + ".species_type' will take precedence.\n"); + } + else { + // TODO: Add ASSERT_WITH_MESSAGE to test if mass is a constant record + auto p_m_ptr = + ps["mass"][openPMD::RecordComponent::SCALAR].loadChunk(); + m_openpmd_input_series->flush(); + amrex::ParticleReal const p_m = p_m_ptr.get()[0]; + auto const mass_unit = static_cast(ps["mass"][openPMD::RecordComponent::SCALAR].unitSI()); + mass = p_m * mass_unit; + } } } // IOProcessor - // Broadcast charge and mass to non-IO processors - if (!charge_is_specified && !species_is_specified) - amrex::ParallelDescriptor::Bcast(&charge, 1, - amrex::ParallelDescriptor::IOProcessorNumber()); - if (!mass_is_specified && !species_is_specified) - amrex::ParallelDescriptor::Bcast(&mass, 1, - amrex::ParallelDescriptor::IOProcessorNumber()); + // Broadcast charge and mass to non-IO processors if read in from the file + if (!charge_is_specified && !species_is_specified) { + // Use ReduceBoolOr since Bcast(bool) doesn't compile + amrex::ParallelDescriptor::ReduceBoolOr(charge_from_source, amrex::ParallelDescriptor::IOProcessorNumber()); + if (charge_from_source) { + amrex::ParallelDescriptor::Bcast(&charge, 1, amrex::ParallelDescriptor::IOProcessorNumber()); + } + } + if (!mass_is_specified && !species_is_specified) { + amrex::ParallelDescriptor::ReduceBoolOr(mass_from_source, amrex::ParallelDescriptor::IOProcessorNumber()); + if (mass_from_source) { + amrex::ParallelDescriptor::Bcast(&mass, 1, amrex::ParallelDescriptor::IOProcessorNumber()); + } + } #else WARPX_ABORT_WITH_MESSAGE( "Plasma injection via external_file requires openPMD support: " @@ -563,70 +521,29 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) #endif // WARPX_USE_OPENPMD } -// Depending on injection type at runtime, initialize inj_rho -// so that inj_rho->getDensity calls -// InjectorPosition[Constant or Predefined or etc.].getDensity. -void PlasmaInjector::parseDensity (const amrex::ParmParse& pp_species_name) -{ - // parse density information - std::string rho_prof_s; - pp_species_name.get("profile", rho_prof_s); - std::transform(rho_prof_s.begin(), rho_prof_s.end(), - rho_prof_s.begin(), ::tolower); - if (rho_prof_s == "constant") { - utils::parser::getWithParser(pp_species_name, "density", density); - // Construct InjectorDensity with InjectorDensityConstant. - h_inj_rho.reset(new InjectorDensity((InjectorDensityConstant*)nullptr, density)); - } else if (rho_prof_s == "predefined") { - // Construct InjectorDensity with InjectorDensityPredefined. - h_inj_rho.reset(new InjectorDensity((InjectorDensityPredefined*)nullptr,species_name)); - } else if (rho_prof_s == "parse_density_function") { - utils::parser::Store_parserString( - pp_species_name, "density_function(x,y,z)", str_density_function); - // Construct InjectorDensity with InjectorDensityParser. - density_parser = std::make_unique( - utils::parser::makeParser(str_density_function,{"x","y","z"})); - h_inj_rho.reset(new InjectorDensity((InjectorDensityParser*)nullptr, - density_parser->compile<3>())); - } else { - StringParseAbortMessage("Density profile type", rho_prof_s); - } - - if (h_inj_rho) { -#ifdef AMREX_USE_GPU - d_inj_rho = static_cast - (amrex::The_Arena()->alloc(sizeof(InjectorDensity))); - amrex::Gpu::htod_memcpy_async(d_inj_rho, h_inj_rho.get(), sizeof(InjectorDensity)); -#else - d_inj_rho = h_inj_rho.get(); -#endif - } -} - // Depending on injection type at runtime, initialize inj_flux // so that inj_flux->getFlux calls // InjectorFlux[Constant or Parser or etc.].getFlux. -void PlasmaInjector::parseFlux (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::parseFlux (amrex::ParmParse const& pp_species) { // parse flux information std::string flux_prof_s; - pp_species_name.get("flux_profile", flux_prof_s); + utils::parser::get(pp_species, source_name, "flux_profile", flux_prof_s); std::transform(flux_prof_s.begin(), flux_prof_s.end(), flux_prof_s.begin(), ::tolower); if (flux_prof_s == "constant") { - utils::parser::getWithParser(pp_species_name, "flux", flux); + utils::parser::getWithParser(pp_species, source_name, "flux", flux); // Construct InjectorFlux with InjectorFluxConstant. h_inj_flux.reset(new InjectorFlux((InjectorFluxConstant*)nullptr, flux)); } else if (flux_prof_s == "parse_flux_function") { - utils::parser::Store_parserString( - pp_species_name, "flux_function(x,y,z,t)", str_flux_function); + utils::parser::Store_parserString(pp_species, source_name, "flux_function(x,y,z,t)", str_flux_function); // Construct InjectorFlux with InjectorFluxParser. flux_parser = std::make_unique( utils::parser::makeParser(str_flux_function,{"x","y","z","t"})); h_inj_flux.reset(new InjectorFlux((InjectorFluxParser*)nullptr, flux_parser->compile<4>())); } else { - StringParseAbortMessage("Flux profile type", flux_prof_s); + SpeciesUtils::StringParseAbortMessage("Flux profile type", flux_prof_s); } if (h_inj_flux) { #ifdef AMREX_USE_GPU @@ -640,138 +557,6 @@ void PlasmaInjector::parseFlux (const amrex::ParmParse& pp_species_name) } -// Depending on injection type at runtime, initialize inj_mom -// so that inj_mom->getMomentum calls -// InjectorMomentum[Constant or Gaussian or etc.].getMomentum. -void PlasmaInjector::parseMomentum (const amrex::ParmParse& pp_species_name, const std::string& style) -{ - using namespace amrex::literals; - - // parse momentum information - std::string mom_dist_s; - pp_species_name.get("momentum_distribution_type", mom_dist_s); - std::transform(mom_dist_s.begin(), - mom_dist_s.end(), - mom_dist_s.begin(), - ::tolower); - if (mom_dist_s == "at_rest") { - constexpr amrex::Real ux = 0._rt; - constexpr amrex::Real uy = 0._rt; - constexpr amrex::Real uz = 0._rt; - // Construct InjectorMomentum with InjectorMomentumConstant. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); - } else if (mom_dist_s == "constant") { - amrex::Real ux = 0._rt; - amrex::Real uy = 0._rt; - amrex::Real uz = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux", ux); - utils::parser::queryWithParser(pp_species_name, "uy", uy); - utils::parser::queryWithParser(pp_species_name, "uz", uz); - // Construct InjectorMomentum with InjectorMomentumConstant. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); - } else if (mom_dist_s == "gaussian") { - amrex::Real ux_m = 0._rt; - amrex::Real uy_m = 0._rt; - amrex::Real uz_m = 0._rt; - amrex::Real ux_th = 0._rt; - amrex::Real uy_th = 0._rt; - amrex::Real uz_th = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); - utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); - utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); - utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); - utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); - utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); - // Construct InjectorMomentum with InjectorMomentumGaussian. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussian*)nullptr, - ux_m, uy_m, uz_m, ux_th, uy_th, uz_th)); - } else if (mom_dist_s == "gaussianflux") { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(style == "nfluxpercell", - "Error: gaussianflux can only be used with injection_style = NFluxPerCell"); - amrex::Real ux_m = 0._rt; - amrex::Real uy_m = 0._rt; - amrex::Real uz_m = 0._rt; - amrex::Real ux_th = 0._rt; - amrex::Real uy_th = 0._rt; - amrex::Real uz_th = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); - utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); - utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); - utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); - utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); - utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); - // Construct InjectorMomentum with InjectorMomentumGaussianFlux. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussianFlux*)nullptr, - ux_m, uy_m, uz_m, ux_th, uy_th, uz_th, - flux_normal_axis, flux_direction)); - } else if (mom_dist_s == "uniform") { - amrex::Real ux_min = 0._rt; - amrex::Real uy_min = 0._rt; - amrex::Real uz_min = 0._rt; - amrex::Real ux_max = 0._rt; - amrex::Real uy_max = 0._rt; - amrex::Real uz_max = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_min", ux_min); - utils::parser::queryWithParser(pp_species_name, "uy_min", uy_min); - utils::parser::queryWithParser(pp_species_name, "uz_min", uz_min); - utils::parser::queryWithParser(pp_species_name, "ux_max", ux_max); - utils::parser::queryWithParser(pp_species_name, "uy_max", uy_max); - utils::parser::queryWithParser(pp_species_name, "uz_max", uz_max); - // Construct InjectorMomentum with InjectorMomentumUniform. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumUniform*)nullptr, - ux_min, uy_min, uz_min, ux_max, uy_max, uz_max)); - } else if (mom_dist_s == "maxwell_boltzmann"){ - h_mom_temp = std::make_unique(pp_species_name); - const GetTemperature getTemp(*h_mom_temp); - h_mom_vel = std::make_unique(pp_species_name); - const GetVelocity getVel(*h_mom_vel); - // Construct InjectorMomentum with InjectorMomentumBoltzmann. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumBoltzmann*)nullptr, getTemp, getVel)); - } else if (mom_dist_s == "maxwell_juttner"){ - h_mom_temp = std::make_unique(pp_species_name); - const GetTemperature getTemp(*h_mom_temp); - h_mom_vel = std::make_unique(pp_species_name); - const GetVelocity getVel(*h_mom_vel); - // Construct InjectorMomentum with InjectorMomentumJuttner. - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumJuttner*)nullptr, getTemp, getVel)); - } else if (mom_dist_s == "radial_expansion") { - amrex::Real u_over_r = 0._rt; - utils::parser::queryWithParser(pp_species_name, "u_over_r", u_over_r); - // Construct InjectorMomentum with InjectorMomentumRadialExpansion. - h_inj_mom.reset(new InjectorMomentum - ((InjectorMomentumRadialExpansion*)nullptr, u_over_r)); - } else if (mom_dist_s == "parse_momentum_function") { - utils::parser::Store_parserString(pp_species_name, "momentum_function_ux(x,y,z)", - str_momentum_function_ux); - utils::parser::Store_parserString(pp_species_name, "momentum_function_uy(x,y,z)", - str_momentum_function_uy); - utils::parser::Store_parserString(pp_species_name, "momentum_function_uz(x,y,z)", - str_momentum_function_uz); - // Construct InjectorMomentum with InjectorMomentumParser. - ux_parser = std::make_unique( - utils::parser::makeParser(str_momentum_function_ux, {"x","y","z"})); - uy_parser = std::make_unique( - utils::parser::makeParser(str_momentum_function_uy, {"x","y","z"})); - uz_parser = std::make_unique( - utils::parser::makeParser(str_momentum_function_uz, {"x","y","z"})); - h_inj_mom.reset(new InjectorMomentum((InjectorMomentumParser*)nullptr, - ux_parser->compile<3>(), - uy_parser->compile<3>(), - uz_parser->compile<3>())); - } else { - StringParseAbortMessage("Momentum distribution type", mom_dist_s); - } - if (h_inj_mom) { -#ifdef AMREX_USE_GPU - d_inj_mom = static_cast - (amrex::The_Arena()->alloc(sizeof(InjectorMomentum))); - amrex::Gpu::htod_memcpy_async(d_inj_mom, h_inj_mom.get(), sizeof(InjectorMomentum)); -#else - d_inj_mom = h_inj_mom.get(); -#endif - } -} - amrex::XDim3 PlasmaInjector::getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept @@ -794,38 +579,56 @@ bool PlasmaInjector::overlapsWith (const amrex::XDim3& lo, || (zmin > hi.z) || (zmax < lo.z) ); } +bool +PlasmaInjector::queryCharge (amrex::ParticleReal& a_charge) const +{ + if (charge_from_source) { + a_charge = charge; + } + return charge_from_source; +} + +bool +PlasmaInjector::queryMass (amrex::ParticleReal& a_mass) const +{ + if (mass_from_source) { + a_mass = mass; + } + return mass_from_source; +} + InjectorPosition* -PlasmaInjector::getInjectorPosition () +PlasmaInjector::getInjectorPosition () const { return d_inj_pos; } InjectorPosition* -PlasmaInjector::getInjectorFluxPosition () +PlasmaInjector::getInjectorFluxPosition () const { return d_flux_pos; } InjectorDensity* -PlasmaInjector::getInjectorDensity () +PlasmaInjector::getInjectorDensity () const { return d_inj_rho; } InjectorFlux* -PlasmaInjector::getInjectorFlux () +PlasmaInjector::getInjectorFlux () const { return d_inj_flux; } InjectorMomentum* -PlasmaInjector::getInjectorMomentumDevice () +PlasmaInjector::getInjectorMomentumDevice () const { return d_inj_mom; } InjectorMomentum* -PlasmaInjector::getInjectorMomentumHost () +PlasmaInjector::getInjectorMomentumHost () const { return h_inj_mom.get(); } diff --git a/Source/Initialization/TemperatureProperties.H b/Source/Initialization/TemperatureProperties.H index d9d9121c846..4d30b67c851 100644 --- a/Source/Initialization/TemperatureProperties.H +++ b/Source/Initialization/TemperatureProperties.H @@ -30,8 +30,9 @@ struct TemperatureProperties * information * * \param[in] pp: Reference to the parameter parser object for the species being initialized + * \param[in] source_name: Optional group name of the input parameters */ - TemperatureProperties (const amrex::ParmParse& pp); + TemperatureProperties (const amrex::ParmParse& pp, std::string const& source_name); /* Type of temperature initialization */ TemperatureInitType m_type; diff --git a/Source/Initialization/TemperatureProperties.cpp b/Source/Initialization/TemperatureProperties.cpp index 34e1d2b5fcc..ff73134a24e 100644 --- a/Source/Initialization/TemperatureProperties.cpp +++ b/Source/Initialization/TemperatureProperties.cpp @@ -17,17 +17,17 @@ * If temperature is a constant, store value. If a parser, make and * store the parser function */ -TemperatureProperties::TemperatureProperties (const amrex::ParmParse& pp) { +TemperatureProperties::TemperatureProperties (const amrex::ParmParse& pp, std::string const& source_name) { // Set defaults amrex::Real theta; std::string temp_dist_s = "constant"; std::string mom_dist_s; - pp.query("theta_distribution_type", temp_dist_s); - pp.query("momentum_distribution_type", mom_dist_s); + utils::parser::query(pp, source_name, "theta_distribution_type", temp_dist_s); + utils::parser::query(pp, source_name, "momentum_distribution_type", mom_dist_s); if (temp_dist_s == "constant") { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - utils::parser::queryWithParser(pp, "theta", theta), + utils::parser::queryWithParser(pp, source_name, "theta", theta), "Temperature parameter theta not specified"); // Do validation on theta value @@ -58,7 +58,7 @@ TemperatureProperties::TemperatureProperties (const amrex::ParmParse& pp) { } else if (temp_dist_s == "parser") { std::string str_theta_function; - utils::parser::Store_parserString(pp, "theta_function(x,y,z)", str_theta_function); + utils::parser::Store_parserString(pp, source_name, "theta_function(x,y,z)", str_theta_function); m_ptr_temperature_parser = std::make_unique( utils::parser::makeParser(str_theta_function,{"x","y","z"})); diff --git a/Source/Initialization/VelocityProperties.H b/Source/Initialization/VelocityProperties.H index 6b981849dab..453d97725e2 100644 --- a/Source/Initialization/VelocityProperties.H +++ b/Source/Initialization/VelocityProperties.H @@ -33,8 +33,9 @@ struct VelocityProperties * store the parser function * * \param[in] pp: Reference to the parameter parser object for the species being initialized + * \param[in] source_name: Optional group name of the input parameters */ - VelocityProperties (const amrex::ParmParse& pp); + VelocityProperties (const amrex::ParmParse& pp, std::string const& source_name); /* Type of velocity initialization */ VelocityInitType m_type; @@ -44,7 +45,7 @@ struct VelocityProperties int m_sign_dir; // Sign of the velocity direction positive=1, negative=-1 /* Constant velocity value, if m_type == VelConstantValue */ - amrex::Real m_velocity; + amrex::Real m_velocity{0}; /* Storage of the parser function, if m_type == VelParserFunction */ std::unique_ptr m_ptr_velocity_parser; }; diff --git a/Source/Initialization/VelocityProperties.cpp b/Source/Initialization/VelocityProperties.cpp index 802eabc40c9..14b48c79524 100644 --- a/Source/Initialization/VelocityProperties.cpp +++ b/Source/Initialization/VelocityProperties.cpp @@ -11,13 +11,13 @@ #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" -VelocityProperties::VelocityProperties (const amrex::ParmParse& pp) { +VelocityProperties::VelocityProperties (const amrex::ParmParse& pp, std::string const& source_name) +{ // Set defaults std::string vel_dist_s = "constant"; std::string vel_dir_s = "x"; - m_velocity = 0; - pp.query("bulk_vel_dir", vel_dir_s); + utils::parser::query(pp, source_name, "bulk_vel_dir", vel_dir_s); if(vel_dir_s[0] == '-'){ m_sign_dir = -1; } @@ -44,9 +44,9 @@ VelocityProperties::VelocityProperties (const amrex::ParmParse& pp) { " other character."); } - pp.query("beta_distribution_type", vel_dist_s); + utils::parser::query(pp, source_name, "beta_distribution_type", vel_dist_s); if (vel_dist_s == "constant") { - utils::parser::queryWithParser(pp, "beta", m_velocity); + utils::parser::queryWithParser(pp, source_name, "beta", m_velocity); m_type = VelConstantValue; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( m_velocity > -1 && m_velocity < 1, @@ -56,7 +56,7 @@ VelocityProperties::VelocityProperties (const amrex::ParmParse& pp) { } else if (vel_dist_s == "parser") { std::string str_beta_function; - utils::parser::Store_parserString(pp, "beta_function(x,y,z)", str_beta_function); + utils::parser::Store_parserString(pp, source_name, "beta_function(x,y,z)", str_beta_function); m_ptr_velocity_parser = std::make_unique( utils::parser::makeParser(str_beta_function,{"x","y","z"})); diff --git a/Source/Initialization/WarpXAMReXInit.H b/Source/Initialization/WarpXAMReXInit.H index 50c90d40ae6..613250b5c2f 100644 --- a/Source/Initialization/WarpXAMReXInit.H +++ b/Source/Initialization/WarpXAMReXInit.H @@ -7,8 +7,6 @@ #ifndef WARPX_AMREX_INIT_H_ #define WARPX_AMREX_INIT_H_ -#include - #include namespace warpx::initialization diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 8502bdc4ae6..d1dd0bbe90b 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -8,6 +8,7 @@ #include "Initialization/WarpXAMReXInit.H" #include +#include #include #include @@ -29,6 +30,10 @@ namespace { bool the_arena_is_managed = false; // AMReX' default: true pp_amrex.queryAdd("the_arena_is_managed", the_arena_is_managed); + // https://amrex-codes.github.io/amrex/docs_html/InputsComputeBackends.html + std::string omp_threads = "nosmt"; // AMReX' default: system + pp_amrex.queryAdd("omp_threads", omp_threads); + // Work-around: // If warpx.numprocs is used for the domain decomposition, we will not use blocking factor // to generate grids. Nonetheless, AMReX has asserts in place that validate that the diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index bfac6db544d..3b36a6336bf 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -20,6 +20,7 @@ #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Filter/BilinearFilter.H" #include "Filter/NCIGodfreyFilter.H" +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Utils/Algorithms/LinearInterpolation.H" #include "Utils/Logo/GetLogo.H" @@ -29,7 +30,7 @@ #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" #include "Utils/WarpXUtil.H" -#include "Python/WarpX_py.H" +#include "Python/callbacks.H" #include #include @@ -119,7 +120,7 @@ WarpX::PostProcessBaseGrids (BoxArray& ba0) const #if defined(WARPX_DIM_3D) for (int k = 0; k < numprocs[2]; ++k) { // The first extra[2] blocks get one extra cell with a total of - // sz[2]+1. The rest get sz[2] cells. The docomposition in y + // sz[2]+1. The rest get sz[2] cells. The decomposition in y // and x directions are similar. int klo = (k < extra[2]) ? k*(sz[2]+1) : (k*sz[2]+extra[2]); int khi = (k < extra[2]) ? klo+(sz[2]+1)-1 : klo+sz[2]-1; @@ -475,8 +476,9 @@ WarpX::InitData () ExecutePythonCallback("beforeInitEsolve"); ComputeSpaceChargeField(reset_fields); ExecutePythonCallback("afterInitEsolve"); - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) + if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { ComputeMagnetostaticField(); + } // Set up an invariant condition through the rest of // execution, that any code besides the field solver that @@ -485,7 +487,7 @@ WarpX::InitData () AddExternalFields(); } - if (restart_chkfile.empty() || write_diagonstics_on_restart) { + if (restart_chkfile.empty() || write_diagnostics_on_restart) { // Write full diagnostics before the first iteration. multi_diags->FilterComputePackFlush(istep[0] - 1); @@ -506,12 +508,12 @@ void WarpX::AddExternalFields () { for (int lev = 0; lev <= finest_level; ++lev) { // FIXME: RZ multimode has more than one component for all these - if (add_external_E_field) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { amrex::MultiFab::Add(*Efield_fp[lev][0], *Efield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][1], *Efield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][2], *Efield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); } - if (add_external_B_field) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { amrex::MultiFab::Add(*Bfield_fp[lev][0], *Bfield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][1], *Bfield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][2], *Bfield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); @@ -551,7 +553,7 @@ WarpX::InitPML () do_pml_Hi[0][idim] = 1; // on level 0 } } - if (finest_level > 0) do_pml = 1; + if (finest_level > 0) { do_pml = 1; } if (do_pml) { #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) @@ -582,10 +584,12 @@ WarpX::InitPML () // Domain box at level, lev const amrex::Box DomainBox = Geom(lev).Domain(); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (levelBox.smallEnd(idim) == DomainBox.smallEnd(idim)) + if (levelBox.smallEnd(idim) == DomainBox.smallEnd(idim)) { do_pml_Lo[lev][idim] = do_pml_Lo[0][idim]; - if (levelBox.bigEnd(idim) == DomainBox.bigEnd(idim)) + } + if (levelBox.bigEnd(idim) == DomainBox.bigEnd(idim)) { do_pml_Hi[lev][idim] = do_pml_Hi[0][idim]; + } } #ifdef WARPX_DIM_RZ @@ -618,8 +622,9 @@ WarpX::ComputePMLFactors () { for (int lev = 0; lev <= finest_level; ++lev) { - if (pml[lev]) + if (pml[lev]) { pml[lev]->ComputePMLFactors(dt[lev]); + } } } } @@ -729,34 +734,6 @@ WarpX::PostRestart () void WarpX::InitLevelData (int lev, Real /*time*/) { - - const ParmParse pp_warpx("warpx"); - - // default values of E_external_grid and B_external_grid - // are used to set the E and B field when "constant" or - // "parser" is not explicitly used in the input. - pp_warpx.query("B_ext_grid_init_style", B_ext_grid_s); - std::transform(B_ext_grid_s.begin(), - B_ext_grid_s.end(), - B_ext_grid_s.begin(), - ::tolower); - - pp_warpx.query("E_ext_grid_init_style", E_ext_grid_s); - std::transform(E_ext_grid_s.begin(), - E_ext_grid_s.end(), - E_ext_grid_s.begin(), - ::tolower); - - // if the input string is "constant", the values for the - // external grid must be provided in the input. - if (B_ext_grid_s == "constant") - utils::parser::getArrWithParser(pp_warpx, "B_external_grid", B_external_grid); - - // if the input string is "constant", the values for the - // external grid must be provided in the input. - if (E_ext_grid_s == "constant") - utils::parser::getArrWithParser(pp_warpx, "E_external_grid", E_external_grid); - // initialize the averaged fields only if the averaged algorithm // is activated ('psatd.do_time_averaging=1') const ParmParse pp_psatd("psatd"); @@ -764,33 +741,46 @@ WarpX::InitLevelData (int lev, Real /*time*/) for (int i = 0; i < 3; ++i) { - if (B_ext_grid_s == "constant" || B_ext_grid_s == "default") { - Bfield_fp[lev][i]->setVal(B_external_grid[i]); - if (fft_do_time_averaging) { - Bfield_avg_fp[lev][i]->setVal(B_external_grid[i]); - } + // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. + // The default maxlevel_extEMfield_init value is the total number of levels in the simulation + const auto is_B_ext_const = + m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant || + m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::default_zero; + if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) + { + Bfield_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + if (fft_do_time_averaging) { + Bfield_avg_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + } if (lev > 0) { - Bfield_aux[lev][i]->setVal(B_external_grid[i]); - Bfield_cp[lev][i]->setVal(B_external_grid[i]); - if (fft_do_time_averaging) { - Bfield_avg_cp[lev][i]->setVal(B_external_grid[i]); - } + Bfield_aux[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + Bfield_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + if (fft_do_time_averaging) { + Bfield_avg_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + } } } - if (E_ext_grid_s == "constant" || E_ext_grid_s == "default") { - Efield_fp[lev][i]->setVal(E_external_grid[i]); - if (fft_do_time_averaging) { - Efield_avg_fp[lev][i]->setVal(E_external_grid[i]); + + // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. + // The default maxlevel_extEMfield_init value is the total number of levels in the simulation + const auto is_E_ext_const = + m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant || + m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::default_zero; + if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) + { + Efield_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + if (fft_do_time_averaging) { + Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); } - if (lev > 0) { - Efield_aux[lev][i]->setVal(E_external_grid[i]); - Efield_cp[lev][i]->setVal(E_external_grid[i]); - if (fft_do_time_averaging) { - Efield_avg_cp[lev][i]->setVal(E_external_grid[i]); - } - } + if (lev > 0) { + Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + Efield_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + if (fft_do_time_averaging) { + Efield_avg_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + } + } } } @@ -801,179 +791,142 @@ WarpX::InitLevelData (int lev, Real /*time*/) // if the input string for the B-field is "parse_b_ext_grid_function", // then the analytical expression or function must be // provided in the input file. - if (B_ext_grid_s == "parse_b_ext_grid_function") { - -#ifdef WARPX_DIM_RZ - WARPX_ABORT_WITH_MESSAGE( - "E and B parser for external fields does not work with RZ -- TO DO"); -#endif - - //! Strings storing parser function to initialize the components of the magnetic field on the grid - std::string str_Bx_ext_grid_function; - std::string str_By_ext_grid_function; - std::string str_Bz_ext_grid_function; - - utils::parser::Store_parserString(pp_warpx, "Bx_external_grid_function(x,y,z)", - str_Bx_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "By_external_grid_function(x,y,z)", - str_By_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "Bz_external_grid_function(x,y,z)", - str_Bz_ext_grid_function); - Bxfield_parser = std::make_unique( - utils::parser::makeParser(str_Bx_ext_grid_function,{"x","y","z"})); - Byfield_parser = std::make_unique( - utils::parser::makeParser(str_By_ext_grid_function,{"x","y","z"})); - Bzfield_parser = std::make_unique( - utils::parser::makeParser(str_Bz_ext_grid_function,{"x","y","z"})); - - // Initialize Bfield_fp with external function - InitializeExternalFieldsOnGridUsingParser(Bfield_fp[lev][0].get(), - Bfield_fp[lev][1].get(), - Bfield_fp[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser(Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser(Bfield_cp[lev][0].get(), - Bfield_cp[lev][1].get(), - Bfield_cp[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::coarse); - } + // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. + // The default maxlevel_extEMfield_init value is the total number of levels in the simulation + if ((m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) + && (lev <= maxlevel_extEMfield_init)) { + + // Initialize Bfield_fp with external function + InitializeExternalFieldsOnGridUsingParser( + Bfield_fp[lev][0].get(), + Bfield_fp[lev][1].get(), + Bfield_fp[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::fine); + + if (lev > 0) { + InitializeExternalFieldsOnGridUsingParser( + Bfield_aux[lev][0].get(), + Bfield_aux[lev][1].get(), + Bfield_aux[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::fine); + + InitializeExternalFieldsOnGridUsingParser( + Bfield_cp[lev][0].get(), + Bfield_cp[lev][1].get(), + Bfield_cp[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::coarse); + } } // if the input string for the E-field is "parse_e_ext_grid_function", // then the analytical expression or function must be // provided in the input file. - if (E_ext_grid_s == "parse_e_ext_grid_function") { - -#ifdef WARPX_DIM_RZ - WARPX_ABORT_WITH_MESSAGE( - "E and B parser for external fields does not work with RZ -- TO DO"); -#endif - - //! Strings storing parser function to initialize the components of the electric field on the grid - std::string str_Ex_ext_grid_function; - std::string str_Ey_ext_grid_function; - std::string str_Ez_ext_grid_function; - - utils::parser::Store_parserString(pp_warpx, "Ex_external_grid_function(x,y,z)", - str_Ex_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "Ey_external_grid_function(x,y,z)", - str_Ey_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "Ez_external_grid_function(x,y,z)", - str_Ez_ext_grid_function); - - Exfield_parser = std::make_unique( - utils::parser::makeParser(str_Ex_ext_grid_function,{"x","y","z"})); - Eyfield_parser = std::make_unique( - utils::parser::makeParser(str_Ey_ext_grid_function,{"x","y","z"})); - Ezfield_parser = std::make_unique( - utils::parser::makeParser(str_Ez_ext_grid_function,{"x","y","z"})); - - // Initialize Efield_fp with external function - InitializeExternalFieldsOnGridUsingParser(Efield_fp[lev][0].get(), - Efield_fp[lev][1].get(), - Efield_fp[lev][2].get(), - Exfield_parser->compile<3>(), - Eyfield_parser->compile<3>(), - Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); + // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. + // The default maxlevel_extEMfield_init value is the total number of levels in the simulation + if ((m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) + && (lev <= maxlevel_extEMfield_init)) { + + // Initialize Efield_fp with external function + InitializeExternalFieldsOnGridUsingParser( + Efield_fp[lev][0].get(), + Efield_fp[lev][1].get(), + Efield_fp[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::fine); #ifdef AMREX_USE_EB // We initialize ECTRhofield consistently with the Efield if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - m_fdtd_solver_fp[lev]->EvolveECTRho(Efield_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); - + m_fdtd_solver_fp[lev]->EvolveECTRho( + Efield_fp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], lev); } #endif - if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser(Efield_aux[lev][0].get(), - Efield_aux[lev][1].get(), - Efield_aux[lev][2].get(), - Exfield_parser->compile<3>(), - Eyfield_parser->compile<3>(), - Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser(Efield_cp[lev][0].get(), - Efield_cp[lev][1].get(), - Efield_cp[lev][2].get(), - Exfield_parser->compile<3>(), - Eyfield_parser->compile<3>(), - Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::coarse); + if (lev > 0) { + InitializeExternalFieldsOnGridUsingParser( + Efield_aux[lev][0].get(), + Efield_aux[lev][1].get(), + Efield_aux[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::fine); + + InitializeExternalFieldsOnGridUsingParser( + Efield_cp[lev][0].get(), + Efield_cp[lev][1].get(), + Efield_cp[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::coarse); #ifdef AMREX_USE_EB - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - // We initialize ECTRhofield consistently with the Efield - m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + // We initialize ECTRhofield consistently with the Efield + m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], lev); - } + } #endif } } // Reading external fields from data file - if (add_external_B_field) { - std::string read_fields_from_path="./"; - pp_warpx.query("read_fields_from_path", read_fields_from_path); + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][0].get(), "B", "r"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][1].get(), "B", "t"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); #else - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][0].get(), "B", "x"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][1].get(), "B", "y"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); #endif } - if (add_external_E_field) { - std::string read_fields_from_path="./"; - pp_warpx.query("read_fields_from_path", read_fields_from_path); + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][0].get(), "E", "r"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][1].get(), "E", "t"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); #else - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][0].get(), "E", "x"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][1].get(), "E", "y"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); #endif } @@ -1011,27 +964,27 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); - const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); - const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); + const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); - auto const& mfxfab = mfx->array(mfi); - auto const& mfyfab = mfy->array(mfi); - auto const& mfzfab = mfz->array(mfi); + auto const& mfxfab = mfx->array(mfi); + auto const& mfyfab = mfy->array(mfi); + auto const& mfzfab = mfz->array(mfi); #ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); - amrex::Array4 const& Sx = face_areas[0]->array(mfi); - amrex::Array4 const& Sy = face_areas[1]->array(mfi); - amrex::Array4 const& Sz = face_areas[2]->array(mfi); + amrex::Array4 const& lx = edge_lengths[0]->array(mfi); + amrex::Array4 const& ly = edge_lengths[1]->array(mfi); + amrex::Array4 const& lz = edge_lengths[2]->array(mfi); + amrex::Array4 const& Sx = face_areas[0]->array(mfi); + amrex::Array4 const& Sy = face_areas[1]->array(mfi); + amrex::Array4 const& Sz = face_areas[2]->array(mfi); #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Dim3 lx_lo = amrex::lbound(lx); - const amrex::Dim3 lx_hi = amrex::ubound(lx); - const amrex::Dim3 lz_lo = amrex::lbound(lz); - const amrex::Dim3 lz_hi = amrex::ubound(lz); + const amrex::Dim3 lx_lo = amrex::lbound(lx); + const amrex::Dim3 lx_hi = amrex::ubound(lx); + const amrex::Dim3 lz_lo = amrex::lbound(lz); + const amrex::Dim3 lz_hi = amrex::ubound(lz); #endif #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) @@ -1041,7 +994,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( #endif #else - amrex::ignore_unused(edge_lengths, face_areas, field); + amrex::ignore_unused(edge_lengths, face_areas, field); #endif amrex::ParallelFor (tbx, tby, tbz, @@ -1405,7 +1358,7 @@ WarpX::ReadExternalFieldFromFile ( "XZ expects axisLabels {x, z}"); #elif defined(WARPX_DIM_1D_Z) WARPX_ABORT_WITH_MESSAGE( - "Reading from openPMD for external fields is not known to work with 1D3V (see #3830)"); + "Reading from openPMD for external fields is not known to work with 1D3V (see #3830)"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(fileGeom == "cartesian", "1D3V can only read from files with cartesian geometry"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(axisLabels[0] == "z"); #elif defined(WARPX_DIM_RZ) @@ -1415,27 +1368,27 @@ WarpX::ReadExternalFieldFromFile ( #endif const auto offset = F.gridGlobalOffset(); - const amrex::Real offset0 = offset[0]; - const amrex::Real offset1 = offset[1]; + const auto offset0 = static_cast(offset[0]); + const auto offset1 = static_cast(offset[1]); #if defined(WARPX_DIM_3D) - const amrex::Real offset2 = offset[2]; + const auto offset2 = static_cast(offset[2]); #endif const auto d = F.gridSpacing(); #if defined(WARPX_DIM_RZ) - const amrex::Real file_dr = d[0]; - const amrex::Real file_dz = d[1]; + const auto file_dr = static_cast(d[0]); + const auto file_dz = static_cast(d[1]); #elif defined(WARPX_DIM_3D) - const amrex::Real file_dx = d[0]; - const amrex::Real file_dy = d[1]; - const amrex::Real file_dz = d[2]; + const auto file_dx = static_cast(d[0]); + const auto file_dy = static_cast(d[1]); + const auto file_dz = static_cast(d[2]); #endif auto FC = F[F_component]; const auto extent = FC.getExtent(); - const int extent0 = extent[0]; - const int extent1 = extent[1]; - const int extent2 = extent[2]; + const auto extent0 = static_cast(extent[0]); + const auto extent1 = static_cast(extent[1]); + const auto extent2 = static_cast(extent[2]); // Determine the chunk data that will be loaded. // Now, the full range of data is loaded. @@ -1446,12 +1399,12 @@ WarpX::ReadExternalFieldFromFile ( auto FC_chunk_data = FC.loadChunk(chunk_offset,chunk_extent); series.flush(); - auto FC_data_host = FC_chunk_data.get(); + auto *FC_data_host = FC_chunk_data.get(); // Load data to GPU const size_t total_extent = size_t(extent[0]) * extent[1] * extent[2]; amrex::Gpu::DeviceVector FC_data_gpu(total_extent); - auto FC_data = FC_data_gpu.data(); + auto *FC_data = FC_data_gpu.data(); amrex::Gpu::copy(amrex::Gpu::hostToDevice, FC_data_host, FC_data_host + total_extent, FC_data); // Loop over boxes @@ -1482,16 +1435,16 @@ WarpX::ReadExternalFieldFromFile ( // 0,1 denote r,z in 2D rz. amrex::Real x0, x1; if ( box.type(0)==amrex::IndexType::CellIndex::NODE ) - { x0 = real_box.lo(0) + ii*dx[0]; } - else { x0 = real_box.lo(0) + ii*dx[0] + 0.5*dx[0]; } + { x0 = static_cast(real_box.lo(0)) + ii*dx[0]; } + else { x0 = static_cast(real_box.lo(0)) + ii*dx[0] + 0.5_rt*dx[0]; } if ( box.type(1)==amrex::IndexType::CellIndex::NODE ) { x1 = real_box.lo(1) + j*dx[1]; } - else { x1 = real_box.lo(1) + j*dx[1] + 0.5*dx[1]; } + else { x1 = real_box.lo(1) + j*dx[1] + 0.5_rt*dx[1]; } #if defined(WARPX_DIM_RZ) // Get index of the external field array - int const ir = floor( (x0-offset0)/file_dr ); - int const iz = floor( (x1-offset1)/file_dz ); + int const ir = std::floor( (x0-offset0)/file_dr ); + int const iz = std::floor( (x1-offset1)/file_dz ); // Get coordinates of external grid point amrex::Real const xx0 = offset0 + ir * file_dr; @@ -1501,12 +1454,12 @@ WarpX::ReadExternalFieldFromFile ( amrex::Real x2; if ( box.type(2)==amrex::IndexType::CellIndex::NODE ) { x2 = real_box.lo(2) + k*dx[2]; } - else { x2 = real_box.lo(2) + k*dx[2] + 0.5*dx[2]; } + else { x2 = real_box.lo(2) + k*dx[2] + 0.5_rt*dx[2]; } // Get index of the external field array - int const ix = floor( (x0-offset0)/file_dx ); - int const iy = floor( (x1-offset1)/file_dy ); - int const iz = floor( (x2-offset2)/file_dz ); + int const ix = std::floor( (x0-offset0)/file_dx ); + int const iy = std::floor( (x1-offset1)/file_dy ); + int const iz = std::floor( (x2-offset2)/file_dz ); // Get coordinates of external grid point amrex::Real const xx0 = offset0 + ix * file_dx; @@ -1521,10 +1474,10 @@ WarpX::ReadExternalFieldFromFile ( f01 = fc_array(0, iz , ir+1), f10 = fc_array(0, iz+1, ir ), f11 = fc_array(0, iz+1, ir+1); - mffab(i,j,k) = utils::algorithms::bilinear_interp + mffab(i,j,k) = static_cast(utils::algorithms::bilinear_interp (xx0, xx0+file_dr, xx1, xx1+file_dz, f00, f01, f10, f11, - x0, x1); + x0, x1)); #elif defined(WARPX_DIM_3D) const amrex::Array4 fc_array(FC_data, {0,0,0}, {extent2, extent1, extent0}, 1); const double @@ -1536,10 +1489,10 @@ WarpX::ReadExternalFieldFromFile ( f101 = fc_array(iz+1, iy , ix+1), f110 = fc_array(iz , iy+1, ix+1), f111 = fc_array(iz+1, iy+1, ix+1); - mffab(i,j,k) = utils::algorithms::trilinear_interp + mffab(i,j,k) = static_cast(utils::algorithms::trilinear_interp (xx0, xx0+file_dx, xx1, xx1+file_dy, xx2, xx2+file_dz, f000, f001, f010, f011, f100, f101, f110, f111, - x0, x1, x2); + x0, x1, x2)); #endif } @@ -1553,7 +1506,7 @@ WarpX::ReadExternalFieldFromFile ( void WarpX::ReadExternalFieldFromFile (std::string , amrex::MultiFab* ,std::string, std::string) { -#if defined(WARPX_DIM_1D) +#if defined(WARPX_DIM_1D_Z) WARPX_ABORT_WITH_MESSAGE("Reading fields from openPMD files is not supported in 1D"); #elif defined(WARPX_DIM_XZ) WARPX_ABORT_WITH_MESSAGE("Reading from openPMD for external fields is not known to work with XZ (see #3828)"); diff --git a/Source/Laser/LaserProfiles.H b/Source/Laser/LaserProfiles.H index 487e2d9e527..3aea88ac956 100644 --- a/Source/Laser/LaserProfiles.H +++ b/Source/Laser/LaserProfiles.H @@ -97,7 +97,13 @@ public: amrex::Real t, amrex::Real* AMREX_RESTRICT amplitude) const = 0; - virtual ~ILaserProfile(){} + ILaserProfile () = default; + virtual ~ILaserProfile() = default; + + ILaserProfile ( ILaserProfile const &) = default; + ILaserProfile& operator= ( ILaserProfile const & ) = default; + ILaserProfile ( ILaserProfile&& ) = default; + ILaserProfile& operator= ( ILaserProfile&& ) = default; }; /** @@ -110,11 +116,11 @@ public: void init ( const amrex::ParmParse& ppl, - CommonLaserParameters params) override final; + CommonLaserParameters params) final; //No update needed void - update (amrex::Real /*t */) override final {} + update (amrex::Real /*t */) final {} void fill_amplitude ( @@ -122,7 +128,7 @@ public: amrex::Real const * AMREX_RESTRICT Xp, amrex::Real const * AMREX_RESTRICT Yp, amrex::Real t, - amrex::Real * AMREX_RESTRICT amplitude) const override final; + amrex::Real * AMREX_RESTRICT amplitude) const final; private: struct { @@ -152,11 +158,11 @@ public: void init ( const amrex::ParmParse& ppl, - CommonLaserParameters params) override final; + CommonLaserParameters params) final; //No update needed void - update (amrex::Real /*t */) override final {} + update (amrex::Real /*t */) final {} void fill_amplitude ( @@ -164,7 +170,7 @@ public: amrex::Real const * AMREX_RESTRICT Xp, amrex::Real const * AMREX_RESTRICT Yp, amrex::Real t, - amrex::Real * AMREX_RESTRICT amplitude) const override final; + amrex::Real * AMREX_RESTRICT amplitude) const final; private: struct{ @@ -187,14 +193,14 @@ public: void init ( const amrex::ParmParse& ppl, - CommonLaserParameters params) override final; + CommonLaserParameters params) final; /** \brief Reads new field data chunk from file if needed * * @param[in] t simulation time (seconds) */ void - update (amrex::Real t) override final; + update (amrex::Real t) final; /** \brief compute field amplitude at particles' position for a laser beam * loaded from an E(x,y,t) file. @@ -215,7 +221,7 @@ public: amrex::Real const * AMREX_RESTRICT Xp, amrex::Real const * AMREX_RESTRICT Yp, amrex::Real t, - amrex::Real * AMREX_RESTRICT amplitude) const override final; + amrex::Real * AMREX_RESTRICT amplitude) const final; /** \brief Function to fill the amplitude in case of a uniform grid and for the lasy format in 3D Cartesian. * This function cannot be private due to restrictions related to @@ -302,7 +308,7 @@ private: * * \param t: simulation time */ - std::pair find_left_right_time_indices(amrex::Real t) const; + [[nodiscard]] std::pair find_left_right_time_indices(amrex::Real t) const; /** \brief Load field data within the temporal range [t_begin, t_end] * diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp index fbac9d9707f..0fdb45c64f8 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp @@ -114,8 +114,9 @@ void WarpXLaserProfiles::FromFileLaserProfile::update (amrex::Real t) { t += m_params.t_min - m_params.t_delay; - if(t >= m_params.t_max) + if(t >= m_params.t_max) { return; + } const auto idx_times = find_left_right_time_indices(t); const auto idx_t_left = idx_times.first; const auto idx_t_right = idx_times.second; @@ -172,7 +173,7 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(E.getAttribute("dataOrder").get() == "C", "Reading from files with non-C dataOrder is not implemented"); - std::string fileGeom = E.getAttribute("geometry").get(); + auto fileGeom = E.getAttribute("geometry").get(); auto E_laser = E[io::RecordComponent::SCALAR]; auto extent = E_laser.getExtent(); // Extract grid offset and grid spacing @@ -183,40 +184,40 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_ //Dimensions of lasy file data: {m,t,r} amrex::Print() << Utils::TextMsg::Info( "Found lasy file in RZ geometry" ); m_params.file_in_cartesian_geom = 0; - m_params.n_rz_azimuthal_components = extent[0]; - m_params.nt = extent[1]; - m_params.nr = extent[2]; - if(m_params.nt <= 1) WARPX_ABORT_WITH_MESSAGE("nt in lasy file must be >=2"); - if(m_params.nr <= 1) WARPX_ABORT_WITH_MESSAGE("nr in lasy file must be >=2"); + m_params.n_rz_azimuthal_components = static_cast(extent[0]); + m_params.nt = static_cast(extent[1]); + m_params.nr = static_cast(extent[2]); + if(m_params.nt <= 1) { WARPX_ABORT_WITH_MESSAGE("nt in lasy file must be >=2"); } + if(m_params.nr <= 1) { WARPX_ABORT_WITH_MESSAGE("nr in lasy file must be >=2"); } // Calculate the min and max of the grid - m_params.t_min = offset[0] + position[0]*spacing[0]; - m_params.t_max = m_params.t_min + (m_params.nt-1)*spacing[0]; - m_params.r_min = offset[1] + position[1]*spacing[1]; - m_params.r_max = m_params.r_min + (m_params.nr-1)*spacing[1]; + m_params.t_min = static_cast(offset[0] + position[0]*spacing[0]); + m_params.t_max = static_cast(m_params.t_min + (m_params.nt-1)*spacing[0]); + m_params.r_min = static_cast(offset[1] + position[1]*spacing[1]); + m_params.r_max = static_cast(m_params.r_min + (m_params.nr-1)*spacing[1]); } else if (fileGeom=="cartesian"){ //Dimensions of lasy file data: {t,y,x} amrex::Print() << Utils::TextMsg::Info( "Found lasy file in 3D cartesian geometry"); m_params.file_in_cartesian_geom = 1; - m_params.nt = extent[0]; - m_params.ny = extent[1]; - m_params.nx = extent[2]; + m_params.nt = static_cast(extent[0]); + m_params.ny = static_cast(extent[1]); + m_params.nx = static_cast(extent[2]); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_params.nt > 1, "nt in lasy file must be >=2"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_params.nx > 1, "nx in lasy file must be >=2"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_params.ny > 1, "ny in lasy file must be >=2 in 3D"); // Calculate the min and max of the grid - m_params.t_min = offset[0] + position[0]*spacing[0]; - m_params.t_max = m_params.t_min + (m_params.nt-1)*spacing[0]; - m_params.y_min = offset[1] + position[1]*spacing[1]; - m_params.y_max = m_params.y_min + (m_params.ny-1)*spacing[1]; - m_params.x_min = offset[2] + position[2]*spacing[2]; - m_params.x_max = m_params.x_min + (m_params.nx-1)*spacing[2]; + m_params.t_min = static_cast(offset[0] + position[0]*spacing[0]); + m_params.t_max = static_cast(m_params.t_min + (m_params.nt-1)*spacing[0]); + m_params.y_min = static_cast(offset[1] + position[1]*spacing[1]); + m_params.y_max = static_cast(m_params.y_min + (m_params.ny-1)*spacing[1]); + m_params.x_min = static_cast(offset[2] + position[2]*spacing[2]); + m_params.x_max = static_cast(m_params.x_min + (m_params.nx-1)*spacing[2]); } else{ WARPX_ABORT_WITH_MESSAGE("The lasy file's geometry has to be in either RZ or 3D cartesian coordinates"); } } //Broadcast parameters - ParallelDescriptor::Bcast(&m_params.file_in_cartesian_geom, 1, ParallelDescriptor::IOProcessorNumber()); + ParallelDescriptor::Bcast(&m_params.file_in_cartesian_geom, 1, ParallelDescriptor::IOProcessorNumber()); ParallelDescriptor::Bcast(&m_params.nt, 1, ParallelDescriptor::IOProcessorNumber()); ParallelDescriptor::Bcast(&m_params.nx, 1, ParallelDescriptor::IOProcessorNumber()); ParallelDescriptor::Bcast(&m_params.ny, 1, ParallelDescriptor::IOProcessorNumber()); @@ -240,24 +241,24 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_ { if(ParallelDescriptor::IOProcessor()){ std::ifstream inp(binary_file_name, std::ios::binary); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); } inp.exceptions(std::ios_base::failbit | std::ios_base::badbit); //Uniform grid flag char flag; inp.read(&flag, 1); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read grid type from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read grid type from binary file"); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flag, "Binary files with non uniform grid are no longer supported"); //Grid points along t, x and y inp.read(reinterpret_cast(&m_params.nt), sizeof(uint32_t)); inp.read(reinterpret_cast(&m_params.nx), sizeof(uint32_t)); inp.read(reinterpret_cast(&m_params.ny), sizeof(uint32_t)); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read sizes from binary file"); - if(m_params.nt <= 1) WARPX_ABORT_WITH_MESSAGE("nt in binary file must be >=2"); - if(m_params.nx <= 1) WARPX_ABORT_WITH_MESSAGE("nx in binary file must be >=2"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read sizes from binary file"); } + if(m_params.nt <= 1) { WARPX_ABORT_WITH_MESSAGE("nt in binary file must be >=2"); } + if(m_params.nx <= 1) { WARPX_ABORT_WITH_MESSAGE("nx in binary file must be >=2"); } #if (defined(WARPX_DIM_3D) || (defined WARPX_DIM_RZ)) - if(m_params.ny <= 1) WARPX_ABORT_WITH_MESSAGE("ny in binary file must be >=2 in 3D"); + if(m_params.ny <= 1) { WARPX_ABORT_WITH_MESSAGE("ny in binary file must be >=2 in 3D"); } #elif defined(WARPX_DIM_XZ) - if(m_params.ny != 1) WARPX_ABORT_WITH_MESSAGE("ny in binary file must be 1 in 2D"); + if(m_params.ny != 1) { WARPX_ABORT_WITH_MESSAGE("ny in binary file must be 1 in 2D"); } #endif //Coordinates Vector dbuf_t, dbuf_x, dbuf_y; @@ -269,12 +270,12 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_ dbuf_y.resize(1); #endif inp.read(reinterpret_cast(dbuf_t.dataPtr()), - dbuf_t.size()*sizeof(double)); + static_cast(dbuf_t.size()*sizeof(double))); inp.read(reinterpret_cast(dbuf_x.dataPtr()), - dbuf_x.size()*sizeof(double)); + static_cast(dbuf_x.size()*sizeof(double))); inp.read(reinterpret_cast(dbuf_y.dataPtr()), - dbuf_y.size()*sizeof(double)); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read coords from binary file"); + static_cast(dbuf_y.size()*sizeof(double))); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read coords from binary file"); } m_params.t_min = static_cast(dbuf_t[0]); m_params.t_max = static_cast(dbuf_t[1]); @@ -313,17 +314,15 @@ WarpXLaserProfiles::FromFileLaserProfile::read_data_t_chunk (int t_begin, int t_ { #ifdef WARPX_USE_OPENPMD //Indices of the first and last timestep to read - std::uint64_t const i_first = max(0, t_begin); - std::uint64_t const i_last = min(t_end-1, m_params.nt-1); + auto const i_first = static_cast(max(0, t_begin)); + auto const i_last = static_cast(min(t_end-1, m_params.nt-1)); amrex::Print() << Utils::TextMsg::Info( "Reading [" + std::to_string(i_first) + ", " + std::to_string(i_last) + "] data chunk from " + m_params.lasy_file_name); - int data_size; - if (m_params.file_in_cartesian_geom==0) { - data_size = m_params.n_rz_azimuthal_components*(i_last-i_first+1)*m_params.nr; - } else { - data_size = (i_last-i_first+1)*m_params.nx*m_params.ny; - } + const auto data_size = + (m_params.file_in_cartesian_geom==0)? + (m_params.n_rz_azimuthal_components*(i_last-i_first+1)*m_params.nr) : + (i_last-i_first+1)*m_params.nx*m_params.ny; m_params.E_lasy_data.resize(data_size); Vector h_E_lasy_data(m_params.E_lasy_data.size()); if(ParallelDescriptor::IOProcessor()){ @@ -335,10 +334,10 @@ WarpXLaserProfiles::FromFileLaserProfile::read_data_t_chunk (int t_begin, int t_ if (m_params.file_in_cartesian_geom==0) { const openPMD::Extent read_extent = { full_extent[0], (i_last - i_first + 1), full_extent[2]}; auto r_data = E_laser.loadChunk< std::complex >(io::Offset{ 0, i_first, 0}, read_extent); - const int read_size = (i_last - i_first + 1)*m_params.nr; + const auto read_size = (i_last - i_first + 1)*m_params.nr; series.flush(); for (int m=0; m(r_data.get()[j+m*read_size].real()), static_cast(r_data.get()[j+m*read_size].imag())}; @@ -347,9 +346,9 @@ WarpXLaserProfiles::FromFileLaserProfile::read_data_t_chunk (int t_begin, int t_ } else{ const openPMD::Extent read_extent = {(i_last - i_first + 1), full_extent[1], full_extent[2]}; auto x_data = E_laser.loadChunk< std::complex >(io::Offset{i_first, 0, 0}, read_extent); - const int read_size = (i_last - i_first + 1)*m_params.nx*m_params.ny; + const auto read_size = (i_last - i_first + 1)*m_params.nx*m_params.ny; series.flush(); - for (int j=0; j(x_data.get()[j].real()), static_cast(x_data.get()[j].imag())}; @@ -362,8 +361,8 @@ WarpXLaserProfiles::FromFileLaserProfile::read_data_t_chunk (int t_begin, int t_ Gpu::copyAsync(Gpu::hostToDevice,h_E_lasy_data.begin(),h_E_lasy_data.end(),m_params.E_lasy_data.begin()); Gpu::synchronize(); //Update first and last indices - m_params.first_time_index = i_first; - m_params.last_time_index = i_last; + m_params.first_time_index = static_cast(i_first); + m_params.last_time_index = static_cast(i_last); #else amrex::ignore_unused(t_begin, t_end); #endif @@ -385,7 +384,7 @@ WarpXLaserProfiles::FromFileLaserProfile::read_binary_data_t_chunk (int t_begin, if(ParallelDescriptor::IOProcessor()){ //Read data chunk std::ifstream inp(m_params.binary_file_name, std::ios::binary); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); } inp.exceptions(std::ios_base::failbit | std::ios_base::badbit); #if (defined(WARPX_DIM_3D)) auto skip_amount = 1 + @@ -402,13 +401,13 @@ WarpXLaserProfiles::FromFileLaserProfile::read_binary_data_t_chunk (int t_begin, 1*sizeof(double) + sizeof(double)*t_begin*m_params.nx*m_params.ny; #endif - inp.seekg(skip_amount); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); + inp.seekg(static_cast(skip_amount)); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); } const int read_size = (i_last - i_first + 1)* m_params.nx*m_params.ny; Vector buf_e(read_size); - inp.read(reinterpret_cast(buf_e.dataPtr()), read_size*sizeof(double)); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); + inp.read(reinterpret_cast(buf_e.dataPtr()), static_cast(read_size*sizeof(double))); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); } std::transform(buf_e.begin(), buf_e.end(), h_E_binary_data.begin(), [](auto x) {return static_cast(x);} ); } @@ -421,8 +420,8 @@ WarpXLaserProfiles::FromFileLaserProfile::read_binary_data_t_chunk (int t_begin, Gpu::synchronize(); //Update first and last indices - m_params.first_time_index = i_first; - m_params.last_time_index = i_last; + m_params.first_time_index = static_cast(i_first); + m_params.last_time_index = static_cast(i_last); } void @@ -434,7 +433,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_cartes { // Copy member variables to tmp copies // and get pointers to underlying data for GPU. - const amrex::Real omega_t = 2.*MathConst::pi*PhysConst::c*t/m_common_params.wavelength; + const amrex::Real omega_t = 2._rt*MathConst::pi*PhysConst::c*t/m_common_params.wavelength; const Complex exp_omega_t = Complex{ std::cos(-omega_t), std::sin(-omega_t) }; const auto tmp_x_min = m_params.x_min; const auto tmp_x_max = m_params.x_max; @@ -442,7 +441,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_cartes const auto tmp_y_max = m_params.y_max; const auto tmp_nx = m_params.nx; const auto tmp_ny = m_params.ny; - const auto p_E_lasy_data = m_params.E_lasy_data.dataPtr(); + const auto *const p_E_lasy_data = m_params.E_lasy_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* @@ -519,13 +518,13 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_cylind { // Copy member variables to tmp copies // and get pointers to underlying data for GPU. - const amrex::Real omega_t = 2.*MathConst::pi*PhysConst::c*t/m_common_params.wavelength; + const amrex::Real omega_t = 2._rt*MathConst::pi*PhysConst::c*t/m_common_params.wavelength; const Complex exp_omega_t = Complex{ std::cos(-omega_t), std::sin(-omega_t) }; const auto tmp_r_min = m_params.r_min; const auto tmp_r_max = m_params.r_max; const auto tmp_nr = m_params.nr; const auto tmp_n_rz_azimuthal_components = m_params.n_rz_azimuthal_components; - const auto p_E_lasy_data = m_params.E_lasy_data.dataPtr(); + const auto *const p_E_lasy_data = m_params.E_lasy_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* @@ -627,7 +626,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_binary const auto tmp_ny = m_params.ny; #endif const auto tmp_nx = m_params.nx; - const auto p_E_binary_data = m_params.E_binary_data.dataPtr(); + const auto *const p_E_binary_data = m_params.E_binary_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp index ef00d95fd43..96d61d920f1 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp @@ -73,10 +73,11 @@ WarpXLaserProfiles::GaussianLaserProfile::init ( m_params.stc_direction[1]*m_common_params.p_X[1] + m_params.stc_direction[2]*m_common_params.p_X[2]; - if (arg < -1.0_rt || arg > 1.0_rt) + if (arg < -1.0_rt || arg > 1.0_rt) { m_params.theta_stc = 0._rt; - else + } else { m_params.theta_stc = std::acos(arg); + } #else m_params.theta_stc = 0.; #endif diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 3f1702a4214..6c55372d199 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -77,6 +77,7 @@ include $(WARPX_HOME)/Source/Diagnostics/Make.package include $(WARPX_HOME)/Source/EmbeddedBoundary/Make.package include $(WARPX_HOME)/Source/FieldSolver/Make.package include $(WARPX_HOME)/Source/Filter/Make.package +include $(WARPX_HOME)/Source/Fluids/Make.package include $(WARPX_HOME)/Source/Initialization/Make.package include $(WARPX_HOME)/Source/Laser/Make.package include $(WARPX_HOME)/Source/Parallelization/Make.package @@ -120,7 +121,7 @@ ifeq ($(QED),TRUE) CFLAGS += -DWARPX_QED_TABLE_GEN FFLAGS += -DWARPX_QED_TABLE_GEN F90FLAGS += -DWARPX_QED_TABLE_GEN - USERSuffix := $(USERSuffix).GENTABLES + USERSuffix := $(USERSuffix).GENTABLES endif endif @@ -148,31 +149,31 @@ ifeq ($(USE_OPENBC_POISSON),TRUE) endif ifeq ($(USE_OPENPMD), TRUE) - # try pkg-config query - ifeq (0, $(shell pkg-config "openPMD >= 0.15.1"; echo $$?)) - CXXFLAGS += $(shell pkg-config --cflags openPMD) - LIBRARY_LOCATIONS += $(shell pkg-config --variable=libdir openPMD) - libraries += $(shell pkg-config --libs-only-l openPMD) - # fallback to manual settings - else - OPENPMD_LIB_PATH ?= NOT_SET - ifneq ($(OPENPMD_LIB_PATH),NOT_SET) - LIBRARY_LOCATIONS += $(OPENPMD_LIB_PATH) - endif - OPENPMD_INCLUDE_PATH ?= NOT_SET - ifneq ($(OPENPMD_INCLUDE_PATH),NOT_SET) - INCLUDE_LOCATIONS += $(OPENPMD_INCLUDE_PATH) - endif - libraries += -lopenPMD - endif - DEFINES += -DWARPX_USE_OPENPMD - USERSuffix := $(USERSuffix).OPMD + # try pkg-config query + ifeq (0, $(shell pkg-config "openPMD >= 0.15.1"; echo $$?)) + CXXFLAGS += $(shell pkg-config --cflags openPMD) + LIBRARY_LOCATIONS += $(shell pkg-config --variable=libdir openPMD) + libraries += $(shell pkg-config --libs-only-l openPMD) + # fallback to manual settings + else + OPENPMD_LIB_PATH ?= NOT_SET + ifneq ($(OPENPMD_LIB_PATH),NOT_SET) + LIBRARY_LOCATIONS += $(OPENPMD_LIB_PATH) + endif + OPENPMD_INCLUDE_PATH ?= NOT_SET + ifneq ($(OPENPMD_INCLUDE_PATH),NOT_SET) + INCLUDE_LOCATIONS += $(OPENPMD_INCLUDE_PATH) + endif + libraries += -lopenPMD + endif + DEFINES += -DWARPX_USE_OPENPMD + USERSuffix := $(USERSuffix).OPMD endif ifeq ($(USE_PSATD),TRUE) USERSuffix := $(USERSuffix).PSATD - DEFINES += -DWARPX_USE_PSATD + DEFINES += -DWARPX_USE_PSATD -DABLASTR_USE_FFT ifeq ($(USE_CUDA),TRUE) # Use cuFFT libraries += -lcufft @@ -182,18 +183,18 @@ ifeq ($(USE_PSATD),TRUE) LIBRARY_LOCATIONS += $(ROC_PATH)/rocfft/lib libraries += -lrocfft else # Running on CPU - # Use FFTW - ifeq ($(PRECISION),FLOAT) - libraries += -lfftw3f_mpi -lfftw3f -lfftw3f_threads - else - libraries += -lfftw3_mpi -lfftw3 -lfftw3_threads - endif - FFTW_HOME ?= NOT_SET - ifneq ($(FFTW_HOME),NOT_SET) - VPATH_LOCATIONS += $(FFTW_HOME)/include - INCLUDE_LOCATIONS += $(FFTW_HOME)/include - LIBRARY_LOCATIONS += $(FFTW_HOME)/lib - endif + # Use FFTW + ifeq ($(PRECISION),FLOAT) + libraries += -lfftw3f_mpi -lfftw3f -lfftw3f_threads + else + libraries += -lfftw3_mpi -lfftw3 -lfftw3_threads + endif + FFTW_HOME ?= NOT_SET + ifneq ($(FFTW_HOME),NOT_SET) + VPATH_LOCATIONS += $(FFTW_HOME)/include + INCLUDE_LOCATIONS += $(FFTW_HOME)/include + LIBRARY_LOCATIONS += $(FFTW_HOME)/lib + endif endif ifeq ($(USE_RZ),TRUE) # Use blas and lapack @@ -210,7 +211,7 @@ ifeq ($(USE_RZ),TRUE) endif ifeq ($(USE_HDF5),TRUE) - DEFINES += -DWARPX_USE_HDF5 + DEFINES += -DWARPX_USE_HDF5 endif ifeq ($(USE_GPUCLOCK),TRUE) diff --git a/Source/Parallelization/CMakeLists.txt b/Source/Parallelization/CMakeLists.txt index 48f0d3a49bd..d38f8d6bbf8 100644 --- a/Source/Parallelization/CMakeLists.txt +++ b/Source/Parallelization/CMakeLists.txt @@ -5,5 +5,6 @@ foreach(D IN LISTS WarpX_DIMS) GuardCellManager.cpp WarpXComm.cpp WarpXRegrid.cpp + WarpXSumGuardCells.cpp ) endforeach() diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index ee8fd044abb..38cef54921b 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -100,7 +100,7 @@ public: amrex::IntVect ng_UpdateAux = amrex::IntVect::TheZeroVector(); // Number of guard cells of all MultiFabs that must exchanged before moving window amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); - // Number of guard cells of E and B that are exchanged immediatly after the main PSATD push + // Number of guard cells of E and B that are exchanged immediately after the main PSATD push amrex::IntVect ng_afterPushPSATD = amrex::IntVect::TheZeroVector(); // Number of guard cells for local deposition of J and rho diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index a4547c25c6d..ae5ab839d10 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -103,7 +103,7 @@ guardCellManager::Init ( if (do_moving_window) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(ref_ratios.size() <= 1, "The number of grow cells for the moving window currently assumes 2 levels max."); - const int nlevs = ref_ratios.size()+1; + const auto nlevs = static_cast(ref_ratios.size()+1); const int max_r = (nlevs > 1) ? ref_ratios[0][moving_window_dir] : 2; ngx = std::max(ngx,max_r); @@ -172,7 +172,7 @@ guardCellManager::Init ( // After pushing particle int ng_alloc_F_int = (do_moving_window) ? 2 : 0; // CKC solver requires one additional guard cell - if (electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) { ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); } ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int)); // Used if warpx.do_divb_cleaning = 1 @@ -208,11 +208,11 @@ guardCellManager::Init ( utils::parser::queryWithParser(pp_psatd, "nz_guard", ngFFt_z); #if defined(WARPX_DIM_3D) - IntVect ngFFT = IntVect(ngFFt_x, ngFFt_y, ngFFt_z); + auto ngFFT = IntVect(ngFFt_x, ngFFt_y, ngFFt_z); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - IntVect ngFFT = IntVect(ngFFt_x, ngFFt_z); + auto ngFFT = IntVect(ngFFt_x, ngFFt_z); #elif defined(WARPX_DIM_1D_Z) - IntVect ngFFT = IntVect(ngFFt_z); + auto ngFFT = IntVect(ngFFt_z); #endif #ifdef WARPX_DIM_RZ @@ -222,7 +222,7 @@ guardCellManager::Init ( } } #else - amrex::ignore_unused(do_pml, do_pml_in_domain, pml_ncell); + amrex::ignore_unused(do_pml, do_pml_in_domain, pml_ncell); #endif // All boxes should have the same number of guard cells, to avoid temporary parallel copies: @@ -309,13 +309,14 @@ guardCellManager::Init ( // factor grows symmetrically by half a cell on each side. So every // +2 orders, one touches one more cell point. int const FGcell = (nox + 1) / 2; // integer division - IntVect ng_FieldGather_noNCI = IntVect(AMREX_D_DECL(FGcell,FGcell,FGcell)); + auto ng_FieldGather_noNCI = IntVect(AMREX_D_DECL(FGcell,FGcell,FGcell)); ng_FieldGather_noNCI = ng_FieldGather_noNCI.min(ng_alloc_EB); // If NCI filter, add guard cells in the z direction IntVect ng_NCIFilter = IntVect::TheZeroVector(); - if (do_fdtd_nci_corr) + if (do_fdtd_nci_corr) { ng_NCIFilter[WARPX_ZINDEX] = NCIGodfreyFilter::m_stencil_width; + } // Note: communications of guard cells for bilinear filter are handled // separately. diff --git a/Source/Parallelization/Make.package b/Source/Parallelization/Make.package index 629cfafea62..7930118a1d2 100644 --- a/Source/Parallelization/Make.package +++ b/Source/Parallelization/Make.package @@ -1,5 +1,6 @@ CEXE_sources += WarpXComm.cpp CEXE_sources += WarpXRegrid.cpp CEXE_sources += GuardCellManager.cpp +CEXE_sources += WarpXSumGuardCells.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Parallelization diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 9b7f16478fc..7d5b25b1560 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -181,6 +181,16 @@ WarpX::UpdateAuxilaryDataStagToNodal () ng_src, ng, WarpX::do_single_precision_comms, cperiod); } + const amrex::IntVect& refinement_ratio = refRatio(lev-1); + + const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); + + const amrex::IntVect& Bx_cp_stag = Bfield_cp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_cp_stag = Bfield_cp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_cp_stag = Bfield_cp[lev][2]->ixType().toIntVect(); + #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif @@ -203,9 +213,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept { - warpx_interp_nd_bfield_x(j,k,l, bx_aux, bx_fp, bx_cp, bx_c); - warpx_interp_nd_bfield_y(j,k,l, by_aux, by_fp, by_cp, by_c); - warpx_interp_nd_bfield_z(j,k,l, bz_aux, bz_fp, bz_cp, bz_c); + warpx_interp(j, k, l, bx_aux, bx_fp, bx_cp, bx_c, Bx_fp_stag, Bx_cp_stag, refinement_ratio); + warpx_interp(j, k, l, by_aux, by_fp, by_cp, by_c, By_fp_stag, By_cp_stag, refinement_ratio); + warpx_interp(j, k, l, bz_aux, bz_fp, bz_cp, bz_c, Bz_fp_stag, Bz_cp_stag, refinement_ratio); }); } } @@ -239,6 +249,16 @@ WarpX::UpdateAuxilaryDataStagToNodal () ng_src, ng, WarpX::do_single_precision_comms, cperiod); } + const amrex::IntVect& refinement_ratio = refRatio(lev-1); + + const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); + + const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_cp_stag = Efield_cp[lev][2]->ixType().toIntVect(); + #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif @@ -261,9 +281,9 @@ WarpX::UpdateAuxilaryDataStagToNodal () amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept { - warpx_interp_nd_efield_x(j,k,l, ex_aux, ex_fp, ex_cp, ex_c); - warpx_interp_nd_efield_y(j,k,l, ey_aux, ey_fp, ey_cp, ey_c); - warpx_interp_nd_efield_z(j,k,l, ez_aux, ez_fp, ez_cp, ez_c); + warpx_interp(j, k, l, ex_aux, ex_fp, ex_cp, ex_c, Ex_fp_stag, Ex_cp_stag, refinement_ratio); + warpx_interp(j, k, l, ey_aux, ey_fp, ey_cp, ey_c, Ey_fp_stag, Ey_cp_stag, refinement_ratio); + warpx_interp(j, k, l, ez_aux, ez_fp, ez_cp, ez_c, Ez_fp_stag, Ez_cp_stag, refinement_ratio); }); } } @@ -531,7 +551,7 @@ void WarpX::FillBoundaryE (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryE(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -561,13 +581,13 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In (patch_type == PatchType::fine) ? pml[lev]->GetE_fp() : pml[lev]->GetE_cp(); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); - pml[lev]->FillBoundaryE(patch_type); + pml[lev]->FillBoundaryE(patch_type, nodal_sync); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) if (pml_rz[lev]) { - pml_rz[lev]->FillBoundaryE(patch_type); + pml_rz[lev]->FillBoundaryE(patch_type, nodal_sync); } #endif } @@ -588,7 +608,7 @@ void WarpX::FillBoundaryB (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryB(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -618,13 +638,13 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In (patch_type == PatchType::fine) ? pml[lev]->GetB_fp() : pml[lev]->GetB_cp(); pml[lev]->Exchange(mf_pml, mf, patch_type, do_pml_in_domain); - pml[lev]->FillBoundaryB(patch_type); + pml[lev]->FillBoundaryB(patch_type, nodal_sync); } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) if (pml_rz[lev]) { - pml_rz[lev]->FillBoundaryB(patch_type); + pml_rz[lev]->FillBoundaryB(patch_type, nodal_sync); } #endif } @@ -645,7 +665,7 @@ void WarpX::FillBoundaryE_avg(int lev, IntVect ng) { FillBoundaryE_avg(lev, PatchType::fine, ng); - if (lev > 0) FillBoundaryE_avg(lev, PatchType::coarse, ng); + if (lev > 0) { FillBoundaryE_avg(lev, PatchType::coarse, ng); } } void @@ -654,9 +674,9 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) if (patch_type == PatchType::fine) { if (do_pml && pml[lev]->ok()) - { + { WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); - } + } const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ){ @@ -674,9 +694,9 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) else if (patch_type == PatchType::coarse) { if (do_pml && pml[lev]->ok()) - { + { WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); - } + } const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ) { @@ -699,7 +719,7 @@ void WarpX::FillBoundaryB_avg (int lev, IntVect ng) { FillBoundaryB_avg(lev, PatchType::fine, ng); - if (lev > 0) FillBoundaryB_avg(lev, PatchType::coarse, ng); + if (lev > 0) { FillBoundaryB_avg(lev, PatchType::coarse, ng); } } void @@ -708,9 +728,9 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) if (patch_type == PatchType::fine) { if (do_pml && pml[lev]->ok()) - { + { WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); - } + } const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ) { const Vector mf{Bfield_avg_fp[lev][0].get(),Bfield_avg_fp[lev][1].get(),Bfield_avg_fp[lev][2].get()}; @@ -727,9 +747,9 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) else if (patch_type == PatchType::coarse) { if (do_pml && pml[lev]->ok()) - { + { WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); - } + } const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ){ @@ -750,7 +770,7 @@ void WarpX::FillBoundaryF (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryF(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -760,8 +780,8 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_fp[lev]) pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); - pml[lev]->FillBoundaryF(patch_type); + if (F_fp[lev]) { pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); } + pml[lev]->FillBoundaryF(patch_type, nodal_sync); } if (F_fp[lev]) @@ -775,8 +795,8 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_cp[lev]) pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); - pml[lev]->FillBoundaryF(patch_type); + if (F_cp[lev]) { pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); } + pml[lev]->FillBoundaryF(patch_type, nodal_sync); } if (F_cp[lev]) @@ -804,8 +824,8 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_fp[lev]) pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); - pml[lev]->FillBoundaryG(patch_type); + if (G_fp[lev]) { pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); } + pml[lev]->FillBoundaryG(patch_type, nodal_sync); } if (G_fp[lev]) @@ -819,8 +839,8 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_cp[lev]) pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); - pml[lev]->FillBoundaryG(patch_type); + if (G_cp[lev]) { pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); } + pml[lev]->FillBoundaryG(patch_type, nodal_sync); } if (G_cp[lev]) @@ -884,7 +904,7 @@ WarpX::SyncCurrent ( // SumBoundary even if there is only a single process, because a process // may have multiple boxes. Furthermore, even if there is only a single // box on a single process, SumBoundary should also be called if there - // are periodic boundaries. So we always call SumBounary even if it + // are periodic boundaries. So we always call SumBoundary even if it // might be a no-op in some cases, because the function does not perform // any communication if not necessary. // @@ -903,13 +923,13 @@ WarpX::SyncCurrent ( // have all been properly filtered and summed. For the current level, if // there are levels below this, we need to process this level's cp data // just like we have done for the finer level. The iteration continues - // until we reache level 0. There are however two additional + // until we reach level 0. There are however two additional // complications. // // The first complication is that simply calling ParallelAdd to add the // finer level's cp data to the current level's fp data does not // work. Suppose there are multiple boxes on the current level (or just - // a single box with periodic bounaries). A given (i,j,k) can be present + // a single box with periodic boundaries). A given (i,j,k) can be present // in more than one box for nodal data in AMReX. // At the time of calling ParallelAdd, the current // level's fp data have not been summed. Because of how ParallelAdd @@ -1028,7 +1048,7 @@ WarpX::SyncRho ( { WARPX_PROFILE("WarpX::SyncRho()"); - if (!charge_fp[0]) return; + if (!charge_fp[0]) { return; } const int ncomp = charge_fp[0]->nComp(); // See comments in WarpX::SyncCurrent for an explanation of the algorithm. @@ -1310,7 +1330,7 @@ void WarpX::ApplyFilterandSumBoundaryRho ( const int glev = (patch_type == PatchType::fine) ? lev : lev-1; const std::unique_ptr& rho = (patch_type == PatchType::fine) ? charge_fp[lev] : charge_cp[lev]; - if (rho == nullptr) return; + if (rho == nullptr) { return; } ApplyFilterandSumBoundaryRho(lev, glev, *rho, icomp, ncomp); } @@ -1353,7 +1373,7 @@ void WarpX::AddRhoFromFineLevelandSumBoundary ( const int icomp, const int ncomp) { - if (!charge_fp[lev]) return; + if (!charge_fp[lev]) { return; } ApplyFilterandSumBoundaryRho(charge_fp, charge_cp, lev, PatchType::fine, icomp, ncomp); @@ -1432,7 +1452,7 @@ void WarpX::NodalSyncJ ( const int lev, PatchType patch_type) { - if (!override_sync_intervals.contains(istep[0])) return; + if (!override_sync_intervals.contains(istep[0])) { return; } if (patch_type == PatchType::fine) { @@ -1458,7 +1478,7 @@ void WarpX::NodalSyncRho ( const int icomp, const int ncomp) { - if (!override_sync_intervals.contains(istep[0])) return; + if (!override_sync_intervals.contains(istep[0])) { return; } if (patch_type == PatchType::fine && charge_fp[lev]) { @@ -1473,62 +1493,3 @@ void WarpX::NodalSyncRho ( ablastr::utils::communication::OverrideSync(rhoc, WarpX::do_single_precision_comms, cperiod); } } - -void WarpX::NodalSyncPML () -{ - for (int lev = 0; lev <= finest_level; lev++) { - NodalSyncPML(lev); - } -} - -void WarpX::NodalSyncPML (int lev) -{ - NodalSyncPML(lev, PatchType::fine); - if (lev > 0) NodalSyncPML(lev, PatchType::coarse); -} - -void WarpX::NodalSyncPML (int lev, PatchType patch_type) -{ - if (pml[lev] && pml[lev]->ok()) - { - const std::array& pml_E = (patch_type == PatchType::fine) ? - pml[lev]->GetE_fp() : pml[lev]->GetE_cp(); - const std::array& pml_B = (patch_type == PatchType::fine) ? - pml[lev]->GetB_fp() : pml[lev]->GetB_cp(); - amrex::MultiFab* const pml_F = (patch_type == PatchType::fine) ? pml[lev]->GetF_fp() : pml[lev]->GetF_cp(); - amrex::MultiFab* const pml_G = (patch_type == PatchType::fine) ? pml[lev]->GetG_fp() : pml[lev]->GetG_cp(); - - // Always synchronize nodal points - const amrex::Periodicity& period = Geom(lev).periodicity(); - ablastr::utils::communication::OverrideSync(*pml_E[0], WarpX::do_single_precision_comms, period); - ablastr::utils::communication::OverrideSync(*pml_E[1], WarpX::do_single_precision_comms, period); - ablastr::utils::communication::OverrideSync(*pml_E[2], WarpX::do_single_precision_comms, period); - ablastr::utils::communication::OverrideSync(*pml_B[0], WarpX::do_single_precision_comms, period); - ablastr::utils::communication::OverrideSync(*pml_B[1], WarpX::do_single_precision_comms, period); - ablastr::utils::communication::OverrideSync(*pml_B[2], WarpX::do_single_precision_comms, period); - if (pml_F) { - ablastr::utils::communication::OverrideSync(*pml_F, WarpX::do_single_precision_comms, period); - } - if (pml_G) { - ablastr::utils::communication::OverrideSync(*pml_G, WarpX::do_single_precision_comms, period); - } - } - -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) - if (pml_rz[lev]) - { - // This is not actually needed with RZ PSATD since the - // arrays are always cell centered. Keep for now since - // it may be useful if the PML is used with FDTD - const std::array pml_rz_E = pml_rz[lev]->GetE_fp(); - const std::array pml_rz_B = pml_rz[lev]->GetB_fp(); - - // Always synchronize nodal points - const amrex::Periodicity& period = Geom(lev).periodicity(); - pml_rz_E[0]->OverrideSync(period); - pml_rz_E[1]->OverrideSync(period); - pml_rz_B[0]->OverrideSync(period); - pml_rz_B[1]->OverrideSync(period); - } -#endif -} diff --git a/Source/Parallelization/WarpXComm_K.H b/Source/Parallelization/WarpXComm_K.H index b97773af251..4cec33bc84f 100644 --- a/Source/Parallelization/WarpXComm_K.H +++ b/Source/Parallelization/WarpXComm_K.H @@ -10,6 +10,20 @@ #include #include +/** + * \brief Interpolation function called within WarpX::UpdateAuxilaryDataSameType + * to interpolate data from the coarse and fine grids to the fine aux grid, + * assuming that all grids have the same staggering (either collocated or staggered). + * + * \param[in] j index along x of the output array + * \param[in] k index along y (in 3D) or z (in 2D) of the output array + * \param[in] l index along z (in 3D, l=0 in 2D) of the output array + * \param[in,out] arr_aux output array where interpolated values are stored + * \param[in] arr_fine input fine-patch array storing the values to interpolate + * \param[in] arr_coarse input coarse-patch array storing the values to interpolate + * \param[in] arr_stag IndexType of the arrays + * \param[in] rr mesh refinement ratios along each direction + */ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void warpx_interp (int j, int k, int l, amrex::Array4 const& arr_aux, @@ -68,561 +82,218 @@ void warpx_interp (int j, int k, int l, arr_aux(j,k,l) = arr_fine(j,k,l) + res; } +/** + * \brief Interpolation function called within WarpX::UpdateAuxilaryDataStagToNodal + * to interpolate data from the coarse and fine grids to the fine aux grid, + * with momentum-conserving field gathering, hence between grids with different staggering, + * and assuming that the aux grid is collocated. + * + * \param[in] j index along x of the output array + * \param[in] k index along y (in 3D) or z (in 2D) of the output array + * \param[in] l index along z (in 3D, l=0 in 2D) of the output array + * \param[in,out] arr_aux output array where interpolated values are stored + * \param[in] arr_fine input fine-patch array storing the values to interpolate + * \param[in] arr_coarse input coarse-patch array storing the values to interpolate + * \param[in] arr_fine_stag IndexType of the fine-patch arrays + * \param[in] arr_coarse_stag IndexType of the coarse-patch arrays + * \param[in] rr mesh refinement ratios along each direction + */ AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void warpx_interp_nd_bfield_x (int j, int k, int l, - amrex::Array4 const& Bxa, - amrex::Array4 const& Bxf, - amrex::Array4 const& Bxc, - amrex::Array4 const& Bxg) +void warpx_interp (int j, int k, int l, + amrex::Array4 const& arr_aux, + amrex::Array4 const& arr_fine, + amrex::Array4 const& arr_coarse, + amrex::Array4 const& arr_tmp, + const amrex::IntVect& arr_fine_stag, + const amrex::IntVect& arr_coarse_stag, + const amrex::IntVect& rr) { using namespace amrex; - // Pad Bxf with zeros beyond ghost cells for out-of-bound accesses - const auto Bxf_zeropad = [Bxf] (const int jj, const int kk, const int ll) noexcept + // Pad input arrays with zeros beyond ghost cells + // for out-of-bound accesses due to large-stencil operations + const auto arr_fine_zeropad = [arr_fine] (const int jj, const int kk, const int ll) noexcept { - return Bxf.contains(jj,kk,ll) ? Bxf(jj,kk,ll) : 0.0_rt; + return arr_fine.contains(jj,kk,ll) ? arr_fine(jj,kk,ll) : 0.0_rt; + }; + const auto arr_coarse_zeropad = [arr_coarse] (const int jj, const int kk, const int ll) noexcept + { + return arr_coarse.contains(jj,kk,ll) ? arr_coarse(jj,kk,ll) : 0.0_rt; + }; + const auto arr_tmp_zeropad = [arr_tmp] (const int jj, const int kk, const int ll) noexcept + { + return arr_tmp.contains(jj,kk,ll) ? arr_tmp(jj,kk,ll) : 0.0_rt; }; - const int jg = amrex::coarsen(j,2); - const Real wx = (j == jg*2) ? 0.0_rt : 0.5_rt; - const Real owx = 1.0_rt-wx; - - const int kg = amrex::coarsen(k,2); - Real wy = 0.0_rt; - Real owy = 0.0_rt; - wy = (k == kg*2) ? 0.0_rt : 0.5_rt; - owy = 1.0_rt-wy; + // NOTE Indices (j,k,l) in the following refer to: + // - (z,-,-) in 1D + // - (x,z,-) in 2D + // - (r,z,-) in RZ + // - (x,y,z) in 3D + // Refinement ratio + const int rj = rr[0]; #if defined(WARPX_DIM_1D_Z) - - // interp from coarse nodal to fine nodal - const Real bg = owx * Bxg(jg ,0,0) - + wx * Bxg(jg+1,0,0); - - // interp from coarse staggered to fine nodal - const Real bc = owx * Bxc(jg ,0,0) - + wx * Bxc(jg+1,0,0); - - // interp from fine staggered to fine nodal - const Real bf = 0.5_rt*(Bxf_zeropad(j-1,0,0) + Bxf_zeropad(j,0,0)); - amrex::ignore_unused(owy); - + constexpr int rk = 1; + constexpr int rl = 1; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - - // interp from coarse nodal to fine nodal - const Real bg = owx * owy * Bxg(jg ,kg ,0) - + owx * wy * Bxg(jg ,kg+1,0) - + wx * owy * Bxg(jg+1,kg ,0) - + wx * wy * Bxg(jg+1,kg+1,0); - - // interp from coarse staggered to fine nodal - wy = 0.5_rt-wy; owy = 1.0_rt-wy; - const Real bc = owx * owy * Bxc(jg ,kg ,0) - + owx * wy * Bxc(jg ,kg-1,0) - + wx * owy * Bxc(jg+1,kg ,0) - + wx * wy * Bxc(jg+1,kg-1,0); - - // interp from fine staggered to fine nodal - const Real bf = 0.5_rt*(Bxf_zeropad(j,k-1,0) + Bxf_zeropad(j,k,0)); - + const int rk = rr[1]; + constexpr int rl = 1; #else - - const int lg = amrex::coarsen(l,2); - Real wz = (l == lg*2) ? 0.0_rt : 0.5_rt; - Real owz = 1.0_rt-wz; - - // interp from coarse nodal to fine nodal - const Real bg = owx * owy * owz * Bxg(jg ,kg ,lg ) - + wx * owy * owz * Bxg(jg+1,kg ,lg ) - + owx * wy * owz * Bxg(jg ,kg+1,lg ) - + wx * wy * owz * Bxg(jg+1,kg+1,lg ) - + owx * owy * wz * Bxg(jg ,kg ,lg+1) - + wx * owy * wz * Bxg(jg+1,kg ,lg+1) - + owx * wy * wz * Bxg(jg ,kg+1,lg+1) - + wx * wy * wz * Bxg(jg+1,kg+1,lg+1); - - // interp from coarse staggered to fine nodal - wy = 0.5_rt-wy; owy = 1.0_rt-wy; - wz = 0.5_rt-wz; owz = 1.0_rt-wz; - const Real bc = owx * owy * owz * Bxc(jg ,kg ,lg ) - + wx * owy * owz * Bxc(jg+1,kg ,lg ) - + owx * wy * owz * Bxc(jg ,kg-1,lg ) - + wx * wy * owz * Bxc(jg+1,kg-1,lg ) - + owx * owy * wz * Bxc(jg ,kg ,lg-1) - + wx * owy * wz * Bxc(jg+1,kg ,lg-1) - + owx * wy * wz * Bxc(jg ,kg-1,lg-1) - + wx * wy * wz * Bxc(jg+1,kg-1,lg-1); - - // interp from fine stagged to fine nodal - const Real bf = 0.25_rt*(Bxf_zeropad(j,k-1,l-1) + Bxf_zeropad(j,k,l-1) + Bxf_zeropad(j,k-1,l) + Bxf_zeropad(j,k,l)); + const int rk = rr[1]; + const int rl = rr[2]; #endif - Bxa(j,k,l) = bg + (bf-bc); -} - -AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void warpx_interp_nd_bfield_y (int j, int k, int l, - amrex::Array4 const& Bya, - amrex::Array4 const& Byf, - amrex::Array4 const& Byc, - amrex::Array4 const& Byg) -{ - using namespace amrex; - - // Pad Byf with zeros beyond ghost cells for out-of-bound accesses - const auto Byf_zeropad = [Byf] (const int jj, const int kk, const int ll) noexcept - { - return Byf.contains(jj,kk,ll) ? Byf(jj,kk,ll) : 0.0_rt; - }; - - const int jg = amrex::coarsen(j,2); - Real wx, owx; - wx = (j == jg*2) ? 0.0_rt : 0.5_rt; - owx = 1.0_rt-wx; - - const int kg = amrex::coarsen(k,2); - Real wy = 0.0_rt; - Real owy = 0.0_rt; - wy = (k == kg*2) ? 0.0_rt : 0.5_rt; - owy = 1.0_rt-wy; + // Staggering of fine array (0: cell-centered; 1: nodal) + const int sj_fp = arr_fine_stag[0]; +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const int sk_fp = arr_fine_stag[1]; +#elif defined(WARPX_DIM_3D) + const int sk_fp = arr_fine_stag[1]; + const int sl_fp = arr_fine_stag[2]; +#endif + // Staggering of coarse array (0: cell-centered; 1: nodal) + const int sj_cp = arr_coarse_stag[0]; #if defined(WARPX_DIM_1D_Z) - - // interp from coarse nodal to fine nodal - const Real bg = owx * Byg(jg ,0,0) - + wx * Byg(jg+1,0,0); - - // interp from coarse staggered to fine nodal - const Real bc = owx * Byc(jg ,0,0) - + wx * Byc(jg+1,0,0); - - // interp from fine staggered to fine nodal - const Real bf = 0.5_rt*(Byf_zeropad(j-1,0,0) + Byf_zeropad(j,0,0)); - amrex::ignore_unused(owy); - + constexpr int sk_cp = 0; + constexpr int sl_cp = 0; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - - // interp from coarse nodal to fine nodal - const Real bg = owx * owy * Byg(jg ,kg ,0) - + owx * wy * Byg(jg ,kg+1,0) - + wx * owy * Byg(jg+1,kg ,0) - + wx * wy * Byg(jg+1,kg+1,0); - - // interp from coarse stagged (cell-centered for By) to fine nodal - wx = 0.5_rt-wx; owx = 1.0_rt-wx; - wy = 0.5_rt-wy; owy = 1.0_rt-wy; - const Real bc = owx * owy * Byc(jg ,kg ,0) - + owx * wy * Byc(jg ,kg-1,0) - + wx * owy * Byc(jg-1,kg ,0) - + wx * wy * Byc(jg-1,kg-1,0); - - // interp form fine stagger (cell-centered for By) to fine nodal - const Real bf = 0.25_rt*(Byf_zeropad(j,k,0) + Byf_zeropad(j-1,k,0) + Byf_zeropad(j,k-1,0) + Byf_zeropad(j-1,k-1,0)); - + const int sk_cp = arr_coarse_stag[1]; + constexpr int sl_cp = 0; #else - - const int lg = amrex::coarsen(l,2); - Real wz = (l == lg*2) ? 0.0_rt : 0.5_rt; - Real owz = 1.0_rt-wz; - - // interp from coarse nodal to fine nodal - const Real bg = owx * owy * owz * Byg(jg ,kg ,lg ) - + wx * owy * owz * Byg(jg+1,kg ,lg ) - + owx * wy * owz * Byg(jg ,kg+1,lg ) - + wx * wy * owz * Byg(jg+1,kg+1,lg ) - + owx * owy * wz * Byg(jg ,kg ,lg+1) - + wx * owy * wz * Byg(jg+1,kg ,lg+1) - + owx * wy * wz * Byg(jg ,kg+1,lg+1) - + wx * wy * wz * Byg(jg+1,kg+1,lg+1); - - // interp from coarse staggered to fine nodal - wx = 0.5_rt-wx; owx = 1.0_rt-wx; - wz = 0.5_rt-wz; owz = 1.0_rt-wz; - const Real bc = owx * owy * owz * Byc(jg ,kg ,lg ) - + wx * owy * owz * Byc(jg-1,kg ,lg ) - + owx * wy * owz * Byc(jg ,kg+1,lg ) - + wx * wy * owz * Byc(jg-1,kg+1,lg ) - + owx * owy * wz * Byc(jg ,kg ,lg-1) - + wx * owy * wz * Byc(jg-1,kg ,lg-1) - + owx * wy * wz * Byc(jg ,kg+1,lg-1) - + wx * wy * wz * Byc(jg-1,kg+1,lg-1); - - // interp from fine stagged to fine nodal - const Real bf = 0.25_rt*(Byf_zeropad(j-1,k,l-1) + Byf_zeropad(j,k,l-1) + Byf_zeropad(j-1,k,l) + Byf_zeropad(j,k,l)); - + const int sk_cp = arr_coarse_stag[1]; + const int sl_cp = arr_coarse_stag[2]; #endif - Bya(j,k,l) = bg + (bf-bc); -} + // Number of points used for interpolation from coarse grid to fine grid + int nj; + int nk; + int nl; -AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void warpx_interp_nd_bfield_z (int j, int k, int l, - amrex::Array4 const& Bza, - amrex::Array4 const& Bzf, - amrex::Array4 const& Bzc, - amrex::Array4 const& Bzg) -{ - using namespace amrex; + int jc = amrex::coarsen(j, rj); + int kc = amrex::coarsen(k, rk); + int lc = amrex::coarsen(l, rl); - // Pad Bzf with zeros beyond ghost cells for out-of-bound accesses - const auto Bzf_zeropad = [Bzf] (const int jj, const int kk, const int ll) noexcept - { - return Bzf.contains(jj,kk,ll) ? Bzf(jj,kk,ll) : 0.0_rt; - }; + amrex::Real tmp = 0.0_rt; + amrex::Real fine = 0.0_rt; + amrex::Real coarse = 0.0_rt; - const int jg = amrex::coarsen(j,2); - Real wx, owx; - wx = (j == jg*2) ? 0.0_rt : 0.5_rt; - owx = 1.0_rt-wx; + amrex::Real wj; + amrex::Real wk; + amrex::Real wl; - const int kg = amrex::coarsen(k,2); - Real wy = 0.0_rt; - Real owy = 0.0_rt; - wy = (k == kg*2) ? 0.0_rt : 0.5_rt; - owy = 1.0_rt-wy; + // 1) Interpolation from coarse nodal to fine nodal + nj = 2; #if defined(WARPX_DIM_1D_Z) - - // interp from coarse nodal to fine nodal - const Real bg = owx * Bzg(jg ,0,0) - + wx * Bzg(jg+1,0,0); - - // interp from coarse staggered to fine nodal - const Real bc = owx * Bzc(jg ,0,0) - + wx * Bzc(jg+1,0,0); - - // interp from fine staggered to fine nodal - const Real bf = Bzf_zeropad(j,0,0); - amrex::ignore_unused(owy); - + nk = 1; + nl = 1; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - - // interp from coarse nodal to fine nodal - const Real bg = owx * owy * Bzg(jg ,kg ,0) - + owx * wy * Bzg(jg ,kg+1,0) - + wx * owy * Bzg(jg+1,kg ,0) - + wx * wy * Bzg(jg+1,kg+1,0); - - // interp from coarse staggered to fine nodal - wx = 0.5_rt-wx; owx = 1.0_rt-wx; - const Real bc = owx * owy * Bzc(jg ,kg ,0) - + owx * wy * Bzc(jg ,kg+1,0) - + wx * owy * Bzc(jg-1,kg ,0) - + wx * wy * Bzc(jg-1,kg+1,0); - - // interp from fine staggered to fine nodal - const Real bf = 0.5_rt*(Bzf_zeropad(j-1,k,0) + Bzf_zeropad(j,k,0)); - + nk = 2; + nl = 1; #else - - const int lg = amrex::coarsen(l,2); - const Real wz = (l == lg*2) ? 0.0_rt : 0.5_rt; - const Real owz = 1.0_rt-wz; - - // interp from coarse nodal to fine nodal - const Real bg = owx * owy * owz * Bzg(jg ,kg ,lg ) - + wx * owy * owz * Bzg(jg+1,kg ,lg ) - + owx * wy * owz * Bzg(jg ,kg+1,lg ) - + wx * wy * owz * Bzg(jg+1,kg+1,lg ) - + owx * owy * wz * Bzg(jg ,kg ,lg+1) - + wx * owy * wz * Bzg(jg+1,kg ,lg+1) - + owx * wy * wz * Bzg(jg ,kg+1,lg+1) - + wx * wy * wz * Bzg(jg+1,kg+1,lg+1); - - // interp from coarse staggered to fine nodal - wx = 0.5_rt-wx; owx = 1.0_rt-wx; - wy = 0.5_rt-wy; owy = 1.0_rt-wy; - const Real bc = owx * owy * owz * Bzc(jg ,kg ,lg ) - + wx * owy * owz * Bzc(jg-1,kg ,lg ) - + owx * wy * owz * Bzc(jg ,kg-1,lg ) - + wx * wy * owz * Bzc(jg-1,kg-1,lg ) - + owx * owy * wz * Bzc(jg ,kg ,lg+1) - + wx * owy * wz * Bzc(jg-1,kg ,lg+1) - + owx * wy * wz * Bzc(jg ,kg-1,lg+1) - + wx * wy * wz * Bzc(jg-1,kg-1,lg+1); - - // interp from fine stagged to fine nodal - const Real bf = 0.25_rt*(Bzf_zeropad(j-1,k-1,l) + Bzf_zeropad(j,k-1,l) + Bzf_zeropad(j-1,k,l) + Bzf_zeropad(j,k,l)); - + nk = 2; + nl = 2; #endif - Bza(j,k,l) = bg + (bf-bc); -} - -AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void warpx_interp_nd_efield_x (int j, int k, int l, - amrex::Array4 const& Exa, - amrex::Array4 const& Exf, - amrex::Array4 const& Exc, - amrex::Array4 const& Exg) -{ - using namespace amrex; - - // Pad Exf with zeros beyond ghost cells for out-of-bound accesses - const auto Exf_zeropad = [Exf] (const int jj, const int kk, const int ll) noexcept - { - return Exf.contains(jj,kk,ll) ? Exf(jj,kk,ll) : 0.0_rt; - }; - - const int jg = amrex::coarsen(j,2); - Real wx, owx; - wx = (j == jg*2) ? 0.0_rt : 0.5_rt; - owx = 1.0_rt-wx; + wj = 1.0_rt; + wk = 1.0_rt; + wl = 1.0_rt; + for (int jj = 0; jj < nj; jj++) { + for (int kk = 0; kk < nk; kk++) { + for (int ll = 0; ll < nl; ll++) { + wj = (rj - amrex::Math::abs(j - (jc + jj) * rj)) / static_cast(rj); +#if (AMREX_SPACEDIM >= 2) + wk = (rk - amrex::Math::abs(k - (kc + kk) * rk)) / static_cast(rk); +#endif +#if (AMREX_SPACEDIM == 3) + wl = (rl - amrex::Math::abs(l - (lc + ll) * rl)) / static_cast(rl); +#endif + tmp += wj * wk * wl * arr_tmp_zeropad(jc+jj,kc+kk,lc+ll); + } + } + } - const int kg = amrex::coarsen(k,2); - Real wy = 0.0_rt; - Real owy =0.0_rt; - wy = (k == kg*2) ? 0.0_rt : 0.5_rt; - owy = 1.0_rt-wy; + // 2) Interpolation from coarse staggered to fine nodal + nj = 2; #if defined(WARPX_DIM_1D_Z) - - // interp from coarse nodal to fine nodal - const Real eg = owx * Exg(jg ,0,0) - + wx * Exg(jg+1,0,0); - - // interp from coarse staggered to fine nodal - const Real ec = owx * Exc(jg ,0,0) - + wx * Exc(jg+1,0,0); - - // interp from fine staggered to fine nodal - const Real ef = Exf_zeropad(j,0,0); - amrex::ignore_unused(owy); - + nk = 1; + nl = 1; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - - // interp from coarse nodal to fine nodal - const Real eg = owx * owy * Exg(jg ,kg ,0) - + owx * wy * Exg(jg ,kg+1,0) - + wx * owy * Exg(jg+1,kg ,0) - + wx * wy * Exg(jg+1,kg+1,0); - - // interp from coarse staggered to fine nodal - wx = 0.5_rt-wx; owx = 1.0_rt-wx; - const Real ec = owx * owy * Exc(jg ,kg ,0) - + owx * wy * Exc(jg ,kg+1,0) - + wx * owy * Exc(jg-1,kg ,0) - + wx * wy * Exc(jg-1,kg+1,0); - - // interp from fine staggered to fine nodal - const Real ef = 0.5_rt*(Exf_zeropad(j-1,k,0) + Exf_zeropad(j,k,0)); - + nk = 2; + nl = 1; #else - - const int lg = amrex::coarsen(l,2); - const Real wz = (l == lg*2) ? 0.0 : 0.5; - const Real owz = 1.0_rt-wz; - - // interp from coarse nodal to fine nodal - const Real eg = owx * owy * owz * Exg(jg ,kg ,lg ) - + wx * owy * owz * Exg(jg+1,kg ,lg ) - + owx * wy * owz * Exg(jg ,kg+1,lg ) - + wx * wy * owz * Exg(jg+1,kg+1,lg ) - + owx * owy * wz * Exg(jg ,kg ,lg+1) - + wx * owy * wz * Exg(jg+1,kg ,lg+1) - + owx * wy * wz * Exg(jg ,kg+1,lg+1) - + wx * wy * wz * Exg(jg+1,kg+1,lg+1); - - // interp from coarse staggered to fine nodal - wx = 0.5_rt-wx; owx = 1.0_rt-wx; - const Real ec = owx * owy * owz * Exc(jg ,kg ,lg ) - + wx * owy * owz * Exc(jg-1,kg ,lg ) - + owx * wy * owz * Exc(jg ,kg+1,lg ) - + wx * wy * owz * Exc(jg-1,kg+1,lg ) - + owx * owy * wz * Exc(jg ,kg ,lg+1) - + wx * owy * wz * Exc(jg-1,kg ,lg+1) - + owx * wy * wz * Exc(jg ,kg+1,lg+1) - + wx * wy * wz * Exc(jg-1,kg+1,lg+1); - - // interp from fine staggered to fine nodal - const Real ef = 0.5_rt*(Exf_zeropad(j-1,k,l) + Exf_zeropad(j,k,l)); - + nk = 2; + nl = 2; #endif - Exa(j,k,l) = eg + (ef-ec); -} - -AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void warpx_interp_nd_efield_y (int j, int k, int l, - amrex::Array4 const& Eya, - amrex::Array4 const& Eyf, - amrex::Array4 const& Eyc, - amrex::Array4 const& Eyg) -{ - using namespace amrex; + const int jn = (sj_cp == 1) ? j : j - rj / 2; + const int kn = (sk_cp == 1) ? k : k - rk / 2; + const int ln = (sl_cp == 1) ? l : l - rl / 2; - // Pad Eyf with zeros beyond ghost cells for out-of-bound accesses - const auto Eyf_zeropad = [Eyf] (const int jj, const int kk, const int ll) noexcept - { - return Eyf.contains(jj,kk,ll) ? Eyf(jj,kk,ll) : 0.0_rt; - }; + jc = amrex::coarsen(jn, rj); + kc = amrex::coarsen(kn, rk); + lc = amrex::coarsen(ln, rl); - const int jg = amrex::coarsen(j,2); - const Real wx = (j == jg*2) ? 0.0_rt : 0.5_rt; - const Real owx = 1.0_rt-wx; + wj = 1.0_rt; + wk = 1.0_rt; + wl = 1.0_rt; + for (int jj = 0; jj < nj; jj++) { + for (int kk = 0; kk < nk; kk++) { + for (int ll = 0; ll < nl; ll++) { + wj = (rj - amrex::Math::abs(jn - (jc + jj) * rj)) / static_cast(rj); +#if (AMREX_SPACEDIM >= 2) + wk = (rk - amrex::Math::abs(kn - (kc + kk) * rk)) / static_cast(rk); +#endif +#if (AMREX_SPACEDIM == 3) + wl = (rl - amrex::Math::abs(ln - (lc + ll) * rl)) / static_cast(rl); +#endif + coarse += wj * wk * wl * arr_coarse_zeropad(jc+jj,kc+kk,lc+ll); + } + } + } - const int kg = amrex::coarsen(k,2); - Real wy = 0.0_rt; - Real owy = 0.0_rt; - wy = (k == kg*2) ? 0.0_rt : 0.5_rt; - owy = 1.0_rt-wy; + // 3) Interpolation from fine staggered to fine nodal + nj = (sj_fp == 0) ? 2 : 1; #if defined(WARPX_DIM_1D_Z) - - // interp from coarse nodal to fine nodal - const Real eg = owx * Eyg(jg ,0,0) - + wx * Eyg(jg+1,0,0); - - // interp from coarse staggered to fine nodal - const Real ec = owx * Eyc(jg ,0,0) - + wx * Eyc(jg+1,0,0); - - // interp from fine staggered to fine nodal - const Real ef = Eyf_zeropad(j,0,0); - amrex::ignore_unused(owy); - + nk = 1; + nl = 1; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - - // interp from coarse nodal to fine nodal - const Real eg = owx * owy * Eyg(jg ,kg ,0) - + owx * wy * Eyg(jg ,kg+1,0) - + wx * owy * Eyg(jg+1,kg ,0) - + wx * wy * Eyg(jg+1,kg+1,0); - - // interp from coarse staggered to fine nodal (Eyc is actually nodal) - const Real ec = owx * owy * Eyc(jg ,kg ,0) - + owx * wy * Eyc(jg ,kg+1,0) - + wx * owy * Eyc(jg+1,kg ,0) - + wx * wy * Eyc(jg+1,kg+1,0); - - // interp from fine staggered to fine nodal - const Real ef = Eyf_zeropad(j,k,0); - + nk = (sk_fp == 0) ? 2 : 1; + nl = 1; #else - - const int lg = amrex::coarsen(l,2); - const Real wz = (l == lg*2) ? 0.0 : 0.5; - const Real owz = 1.0_rt-wz; - - // interp from coarse nodal to fine nodal - const Real eg = owx * owy * owz * Eyg(jg ,kg ,lg ) - + wx * owy * owz * Eyg(jg+1,kg ,lg ) - + owx * wy * owz * Eyg(jg ,kg+1,lg ) - + wx * wy * owz * Eyg(jg+1,kg+1,lg ) - + owx * owy * wz * Eyg(jg ,kg ,lg+1) - + wx * owy * wz * Eyg(jg+1,kg ,lg+1) - + owx * wy * wz * Eyg(jg ,kg+1,lg+1) - + wx * wy * wz * Eyg(jg+1,kg+1,lg+1); - - // interp from coarse staggered to fine nodal - wy = 0.5_rt-wy; owy = 1.0_rt-wy; - const Real ec = owx * owy * owz * Eyc(jg ,kg ,lg ) - + wx * owy * owz * Eyc(jg+1,kg ,lg ) - + owx * wy * owz * Eyc(jg ,kg-1,lg ) - + wx * wy * owz * Eyc(jg+1,kg-1,lg ) - + owx * owy * wz * Eyc(jg ,kg ,lg+1) - + wx * owy * wz * Eyc(jg+1,kg ,lg+1) - + owx * wy * wz * Eyc(jg ,kg-1,lg+1) - + wx * wy * wz * Eyc(jg+1,kg-1,lg+1); - - // interp from fine staggered to fine nodal - const Real ef = 0.5_rt*(Eyf_zeropad(j,k-1,l) + Eyf_zeropad(j,k,l)); - + nk = (sk_fp == 0) ? 2 : 1; + nl = (sl_fp == 0) ? 2 : 1; #endif - Eya(j,k,l) = eg + (ef-ec); -} - -AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void warpx_interp_nd_efield_z (int j, int k, int l, - amrex::Array4 const& Eza, - amrex::Array4 const& Ezf, - amrex::Array4 const& Ezc, - amrex::Array4 const& Ezg) -{ - using namespace amrex; - - // Pad Ezf with zeros beyond ghost cells for out-of-bound accesses - const auto Ezf_zeropad = [Ezf] (const int jj, const int kk, const int ll) noexcept - { - return Ezf.contains(jj,kk,ll) ? Ezf(jj,kk,ll) : 0.0_rt; - }; - - const int jg = amrex::coarsen(j,2); - const Real wx = (j == jg*2) ? 0.0_rt : 0.5_rt; - const Real owx = 1.0_rt-wx; - - const int kg = amrex::coarsen(k,2); - Real wy = 0.0_rt; - Real owy = 0.0_rt; - wy = (k == kg*2) ? 0.0_rt : 0.5_rt; - owy = 1.0_rt-wy; - + const int jm = (sj_fp == 0) ? j-1 : j; #if defined(WARPX_DIM_1D_Z) - - // interp from coarse nodal to fine nodal - const Real eg = owx * Ezg(jg ,0,0) - + wx * Ezg(jg+1,0,0); - - // interp from coarse staggered to fine nodal - const Real ec = owx * Ezc(jg ,0,0) - + wx * Ezc(jg+1,0,0); - - // interp from fine staggered to fine nodal - const Real ef = 0.5_rt*(Ezf_zeropad(j-1,0,0) + Ezf_zeropad(j,0,0)); - amrex::ignore_unused(owy); - + const int km = k; + const int lm = l; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - - // interp from coarse nodal to fine nodal - const Real eg = owx * owy * Ezg(jg ,kg ,0) - + owx * wy * Ezg(jg ,kg+1,0) - + wx * owy * Ezg(jg+1,kg ,0) - + wx * wy * Ezg(jg+1,kg+1,0); - - // interp from coarse stagged to fine nodal - wy = 0.5_rt-wy; owy = 1.0_rt-wy; - const Real ec = owx * owy * Ezc(jg ,kg ,0) - + owx * wy * Ezc(jg ,kg-1,0) - + wx * owy * Ezc(jg+1,kg ,0) - + wx * wy * Ezc(jg+1,kg-1,0); - - // interp from fine staggered to fine nodal - const Real ef = 0.5_rt*(Ezf_zeropad(j,k-1,0) + Ezf_zeropad(j,k,0)); - + const int km = (sk_fp == 0) ? k-1 : k; + const int lm = l; #else - - const int lg = amrex::coarsen(l,2); - Real wz = (l == lg*2) ? 0.0_rt : 0.5_rt; - Real owz = 1.0_rt-wz; - - // interp from coarse nodal to fine nodal - const Real eg = owx * owy * owz * Ezg(jg ,kg ,lg ) - + wx * owy * owz * Ezg(jg+1,kg ,lg ) - + owx * wy * owz * Ezg(jg ,kg+1,lg ) - + wx * wy * owz * Ezg(jg+1,kg+1,lg ) - + owx * owy * wz * Ezg(jg ,kg ,lg+1) - + wx * owy * wz * Ezg(jg+1,kg ,lg+1) - + owx * wy * wz * Ezg(jg ,kg+1,lg+1) - + wx * wy * wz * Ezg(jg+1,kg+1,lg+1); - - // interp from coarse staggered to fine nodal - wz = 0.5_rt-wz; owz = 1.0_rt-wz; - const Real ec = owx * owy * owz * Ezc(jg ,kg ,lg ) - + wx * owy * owz * Ezc(jg+1,kg ,lg ) - + owx * wy * owz * Ezc(jg ,kg+1,lg ) - + wx * wy * owz * Ezc(jg+1,kg+1,lg ) - + owx * owy * wz * Ezc(jg ,kg ,lg-1) - + wx * owy * wz * Ezc(jg+1,kg ,lg-1) - + owx * wy * wz * Ezc(jg ,kg+1,lg-1) - + wx * wy * wz * Ezc(jg+1,kg+1,lg-1); - - // interp from fine staggered to fine nodal - const Real ef = 0.5_rt*(Ezf_zeropad(j,k,l-1) + Ezf_zeropad(j,k,l)); - + const int km = (sk_fp == 0) ? k-1 : k; + const int lm = (sl_fp == 0) ? l-1 : l; #endif - Eza(j,k,l) = eg + (ef-ec); + for (int jj = 0; jj < nj; jj++) { + for (int kk = 0; kk < nk; kk++) { + for (int ll = 0; ll < nl; ll++) { + wj = 1.0_rt / static_cast(nj); + wk = 1.0_rt / static_cast(nk); + wl = 1.0_rt / static_cast(nl); + fine += wj * wk * wl * arr_fine_zeropad(jm+jj,km+kk,lm+ll); + } + } + } + + // Final result + arr_aux(j,k,l) = tmp + (fine - coarse); } /** diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index d890046dac6..dcea3cab58a 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -11,6 +11,7 @@ #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Particles/ParticleBoundaryBuffer.H" #include "Particles/WarpXParticleContainer.H" @@ -150,11 +151,11 @@ template void RemakeMultiFab (std::unique_ptr& mf, const DistributionMapping& dm, const bool redistribute, const int lev) { - if (mf == nullptr) return; + if (mf == nullptr) { return; } const IntVect& ng = mf->nGrowVect(); std::unique_ptr pmf; WarpX::AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); - if (redistribute) pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); + if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } mf = std::move(pmf); } @@ -163,17 +164,17 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi { if (ba == boxArray(lev)) { - if (ParallelDescriptor::NProcs() == 1) return; + if (ParallelDescriptor::NProcs() == 1) { return; } // Fine patch for (int idim=0; idim < 3; ++idim) { RemakeMultiFab(Bfield_fp[lev][idim], dm, true ,lev); RemakeMultiFab(Efield_fp[lev][idim], dm, true ,lev); - if (add_external_B_field) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { RemakeMultiFab(Bfield_fp_external[lev][idim], dm, true ,lev); } - if (add_external_E_field) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { RemakeMultiFab(Efield_fp_external[lev][idim], dm, true ,lev); } RemakeMultiFab(current_fp[lev][idim], dm, false ,lev); @@ -334,8 +335,9 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(current_buffer_masks[lev], dm, false ,lev); RemakeMultiFab(gather_buffer_masks[lev], dm, false ,lev); - if (current_buffer_masks[lev] || gather_buffer_masks[lev]) + if (current_buffer_masks[lev] || gather_buffer_masks[lev]) { BuildBufferMasks(); + } } // Re-initialize the lattice element finder with the new ba and dm. diff --git a/Source/Parallelization/WarpXSumGuardCells.H b/Source/Parallelization/WarpXSumGuardCells.H index 425ce320856..260ebb3871a 100644 --- a/Source/Parallelization/WarpXSumGuardCells.H +++ b/Source/Parallelization/WarpXSumGuardCells.H @@ -8,10 +8,6 @@ #ifndef WARPX_SUM_GUARD_CELLS_H_ #define WARPX_SUM_GUARD_CELLS_H_ -#include "Utils/WarpXAlgorithmSelection.H" - -#include - #include /** \brief Sum the values of `mf`, where the different boxes overlap @@ -26,20 +22,10 @@ * updates both the *valid* cells and *guard* cells. (This is because a * spectral solver requires the value of the sources over a large stencil.) */ -inline void +void WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, - const int icomp=0, const int ncomp=1) -{ - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) - n_updated_guards = mf.nGrowVect(); - else // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); -} + int icomp=0, int ncomp=1); /** \brief Sum the values of `src` where the different boxes overlap * (i.e. in the guard cells) and copy them into `dst` @@ -56,23 +42,10 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, * Note: `i_comp` is the component where the results will be stored in `dst`; * The component from which we copy in `src` is always 0. */ -inline void +void WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, - const int icomp=0, const int ncomp=1) -{ - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) - n_updated_guards = dst.nGrowVect(); - else // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - - dst.setVal(0., icomp, ncomp, n_updated_guards); -// ablastr::utils::communication::ParallelAdd(dst, src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); - dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); -} + int icomp=0, int ncomp=1); #endif // WARPX_SUM_GUARD_CELLS_H_ diff --git a/Source/Parallelization/WarpXSumGuardCells.cpp b/Source/Parallelization/WarpXSumGuardCells.cpp new file mode 100644 index 00000000000..20ac73cab50 --- /dev/null +++ b/Source/Parallelization/WarpXSumGuardCells.cpp @@ -0,0 +1,51 @@ +/* Copyright 2019 Maxence Thevenet, Remi Lehe, Weiqun Zhang + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpXSumGuardCells.H" + +#include "Utils/WarpXAlgorithmSelection.H" + +#include "WarpX.H" + +#include + +void +WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, + const amrex::IntVect& src_ngrow, + const int icomp, const int ncomp) +{ + amrex::IntVect n_updated_guards; + + // Update both valid cells and guard cells + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + n_updated_guards = mf.nGrowVect(); + } else { // Update only the valid cells + n_updated_guards = amrex::IntVect::TheZeroVector(); + } + ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); +} + + +void +WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, + const amrex::Periodicity& period, + const amrex::IntVect& src_ngrow, + const int icomp, const int ncomp) +{ + amrex::IntVect n_updated_guards; + + // Update both valid cells and guard cells + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + n_updated_guards = dst.nGrowVect(); + } else { // Update only the valid cells + n_updated_guards = amrex::IntVect::TheZeroVector(); + } + + dst.setVal(0., icomp, ncomp, n_updated_guards); + dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); +} diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H index d47234d4571..1db3922d178 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H @@ -7,9 +7,9 @@ #ifndef WARPX_PARTICLES_COLLISION_BACKGROUNDMCCCOLLISION_H_ #define WARPX_PARTICLES_COLLISION_BACKGROUNDMCCCOLLISION_H_ -#include "MCCProcess.H" #include "Particles/MultiParticleContainer.H" #include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" #include #include @@ -25,9 +25,14 @@ class BackgroundMCCCollision final public: BackgroundMCCCollision (std::string collision_name); - virtual ~BackgroundMCCCollision () = default; + ~BackgroundMCCCollision () override = default; - amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes); + BackgroundMCCCollision ( BackgroundMCCCollision const &) = delete; + BackgroundMCCCollision& operator= ( BackgroundMCCCollision const & ) = delete; + BackgroundMCCCollision ( BackgroundMCCCollision&& ) = delete; + BackgroundMCCCollision& operator= ( BackgroundMCCCollision&& ) = delete; + + amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes); /** Perform the collisions * @@ -65,10 +70,10 @@ public: private: - amrex::Vector m_scattering_processes; - amrex::Vector m_ionization_processes; - amrex::Gpu::DeviceVector m_scattering_processes_exe; - amrex::Gpu::DeviceVector m_ionization_processes_exe; + amrex::Vector m_scattering_processes; + amrex::Vector m_ionization_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; + amrex::Gpu::DeviceVector m_ionization_processes_exe; bool init_flag = false; bool ionization_flag = false; diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 3531afb7240..672c6456ec7 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -90,7 +90,7 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name amrex::Vector scattering_process_names; pp_collision_name.queryarr("scattering_processes", scattering_process_names); - // create a vector of MCCProcess objects from each scattering + // create a vector of ScatteringProcess objects from each scattering // process name for (const auto& scattering_process : scattering_process_names) { const std::string kw_cross_section = scattering_process + "_cross_section"; @@ -107,17 +107,17 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name pp_collision_name, kw_energy.c_str(), energy); } - MCCProcess process(scattering_process, cross_section_file, energy); + ScatteringProcess process(scattering_process, cross_section_file, energy); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != MCCProcessType::INVALID, - "Cannot add an unknown MCC process type"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); // if the scattering process is ionization get the secondary species // only one ionization process is supported, the vector // m_ionization_processes is only used to make it simple to calculate // the maximum collision frequency with the same function used for // particle conserving processes - if (process.type() == MCCProcessType::IONIZATION) { + if (process.type() == ScatteringProcessType::IONIZATION) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!ionization_flag, "Background MCC only supports a single ionization process"); ionization_flag = true; @@ -133,8 +133,8 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name } #ifdef AMREX_USE_GPU - amrex::Gpu::HostVector h_scattering_processes_exe; - amrex::Gpu::HostVector h_ionization_processes_exe; + amrex::Gpu::HostVector h_scattering_processes_exe; + amrex::Gpu::HostVector h_ionization_processes_exe; for (auto const& p : m_scattering_processes) { h_scattering_processes_exe.push_back(p.executor()); } @@ -162,7 +162,7 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name * ranges from 1e-4 to 5000 eV in 0.2 eV increments */ amrex::ParticleReal -BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) +BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) { using namespace amrex::literals; amrex::ParticleReal nu, nu_max = 0.0; @@ -181,7 +181,8 @@ BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processe E_step = (energy_step < E_step) ? energy_step : E_step; } - for (amrex::ParticleReal E = E_start; E < E_end; E+=E_step) { + amrex::ParticleReal E = E_start; + while(E < E_end){ amrex::ParticleReal sigma_E = 0.0; // loop through all collision pathways @@ -199,6 +200,8 @@ BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processe if (nu > nu_max) { nu_max = nu; } + + E+=E_step; } return nu_max; } @@ -273,7 +276,7 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult auto const flvl = species1.finestLevel(); for (int lev = 0; lev <= flvl; ++lev) { - auto cost = WarpX::getCosts(lev); + auto *cost = WarpX::getCosts(lev); // firstly loop over particles box by box and do all particle conserving // scattering @@ -285,14 +288,14 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); doBackgroundCollisionsWithinTile(pti, cur_time); if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[pti.index()], wt); } } @@ -321,8 +324,8 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile auto T_a_func = m_background_temperature_func; // get collision parameters - auto scattering_processes = m_scattering_processes_exe.data(); - int const process_count = m_scattering_processes_exe.size(); + auto *scattering_processes = m_scattering_processes_exe.data(); + auto const process_count = static_cast(m_scattering_processes_exe.size()); auto const total_collision_prob = m_total_collision_prob; auto const nu_max = m_nu_max; @@ -337,7 +340,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile // we need particle positions in order to calculate the local density // and temperature - auto GetPosition = GetParticlePosition(pti); + auto GetPosition = GetParticlePosition(pti); // get Struct-Of-Array particle data, also called attribs auto& attribs = pti.GetAttribs(); @@ -349,7 +352,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile [=] AMREX_GPU_HOST_DEVICE (long ip, amrex::RandomEngine const& engine) { // determine if this particle should collide - if (amrex::Random(engine) > total_collision_prob) return; + if (amrex::Random(engine) > total_collision_prob) { return; } amrex::ParticleReal x, y, z; GetPosition.AsStored(ip, x, y, z); @@ -379,7 +382,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile vy = uy[ip] - ua_y; vz = uz[ip] - ua_z; v_coll2 = (vx*vx + vy*vy + vz*vz); - v_coll = sqrt(v_coll2); + v_coll = std::sqrt(v_coll2); // calculate the collision energy in eV ParticleUtils::getCollisionEnergy(v_coll2, m, M, gamma, E_coll); @@ -389,19 +392,19 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile auto const& scattering_process = *(scattering_processes + i); // get collision cross-section - sigma_E = scattering_process.getCrossSection(E_coll); + sigma_E = scattering_process.getCrossSection(static_cast(E_coll)); // calculate normalized collision frequency nu_i += n_a * sigma_E * v_coll / nu_max; // check if this collision should be performed - if (col_select > nu_i) continue; + if (col_select > nu_i) { continue; } // charge exchange is implemented as a simple swap of the projectile // and target velocities which doesn't require any of the Lorentz // transformations below; note that if the projectile and target // have the same mass this is identical to back scattering - if (scattering_process.m_type == MCCProcessType::CHARGE_EXCHANGE) { + if (scattering_process.m_type == ScatteringProcessType::CHARGE_EXCHANGE) { ux[ip] = ua_x; uy[ip] = ua_y; uz[ip] = ua_z; @@ -411,16 +414,17 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile // At this point the given particle has been chosen for a collision // and so we perform the needed calculations to transform to the // COM frame. - uCOM_x = m * vx / (gamma * m + M); - uCOM_y = m * vy / (gamma * m + M); - uCOM_z = m * vz / (gamma * m + M); + uCOM_x = static_cast(m * vx / (gamma * m + M)); + uCOM_y = static_cast(m * vy / (gamma * m + M)); + uCOM_z = static_cast(m * vz / (gamma * m + M)); // subtract any energy penalty of the collision from the // projectile energy if (scattering_process.m_energy_penalty > 0.0_prt) { ParticleUtils::getEnergy(v_coll2, m, E_coll); E_coll = (E_coll - scattering_process.m_energy_penalty) * PhysConst::q_e; - auto scale_fac = sqrt(E_coll * (E_coll + 2.0_prt*mc2) / c2) / m / v_coll; + const auto scale_fac = static_cast( + std::sqrt(E_coll * (E_coll + 2.0_prt*mc2) / c2) / m / v_coll); vx *= scale_fac; vy *= scale_fac; vz *= scale_fac; @@ -429,13 +433,13 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile // transform to COM frame ParticleUtils::doLorentzTransform(vx, vy, vz, uCOM_x, uCOM_y, uCOM_z); - if ((scattering_process.m_type == MCCProcessType::ELASTIC) - || (scattering_process.m_type == MCCProcessType::EXCITATION)) { + if ((scattering_process.m_type == ScatteringProcessType::ELASTIC) + || (scattering_process.m_type == ScatteringProcessType::EXCITATION)) { ParticleUtils::RandomizeVelocity( vx, vy, vz, sqrt(vx*vx + vy*vy + vz*vz), engine ); } - else if (scattering_process.m_type == MCCProcessType::BACK) { + else if (scattering_process.m_type == ScatteringProcessType::BACK) { // elastic scattering with cos(chi) = -1 (i.e. 180 degrees) vx *= -1.0_prt; vy *= -1.0_prt; @@ -484,7 +488,7 @@ void BackgroundMCCCollision::doBackgroundIonization { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); auto& elec_tile = species1.ParticlesAt(lev, pti); auto& ion_tile = species2.ParticlesAt(lev, pti); @@ -497,7 +501,7 @@ void BackgroundMCCCollision::doBackgroundIonization m_mass1, sqrt_kb_m, m_background_temperature_func, t ); - const auto num_added = filterCopyTransformParticles<1>( + const auto num_added = filterCopyTransformParticles<1>(species1, species2, elec_tile, ion_tile, elec_tile, np_elec, np_ion, Filter, CopyElec, CopyIon, Transform ); @@ -508,7 +512,7 @@ void BackgroundMCCCollision::doBackgroundIonization if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[pti.index()], wt); } } diff --git a/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt b/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt index 15742af465f..ca8db0c53ec 100644 --- a/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt +++ b/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt @@ -3,6 +3,5 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE BackgroundMCCCollision.cpp - MCCProcess.cpp ) endforeach() diff --git a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H index 6e9827d8ab3..73ed60d0ad7 100644 --- a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H +++ b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H @@ -4,10 +4,10 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ -#define WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ +#ifndef WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ +#define WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ -#include "MCCProcess.H" +#include "Particles/Collision/ScatteringProcess.H" #include "Utils/ParticleUtils.H" #include "Utils/WarpXConst.H" @@ -39,7 +39,7 @@ public: * ensure accurate energy calculations (otherwise errors occur with single * or mixed precision builds of WarpX). * - * @param[in] mcc_process an MCCProcess object associated with the ionization + * @param[in] mcc_process an ScatteringProcess object associated with the ionization * @param[in] mass colliding particle's mass (could also assume electron) * @param[in] total_collision_prob total probability for a collision to occur * @param[in] nu_max maximum collision frequency @@ -48,7 +48,7 @@ public: * @param[in] t the current simulation time */ ImpactIonizationFilterFunc( - MCCProcess const& mcc_process, + ScatteringProcess const& mcc_process, double const mass, amrex::ParticleReal const total_collision_prob, amrex::ParticleReal const nu_max, @@ -77,7 +77,7 @@ public: using std::sqrt; // determine if this particle should collide - if (Random(engine) > m_total_collision_prob) return false; + if (Random(engine) > m_total_collision_prob) { return false; } // get references to the particle to get its position const auto& p = ptd.getSuperParticle(i); @@ -98,7 +98,7 @@ public: ParticleUtils::getEnergy(u_coll2, m_mass, E_coll); // get collision cross-section - const ParticleReal sigma_E = m_mcc_process.getCrossSection(E_coll); + const ParticleReal sigma_E = m_mcc_process.getCrossSection(static_cast(E_coll)); // calculate normalized collision frequency const ParticleReal nu_i = n_a * sigma_E * sqrt(u_coll2) / m_nu_max; @@ -108,7 +108,7 @@ public: } private: - MCCProcess::Executor m_mcc_process; + ScatteringProcess::Executor m_mcc_process; double m_mass; amrex::ParticleReal m_total_collision_prob = 0; amrex::ParticleReal m_nu_max; @@ -182,7 +182,7 @@ public: // calculate standard deviation in neutral velocity distribution using // the local temperature - const ParticleReal ion_vel_std = m_sqrt_kb_m * sqrt(m_T_a_func(x, y, z, m_t)); + const ParticleReal ion_vel_std = m_sqrt_kb_m * std::sqrt(m_T_a_func(x, y, z, m_t)); // get references to the original particle's velocity auto& ux = src.m_rdata[PIdx::ux][i_src]; @@ -202,7 +202,7 @@ public: ParticleUtils::getEnergy(u_coll2, m_mass1, E_coll); // each electron gets half the energy (could change this later) - const amrex::ParticleReal E_out = (E_coll - m_energy_cost) / 2.0_prt * PhysConst::q_e; + const auto E_out = static_cast((E_coll - m_energy_cost) / 2.0_prt * PhysConst::q_e); // precalculate often used value constexpr auto c2 = PhysConst::c * PhysConst::c; @@ -227,4 +227,4 @@ private: amrex::ParserExecutor<4> m_T_a_func; amrex::Real m_t; }; -#endif // WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ +#endif // WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ diff --git a/Source/Particles/Collision/BackgroundMCC/Make.package b/Source/Particles/Collision/BackgroundMCC/Make.package index 0b007e64e6d..242e85a8764 100644 --- a/Source/Particles/Collision/BackgroundMCC/Make.package +++ b/Source/Particles/Collision/BackgroundMCC/Make.package @@ -1,4 +1,3 @@ CEXE_sources += BackgroundMCCCollision.cpp -CEXE_sources += MCCProcess.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BackgroundMCC diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H index ee4b3b01653..55fa4b9e1e3 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H @@ -26,7 +26,12 @@ class BackgroundStopping final public: BackgroundStopping (std::string collision_name); - virtual ~BackgroundStopping () = default; + ~BackgroundStopping () override = default; + + BackgroundStopping ( BackgroundStopping const &) = delete; + BackgroundStopping& operator= ( BackgroundStopping const & ) = delete; + BackgroundStopping ( BackgroundStopping&& ) = delete; + BackgroundStopping& operator= ( BackgroundStopping&& ) = delete; /** Perform the stopping calculation * diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp index e832020683d..8122d7225b4 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp @@ -104,7 +104,7 @@ BackgroundStopping::doCollisions (amrex::Real cur_time, amrex::Real dt, MultiPar auto const flvl = species.finestLevel(); for (int lev = 0; lev <= flvl; ++lev) { - auto cost = WarpX::getCosts(lev); + auto *cost = WarpX::getCosts(lev); // loop over particles box by box #ifdef _OPENMP @@ -115,7 +115,7 @@ BackgroundStopping::doCollisions (amrex::Real cur_time, amrex::Real dt, MultiPar { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); if (background_type == BackgroundStoppingType::ELECTRONS) { doBackgroundStoppingOnElectronsWithinTile(pti, dt, cur_time, species_mass, species_charge); @@ -126,7 +126,7 @@ BackgroundStopping::doCollisions (amrex::Real cur_time, amrex::Real dt, MultiPar if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add(&(*cost)[pti.index()], wt); } } @@ -159,7 +159,7 @@ void BackgroundStopping::doBackgroundStoppingOnElectronsWithinTile (WarpXParIter amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); // May be needed to evaluate the density and/or temperature functions - auto const GetPosition = GetParticlePosition(pti); + auto const GetPosition = GetParticlePosition(pti); amrex::ParallelFor(np, [=] AMREX_GPU_HOST_DEVICE (long ip) @@ -234,7 +234,7 @@ void BackgroundStopping::doBackgroundStoppingOnIonsWithinTile (WarpXParIter& pti amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); // May be needed to evaluate the density function - auto const GetPosition = GetParticlePosition(pti); + auto const GetPosition = GetParticlePosition(pti); amrex::ParallelFor(np, [=] AMREX_GPU_HOST_DEVICE (long ip) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 5473f0093f0..5c90dab25e6 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -87,8 +87,9 @@ public: BinaryCollision (std::string collision_name, MultiParticleContainer const * const mypc) : CollisionBase(collision_name) { - if(m_species_names.size() != 2) + if(m_species_names.size() != 2) { WARPX_ABORT_WITH_MESSAGE("Binary collision " + collision_name + " must have exactly two species."); + } m_isSameSpecies = (m_species_names[0] == m_species_names[1]); @@ -97,10 +98,19 @@ public: const amrex::ParmParse pp_collision_name(collision_name); pp_collision_name.queryarr("product_species", m_product_species); m_have_product_species = !m_product_species.empty(); + if ((std::is_same::value) & (m_have_product_species)) { + WARPX_ABORT_WITH_MESSAGE( "Binary collision " + collision_name + + " does not produce species. Thus, `product_species` should not be specified in the input script." ); + } m_copy_transform_functor = CopyTransformFunctorType(collision_name, mypc); } - virtual ~BinaryCollision () = default; + ~BinaryCollision () override = default; + + BinaryCollision ( BinaryCollision const &) = default; + BinaryCollision& operator= ( BinaryCollision const & ) = default; + BinaryCollision ( BinaryCollision&& ) noexcept = default; + BinaryCollision& operator= ( BinaryCollision&& ) noexcept = default; /** Perform the collisions * @@ -147,8 +157,8 @@ public: auto copy_species1_data = device_copy_species1.data(); auto copy_species2_data = device_copy_species2.data(); #else - auto copy_species1_data = copy_species1.data(); - auto copy_species2_data = copy_species2.data(); + auto *copy_species1_data = copy_species1.data(); + auto *copy_species2_data = copy_species2.data(); #endif if (m_have_product_species){ species1.defineAllParticleTiles(); @@ -157,14 +167,14 @@ public: // Enable tiling amrex::MFItInfo info; - if (amrex::Gpu::notInLaunchRegion()) info.EnableTiling(species1.tile_size); + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(species1.tile_size); } // Loop over refinement levels for (int lev = 0; lev <= species1.finestLevel(); ++lev){ amrex::LayoutData* cost = WarpX::getCosts(lev); - // Loop over all grids/tiles at this level + // Loop over all grids/tiles at this level #ifdef AMREX_USE_OMP info.SetDynamic(true); #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -174,7 +184,7 @@ public: { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); doCollisionsWithinTile( dt, lev, mfi, species1, species2, product_species_vector, copy_species1_data, copy_species2_data); @@ -182,7 +192,7 @@ public: if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -217,7 +227,7 @@ public: // Store product species data in vectors const int n_product_species = m_product_species.size(); amrex::Vector tile_products; - amrex::Vector get_position_products; + amrex::Vector> get_position_products; amrex::Vector products_np; amrex::Vector products_mass; constexpr int getpos_offset = 0; @@ -225,12 +235,12 @@ public: { ParticleTileType& ptile_product = product_species_vector[i]->ParticlesAt(lev, mfi); tile_products.push_back(&ptile_product); - get_position_products.push_back(GetParticlePosition(ptile_product, + get_position_products.push_back(GetParticlePosition(ptile_product, getpos_offset)); products_np.push_back(ptile_product.numParticles()); products_mass.push_back(product_species_vector[i]->getMass()); } - auto tile_products_data = tile_products.data(); + auto *tile_products_data = tile_products.data(); if ( m_isSameSpecies ) // species_1 == species_2 { @@ -243,14 +253,14 @@ public: // Loop over cells, and collide the particles in each cell // Extract low-level data - int const n_cells = bins_1.numBins(); + auto const n_cells = static_cast(bins_1.numBins()); // - Species 1 const auto soa_1 = ptile_1.getParticleTileData(); index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); - auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); + auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); // Needed to access the particle id ParticleType * AMREX_RESTRICT particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); @@ -332,7 +342,7 @@ public: p_pair_offsets[i_cell] : 0; // Do not collide if there is only one particle in the cell - if ( cell_stop_1 - cell_start_1 <= 1 ) return; + if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } // shuffle ShuffleFisherYates( @@ -358,7 +368,9 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_1, tile_products_data, + soa_1, soa_1, + product_species_vector, + tile_products_data, particle_ptr_1, particle_ptr_1, m1, m1, products_mass, p_mask, products_np, copy_species1, copy_species2, @@ -367,7 +379,7 @@ public: for (int i = 0; i < n_product_species; i++) { - setNewParticleIDs(*(tile_products_data[i]), products_np[i], num_added[i]); + setNewParticleIDs(*(tile_products_data[i]), static_cast(products_np[i]), num_added[i]); } } else // species_1 != species_2 @@ -383,14 +395,14 @@ public: // Loop over cells, and collide the particles in each cell // Extract low-level data - int const n_cells = bins_1.numBins(); + auto const n_cells = static_cast(bins_1.numBins()); // - Species 1 const auto soa_1 = ptile_1.getParticleTileData(); index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); - auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); + auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); // Needed to access the particle id ParticleType * AMREX_RESTRICT particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); @@ -400,7 +412,7 @@ public: index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); const amrex::ParticleReal q2 = species_2.getCharge(); const amrex::ParticleReal m2 = species_2.getMass(); - auto get_position_2 = GetParticlePosition(ptile_2, getpos_offset); + auto get_position_2 = GetParticlePosition(ptile_2, getpos_offset); // Needed to access the particle id ParticleType * AMREX_RESTRICT particle_ptr_2 = ptile_2.GetArrayOfStructs()().data(); @@ -438,11 +450,12 @@ public: const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; // Particular case: no pair if a species has no particle in that cell - if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) + if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) { p_n_pairs_in_each_cell[i_cell] = 0; - else + } else { p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); + } } ); @@ -493,7 +506,7 @@ public: // Do not collide if one species is missing in the cell if ( cell_stop_1 - cell_start_1 < 1 || - cell_stop_2 - cell_start_2 < 1 ) return; + cell_stop_2 - cell_start_2 < 1 ) { return; } // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); @@ -519,7 +532,9 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_2, tile_products_data, + soa_1, soa_2, + product_species_vector, + tile_products_data, particle_ptr_1, particle_ptr_2, m1, m2, products_mass, p_mask, products_np, copy_species1, copy_species2, @@ -528,7 +543,7 @@ public: for (int i = 0; i < n_product_species; i++) { - setNewParticleIDs(*(tile_products_data[i]), products_np[i], num_added[i]); + setNewParticleIDs(*(tile_products_data[i]), static_cast(products_np[i]), num_added[i]); } } // end if ( m_isSameSpecies) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index 0a7aae9835e..b8a390ddb93 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -12,6 +12,8 @@ #include "Particles/MultiParticleContainer.H" +#include + enum struct CollisionType { DeuteriumTritiumToNeutronHeliumFusion, DeuteriumDeuteriumToProtonTritiumFusion, DeuteriumDeuteriumToNeutronHeliumFusion, @@ -36,6 +38,92 @@ namespace BinaryCollisionUtils{ MultiParticleContainer const * mypc); CollisionType nuclear_fusion_type_to_collision_type (NuclearFusionType fusion_type); + + /** + * \brief Return (relativistic) collision energy, collision speed and + * Lorentz factor for transforming between the lab and center-of-momentum + * frames. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void get_collision_parameters ( + const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, + const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, + const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, + const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, + amrex::ParticleReal& E_kin_COM, amrex::ParticleReal& v_rel_COM, + amrex::ParticleReal& lab_to_COM_lorentz_factor ) + { + // General notations in this function: + // x_sq denotes the square of x + // x_star denotes the value of x in the center of mass frame + + using namespace amrex::literals; + using namespace amrex::Math; + + constexpr auto one_pr = amrex::ParticleReal(1.); + constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); + constexpr double c_sq = PhysConst::c * PhysConst::c; + constexpr double inv_csq = 1.0 / c_sq; + + const amrex::ParticleReal m1_sq = m1*m1; + const amrex::ParticleReal m2_sq = m2*m2; + + // Compute Lorentz factor gamma in the lab frame + const double g1 = std::sqrt( 1.0 + static_cast(u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); + const double g2 = std::sqrt( 1.0 + static_cast(u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); + + // Compute momenta + const amrex::ParticleReal p1x = u1x * m1; + const amrex::ParticleReal p1y = u1y * m1; + const amrex::ParticleReal p1z = u1z * m1; + const amrex::ParticleReal p2x = u2x * m2; + const amrex::ParticleReal p2y = u2y * m2; + const amrex::ParticleReal p2z = u2z * m2; + // Square norm of the total (sum between the two particles) momenta in the lab frame + const auto p_total_sq = static_cast( + powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z) + ); + + // Total energy in the lab frame + // Note the use of `double` for energy since this calculation is + // prone to error with single precision. + const auto m1_dbl = static_cast(m1); + const auto m2_dbl = static_cast(m2); + const double E_lab = (m1_dbl * g1 + m2_dbl * g2) * c_sq; + // Total energy squared in the center of mass frame, calculated using the Lorentz invariance + // of the four-momentum norm + const double E_star_sq = E_lab*E_lab - c_sq*p_total_sq; + + // Kinetic energy in the center of mass frame + const double E_star = std::sqrt(E_star_sq); + E_kin_COM = static_cast(E_star - (m1_dbl + m2_dbl)*c_sq); + + // Square of the norm of the momentum of one of the particles in the center of mass frame + // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle + // The expression below is specifically written in a form that avoids returning + // small negative numbers due to machine precision errors, for low-energy particles + const auto E_ratio = static_cast(E_star/((m1 + m2)*c_sq)); + const auto p_star_sq = static_cast( + m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) + + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio) + ); + + // Lorentz factors in the center of mass frame + const auto g1_star = std::sqrt(one_pr + p_star_sq / static_cast(m1_sq*c_sq)); + const auto g2_star = std::sqrt(one_pr + p_star_sq / static_cast(m2_sq*c_sq)); + + // relative velocity in the center of mass frame + v_rel_COM = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + one_pr/(m2*g2_star)); + + // Cross sections and relative velocity are computed in the center of mass frame. + // On the other hand, the particle densities (weight over volume) in the lab frame are used. + // To take this disrepancy into account, it is needed to multiply the + // collision probability by the ratio between the Lorentz factors in the + // COM frame and the Lorentz factors in the lab frame (see + // Perez et al., Phys.Plasmas.19.083104 (2012)). The correction factor + // is calculated here. + lab_to_COM_lorentz_factor = g1_star*g2_star/static_cast(g1*g2); + } } #endif // BINARY_COLLISION_UTILS_H_ diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 8eddad7c496..430bac5e548 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -57,13 +57,11 @@ namespace BinaryCollisionUtils{ ||(product_species1.AmIA() && product_species2.AmIA())){ return NuclearFusionType::DeuteriumDeuteriumToNeutronHelium; } else if ( - (product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA()) + (product_species1.AmIA() && product_species2.AmIA()) ||(product_species1.AmIA() && product_species2.AmIA())){ return NuclearFusionType::DeuteriumDeuteriumToProtonTritium; } else { - WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or tritium and proton"); + WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or hydrogen3 and hydrogen1"); } } else if ((species1.AmIA() && species2.AmIA()) @@ -77,15 +75,15 @@ namespace BinaryCollisionUtils{ auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (product_species1.AmIA() && product_species2.AmIA()) + (product_species1.AmIA() && product_species2.AmIA()) || - (product_species1.AmIA() && product_species2.AmIA()), - "ERROR: Product species of deuterium-helium fusion must be of type proton and helium4"); + (product_species1.AmIA() && product_species2.AmIA()), + "ERROR: Product species of deuterium-helium fusion must be of type hydrogen1 and helium4"); return NuclearFusionType::DeuteriumHeliumToProtonHelium; } - else if ((species1.AmIA() && species2.AmIA()) + else if ((species1.AmIA() && species2.AmIA()) || - (species1.AmIA() && species2.AmIA()) + (species1.AmIA() && species2.AmIA()) ) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -93,8 +91,8 @@ namespace BinaryCollisionUtils{ "ERROR: Proton-boron must contain exactly one product species"); auto& product_species = mypc->GetParticleContainerFromName(product_species_name[0]); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species.AmIA(), - "ERROR: Product species of proton-boron fusion must be of type alpha"); + product_species.AmIA(), + "ERROR: Product species of proton-boron fusion must be of type helium4"); return NuclearFusionType::ProtonBoronToAlphas; } WARPX_ABORT_WITH_MESSAGE("Binary nuclear fusion not implemented between species " + @@ -120,16 +118,21 @@ namespace BinaryCollisionUtils{ CollisionType nuclear_fusion_type_to_collision_type (const NuclearFusionType fusion_type) { - if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) + if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; - if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) + } + if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { return CollisionType::DeuteriumHeliumToProtonHeliumFusion; - if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) + } + if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { return CollisionType::ProtonBoronToAlphasFusion; + } WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); return CollisionType::Undefined; } diff --git a/Source/Particles/Collision/BinaryCollision/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/CMakeLists.txt index 55bbf21e310..60933c83a21 100644 --- a/Source/Particles/Collision/BinaryCollision/CMakeLists.txt +++ b/Source/Particles/Collision/BinaryCollision/CMakeLists.txt @@ -6,3 +6,5 @@ foreach(D IN LISTS WarpX_DIMS) ParticleCreationFunc.cpp ) endforeach() + +add_subdirectory(DSMC) diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index 239f63c7f23..d7403eeacf9 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -54,8 +54,8 @@ void ElasticCollisionPerez ( amrex::RandomEngine const& engine, bool const isSameSpecies) { - const int NI1 = I1e - I1s; - const int NI2 = I2e - I2s; + const T_index NI1 = I1e - I1s; + const T_index NI2 = I2e - I2s; T_PR * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; T_PR * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; @@ -84,8 +84,8 @@ void ElasticCollisionPerez ( T_PR n1 = T_PR(0.0); T_PR n2 = T_PR(0.0); T_PR n12 = T_PR(0.0); - for (int i1=I1s; i1(I1e); ++i1) { n1 += w1[ I1[i1] ]; } - for (int i2=I2s; i2(I2e); ++i2) { n2 += w2[ I2[i2] ]; } + for (T_index i1=I1s; i1(I1e) ) { i1 = I1s; } - ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } + ++i1; if ( i1 == I1e ) { i1 = I1s; } + ++i2; if ( i2 == I2e ) { i2 = I2s; } } n12 = n12 / dV; } // Intra-species: Apply correction in collision rate - if (isSameSpecies) n12 *= T_PR(2.0); + if (isSameSpecies) { n12 *= T_PR(2.0); } // compute Debye length lmdD T_PR lmdD; @@ -126,8 +126,8 @@ void ElasticCollisionPerez ( // call UpdateMomentumPerezElastic() { - int i1 = I1s; int i2 = I2s; - for (int k = 0; k < amrex::max(NI1,NI2); ++k) + T_index i1 = I1s; T_index i2 = I2s; + for (T_index k = 0; k < amrex::max(NI1,NI2); ++k) { #if (defined WARPX_DIM_RZ) @@ -159,8 +159,8 @@ void ElasticCollisionPerez ( u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); #endif - ++i1; if ( i1 == static_cast(I1e) ) { i1 = I1s; } - ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } + ++i1; if ( i1 == I1e ) { i1 = I1s; } + ++i2; if ( i2 == I2e ) { i2 = I2s; } } } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index 3c5f3abd841..feb7acf81d3 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -45,15 +45,16 @@ public: */ PairWiseCoulombCollisionFunc (const std::string collision_name, [[maybe_unused]] MultiParticleContainer const * const mypc, - const bool isSameSpecies) + const bool isSameSpecies): + m_isSameSpecies{isSameSpecies} { using namespace amrex::literals; const amrex::ParmParse pp_collision_name(collision_name); // default Coulomb log, if < 0, will be computed automatically - m_CoulombLog = -1.0_prt; + amrex::ParticleReal CoulombLog = -1.0_prt; utils::parser::queryWithParser( - pp_collision_name, "CoulombLog", m_CoulombLog); - m_isSameSpecies = isSameSpecies; + pp_collision_name, "CoulombLog", CoulombLog); + m_CoulombLog = CoulombLog; } /** @@ -77,7 +78,7 @@ public: index_type const* AMREX_RESTRICT I1, index_type const* AMREX_RESTRICT I2, SoaData_type soa_1, SoaData_type soa_2, - GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const q1, amrex::ParticleReal const q2, amrex::ParticleReal const m1, amrex::ParticleReal const m2, amrex::Real const dt, amrex::Real const dV, diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H index 7c2432f6e1b..3101047e211 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H @@ -102,184 +102,193 @@ void UpdateMomentumPerezElastic ( T_PR const g1s = ( T_PR(1.0) - vcDv1*inv_c2 )*gc*g1; T_PR const g2s = ( T_PR(1.0) - vcDv2*inv_c2 )*gc*g2; - // Compute the Coulomb log lnLmd - T_PR lnLmd; - if ( L > T_PR(0.0) ) { lnLmd = L; } - else - { - // Compute b0 according to eq (22) from Perez et al., Phys.Plasmas.19.083104 (2012) - // Note: there is a typo in the equation, the last square is incorrect! - // See the SMILEI documentation: https://smileipic.github.io/Smilei/Understand/collisions.html - // and https://github.com/ECP-WarpX/WarpX/files/3799803/main.pdf from GitHub #429 - T_PR const b0 = amrex::Math::abs(q1*q2) * inv_c2 / - (T_PR(4.0)*MathConst::pi*PhysConst::ep0) * gc/mass_g * - ( m1*g1s*m2*g2s/(p1sm*p1sm*inv_c2) + T_PR(1.0) ); - - // Compute the minimal impact parameter - constexpr T_PR hbar_pi = static_cast(PhysConst::hbar*MathConst::pi); - const T_PR bmin = amrex::max(hbar_pi/p1sm, b0); - - // Compute the Coulomb log lnLmd - lnLmd = amrex::max( T_PR(2.0), - T_PR(0.5)*std::log(T_PR(1.0)+lmdD*lmdD/(bmin*bmin)) ); - } - // Compute s - const auto tts = m1*g1s*m2*g2s/(inv_c2*p1sm*p1sm) + T_PR(1.0); - const auto tts2 = tts*tts; - T_PR s = n1*n2/n12 * dt*lnLmd*q1*q1*q2*q2 / - ( T_PR(4.0) * MathConst::pi * PhysConst::ep0 * PhysConst::ep0 * - m1*g1*m2*g2/(inv_c2*inv_c2) ) * gc*p1sm/mass_g * tts2; - - // Compute s' - const auto cbrt_n1 = std::cbrt(n1); - const auto cbrt_n2 = std::cbrt(n2); - const auto coeff = static_cast( - std::pow(4.0*MathConst::pi/3.0,1.0/3.0)); - T_PR const vrel = mass_g*p1sm/(m1*g1s*m2*g2s*gc); - T_PR const sp = coeff * n1*n2/n12 * dt * vrel * (m1+m2) / - amrex::max( m1*cbrt_n1*cbrt_n1, - m2*cbrt_n2*cbrt_n2); - - // Determine s - s = amrex::min(s,sp); - - // Get random numbers - T_PR r = amrex::Random(engine); - - // Compute scattering angle - T_PR cosXs; - T_PR sinXs; - if ( s <= T_PR(0.1) ) - { - while ( true ) + T_PR s = 0; + if (p1sm > std::numeric_limits::min()) { + + // s is non-zero (i.e. particles scatter) only if the relative + // motion between particles is not negligible (p1sm non-zero) + + // Compute the Coulomb log lnLmd first + T_PR lnLmd; + if ( L > T_PR(0.0) ) { lnLmd = L; } + else { - cosXs = T_PR(1.0) + s * std::log(r); - // Avoid the bug when r is too small such that cosXs < -1 - if ( cosXs >= T_PR(-1.0) ) { break; } - r = amrex::Random(engine); + // Compute b0 according to eq (22) from Perez et al., Phys.Plasmas.19.083104 (2012) + // Note: there is a typo in the equation, the last square is incorrect! + // See the SMILEI documentation: https://smileipic.github.io/Smilei/Understand/collisions.html + // and https://github.com/ECP-WarpX/WarpX/files/3799803/main.pdf from GitHub #429 + T_PR const b0 = amrex::Math::abs(q1*q2) * inv_c2 / + (T_PR(4.0)*MathConst::pi*PhysConst::ep0) * gc/mass_g * + ( m1*g1s*m2*g2s/(p1sm*p1sm*inv_c2) + T_PR(1.0) ); + + // Compute the minimal impact parameter + constexpr T_PR hbar_pi = static_cast(PhysConst::hbar*MathConst::pi); + const T_PR bmin = amrex::max(hbar_pi/p1sm, b0); + + // Compute the Coulomb log lnLmd + lnLmd = amrex::max( T_PR(2.0), + T_PR(0.5)*std::log(T_PR(1.0)+lmdD*lmdD/(bmin*bmin)) ); } - } - else if ( s > T_PR(0.1) && s <= T_PR(3.0) ) - { - T_PR const Ainv = static_cast( - 0.0056958 + 0.9560202*s - 0.508139*s*s + - 0.47913906*s*s*s - 0.12788975*s*s*s*s + 0.02389567*s*s*s*s*s); - cosXs = Ainv * std::log( std::exp(T_PR(-1.0)/Ainv) + - T_PR(2.0) * r * std::sinh(T_PR(1.0)/Ainv) ); - } - else if ( s > T_PR(3.0) && s <= T_PR(6.0) ) - { - T_PR const A = T_PR(3.0) * std::exp(-s); - cosXs = T_PR(1.0)/A * std::log( std::exp(-A) + - T_PR(2.0) * r * std::sinh(A) ); - } - else - { - cosXs = T_PR(2.0) * r - T_PR(1.0); - } - sinXs = std::sqrt(T_PR(1.0) - cosXs*cosXs); - - // Get random azimuthal angle - T_PR const phis = amrex::Random(engine) * T_PR(2.0) * MathConst::pi; - T_PR const cosphis = std::cos(phis); - T_PR const sinphis = std::sin(phis); - - // Compute post-collision momenta pfs in COM - T_PR p1fsx; - T_PR p1fsy; - T_PR p1fsz; - // p1sp is the p1s perpendicular - T_PR p1sp = std::sqrt( p1sx*p1sx + p1sy*p1sy ); - // Make sure p1sp is not almost zero - if ( p1sp > std::numeric_limits::min() ) - { - p1fsx = ( p1sx*p1sz/p1sp ) * sinXs*cosphis + - ( p1sy*p1sm/p1sp ) * sinXs*sinphis + - ( p1sx ) * cosXs; - p1fsy = ( p1sy*p1sz/p1sp ) * sinXs*cosphis + - (-p1sx*p1sm/p1sp ) * sinXs*sinphis + - ( p1sy ) * cosXs; - p1fsz = (-p1sp ) * sinXs*cosphis + - ( T_PR(0.0) ) * sinXs*sinphis + - ( p1sz ) * cosXs; - // Note a negative sign is different from - // Eq. (12) in Perez's paper, - // but they are the same due to the random nature of phis. - } - else - { - // If the previous p1sp is almost zero - // x->y y->z z->x - // This set is equivalent to the one in Nanbu's paper - p1sp = std::sqrt( p1sy*p1sy + p1sz*p1sz ); - p1fsy = ( p1sy*p1sx/p1sp ) * sinXs*cosphis + - ( p1sz*p1sm/p1sp ) * sinXs*sinphis + - ( p1sy ) * cosXs; - p1fsz = ( p1sz*p1sx/p1sp ) * sinXs*cosphis + - (-p1sy*p1sm/p1sp ) * sinXs*sinphis + - ( p1sz ) * cosXs; - p1fsx = (-p1sp ) * sinXs*cosphis + - ( T_PR(0.0) ) * sinXs*sinphis + - ( p1sx ) * cosXs; - } - T_PR const p2fsx = -p1fsx; - T_PR const p2fsy = -p1fsy; - T_PR const p2fsz = -p1fsz; + // Compute s + const auto tts = m1*g1s*m2*g2s/(inv_c2*p1sm*p1sm) + T_PR(1.0); + const auto tts2 = tts*tts; + s = n1*n2/n12 * dt*lnLmd*q1*q1*q2*q2 / + ( T_PR(4.0) * MathConst::pi * PhysConst::ep0 * PhysConst::ep0 * + m1*g1*m2*g2/(inv_c2*inv_c2) ) * gc*p1sm/mass_g * tts2; - // Transform from COM to lab frame - T_PR p1fx; T_PR p2fx; - T_PR p1fy; T_PR p2fy; - T_PR p1fz; T_PR p2fz; - if ( vcms > std::numeric_limits::min() ) - { - T_PR const vcDp1fs = vcx*p1fsx + vcy*p1fsy + vcz*p1fsz; - T_PR const vcDp2fs = vcx*p2fsx + vcy*p2fsy + vcz*p2fsz; - /* factor = (gc-1.0)/vcms; Rewrite to avoid subtraction losing precision when gc is close to 1 */ - T_PR const factor = gc*gc*inv_c2/(gc+T_PR(1.0)); - T_PR const factor1 = factor*vcDp1fs + m1*g1s*gc; - T_PR const factor2 = factor*vcDp2fs + m2*g2s*gc; - p1fx = p1fsx + vcx * factor1; - p1fy = p1fsy + vcy * factor1; - p1fz = p1fsz + vcz * factor1; - p2fx = p2fsx + vcx * factor2; - p2fy = p2fsy + vcy * factor2; - p2fz = p2fsz + vcz * factor2; - } - else // If vcms = 0, don't do Lorentz-transform. - { - p1fx = p1fsx; - p1fy = p1fsy; - p1fz = p1fsz; - p2fx = p2fsx; - p2fy = p2fsy; - p2fz = p2fsz; - } + // Compute s' + const auto cbrt_n1 = std::cbrt(n1); + const auto cbrt_n2 = std::cbrt(n2); + const auto coeff = static_cast( + std::pow(4.0*MathConst::pi/3.0,1.0/3.0)); + T_PR const vrel = mass_g*p1sm/(m1*g1s*m2*g2s*gc); + T_PR const sp = coeff * n1*n2/n12 * dt * vrel * (m1+m2) / + amrex::max( m1*cbrt_n1*cbrt_n1, + m2*cbrt_n2*cbrt_n2); - // Rejection method - r = amrex::Random(engine); - if ( w2 > r*amrex::max(w1, w2) ) - { - u1x = p1fx / m1; - u1y = p1fy / m1; - u1z = p1fz / m1; -#ifndef AMREX_USE_DPCPP - AMREX_ASSERT(!std::isnan(u1x+u1y+u1z+u2x+u2y+u2z)); - AMREX_ASSERT(!std::isinf(u1x+u1y+u1z+u2x+u2y+u2z)); -#endif + // Determine s + s = amrex::min(s,sp); } - r = amrex::Random(engine); - if ( w1 > r*amrex::max(w1, w2) ) - { - u2x = p2fx / m2; - u2y = p2fy / m2; - u2z = p2fz / m2; + + // Only modify momenta if is s is non-zero + if (s > std::numeric_limits::min()) { + + // Get random numbers + T_PR r = amrex::Random(engine); + + // Compute scattering angle + T_PR cosXs; + T_PR sinXs; + if ( s <= T_PR(0.1) ) + { + while ( true ) + { + cosXs = T_PR(1.0) + s * std::log(r); + // Avoid the bug when r is too small such that cosXs < -1 + if ( cosXs >= T_PR(-1.0) ) { break; } + r = amrex::Random(engine); + } + } + else if ( s > T_PR(0.1) && s <= T_PR(3.0) ) + { + T_PR const Ainv = static_cast( + 0.0056958 + 0.9560202*s - 0.508139*s*s + + 0.47913906*s*s*s - 0.12788975*s*s*s*s + 0.02389567*s*s*s*s*s); + cosXs = Ainv * std::log( std::exp(T_PR(-1.0)/Ainv) + + T_PR(2.0) * r * std::sinh(T_PR(1.0)/Ainv) ); + } + else if ( s > T_PR(3.0) && s <= T_PR(6.0) ) + { + T_PR const A = T_PR(3.0) * std::exp(-s); + cosXs = T_PR(1.0)/A * std::log( std::exp(-A) + + T_PR(2.0) * r * std::sinh(A) ); + } + else + { + cosXs = T_PR(2.0) * r - T_PR(1.0); + } + sinXs = std::sqrt(T_PR(1.0) - cosXs*cosXs); + + // Get random azimuthal angle + T_PR const phis = amrex::Random(engine) * T_PR(2.0) * MathConst::pi; + T_PR const cosphis = std::cos(phis); + T_PR const sinphis = std::sin(phis); + + // Compute post-collision momenta pfs in COM + T_PR p1fsx; + T_PR p1fsy; + T_PR p1fsz; + // p1sp is the p1s perpendicular + T_PR p1sp = std::sqrt( p1sx*p1sx + p1sy*p1sy ); + // Make sure p1sp is not almost zero + if ( p1sp > std::numeric_limits::min() ) + { + p1fsx = ( p1sx*p1sz/p1sp ) * sinXs*cosphis + + ( p1sy*p1sm/p1sp ) * sinXs*sinphis + + ( p1sx ) * cosXs; + p1fsy = ( p1sy*p1sz/p1sp ) * sinXs*cosphis + + (-p1sx*p1sm/p1sp ) * sinXs*sinphis + + ( p1sy ) * cosXs; + p1fsz = (-p1sp ) * sinXs*cosphis + + ( T_PR(0.0) ) * sinXs*sinphis + + ( p1sz ) * cosXs; + // Note a negative sign is different from + // Eq. (12) in Perez's paper, + // but they are the same due to the random nature of phis. + } + else + { + // If the previous p1sp is almost zero + // x->y y->z z->x + // This set is equivalent to the one in Nanbu's paper + p1sp = std::sqrt( p1sy*p1sy + p1sz*p1sz ); + p1fsy = ( p1sy*p1sx/p1sp ) * sinXs*cosphis + + ( p1sz*p1sm/p1sp ) * sinXs*sinphis + + ( p1sy ) * cosXs; + p1fsz = ( p1sz*p1sx/p1sp ) * sinXs*cosphis + + (-p1sy*p1sm/p1sp ) * sinXs*sinphis + + ( p1sz ) * cosXs; + p1fsx = (-p1sp ) * sinXs*cosphis + + ( T_PR(0.0) ) * sinXs*sinphis + + ( p1sx ) * cosXs; + } + + T_PR const p2fsx = -p1fsx; + T_PR const p2fsy = -p1fsy; + T_PR const p2fsz = -p1fsz; + + // Transform from COM to lab frame + T_PR p1fx; T_PR p2fx; + T_PR p1fy; T_PR p2fy; + T_PR p1fz; T_PR p2fz; + if ( vcms > std::numeric_limits::min() ) + { + T_PR const vcDp1fs = vcx*p1fsx + vcy*p1fsy + vcz*p1fsz; + T_PR const vcDp2fs = vcx*p2fsx + vcy*p2fsy + vcz*p2fsz; + /* factor = (gc-1.0)/vcms; Rewrite to avoid subtraction losing precision when gc is close to 1 */ + T_PR const factor = gc*gc*inv_c2/(gc+T_PR(1.0)); + T_PR const factor1 = factor*vcDp1fs + m1*g1s*gc; + T_PR const factor2 = factor*vcDp2fs + m2*g2s*gc; + p1fx = p1fsx + vcx * factor1; + p1fy = p1fsy + vcy * factor1; + p1fz = p1fsz + vcz * factor1; + p2fx = p2fsx + vcx * factor2; + p2fy = p2fsy + vcy * factor2; + p2fz = p2fsz + vcz * factor2; + } + else // If vcms = 0, don't do Lorentz-transform. + { + p1fx = p1fsx; + p1fy = p1fsy; + p1fz = p1fsz; + p2fx = p2fsx; + p2fy = p2fsy; + p2fz = p2fsz; + } + + // Rejection method + r = amrex::Random(engine); + if ( w2 > r*amrex::max(w1, w2) ) + { + u1x = p1fx / m1; + u1y = p1fy / m1; + u1z = p1fz / m1; + } + r = amrex::Random(engine); + if ( w1 > r*amrex::max(w1, w2) ) + { + u2x = p2fx / m2; + u2y = p2fy / m2; + u2z = p2fz / m2; + } #ifndef AMREX_USE_DPCPP AMREX_ASSERT(!std::isnan(u1x+u1y+u1z+u2x+u2y+u2z)); AMREX_ASSERT(!std::isinf(u1x+u1y+u1z+u2x+u2y+u2z)); #endif - } + + } // if s > std::numeric_limits::min() } diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt new file mode 100644 index 00000000000..3575d53ad29 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt @@ -0,0 +1,7 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + DSMC.cpp + ) +endforeach() diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H new file mode 100644 index 00000000000..c714cfdb133 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -0,0 +1,222 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies), Neil Zaim + * + * License: BSD-3-Clause-LBNL + */ +#ifndef COLLISION_FILTER_FUNC_H_ +#define COLLISION_FILTER_FUNC_H_ + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/ScatteringProcess.H" + +#include + +/** + * \brief This function determines whether a collision occurs for a given + * pair of particles. + * + * @param[in] u1x,u1y,u1z momenta of the first colliding particle + * @param[in] u2x,u2y,u2z momenta of the second colliding particle + * @param[in] m1,m2 masses + * @param[in] w1,w2 effective weight of the colliding particles + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] pair_index is the index of the colliding pair + * @param[out] p_mask is a mask that will be set to a non-zero integer if a + * collision occurs. The integer encodes the scattering process. + * @param[out] p_pair_reaction_weight stores the weight of the product particles + * @param[in] process_count number of scattering processes to consider + * @param[in] scattering processes an array of scattering processes included for consideration + * @param[in] engine the random engine. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, + const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, + const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, + const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, + amrex::ParticleReal w1, amrex::ParticleReal w2, + const amrex::Real& dt, const amrex::ParticleReal& dV, const int& pair_index, + index_type* AMREX_RESTRICT p_mask, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + const int& multiplier_ratio, + int const process_count, + ScatteringProcess::Executor* scattering_processes, + const amrex::RandomEngine& engine) +{ + using namespace amrex::literals; + + amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; + + const amrex::ParticleReal w_min = amrex::min(w1, w2); + const amrex::ParticleReal w_max = amrex::max(w1, w2); + + BinaryCollisionUtils::get_collision_parameters( + u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, + E_coll, v_coll, lab_to_COM_factor); + + // convert E_coll to eV + E_coll /= PhysConst::q_e; + + amrex::ParticleReal sigma_tot = 0._prt; + for (int ii = 0; ii < process_count; ii++) { + auto const& scattering_process = *(scattering_processes + ii); + sigma_tot += scattering_process.getCrossSection(E_coll); + } + + // calculate total collision probability + amrex::ParticleReal exponent = ( + lab_to_COM_factor * multiplier_ratio * w_max + * sigma_tot * v_coll * dt / dV + ); + + // Compute actual collision probability that is always between zero and one + // In principle this is obtained by computing 1 - exp(-probability_estimate) + // However, the computation of this quantity can fail numerically when probability_estimate is + // too small (e.g. exp(-probability_estimate) returns 1 and the computation returns 0). + // std::expm1 is used since it maintains correctness for small exponent. + const amrex::ParticleReal probability = -std::expm1(-exponent); + + // Now we determine if a collision should occur + if (amrex::Random(engine) < probability) + { + const amrex::ParticleReal random_number = amrex::Random(engine); + amrex::ParticleReal sigma = 0._prt; + for (int ii = 0; ii < process_count; ii++) { + auto const& scattering_process = *(scattering_processes + ii); + sigma += scattering_process.getCrossSection(E_coll); + if (random_number <= sigma / sigma_tot) + { + p_mask[pair_index] = int(scattering_process.m_type); + p_pair_reaction_weight[pair_index] = w_min; + break; + } + } + } + else + { + p_mask[pair_index] = false; + } +} + +/** + * \brief Function that determines if a collision occurs and if so, what + * type. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] ptd_1,ptd_2 contain the particle data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to a non-zero integer if a + * collision occurs. The integer encodes the scattering process. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] process_count number of scattering processes to consider + * @param[in] scattering processes an array of scattering processes included for consideration + * @param[in] engine the random engine. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void CollisionFilter ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + PData ptd_1, PData ptd_2, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + int const process_count, + ScatteringProcess::Executor* scattering_processes, + amrex::RandomEngine const& engine) +{ + + amrex::ParticleReal * const AMREX_RESTRICT w1 = ptd_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u1x = ptd_1.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u1y = ptd_1.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u1z = ptd_1.m_rdata[PIdx::uz]; + + amrex::ParticleReal * const AMREX_RESTRICT w2 = ptd_2.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u2x = ptd_2.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u2y = ptd_2.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u2z = ptd_2.m_rdata[PIdx::uz]; + + // Number of macroparticles of each species + const int NI1 = I1e - I1s; + const int NI2 = I2e - I2s; + const int max_N = amrex::max(NI1,NI2); + + int i1 = I1s; + int i2 = I2s; + int pair_index = cell_start_pair; + + // Because the number of particles of each species is not always equal (NI1 != NI2 + // in general), some macroparticles will be paired with multiple macroparticles of the + // other species and we need to decrease their weight accordingly. + // c1 corresponds to the minimum number of times a particle of species 1 will be paired + // with a particle of species 2. Same for c2. + const int c1 = amrex::max(NI2/NI1,1); + const int c2 = amrex::max(NI1/NI2,1); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal * const AMREX_RESTRICT theta1 = ptd_1.m_rdata[PIdx::theta]; + amrex::ParticleReal * const AMREX_RESTRICT theta2 = ptd_2.m_rdata[PIdx::theta]; +#endif + + for (int k = 0; k < max_N; ++k) + { + // c1k : how many times the current particle of species 1 is paired with a particle + // of species 2. Same for c2k. + const int c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; + const int c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + +#if (defined WARPX_DIM_RZ) + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) */ + amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; + amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); + u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); +#endif + + CollisionPairFilter( + u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], + u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], + m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, + dt, dV, pair_index, p_mask, p_pair_reaction_weight, + max_N, process_count, scattering_processes, engine); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); + u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); +#endif + + p_pair_indices_1[pair_index] = I1[i1]; + p_pair_indices_2[pair_index] = I2[i2]; + ++i1; if ( i1 == static_cast(I1e) ) { i1 = I1s; } + ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } + ++pair_index; + } +} + +#endif // COLLISION_FILTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H new file mode 100644 index 00000000000..c1be307b811 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H @@ -0,0 +1,85 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef DSMC_H_ +#define DSMC_H_ + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" +#include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/ParticleCreation/SmartUtils.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include +#include + + +/** + * \brief This class performs DSMC (direct simulation Monte Carlo) collisions + * within a cell. Particles are paired up and for each pair a stochastic process + * determines whether a collision occurs. The algorithm is similar to the one + * used for binary Coulomb collisions and the nuclear fusion module. + */ +class DSMC final + : public CollisionBase +{ + // Define shortcuts for frequently-used type names + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleBins = amrex::DenseBins; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using index_type = ParticleBins::index_type; + +public: + + /** + * \brief Constructor of the DSMC class + * + * @param[in] collision_name the name of the collision + */ + DSMC (std::string collision_name); + + /** Perform the collisions + * + * @param cur_time Current time + * @param dt Time step size + * @param mypc Container of species involved + * + */ + void doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) override; + + /** Perform all binary collisions within a tile + * + * \param[in] lev the mesh-refinement level + * \param[in] mfi iterator for multifab + * \param species_1 first species container + * \param species_2 second species container + * \param copy_species1 SmartCopy for species_1 + * \param copy_species2 SmartCopy for species_2 + * + */ + void doCollisionsWithinTile ( + amrex::Real dt, int lev, amrex::MFIter const& mfi, + WarpXParticleContainer& species_1, + WarpXParticleContainer& species_2, + SmartCopy& copy_species1, + SmartCopy& copy_species2 ); + +private: + amrex::Vector m_scattering_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; +}; + +#endif // DSMC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp new file mode 100644 index 00000000000..70a14712a0c --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp @@ -0,0 +1,300 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#include "CollisionFilterFunc.H" +#include "DSMC.H" +#include "SplitAndScatterFunc.H" + + +DSMC::DSMC (const std::string collision_name) + : CollisionBase(collision_name) +{ + using namespace amrex::literals; + amrex::ParmParse pp_collision_name(collision_name); + +#if defined WARPX_DIM_RZ + amrex::Abort("DSMC collisions are only implemented for Cartesian coordinates."); +#endif + + if(m_species_names.size() != 2) + { + amrex::Abort("DSMC collision " + collision_name + " must have exactly two species."); + } + + // query for a list of collision processes + // these could be elastic, excitation, charge_exchange, back, etc. + amrex::Vector scattering_process_names; + pp_collision_name.queryarr("scattering_processes", scattering_process_names); + + // create a vector of ScatteringProcess objects from each scattering + // process name + for (const auto& scattering_process : scattering_process_names) { + std::string kw_cross_section = scattering_process + "_cross_section"; + std::string cross_section_file; + pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); + + // if the scattering process is excitation or ionization get the + // energy associated with that process + amrex::ParticleReal energy = 0._prt; + if (scattering_process.find("excitation") != std::string::npos || + scattering_process.find("ionization") != std::string::npos) { + std::string kw_energy = scattering_process + "_energy"; + utils::parser::getWithParser( + pp_collision_name, kw_energy.c_str(), energy); + } + + ScatteringProcess process(scattering_process, cross_section_file, energy); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); + + m_scattering_processes.push_back(std::move(process)); + } + +#ifdef AMREX_USE_GPU + amrex::Gpu::HostVector h_scattering_processes_exe; + for (auto const& p : m_scattering_processes) { + h_scattering_processes_exe.push_back(p.executor()); + } + m_scattering_processes_exe.resize(h_scattering_processes_exe.size()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), + h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); + amrex::Gpu::streamSynchronize(); +#else + for (auto const& p : m_scattering_processes) { + m_scattering_processes_exe.push_back(p.executor()); + } +#endif +} + +void +DSMC::doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) +{ + WARPX_PROFILE("DSMC::doCollisions()"); + + auto& species1 = mypc->GetParticleContainerFromName(m_species_names[0]); + auto& species2 = mypc->GetParticleContainerFromName(m_species_names[1]); + + // SmartCopy objects are created that will facilitate the particle splitting + // operation involved in DSMC collisions between particles with arbitrary + // weights. + SmartCopyFactory copy_factory_species1(species1, species1); + SmartCopyFactory copy_factory_species2(species2, species2); + auto copy_species1 = copy_factory_species1.getSmartCopy(); + auto copy_species2 = copy_factory_species2.getSmartCopy(); + + species1.defineAllParticleTiles(); + species2.defineAllParticleTiles(); + + // Enable tiling + amrex::MFItInfo info; + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(WarpXParticleContainer::tile_size); } + + // Loop over refinement levels + for (int lev = 0; lev <= species1.finestLevel(); ++lev){ + + amrex::LayoutData* cost = WarpX::getCosts(lev); + + // Loop over all grids/tiles at this level +#ifdef AMREX_USE_OMP + info.SetDynamic(true); +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi = species1.MakeMFIter(lev, info); mfi.isValid(); ++mfi){ + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + } + auto wt = static_cast(amrex::second()); + + doCollisionsWithinTile(dt, lev, mfi, species1, species2, + copy_species1, copy_species2); + + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + wt = static_cast(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } + + // Call redistribute to remove particles with negative ids + species1.Redistribute(lev, lev, 0, true, true); + species2.Redistribute(lev, lev, 0, true, true); + } +} + +void +DSMC::doCollisionsWithinTile( + amrex::Real dt, int const lev, amrex::MFIter const& mfi, + WarpXParticleContainer& species_1, + WarpXParticleContainer& species_2, + SmartCopy& copy_species1, + SmartCopy& copy_species2) +{ + using namespace ParticleUtils; + using namespace amrex::literals; + + // get collision processes + auto *scattering_processes = m_scattering_processes_exe.data(); + int const process_count = static_cast(m_scattering_processes_exe.size()); + + // Extract particles in the tile that `mfi` points to + ParticleTileType& ptile_1 = species_1.ParticlesAt(lev, mfi); + ParticleTileType& ptile_2 = species_2.ParticlesAt(lev, mfi); + + // Find the particles that are in each cell of this tile + ParticleBins bins_1 = findParticlesInEachCell( lev, mfi, ptile_1 ); + ParticleBins bins_2 = findParticlesInEachCell( lev, mfi, ptile_2 ); + + // Extract low-level data + int const n_cells = static_cast(bins_1.numBins()); + + // - Species 1 + index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); + amrex::ParticleReal m1 = species_1.getMass(); + const auto ptd_1 = ptile_1.getParticleTileData(); + + // - Species 2 + index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); + amrex::ParticleReal m2 = species_2.getMass(); + const auto ptd_2 = ptile_2.getParticleTileData(); + + amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); +#if defined WARPX_DIM_1D_Z + auto dV = geom.CellSize(0); +#elif defined WARPX_DIM_XZ + auto dV = geom.CellSize(0) * geom.CellSize(1); +#elif defined WARPX_DIM_RZ + amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box + const auto lo = lbound(cbx); + const auto hi = ubound(cbx); + int nz = hi.y-lo.y+1; + auto dr = geom.CellSize(0); + auto dz = geom.CellSize(1); +#elif defined(WARPX_DIM_3D) + auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); +#endif + + // In the following we set up the "mask" used for creating new particles + // (from splitting events). There is a mask index for every collision + // pair. Below we find the size of the mask based on the greater of the + // number of each species' particle in each cell. + amrex::Gpu::DeviceVector n_pairs_in_each_cell(n_cells); + index_type* AMREX_RESTRICT p_n_pairs_in_each_cell = n_pairs_in_each_cell.dataPtr(); + + // Compute how many pairs in each cell and store in n_pairs_in_each_cell array + // For different species, the number of pairs in a cell is the number of particles of + // the species that has the most particles in that cell + amrex::ParallelFor( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; + const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; + // Particular case: no pair if a species has no particle in that cell + if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) + { + p_n_pairs_in_each_cell[i_cell] = 0; + } + else + { + p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); + } + } + ); + + // Start indices of the pairs in a cell. Will be used for particle creation + amrex::Gpu::DeviceVector pair_offsets(n_cells); + const index_type n_total_pairs = (n_cells == 0) ? 0: + amrex::Scan::ExclusiveSum(n_cells, + p_n_pairs_in_each_cell, pair_offsets.data()); + index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + + // Now we create the mask. In the DSMC scheme the mask will serve two + // purposes, 1) record whether a given pair should collide and 2) record + // the scattering process that should occur. + amrex::Gpu::DeviceVector mask(n_total_pairs); + index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); + + // Will be filled with the index of the first particle of a given pair + amrex::Gpu::DeviceVector pair_indices_1(n_total_pairs); + index_type* AMREX_RESTRICT p_pair_indices_1 = pair_indices_1.dataPtr(); + // Will be filled with the index of the second particle of a given pair + amrex::Gpu::DeviceVector pair_indices_2(n_total_pairs); + index_type* AMREX_RESTRICT p_pair_indices_2 = pair_indices_2.dataPtr(); + + // How much weight should be given to the produced particle - based on the + // weight of the collision partner which is not split + amrex::Gpu::DeviceVector pair_reaction_weight(n_total_pairs); + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight = + pair_reaction_weight.dataPtr(); + + // Loop over cells + amrex::ParallelForRNG( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept + { + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + // Same for species 2 + index_type const cell_start_2 = cell_offsets_2[i_cell]; + index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; + // Same but for the pairs + index_type const cell_start_pair = p_pair_offsets[i_cell]; + + // ux from species1 can be accessed like this: + // ux_1[ indices_1[i] ], where i is between + // cell_start_1 (inclusive) and cell_start_2 (exclusive) + + // Do not collide if one species is missing in the cell + if ( cell_stop_1 - cell_start_1 < 1 || + cell_stop_2 - cell_start_2 < 1 ) { return; } + + // shuffle + ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); + ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); +#if defined WARPX_DIM_RZ + int ri = (i_cell - i_cell%nz) / nz; + auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; +#endif + // Call the function in order to perform collisions + // If there are product species, p_mask, p_pair_indices_1/2, and + // p_pair_reaction_weight are filled here + CollisionFilter( + cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, + indices_1, indices_2, + ptd_1, ptd_2, + m1, m2, dt, dV, + cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, + p_pair_reaction_weight, + process_count, scattering_processes, engine ); + } + ); + + const auto num_p_tile1 = ptile_1.numParticles(); + const auto num_p_tile2 = ptile_2.numParticles(); + + // Create the new product particles and define their initial values + // num_added: how many particles of each product species have been created + const int num_added = splitScatteringParticles( + n_total_pairs, + ptile_1, ptile_2, + p_mask, + copy_species1, copy_species2, + m1, m2, + p_pair_indices_1, p_pair_indices_2, + p_pair_reaction_weight); + + if (num_added > 0) { + setNewParticleIDs(ptile_1, num_p_tile1, num_added); + setNewParticleIDs(ptile_2, num_p_tile2, num_added); + } +} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package new file mode 100644 index 00000000000..b4cfab89c64 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package @@ -0,0 +1,3 @@ +CEXE_sources += DSMC.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H new file mode 100644 index 00000000000..c1fb7ee7e38 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -0,0 +1,165 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef SPLIT_AND_SCATTER_FUNC_H_ +#define SPLIT_AND_SCATTER_FUNC_H_ + +#include "Particles/Collision/ScatteringProcess.H" + +/** + * \brief Function that performs the particle scattering and injection due + * to binary collisions. + * + * \return num_added the number of particles added to each species. + */ +template ::value, int> foo = 0> +int splitScatteringParticles ( + const index_type& n_total_pairs, + Tile& ptile1, Tile& ptile2, + const Index* AMREX_RESTRICT mask, + CopyFunc&& copy1, CopyFunc&& copy2, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const index_type* AMREX_RESTRICT p_pair_indices_1, + const index_type* AMREX_RESTRICT p_pair_indices_2, + const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) noexcept +{ + using namespace amrex; + + if (n_total_pairs == 0) { + return 0; + } + + Gpu::DeviceVector offsets(n_total_pairs); + Index* AMREX_RESTRICT p_offsets = offsets.dataPtr(); + + // The following is used to calculate the appropriate offsets. Note that + // a standard cummulative sum is not appropriate since the mask is also + // used to specify the type of collision and can therefore have values >1 + auto const num_added = amrex::Scan::PrefixSum(n_total_pairs, + [=] AMREX_GPU_DEVICE (Index i) -> Index { return mask[i] ? 1 : 0; }, + [=] AMREX_GPU_DEVICE (Index i, Index s) { p_offsets[i] = s; }, + amrex::Scan::Type::exclusive, amrex::Scan::retSum + ); + + const auto ptile1_index = ptile1.numParticles(); + const auto ptile2_index = ptile2.numParticles(); + ptile1.resize(ptile1_index + num_added); + ptile2.resize(ptile2_index + num_added); + + const auto ptile1_data = ptile1.getParticleTileData(); + const auto ptile2_data = ptile2.getParticleTileData(); + + const Long minus_one_long = -1; + + ParallelForRNG(n_total_pairs, + [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept + { + if (mask[i]) + { + // First, make copies of the colliding particles + copy1(ptile1_data, ptile1_data, p_pair_indices_1[i], p_offsets[i] + ptile1_index, engine); + copy2(ptile2_data, ptile2_data, p_pair_indices_2[i], p_offsets[i] + ptile2_index, engine); + + // Now we adjust the properties of the original and child particles, + // starting with the parent particles + auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; + auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; + + // Remove p_pair_reaction_weight[i] from the colliding particles' weights. + // If the colliding particle weight decreases to zero, remove particle by + // setting its id to -1. + Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); + if (w1 <= 0._prt) { + auto& p = ptile1_data.m_aos[p_pair_indices_1[i]]; + p.atomicSetID(minus_one_long); + } + + Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); + if (w2 <= 0._prt) { + auto& p = ptile2_data.m_aos[p_pair_indices_2[i]]; + p.atomicSetID(minus_one_long); + } + + // Set the child particle properties appropriately + ptile1_data.m_rdata[PIdx::w][p_offsets[i] + ptile1_index] = p_pair_reaction_weight[i]; + ptile2_data.m_rdata[PIdx::w][p_offsets[i] + ptile2_index] = p_pair_reaction_weight[i]; + + auto& ux1 = ptile1_data.m_rdata[PIdx::ux][p_offsets[i] + ptile1_index]; + auto& uy1 = ptile1_data.m_rdata[PIdx::uy][p_offsets[i] + ptile1_index]; + auto& uz1 = ptile1_data.m_rdata[PIdx::uz][p_offsets[i] + ptile1_index]; + auto& ux2 = ptile2_data.m_rdata[PIdx::ux][p_offsets[i] + ptile2_index]; + auto& uy2 = ptile2_data.m_rdata[PIdx::uy][p_offsets[i] + ptile2_index]; + auto& uz2 = ptile2_data.m_rdata[PIdx::uz][p_offsets[i] + ptile2_index]; + + // for simplicity (for now) we assume non-relativistic particles + // and simply calculate the center-of-momentum velocity from the + // rest masses + auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); + auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); + auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); + + // transform to COM frame + ux1 -= uCOM_x; + uy1 -= uCOM_y; + uz1 -= uCOM_z; + ux2 -= uCOM_x; + uy2 -= uCOM_y; + uz2 -= uCOM_z; + + if (mask[i] == int(ScatteringProcessType::ELASTIC)) { + // randomly rotate the velocity vector for the first particle + ParticleUtils::RandomizeVelocity( + ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine + ); + // set the second particles velocity so that the total momentum + // is zero + ux2 = -ux1 * m1 / m2; + uy2 = -uy1 * m1 / m2; + uz2 = -uz1 * m1 / m2; + } else if (mask[i] == int(ScatteringProcessType::BACK)) { + // reverse the velocity vectors of both particles + ux1 *= -1.0_prt; + uy1 *= -1.0_prt; + uz1 *= -1.0_prt; + ux2 *= -1.0_prt; + uy2 *= -1.0_prt; + uz2 *= -1.0_prt; + } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { + if (m1 == m2) { + auto const temp_ux = ux1; + auto const temp_uy = uy1; + auto const temp_uz = uz1; + ux1 = ux2; + uy1 = uy2; + uz1 = uz2; + ux2 = temp_ux; + uy2 = temp_uy; + uz2 = temp_uz; + } + else { + Abort("Uneven mass charge-exchange not implemented yet."); + } + } + else { + Abort("Unknown scattering process."); + } + // transform back to labframe + ux1 += uCOM_x; + uy1 += uCOM_y; + uz1 += uCOM_z; + ux2 += uCOM_x; + uy2 += uCOM_y; + uz2 += uCOM_z; + } + }); + + Gpu::synchronize(); + return static_cast(num_added); +} +#endif // SPLIT_AND_SCATTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/Make.package b/Source/Particles/Collision/BinaryCollision/Make.package index 92e2f0b5dd1..393e6270345 100644 --- a/Source/Particles/Collision/BinaryCollision/Make.package +++ b/Source/Particles/Collision/BinaryCollision/Make.package @@ -1,4 +1,6 @@ CEXE_sources += BinaryCollisionUtils.cpp CEXE_sources += ParticleCreationFunc.cpp +include $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC/Make.package + VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index 23939bbc2ab..397536b67bf 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -54,9 +54,12 @@ public: * @param[in] isSameSpecies whether the two colliding species are the same */ NuclearFusionFunc (const std::string collision_name, MultiParticleContainer const * const mypc, - const bool isSameSpecies) : m_isSameSpecies(isSameSpecies) + const bool isSameSpecies): + m_fusion_multiplier{amrex::ParticleReal{1.0}}, // default fusion multiplier + m_probability_threshold{amrex::ParticleReal{0.02}}, // default fusion probability threshold + m_probability_target_value{amrex::ParticleReal{0.002}}, // default fusion probability target_value + m_isSameSpecies{isSameSpecies} { - using namespace amrex::literals; #ifdef AMREX_SINGLE_PRECISION_PARTICLES WARPX_ABORT_WITH_MESSAGE("Nuclear fusion module does not currently work with single precision"); @@ -65,16 +68,10 @@ public: m_fusion_type = BinaryCollisionUtils::get_nuclear_fusion_type(collision_name, mypc); const amrex::ParmParse pp_collision_name(collision_name); - // default fusion multiplier - m_fusion_multiplier = 1.0_prt; utils::parser::queryWithParser( pp_collision_name, "fusion_multiplier", m_fusion_multiplier); - // default fusion probability threshold - m_probability_threshold = 0.02_prt; utils::parser::queryWithParser( pp_collision_name, "fusion_probability_threshold", m_probability_threshold); - // default fusion probability target_value - m_probability_target_value = 0.002_prt; utils::parser::queryWithParser( pp_collision_name, "fusion_probability_target_value", m_probability_target_value); @@ -123,7 +120,7 @@ public: index_type const* AMREX_RESTRICT I1, index_type const* AMREX_RESTRICT I2, SoaData_type soa_1, SoaData_type soa_2, - GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, amrex::Real const dt, amrex::Real const dV, @@ -144,36 +141,37 @@ public: amrex::ParticleReal * const AMREX_RESTRICT u2z = soa_2.m_rdata[PIdx::uz]; // Number of macroparticles of each species - const int NI1 = I1e - I1s; - const int NI2 = I2e - I2s; - const int max_N = amrex::max(NI1,NI2); + const index_type NI1 = I1e - I1s; + const index_type NI2 = I2e - I2s; + const index_type max_N = amrex::max(NI1,NI2); - int i1 = I1s; - int i2 = I2s; - int pair_index = cell_start_pair; + index_type i1 = I1s; + index_type i2 = I2s; + index_type pair_index = cell_start_pair; // Because the number of particles of each species is not always equal (NI1 != NI2 // in general), some macroparticles will be paired with multiple macroparticles of the // other species and we need to decrease their weight accordingly. // c1 corresponds to the minimum number of times a particle of species 1 will be paired // with a particle of species 2. Same for c2. - const int c1 = amrex::max(NI2/NI1,1); - const int c2 = amrex::max(NI1/NI2,1); + const index_type c1 = amrex::max(NI2/NI1,1u); + const index_type c2 = amrex::max(NI1/NI2,1u); // multiplier ratio to take into account unsampled pairs - const int multiplier_ratio = (m_isSameSpecies)?(2*max_N - 1):(max_N); + const auto multiplier_ratio = static_cast( + (m_isSameSpecies)?(2u*max_N - 1):(max_N)); #if (defined WARPX_DIM_RZ) amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; amrex::ParticleReal * const AMREX_RESTRICT theta2 = soa_2.m_rdata[PIdx::theta]; #endif - for (int k = 0; k < max_N; ++k) + for (index_type k = 0; k < max_N; ++k) { // c1k : how many times the current particle of species 1 is paired with a particle // of species 2. Same for c2k. - const int c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; - const int c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + const index_type c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; + const index_type c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; #if (defined WARPX_DIM_RZ) /* In RZ geometry, macroparticles can collide with other macroparticles @@ -194,7 +192,7 @@ public: u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, - dt, dV, pair_index, p_mask, p_pair_reaction_weight, + dt, dV, static_cast(pair_index), p_mask, p_pair_reaction_weight, m_fusion_multiplier, multiplier_ratio, m_probability_threshold, m_probability_target_value, @@ -208,8 +206,8 @@ public: p_pair_indices_1[pair_index] = I1[i1]; p_pair_indices_2[pair_index] = I2[i2]; - ++i1; if ( i1 == static_cast(I1e) ) { i1 = I1s; } - ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } + ++i1; if ( i1 == I1e ) { i1 = I1s; } + ++i2; if ( i2 == I2e ) { i2 = I2s; } ++pair_index; } diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H index 4a5d518b750..ec9a1989189 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H @@ -42,8 +42,9 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe // Compute Gamow factor, in MeV constexpr auto one_pr = 1._prt; constexpr auto Z_boron = 5._prt; - constexpr amrex::ParticleReal m_boron = 10.7319_prt * PhysConst::m_p; - constexpr amrex::ParticleReal m_reduced = m_boron / (one_pr + m_boron/PhysConst::m_p); + constexpr amrex::ParticleReal m_boron = 11.00930536_prt * PhysConst::m_u; + constexpr amrex::ParticleReal m_hydrogen = 1.00782503223 * PhysConst::m_u; + constexpr amrex::ParticleReal m_reduced = m_boron / (one_pr + m_boron/m_hydrogen); constexpr amrex::ParticleReal gamow_factor = m_reduced / 2._prt * (PhysConst::q_e*PhysConst::q_e * Z_boron / (2._prt*PhysConst::ep0*PhysConst::hbar)) * diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 085fc597f58..7b29267ec32 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -81,7 +81,7 @@ namespace { // which can cause compilation to fail or generate a warning, so we're explicitly setting // them as double. Note that nuclear fusion module does not currently work with single // precision anyways. - constexpr double m_alpha = PhysConst::m_u * 4.002602_prt; + constexpr double m_alpha = PhysConst::m_u * 4.00260325413_prt; constexpr double m_beryllium = PhysConst::m_p * 7.94748_prt; constexpr double mBe_sq = m_beryllium*m_beryllium; constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; @@ -97,20 +97,20 @@ namespace { soa_2.m_rdata[PIdx::ux][idx_2], soa_2.m_rdata[PIdx::uy][idx_2], soa_2.m_rdata[PIdx::uz][idx_2], m2, - ux_alpha1, uy_alpha1, uz_alpha1, m_alpha, - ux_Be, uy_Be, uz_Be, m_beryllium, + ux_alpha1, uy_alpha1, uz_alpha1, static_cast(m_alpha), + ux_Be, uy_Be, uz_Be, static_cast(m_beryllium), E_fusion, engine); // Compute momentum of beryllium in lab frame - const amrex::ParticleReal px_Be = m_beryllium * ux_Be; - const amrex::ParticleReal py_Be = m_beryllium * uy_Be; - const amrex::ParticleReal pz_Be = m_beryllium * uz_Be; + const amrex::ParticleReal px_Be = static_cast(m_beryllium) * ux_Be; + const amrex::ParticleReal py_Be = static_cast(m_beryllium) * uy_Be; + const amrex::ParticleReal pz_Be = static_cast(m_beryllium) * uz_Be; // Compute momentum norm of second and third alphas in Beryllium rest frame // Factor 0.5 is here because each alpha only gets half of the decay energy constexpr amrex::ParticleReal gamma_Bestar = (1._prt + 0.5_prt*E_decay/(m_alpha*c_sq)); constexpr amrex::ParticleReal gamma_Bestar_sq_minus_one = gamma_Bestar*gamma_Bestar - 1._prt; - const amrex::ParticleReal p_Bestar = m_alpha*PhysConst::c*std::sqrt(gamma_Bestar_sq_minus_one); + const amrex::ParticleReal p_Bestar = static_cast(m_alpha)*PhysConst::c*std::sqrt(gamma_Bestar_sq_minus_one); // Compute momentum of second alpha in Beryllium rest frame, assuming isotropic distribution amrex::ParticleReal px_Bestar, py_Bestar, pz_Bestar; @@ -120,8 +120,8 @@ namespace { amrex::ParticleReal px_alpha2, py_alpha2, pz_alpha2; // Preliminary calculation: compute Beryllium velocity v_Be const amrex::ParticleReal p_Be_sq = px_Be*px_Be + py_Be*py_Be + pz_Be*pz_Be; - const amrex::ParticleReal g_Be = std::sqrt(1._prt + p_Be_sq / (mBe_sq*c_sq)); - const amrex::ParticleReal mg_Be = m_beryllium*g_Be; + const amrex::ParticleReal g_Be = std::sqrt(1._prt + p_Be_sq / (static_cast(mBe_sq)*c_sq)); + const amrex::ParticleReal mg_Be = static_cast(m_beryllium)*g_Be; const amrex::ParticleReal v_Bex = px_Be / mg_Be; const amrex::ParticleReal v_Bey = py_Be / mg_Be; const amrex::ParticleReal v_Bez = pz_Be / mg_Be; @@ -133,7 +133,7 @@ namespace { { const amrex::ParticleReal vcDps = v_Bex*px_Bestar + v_Bey*py_Bestar + v_Bez*pz_Bestar; const amrex::ParticleReal factor0 = (g_Be-1._prt)/v_Be_sq; - const amrex::ParticleReal factor = factor0*vcDps + m_alpha*gamma_Bestar*g_Be; + const amrex::ParticleReal factor = factor0*vcDps + static_cast(m_alpha)*gamma_Bestar*g_Be; px_alpha2 = px_Bestar + v_Bex * factor; py_alpha2 = py_Bestar + v_Bey * factor; pz_alpha2 = pz_Bestar + v_Bez * factor; @@ -159,18 +159,18 @@ namespace { soa_alpha.m_rdata[PIdx::ux][idx_alpha_start + 1] = ux_alpha1; soa_alpha.m_rdata[PIdx::uy][idx_alpha_start + 1] = uy_alpha1; soa_alpha.m_rdata[PIdx::uz][idx_alpha_start + 1] = uz_alpha1; - soa_alpha.m_rdata[PIdx::ux][idx_alpha_start + 2] = px_alpha2/m_alpha; - soa_alpha.m_rdata[PIdx::uy][idx_alpha_start + 2] = py_alpha2/m_alpha; - soa_alpha.m_rdata[PIdx::uz][idx_alpha_start + 2] = pz_alpha2/m_alpha; - soa_alpha.m_rdata[PIdx::ux][idx_alpha_start + 3] = px_alpha2/m_alpha; - soa_alpha.m_rdata[PIdx::uy][idx_alpha_start + 3] = py_alpha2/m_alpha; - soa_alpha.m_rdata[PIdx::uz][idx_alpha_start + 3] = pz_alpha2/m_alpha; - soa_alpha.m_rdata[PIdx::ux][idx_alpha_start + 4] = px_alpha3/m_alpha; - soa_alpha.m_rdata[PIdx::uy][idx_alpha_start + 4] = py_alpha3/m_alpha; - soa_alpha.m_rdata[PIdx::uz][idx_alpha_start + 4] = pz_alpha3/m_alpha; - soa_alpha.m_rdata[PIdx::ux][idx_alpha_start + 5] = px_alpha3/m_alpha; - soa_alpha.m_rdata[PIdx::uy][idx_alpha_start + 5] = py_alpha3/m_alpha; - soa_alpha.m_rdata[PIdx::uz][idx_alpha_start + 5] = pz_alpha3/m_alpha; + soa_alpha.m_rdata[PIdx::ux][idx_alpha_start + 2] = px_alpha2/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::uy][idx_alpha_start + 2] = py_alpha2/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::uz][idx_alpha_start + 2] = pz_alpha2/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::ux][idx_alpha_start + 3] = px_alpha2/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::uy][idx_alpha_start + 3] = py_alpha2/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::uz][idx_alpha_start + 3] = pz_alpha2/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::ux][idx_alpha_start + 4] = px_alpha3/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::uy][idx_alpha_start + 4] = py_alpha3/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::uz][idx_alpha_start + 4] = pz_alpha3/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::ux][idx_alpha_start + 5] = px_alpha3/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::uy][idx_alpha_start + 5] = py_alpha3/static_cast(m_alpha); + soa_alpha.m_rdata[PIdx::uz][idx_alpha_start + 5] = pz_alpha3/static_cast(m_alpha); } } diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H index 8b0e71763bc..07e0174438b 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H @@ -15,7 +15,6 @@ #include "Utils/WarpXConst.H" #include -#include #include #include @@ -67,89 +66,33 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part const NuclearFusionType& fusion_type, const amrex::RandomEngine& engine) { - // General notations in this function: - // x_sq denotes the square of x - // x_star denotes the value of x in the center of mass frame + amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; + + BinaryCollisionUtils::get_collision_parameters( + u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, + E_coll, v_coll, lab_to_COM_factor); using namespace amrex::literals; - using namespace amrex::Math; const amrex::ParticleReal w_min = amrex::min(w1, w2); const amrex::ParticleReal w_max = amrex::max(w1, w2); - constexpr auto one_pr = amrex::ParticleReal(1.); - constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); - constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; - constexpr amrex::ParticleReal inv_csq = one_pr / ( c_sq ); - - const amrex::ParticleReal m1_sq = m1*m1; - const amrex::ParticleReal m2_sq = m2*m2; - - // Compute Lorentz factor gamma in the lab frame - const amrex::ParticleReal g1 = std::sqrt( one_pr + (u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); - const amrex::ParticleReal g2 = std::sqrt( one_pr + (u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); - - // Compute momenta - const amrex::ParticleReal p1x = u1x * m1; - const amrex::ParticleReal p1y = u1y * m1; - const amrex::ParticleReal p1z = u1z * m1; - const amrex::ParticleReal p2x = u2x * m2; - const amrex::ParticleReal p2y = u2y * m2; - const amrex::ParticleReal p2z = u2z * m2; - // Square norm of the total (sum between the two particles) momenta in the lab frame - const amrex::ParticleReal p_total_sq = powi<2>(p1x + p2x) + - powi<2>(p1y+p2y) + - powi<2>(p1z+p2z); - - // Total energy in the lab frame - const amrex::ParticleReal E_lab = (m1 * g1 + m2 * g2) * c_sq; - // Total energy squared in the center of mass frame, calculated using the Lorentz invariance - // of the four-momentum norm - const amrex::ParticleReal E_star_sq = E_lab*E_lab - c_sq*p_total_sq; - - // Kinetic energy in the center of mass frame - const amrex::ParticleReal E_star = std::sqrt(E_star_sq); - const amrex::ParticleReal E_kin_star = E_star - (m1 + m2)*c_sq; - // Compute fusion cross section as a function of kinetic energy in the center of mass frame auto fusion_cross_section = amrex::ParticleReal(0.); if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { - fusion_cross_section = ProtonBoronFusionCrossSection(E_kin_star); + fusion_cross_section = ProtonBoronFusionCrossSection(E_coll); } else if ((fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) || (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) || (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium)) { - fusion_cross_section = BoschHaleFusionCrossSection(E_kin_star, fusion_type, m1, m2); + fusion_cross_section = BoschHaleFusionCrossSection(E_coll, fusion_type, m1, m2); } - // Square of the norm of the momentum of one of the particles in the center of mass frame - // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle - // The expression below is specifically written in a form that avoids returning - // small negative numbers due to machine precision errors, for low-energy particles - const amrex::ParticleReal E_ratio = E_star/((m1 + m2)*c_sq); - const amrex::ParticleReal p_star_sq = m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) - + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio ); - - // Lorentz factors in the center of mass frame - const amrex::ParticleReal g1_star = std::sqrt(one_pr + p_star_sq / (m1_sq*c_sq)); - const amrex::ParticleReal g2_star = std::sqrt(one_pr + p_star_sq / (m2_sq*c_sq)); - - // relative velocity in the center of mass frame - const amrex::ParticleReal v_rel = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + - one_pr/(m2*g2_star)); - - // Fusion cross section and relative velocity are computed in the center of mass frame. - // On the other hand, the particle densities (weight over volume) in the lab frame are used. To - // take into account this discrepancy, we need to multiply the fusion probability by the ratio - // between the Lorentz factors in the COM frame and the Lorentz factors in the lab frame - // (see Perez et al., Phys.Plasmas.19.083104 (2012)) - const amrex::ParticleReal lab_to_COM_factor = g1_star*g2_star/(g1*g2); - // First estimate of probability to have fusion reaction amrex::ParticleReal probability_estimate = multiplier_ratio * fusion_multiplier * - lab_to_COM_factor * w_max * fusion_cross_section * v_rel * dt / dV; + lab_to_COM_factor * w_max * fusion_cross_section * v_coll * dt / dV; // Effective fusion multiplier amrex::ParticleReal fusion_multiplier_eff = fusion_multiplier; @@ -162,7 +105,7 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part // We aim for a fusion probability of probability_target_value but take into account // the constraint that the fusion_multiplier cannot be smaller than one fusion_multiplier_eff = amrex::max(fusion_multiplier * - probability_target_value / probability_estimate , one_pr); + probability_target_value / probability_estimate , 1._prt); probability_estimate *= fusion_multiplier_eff/fusion_multiplier; } @@ -170,18 +113,8 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part // In principle this is obtained by computing 1 - exp(-probability_estimate) // However, the computation of this quantity can fail numerically when probability_estimate is // too small (e.g. exp(-probability_estimate) returns 1 and the computation returns 0). - // In this case, we simply use "probability_estimate" instead of 1 - exp(-probability_estimate) - // The threshold exp_threshold at which we switch between the two formulas is determined by the - // fact that computing the exponential is only useful if it can resolve the x^2/2 term of its - // Taylor expansion, i.e. the square of probability_estimate should be greater than the - // machine epsilon. -#ifdef AMREX_SINGLE_PRECISION_PARTICLES - constexpr auto exp_threshold = amrex::ParticleReal(1.e-3); -#else - constexpr auto exp_threshold = amrex::ParticleReal(5.e-8); -#endif - const amrex::ParticleReal probability = (probability_estimate < exp_threshold) ? - probability_estimate: one_pr - std::exp(-probability_estimate); + // std::expm1 is used since it maintains correctness for small exponent. + const amrex::ParticleReal probability = -std::expm1(-probability_estimate); // Get a random number const amrex::ParticleReal random_number = amrex::Random(engine); @@ -198,6 +131,4 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part } } - - #endif // SINGLE_NUCLEAR_FUSION_EVENT_H_ diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 2b43f76e89f..dc830b477df 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -100,6 +100,7 @@ public: amrex::Vector operator() ( const index_type& n_total_pairs, const SoaData_type& soa_1, const SoaData_type& soa_2, + const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, ParticleType* particle_ptr_1, ParticleType* particle_ptr_2, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, @@ -115,7 +116,7 @@ public: { using namespace amrex::literals; - if (n_total_pairs == 0) return {m_num_product_species, 0}; + if (n_total_pairs == 0) { return amrex::Vector(m_num_product_species, 0); } // Compute offset array and allocate memory for the produced species amrex::Gpu::DeviceVector offsets(n_total_pairs); @@ -130,7 +131,7 @@ public: // one electron, we create two electrons, one at the position of each particle that // collided. This allows for exact charge conservation. const index_type num_added = total * m_num_products_host[i] * 2; - num_added_vec[i] = num_added; + num_added_vec[i] = static_cast(num_added); tile_products[i]->resize(products_np[i] + num_added); } @@ -153,7 +154,7 @@ public: amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, products_np.begin(), products_np.end(), device_products_np.begin()); - amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, products_mass.begin(), + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, products_mass.begin(), products_mass.end(), device_products_mass.begin()); amrex::Gpu::streamSynchronize(); @@ -184,11 +185,11 @@ public: const auto product_index = products_np_data[j] + 2*(p_offsets[i]*p_num_products_device[j] + k); // Create product particle at position of particle 1 - copy_species1[j](soa_products_data[j], soa_1, p_pair_indices_1[i], - product_index, engine); + copy_species1[j](soa_products_data[j], soa_1, static_cast(p_pair_indices_1[i]), + static_cast(product_index), engine); // Create another product particle at position of particle 2 - copy_species2[j](soa_products_data[j], soa_2, p_pair_indices_2[i], - product_index + 1, engine); + copy_species2[j](soa_products_data[j], soa_2, static_cast(p_pair_indices_2[i]), + static_cast(product_index + 1), engine); // Set the weight of the new particles to p_pair_reaction_weight[i]/2 soa_products_data[j].m_rdata[PIdx::w][product_index] = @@ -251,6 +252,27 @@ public: } }); + // Initialize the user runtime components + for (int i = 0; i < m_num_product_species; i++) + { + int start_index = int(products_np[i]); + int stop_index = int(products_np[i] + num_added_vec[i]); + ParticleCreation::DefaultInitializeRuntimeAttributes(*tile_products[i], + 0, 0, + pc_products[i]->getUserRealAttribs(), pc_products[i]->getUserIntAttribs(), + pc_products[i]->getParticleComps(), pc_products[i]->getParticleiComps(), + pc_products[i]->getUserRealAttribParser(), + pc_products[i]->getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the SmartCopy functors + pc_products[i]->get_breit_wheeler_engine_ptr(), + pc_products[i]->get_quantum_sync_engine_ptr(), +#endif + pc_products[i]->getIonizationInitialLevel(), + start_index, stop_index); + } + amrex::Gpu::synchronize(); return num_added_vec; @@ -289,6 +311,7 @@ public: amrex::Vector operator() ( const index_type& /*n_total_pairs*/, const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, ParticleType* /*particle_ptr_1*/, ParticleType* /*particle_ptr_2*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp index 1ef5ecad229..33629cd530f 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp @@ -19,45 +19,45 @@ ParticleCreationFunc::ParticleCreationFunc (const std::string collision_name, MultiParticleContainer const * const mypc) - { - const amrex::ParmParse pp_collision_name(collision_name); +{ + const amrex::ParmParse pp_collision_name(collision_name); - m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); + m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); - if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) - { - // Proton-Boron fusion only produces alpha particles - m_num_product_species = 1; - // Proton-Boron fusion produces 3 alpha particles per fusion reaction - m_num_products_host.push_back(3); + if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) + { + // Proton-Boron fusion only produces alpha particles + m_num_product_species = 1; + // Proton-Boron fusion produces 3 alpha particles per fusion reaction + m_num_products_host.push_back(3); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediatly - m_num_products_device.push_back(3); + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(3); #endif - } - else if ((m_collision_type == CollisionType::DeuteriumTritiumToNeutronHeliumFusion) - || (m_collision_type == CollisionType::DeuteriumDeuteriumToProtonTritiumFusion) - || (m_collision_type == CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion)) - { - m_num_product_species = 2; - m_num_products_host.push_back(1); - m_num_products_host.push_back(1); + } + else if ((m_collision_type == CollisionType::DeuteriumTritiumToNeutronHeliumFusion) + || (m_collision_type == CollisionType::DeuteriumDeuteriumToProtonTritiumFusion) + || (m_collision_type == CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion)) + { + m_num_product_species = 2; + m_num_products_host.push_back(1); + m_num_products_host.push_back(1); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediatly - m_num_products_device.push_back(1); - m_num_products_device.push_back(1); + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(1); + m_num_products_device.push_back(1); #endif - } - else - { - WARPX_ABORT_WITH_MESSAGE("Unknown collision type in ParticleCreationFunc"); - } + } + else + { + WARPX_ABORT_WITH_MESSAGE("Unknown collision type in ParticleCreationFunc"); + } #ifdef AMREX_USE_GPU - m_num_products_device.resize(m_num_product_species); - amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), - m_num_products_host.end(), - m_num_products_device.begin()); - amrex::Gpu::streamSynchronize(); + m_num_products_device.resize(m_num_product_species); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), + m_num_products_host.end(), + m_num_products_device.begin()); + amrex::Gpu::streamSynchronize(); #endif - } +} diff --git a/Source/Particles/Collision/CMakeLists.txt b/Source/Particles/Collision/CMakeLists.txt index f3183032179..62eb3b3ad9c 100644 --- a/Source/Particles/Collision/CMakeLists.txt +++ b/Source/Particles/Collision/CMakeLists.txt @@ -4,6 +4,7 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE CollisionHandler.cpp CollisionBase.cpp + ScatteringProcess.cpp ) endforeach() diff --git a/Source/Particles/Collision/CollisionBase.H b/Source/Particles/Collision/CollisionBase.H index 517f5e414c0..272cc056b1a 100644 --- a/Source/Particles/Collision/CollisionBase.H +++ b/Source/Particles/Collision/CollisionBase.H @@ -25,6 +25,7 @@ public: CollisionBase(CollisionBase const &) = delete; CollisionBase(CollisionBase &&) = delete; CollisionBase & operator=(CollisionBase const &) = delete; + CollisionBase & operator=(CollisionBase const &&) = delete; virtual ~CollisionBase() = default; diff --git a/Source/Particles/Collision/CollisionBase.cpp b/Source/Particles/Collision/CollisionBase.cpp index 701d5ff6bc6..8f799c83013 100644 --- a/Source/Particles/Collision/CollisionBase.cpp +++ b/Source/Particles/Collision/CollisionBase.cpp @@ -18,8 +18,8 @@ CollisionBase::CollisionBase (std::string collision_name) pp_collision_name.getarr("species", m_species_names); // number of time steps between collisions - m_ndt = 1; + int ndt = 1; utils::parser::queryWithParser( - pp_collision_name, "ndt", m_ndt); - + pp_collision_name, "ndt", ndt); + m_ndt = ndt; } diff --git a/Source/Particles/Collision/CollisionHandler.cpp b/Source/Particles/Collision/CollisionHandler.cpp index 213e5edd577..5abee34c86a 100644 --- a/Source/Particles/Collision/CollisionHandler.cpp +++ b/Source/Particles/Collision/CollisionHandler.cpp @@ -10,6 +10,7 @@ #include "Particles/Collision/BackgroundStopping/BackgroundStopping.H" #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" #include "Particles/Collision/BinaryCollision/BinaryCollision.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMC.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Utils/TextMsg.H" @@ -52,6 +53,9 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) else if (type == "background_stopping") { allcollisions[i] = std::make_unique(collision_names[i]); } + else if (type == "dsmc") { + allcollisions[i] = std::make_unique(collision_names[i]); + } else if (type == "nuclearfusion") { allcollisions[i] = std::make_unique>( diff --git a/Source/Particles/Collision/Make.package b/Source/Particles/Collision/Make.package index e9aad3691a3..9ec153879f4 100644 --- a/Source/Particles/Collision/Make.package +++ b/Source/Particles/Collision/Make.package @@ -1,5 +1,6 @@ CEXE_sources += CollisionHandler.cpp CEXE_sources += CollisionBase.cpp +CEXE_sources += ScatteringProcess.cpp include $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/Make.package include $(WARPX_HOME)/Source/Particles/Collision/BackgroundMCC/Make.package diff --git a/Source/Particles/Collision/BackgroundMCC/MCCProcess.H b/Source/Particles/Collision/ScatteringProcess.H similarity index 71% rename from Source/Particles/Collision/BackgroundMCC/MCCProcess.H rename to Source/Particles/Collision/ScatteringProcess.H index 31f790153bd..d05ea32e2fe 100644 --- a/Source/Particles/Collision/BackgroundMCC/MCCProcess.H +++ b/Source/Particles/Collision/ScatteringProcess.H @@ -1,18 +1,20 @@ -/* Copyright 2021 Modern Electron +/* Copyright 2021-2023 The WarpX Community * * This file is part of WarpX. * + * Authors: Modern Electron, Roelof Groenewald (TAE Technologies) + * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ -#define WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ +#ifndef WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ +#define WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ #include #include #include #include -enum class MCCProcessType { +enum class ScatteringProcessType { INVALID, ELASTIC, BACK, @@ -21,28 +23,29 @@ enum class MCCProcessType { IONIZATION, }; -class MCCProcess +class ScatteringProcess { public: - MCCProcess ( + ScatteringProcess ( const std::string& scattering_process, const std::string& cross_section_file, amrex::ParticleReal energy ); template - MCCProcess ( + ScatteringProcess ( const std::string& scattering_process, const InputVector&& energies, const InputVector&& sigmas, amrex::ParticleReal energy ); - MCCProcess (MCCProcess const&) = delete; - MCCProcess& operator= (MCCProcess const&) = delete; + ~ScatteringProcess() = default; - MCCProcess (MCCProcess &&) = default; - MCCProcess& operator= (MCCProcess &&) = default; + ScatteringProcess (ScatteringProcess const&) = delete; + ScatteringProcess& operator= (ScatteringProcess const&) = delete; + ScatteringProcess (ScatteringProcess &&) = default; + ScatteringProcess& operator= (ScatteringProcess &&) = default; /** Read the given cross-section data file to memory. * @@ -73,6 +76,7 @@ public: * @param E_coll collision energy in eV * */ + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::ParticleReal getCrossSection (amrex::ParticleReal E_coll) const { @@ -97,9 +101,10 @@ public: amrex::ParticleReal* m_sigmas_data = nullptr; amrex::ParticleReal m_energy_lo, m_energy_hi, m_sigma_lo, m_sigma_hi, m_dE; amrex::ParticleReal m_energy_penalty; - MCCProcessType m_type; + ScatteringProcessType m_type; }; + [[nodiscard]] Executor const& executor () const { #ifdef AMREX_USE_GPU return m_exe_d; @@ -108,22 +113,22 @@ public: #endif } - amrex::ParticleReal getCrossSection (amrex::ParticleReal E_coll) const + [[nodiscard]] amrex::ParticleReal getCrossSection (amrex::ParticleReal E_coll) const { return m_exe_h.getCrossSection(E_coll); } - amrex::ParticleReal getEnergyPenalty () const { return m_exe_h.m_energy_penalty; } - amrex::ParticleReal getMinEnergyInput () const { return m_exe_h.m_energy_lo; } - amrex::ParticleReal getMaxEnergyInput () const { return m_exe_h.m_energy_hi; } - amrex::ParticleReal getEnergyInputStep () const { return m_exe_h.m_dE; } + [[nodiscard]] amrex::ParticleReal getEnergyPenalty () const { return m_exe_h.m_energy_penalty; } + [[nodiscard]] amrex::ParticleReal getMinEnergyInput () const { return m_exe_h.m_energy_lo; } + [[nodiscard]] amrex::ParticleReal getMaxEnergyInput () const { return m_exe_h.m_energy_hi; } + [[nodiscard]] amrex::ParticleReal getEnergyInputStep () const { return m_exe_h.m_dE; } - MCCProcessType type () const { return m_exe_h.m_type; } + [[nodiscard]] ScatteringProcessType type () const { return m_exe_h.m_type; } private: static - MCCProcessType parseProcessType(const std::string& process); + ScatteringProcessType parseProcessType(const std::string& process); void init (const std::string& scattering_process, amrex::ParticleReal energy); @@ -139,4 +144,4 @@ private: int m_grid_size; }; -#endif // WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ +#endif // WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ diff --git a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp b/Source/Particles/Collision/ScatteringProcess.cpp similarity index 78% rename from Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp rename to Source/Particles/Collision/ScatteringProcess.cpp index 0451f62be60..c15d11eea07 100644 --- a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp +++ b/Source/Particles/Collision/ScatteringProcess.cpp @@ -1,15 +1,17 @@ -/* Copyright 2021 Modern Electron +/* Copyright 2021-2023 The WarpX Community * * This file is part of WarpX. * + * Authors: Modern Electron, Roelof Groenewald (TAE Technologies) + * * License: BSD-3-Clause-LBNL */ -#include "MCCProcess.H" +#include "ScatteringProcess.H" #include "Utils/TextMsg.H" #include "WarpX.H" -MCCProcess::MCCProcess ( +ScatteringProcess::ScatteringProcess ( const std::string& scattering_process, const std::string& cross_section_file, const amrex::ParticleReal energy ) @@ -21,7 +23,7 @@ MCCProcess::MCCProcess ( } template -MCCProcess::MCCProcess ( +ScatteringProcess::ScatteringProcess ( const std::string& scattering_process, const InputVector&& energies, const InputVector&& sigmas, @@ -34,13 +36,13 @@ MCCProcess::MCCProcess ( } void -MCCProcess::init (const std::string& scattering_process, const amrex::ParticleReal energy) +ScatteringProcess::init (const std::string& scattering_process, const amrex::ParticleReal energy) { using namespace amrex::literals; m_exe_h.m_sigmas_data = m_sigmas_h.data(); // save energy grid parameters for easy use - m_grid_size = m_energies.size(); + m_grid_size = static_cast(m_energies.size()); m_exe_h.m_energy_lo = m_energies[0]; m_exe_h.m_energy_hi = m_energies[m_grid_size-1]; m_exe_h.m_sigma_lo = m_sigmas_h[0]; @@ -72,44 +74,44 @@ MCCProcess::init (const std::string& scattering_process, const amrex::ParticleRe #endif } -MCCProcessType -MCCProcess::parseProcessType(const std::string& scattering_process) +ScatteringProcessType +ScatteringProcess::parseProcessType(const std::string& scattering_process) { if (scattering_process == "elastic") { - return MCCProcessType::ELASTIC; + return ScatteringProcessType::ELASTIC; } else if (scattering_process == "back") { - return MCCProcessType::BACK; + return ScatteringProcessType::BACK; } else if (scattering_process == "charge_exchange") { - return MCCProcessType::CHARGE_EXCHANGE; + return ScatteringProcessType::CHARGE_EXCHANGE; } else if (scattering_process == "ionization") { - return MCCProcessType::IONIZATION; + return ScatteringProcessType::IONIZATION; } else if (scattering_process.find("excitation") != std::string::npos) { - return MCCProcessType::EXCITATION; + return ScatteringProcessType::EXCITATION; } else { - return MCCProcessType::INVALID; + return ScatteringProcessType::INVALID; } } void -MCCProcess::readCrossSectionFile ( +ScatteringProcess::readCrossSectionFile ( const std::string cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ) { std::ifstream infile(cross_section_file); - if(!infile.is_open()) WARPX_ABORT_WITH_MESSAGE("Failed to open cross-section data file"); + if(!infile.is_open()) { WARPX_ABORT_WITH_MESSAGE("Failed to open cross-section data file"); } amrex::ParticleReal energy, sigma; while (infile >> energy >> sigma) { energies.push_back(energy); sigmas.push_back(sigma); } - if (infile.bad()) WARPX_ABORT_WITH_MESSAGE("Failed to read cross-section data from file."); + if (infile.bad()) { WARPX_ABORT_WITH_MESSAGE("Failed to read cross-section data from file."); } infile.close(); } void -MCCProcess::sanityCheckEnergyGrid ( +ScatteringProcess::sanityCheckEnergyGrid ( const amrex::Vector& energies, amrex::ParticleReal dE ) diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index 20ec4c2e34d..d0db678dfda 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -27,7 +27,7 @@ since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param rho_fab FArrayBox of charge density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param dx 3D cell size * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. @@ -37,11 +37,11 @@ * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template -void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, +void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, const int* ion_lev, amrex::FArrayBox& rho_fab, - long np_to_depose, + long np_to_deposit, const std::array& dx, const std::array xyzmin, amrex::Dim3 lo, @@ -95,8 +95,8 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_depose, - [=] AMREX_GPU_DEVICE (long ip) { + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), @@ -239,7 +239,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, * \param bin_size tile size to use for shared current deposition operations */ template -void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPosition, +void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, const int* ion_lev, amrex::FArrayBox& rho_fab, @@ -261,7 +261,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPosition, { using namespace amrex; - auto permutation = a_bins.permutationPtr(); + const auto *permutation = a_bins.permutationPtr(); #if !defined(AMREX_USE_GPU) amrex::ignore_unused(ix_type, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size, bin_size); diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 49a7c46a3e8..efe6efcc788 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -26,6 +26,246 @@ #include #include +/** + * \brief Kernel for the direct current deposition for thread thread_num + * \tparam depos_order deposition order + * \param xp, yp, zp The particle positions. + * \param wq The charge of the macroparticle + * \param vx,vy,vz The particle velocities + * \param jx_arr,jy_arr,jz_arr Array4 of current density, either full array or tile. + * \param jx_type,jy_type,jz_type The grid types along each direction, either NODE or CELL + * \param relative_time Time at which to deposit J, relative to the time of the + * current positions of the particles. When different than 0, + * the particle position will be temporarily modified to match + * the time of the deposition. + * \param dzi, dxi, dyi The inverse cell sizes + * \param zmin, xmin, ymin The lower bounds of the domain + * \param invvol The inverse volume of a grid cell + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void doDepositionShapeNKernel(const amrex::ParticleReal xp, + const amrex::ParticleReal yp, + const amrex::ParticleReal zp, + const amrex::ParticleReal wq, + const amrex::ParticleReal vx, + const amrex::ParticleReal vy, + const amrex::ParticleReal vz, + amrex::Array4 const& jx_arr, + amrex::Array4 const& jy_arr, + amrex::Array4 const& jz_arr, + amrex::IntVect const& jx_type, + amrex::IntVect const& jy_type, + amrex::IntVect const& jz_type, + const amrex::Real relative_time, + AMREX_D_DECL(const amrex::Real dzi, + const amrex::Real dxi, + const amrex::Real dyi), + AMREX_D_DECL(const amrex::Real zmin, + const amrex::Real xmin, + const amrex::Real ymin), + const amrex::Real invvol, + const amrex::Dim3 lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex::literals; +#if !defined(WARPX_DIM_RZ) + amrex::ignore_unused(n_rz_azimuthal_modes); +#endif +#if defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(xp, yp); +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ignore_unused(yp); +#endif + + constexpr int zdir = WARPX_ZINDEX; + constexpr int NODE = amrex::IndexType::NODE; + constexpr int CELL = amrex::IndexType::CELL; + + // wqx, wqy wqz are particle current in each direction +#if defined(WARPX_DIM_RZ) + // In RZ, wqx is actually wqr, and wqy is wqtheta + // Convert to cylindrical at the mid point + const amrex::Real xpmid = xp + relative_time*vx; + const amrex::Real ypmid = yp + relative_time*vy; + const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); + amrex::Real costheta; + amrex::Real sintheta; + if (rpmid > 0._rt) { + costheta = xpmid/rpmid; + sintheta = ypmid/rpmid; + } else { + costheta = 1._rt; + sintheta = 0._rt; + } + const Complex xy0 = Complex{costheta, sintheta}; + const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); + const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); +#else + const amrex::Real wqx = wq*invvol*vx; + const amrex::Real wqy = wq*invvol*vy; +#endif + const amrex::Real wqz = wq*invvol*vz; + + // --- Compute shape factors + Compute_shape_factor< depos_order > const compute_shape_factor; +#if (AMREX_SPACEDIM >= 2) + // x direction + // Get particle position after 1/2 push back in position +#if defined(WARPX_DIM_RZ) + // Keep these double to avoid bug in single precision + const double xmid = (rpmid - xmin)*dxi; +#else + const double xmid = ((xp - xmin) + relative_time*vx)*dxi; +#endif + // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current + // sx_j[xyz] shape factor along x for the centering of each current + // There are only two possible centerings, node or cell centered, so at most only two shape factor + // arrays will be needed. + // Keep these double to avoid bug in single precision + double sx_node[depos_order + 1] = {0.}; + double sx_cell[depos_order + 1] = {0.}; + int j_node = 0; + int j_cell = 0; + if (jx_type[0] == NODE || jy_type[0] == NODE || jz_type[0] == NODE) { + j_node = compute_shape_factor(sx_node, xmid); + } + if (jx_type[0] == CELL || jy_type[0] == CELL || jz_type[0] == CELL) { + j_cell = compute_shape_factor(sx_cell, xmid - 0.5); + } + + amrex::Real sx_jx[depos_order + 1] = {0._rt}; + amrex::Real sx_jy[depos_order + 1] = {0._rt}; + amrex::Real sx_jz[depos_order + 1] = {0._rt}; + for (int ix=0; ix<=depos_order; ix++) + { + sx_jx[ix] = ((jx_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + sx_jy[ix] = ((jy_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + sx_jz[ix] = ((jz_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + } + + int const j_jx = ((jx_type[0] == NODE) ? j_node : j_cell); + int const j_jy = ((jy_type[0] == NODE) ? j_node : j_cell); + int const j_jz = ((jz_type[0] == NODE) ? j_node : j_cell); +#endif //AMREX_SPACEDIM >= 2 + +#if defined(WARPX_DIM_3D) + // y direction + // Keep these double to avoid bug in single precision + const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + double sy_node[depos_order + 1] = {0.}; + double sy_cell[depos_order + 1] = {0.}; + int k_node = 0; + int k_cell = 0; + if (jx_type[1] == NODE || jy_type[1] == NODE || jz_type[1] == NODE) { + k_node = compute_shape_factor(sy_node, ymid); + } + if (jx_type[1] == CELL || jy_type[1] == CELL || jz_type[1] == CELL) { + k_cell = compute_shape_factor(sy_cell, ymid - 0.5); + } + amrex::Real sy_jx[depos_order + 1] = {0._rt}; + amrex::Real sy_jy[depos_order + 1] = {0._rt}; + amrex::Real sy_jz[depos_order + 1] = {0._rt}; + for (int iy=0; iy<=depos_order; iy++) + { + sy_jx[iy] = ((jx_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + sy_jy[iy] = ((jy_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + sy_jz[iy] = ((jz_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + } + int const k_jx = ((jx_type[1] == NODE) ? k_node : k_cell); + int const k_jy = ((jy_type[1] == NODE) ? k_node : k_cell); + int const k_jz = ((jz_type[1] == NODE) ? k_node : k_cell); +#endif + + // z direction + // Keep these double to avoid bug in single precision + const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + double sz_node[depos_order + 1] = {0.}; + double sz_cell[depos_order + 1] = {0.}; + int l_node = 0; + int l_cell = 0; + if (jx_type[zdir] == NODE || jy_type[zdir] == NODE || jz_type[zdir] == NODE) { + l_node = compute_shape_factor(sz_node, zmid); + } + if (jx_type[zdir] == CELL || jy_type[zdir] == CELL || jz_type[zdir] == CELL) { + l_cell = compute_shape_factor(sz_cell, zmid - 0.5); + } + amrex::Real sz_jx[depos_order + 1] = {0._rt}; + amrex::Real sz_jy[depos_order + 1] = {0._rt}; + amrex::Real sz_jz[depos_order + 1] = {0._rt}; + for (int iz=0; iz<=depos_order; iz++) + { + sz_jx[iz] = ((jx_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + sz_jy[iz] = ((jy_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + sz_jz[iz] = ((jz_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + } + int const l_jx = ((jx_type[zdir] == NODE) ? l_node : l_cell); + int const l_jy = ((jy_type[zdir] == NODE) ? l_node : l_cell); + int const l_jz = ((jz_type[zdir] == NODE) ? l_node : l_cell); + + // Deposit current into jx_arr, jy_arr and jz_arr +#if defined(WARPX_DIM_1D_Z) + for (int iz=0; iz<=depos_order; iz++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+l_jx+iz, 0, 0, 0), + sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+l_jy+iz, 0, 0, 0), + sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+l_jz+iz, 0, 0, 0), + sz_jz[iz]*wqz); + } +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + for (int iz=0; iz<=depos_order; iz++){ + for (int ix=0; ix<=depos_order; ix++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 0), + sx_jx[ix]*sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 0), + sx_jy[ix]*sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 0), + sx_jz[ix]*sz_jz[iz]*wqz); +#if defined(WARPX_DIM_RZ) + Complex xy = xy0; // Note that xy is equal to e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 on the weighting comes from the normalization of the modes + amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode-1), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode ), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.imag()); + amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode-1), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode ), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.imag()); + amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode-1), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode ), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.imag()); + xy = xy*xy0; + } +#endif + } + } +#elif defined(WARPX_DIM_3D) + for (int iz=0; iz<=depos_order; iz++){ + for (int iy=0; iy<=depos_order; iy++){ + for (int ix=0; ix<=depos_order; ix++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+j_jx+ix, lo.y+k_jx+iy, lo.z+l_jx+iz), + sx_jx[ix]*sy_jx[iy]*sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+j_jy+ix, lo.y+k_jy+iy, lo.z+l_jy+iz), + sx_jy[ix]*sy_jy[iy]*sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+j_jz+ix, lo.y+k_jz+iy, lo.z+l_jz+iz), + sx_jz[ix]*sy_jz[iy]*sz_jz[iz]*wqz); + } + } + } +#endif +} + /** * \brief Current Deposition for thread thread_num * \tparam depos_order deposition order @@ -37,7 +277,7 @@ since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match @@ -51,7 +291,7 @@ * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template -void doDepositionShapeN (const GetParticlePosition& GetPosition, +void doDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, const amrex::ParticleReal * const uxp, const amrex::ParticleReal * const uyp, @@ -60,7 +300,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jx_fab, amrex::FArrayBox& jy_fab, amrex::FArrayBox& jz_fab, - long np_to_depose, + long np_to_deposit, amrex::Real relative_time, const std::array& dx, const std::array& xyzmin, @@ -113,10 +353,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::IntVect const jy_type = jy_fab.box().type(); amrex::IntVect const jz_type = jz_fab.box().type(); - constexpr int zdir = WARPX_ZINDEX; - constexpr int NODE = amrex::IndexType::NODE; - constexpr int CELL = amrex::IndexType::CELL; - // Loop over particles and deposit into jx_fab, jy_fab and jz_fab #if defined(WARPX_USE_GPUCLOCK) amrex::Real* cost_real = nullptr; @@ -126,7 +362,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_depose, + np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( @@ -134,200 +370,173 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, cost_real); #endif + amrex::ParticleReal xp, yp, zp; + GetPosition(ip, xp, yp, zp); + // --- Get particle quantities const amrex::Real gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq + uzp[ip]*uzp[ip]*clightsq); + const amrex::Real vx = uxp[ip]*gaminv; + const amrex::Real vy = uyp[ip]*gaminv; + const amrex::Real vz = uzp[ip]*gaminv; + amrex::Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; } - amrex::ParticleReal xp, yp, zp; - GetPosition(ip, xp, yp, zp); + doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, + jx_type, jy_type, jz_type, + relative_time, + AMREX_D_DECL(dzi, dxi, dyi), + AMREX_D_DECL(zmin, xmin, ymin), + invvol, lo, n_rz_azimuthal_modes); - const amrex::Real vx = uxp[ip]*gaminv; - const amrex::Real vy = uyp[ip]*gaminv; - const amrex::Real vz = uzp[ip]*gaminv; - // wqx, wqy wqz are particle current in each direction -#if defined(WARPX_DIM_RZ) - // In RZ, wqx is actually wqr, and wqy is wqtheta - // Convert to cylinderical at the mid point - const amrex::Real xpmid = xp + relative_time*vx; - const amrex::Real ypmid = yp + relative_time*vy; - const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); - amrex::Real costheta; - amrex::Real sintheta; - if (rpmid > 0._rt) { - costheta = xpmid/rpmid; - sintheta = ypmid/rpmid; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } - const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); - const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); -#else - const amrex::Real wqx = wq*invvol*vx; - const amrex::Real wqy = wq*invvol*vy; + } + ); +#if defined(WARPX_USE_GPUCLOCK) + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + amrex::Gpu::streamSynchronize(); + *cost += *cost_real; + amrex::The_Managed_Arena()->free(cost_real); + } #endif - const amrex::Real wqz = wq*invvol*vz; +} - // --- Compute shape factors - Compute_shape_factor< depos_order > const compute_shape_factor; -#if (AMREX_SPACEDIM >= 2) - // x direction - // Get particle position after 1/2 push back in position -#if defined(WARPX_DIM_RZ) - // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; -#else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; +/** + * \brief Direct current deposition for thread thread_num for the implicit scheme + * The only difference from doDepositionShapeN is in how the particle gamma + * is calculated. + * \tparam depos_order deposition order + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp_n,uyp_n,uzp_n Pointer to arrays of particle momentum at time n. + * \param uxp,uyp,uzp Pointer to arrays of particle momentum at time n+1/2. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + const amrex::ParticleReal * const uxp_n, + const amrex::ParticleReal * const uyp_n, + const amrex::ParticleReal * const uzp_n, + const amrex::ParticleReal * const uxp, + const amrex::ParticleReal * const uyp, + const amrex::ParticleReal * const uzp, + const int * const ion_lev, + amrex::FArrayBox& jx_fab, + amrex::FArrayBox& jy_fab, + amrex::FArrayBox& jz_fab, + const long np_to_deposit, + const std::array& dx, + const std::array& xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real* cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex::literals; +#if !defined(WARPX_DIM_RZ) + amrex::ignore_unused(n_rz_azimuthal_modes); #endif - // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current - // sx_j[xyz] shape factor along x for the centering of each current - // There are only two possible centerings, node or cell centered, so at most only two shape factor - // arrays will be needed. - // Keep these double to avoid bug in single precision - double sx_node[depos_order + 1] = {0.}; - double sx_cell[depos_order + 1] = {0.}; - int j_node = 0; - int j_cell = 0; - if (jx_type[0] == NODE || jy_type[0] == NODE || jz_type[0] == NODE) { - j_node = compute_shape_factor(sx_node, xmid); - } - if (jx_type[0] == CELL || jy_type[0] == CELL || jz_type[0] == CELL) { - j_cell = compute_shape_factor(sx_cell, xmid - 0.5); - } - amrex::Real sx_jx[depos_order + 1] = {0._rt}; - amrex::Real sx_jy[depos_order + 1] = {0._rt}; - amrex::Real sx_jz[depos_order + 1] = {0._rt}; - for (int ix=0; ix<=depos_order; ix++) - { - sx_jx[ix] = ((jx_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - sx_jy[ix] = ((jy_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - sx_jz[ix] = ((jz_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - } +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif - int const j_jx = ((jx_type[0] == NODE) ? j_node : j_cell); - int const j_jy = ((jy_type[0] == NODE) ? j_node : j_cell); - int const j_jz = ((jz_type[0] == NODE) ? j_node : j_cell); -#endif //AMREX_SPACEDIM >= 2 + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + const bool do_ionization = ion_lev; + const amrex::Real dzi = 1.0_rt/dx[2]; +#if defined(WARPX_DIM_1D_Z) + const amrex::Real invvol = dzi; +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real dxi = 1.0_rt/dx[0]; + const amrex::Real invvol = dxi*dzi; +#elif defined(WARPX_DIM_3D) + const amrex::Real dxi = 1.0_rt/dx[0]; + const amrex::Real dyi = 1.0_rt/dx[1]; + const amrex::Real invvol = dxi*dyi*dzi; +#endif +#if (AMREX_SPACEDIM >= 2) + const amrex::Real xmin = xyzmin[0]; +#endif #if defined(WARPX_DIM_3D) - // y direction - // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; - double sy_node[depos_order + 1] = {0.}; - double sy_cell[depos_order + 1] = {0.}; - int k_node = 0; - int k_cell = 0; - if (jx_type[1] == NODE || jy_type[1] == NODE || jz_type[1] == NODE) { - k_node = compute_shape_factor(sy_node, ymid); - } - if (jx_type[1] == CELL || jy_type[1] == CELL || jz_type[1] == CELL) { - k_cell = compute_shape_factor(sy_cell, ymid - 0.5); - } - amrex::Real sy_jx[depos_order + 1] = {0._rt}; - amrex::Real sy_jy[depos_order + 1] = {0._rt}; - amrex::Real sy_jz[depos_order + 1] = {0._rt}; - for (int iy=0; iy<=depos_order; iy++) - { - sy_jx[iy] = ((jx_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - sy_jy[iy] = ((jy_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - sy_jz[iy] = ((jz_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - } - int const k_jx = ((jx_type[1] == NODE) ? k_node : k_cell); - int const k_jy = ((jy_type[1] == NODE) ? k_node : k_cell); - int const k_jz = ((jz_type[1] == NODE) ? k_node : k_cell); + const amrex::Real ymin = xyzmin[1]; #endif + const amrex::Real zmin = xyzmin[2]; - // z direction - // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; - double sz_node[depos_order + 1] = {0.}; - double sz_cell[depos_order + 1] = {0.}; - int l_node = 0; - int l_cell = 0; - if (jx_type[zdir] == NODE || jy_type[zdir] == NODE || jz_type[zdir] == NODE) { - l_node = compute_shape_factor(sz_node, zmid); - } - if (jx_type[zdir] == CELL || jy_type[zdir] == CELL || jz_type[zdir] == CELL) { - l_cell = compute_shape_factor(sz_cell, zmid - 0.5); - } - amrex::Real sz_jx[depos_order + 1] = {0._rt}; - amrex::Real sz_jy[depos_order + 1] = {0._rt}; - amrex::Real sz_jz[depos_order + 1] = {0._rt}; - for (int iz=0; iz<=depos_order; iz++) - { - sz_jx[iz] = ((jx_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - sz_jy[iz] = ((jy_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - sz_jz[iz] = ((jz_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - } - int const l_jx = ((jx_type[zdir] == NODE) ? l_node : l_cell); - int const l_jy = ((jy_type[zdir] == NODE) ? l_node : l_cell); - int const l_jz = ((jz_type[zdir] == NODE) ? l_node : l_cell); + amrex::Array4 const& jx_arr = jx_fab.array(); + amrex::Array4 const& jy_arr = jy_fab.array(); + amrex::Array4 const& jz_arr = jz_fab.array(); + amrex::IntVect const jx_type = jx_fab.box().type(); + amrex::IntVect const jy_type = jy_fab.box().type(); + amrex::IntVect const jz_type = jz_fab.box().type(); - // Deposit current into jx_arr, jy_arr and jz_arr -#if defined(WARPX_DIM_1D_Z) - for (int iz=0; iz<=depos_order; iz++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+l_jx+iz, 0, 0, 0), - sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+l_jy+iz, 0, 0, 0), - sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+l_jz+iz, 0, 0, 0), - sz_jz[iz]*wqz); - } + // Loop over particles and deposit into jx_fab, jy_fab and jz_fab +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } #endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - for (int iz=0; iz<=depos_order; iz++){ - for (int ix=0; ix<=depos_order; ix++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 0), - sx_jx[ix]*sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 0), - sx_jy[ix]*sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 0), - sx_jz[ix]*sz_jz[iz]*wqz); -#if defined(WARPX_DIM_RZ) - Complex xy = xy0; // Note that xy is equal to e^{i m theta} - for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { - // The factor 2 on the weighting comes from the normalization of the modes - amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode-1), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode ), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.imag()); - amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode-1), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode ), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.imag()); - amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode-1), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode ), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.imag()); - xy = xy*xy0; - } + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); #endif - } - } -#elif defined(WARPX_DIM_3D) - for (int iz=0; iz<=depos_order; iz++){ - for (int iy=0; iy<=depos_order; iy++){ - for (int ix=0; ix<=depos_order; ix++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+j_jx+ix, lo.y+k_jx+iy, lo.z+l_jx+iz), - sx_jx[ix]*sy_jx[iy]*sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+j_jy+ix, lo.y+k_jy+iy, lo.z+l_jy+iz), - sx_jy[ix]*sy_jy[iy]*sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+j_jz+ix, lo.y+k_jz+iy, lo.z+l_jz+iz), - sx_jz[ix]*sy_jz[iy]*sz_jz[iz]*wqz); - } - } + + amrex::ParticleReal xp, yp, zp; + GetPosition(ip, xp, yp, zp); + + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); + + const amrex::Real vx = uxp[ip]*gaminv; + const amrex::Real vy = uyp[ip]*gaminv; + const amrex::Real vz = uzp[ip]*gaminv; + + amrex::Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; } -#endif + + const amrex::Real relative_time = 0._rt; + doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, + jx_type, jy_type, jz_type, + relative_time, + AMREX_D_DECL(dzi, dxi, dyi), + AMREX_D_DECL(zmin, xmin, ymin), + invvol, lo, n_rz_azimuthal_modes); + } ); #if defined(WARPX_USE_GPUCLOCK) @@ -350,7 +559,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level * \param relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, @@ -365,7 +574,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template -void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, +void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, const amrex::ParticleReal * const uxp, const amrex::ParticleReal * const uyp, @@ -374,7 +583,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jx_fab, amrex::FArrayBox& jy_fab, amrex::FArrayBox& jz_fab, - long np_to_depose, + long np_to_deposit, const amrex::Real relative_time, const std::array& dx, const std::array& xyzmin, @@ -437,7 +646,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, WARPX_ALWAYS_ASSERT_WITH_MESSAGE(shared_mem_bytes <= max_shared_mem_bytes, "Tile size too big for GPU shared memory current deposition"); - amrex::ignore_unused(np_to_depose); + amrex::ignore_unused(np_to_deposit); // Launch one thread-block per bin amrex::launch( nblocks, threads_per_block, shared_mem_bytes, amrex::Gpu::gpuStream(), @@ -545,7 +754,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, // Note, you should never reach this part of the code. This funcion cannot be called unless // using HIP/CUDA, and those things are checked prior //don't use any args - ignore_unused( GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size); + ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size); WARPX_ABORT_WITH_MESSAGE("Shared memory only implemented for HIP/CUDA"); #endif } @@ -562,7 +771,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level * \param[in] relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, @@ -577,7 +786,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template -void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, +void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, const amrex::ParticleReal * const uxp, const amrex::ParticleReal * const uyp, @@ -586,7 +795,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::Array4& Jx_arr, const amrex::Array4& Jy_arr, const amrex::Array4& Jz_arr, - long np_to_depose, + long np_to_deposit, amrex::Real dt, amrex::Real relative_time, const std::array& dx, @@ -656,7 +865,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_depose, + np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( @@ -790,17 +999,17 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // computes min/max positions of current contributions #if !defined(WARPX_DIM_1D_Z) int dil = 1, diu = 1; - if (i_old < i_new) dil = 0; - if (i_old > i_new) diu = 0; + if (i_old < i_new) { dil = 0; } + if (i_old > i_new) { diu = 0; } #endif #if defined(WARPX_DIM_3D) int djl = 1, dju = 1; - if (j_old < j_new) djl = 0; - if (j_old > j_new) dju = 0; + if (j_old < j_new) { djl = 0; } + if (j_old > j_new) { dju = 0; } #endif int dkl = 1, dku = 1; - if (k_old < k_new) dkl = 0; - if (k_old > k_new) dku = 0; + if (k_old < k_new) { dkl = 0; } + if (k_old > k_new) { dku = 0; } #if defined(WARPX_DIM_3D) @@ -927,6 +1136,403 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif } +/** + * \brief Esirkepov Current Deposition for thread thread_num for implicit scheme + * The difference from doEsirkepovDepositionShapeN is in how the old and new + * particles positions are determined and in how the particle gamma is calculated. + * + * \tparam depos_order deposition order + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp,uyp,uzp Pointer to arrays of particle momentum. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dt Time step for particle level + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, + const amrex::ParticleReal * const yp_n, + const amrex::ParticleReal * const zp_n, + const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + [[maybe_unused]]const amrex::ParticleReal * const uxp_n, + [[maybe_unused]]const amrex::ParticleReal * const uyp_n, + [[maybe_unused]]const amrex::ParticleReal * const uzp_n, + [[maybe_unused]]const amrex::ParticleReal * const uxp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uyp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uzp_nph, + const int * const ion_lev, + const amrex::Array4& Jx_arr, + const amrex::Array4& Jy_arr, + const amrex::Array4& Jz_arr, + const long np_to_deposit, + const amrex::Real dt, + const std::array& dx, + const std::array xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real * const cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif + + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + bool const do_ionization = ion_lev; +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if defined(WARPX_DIM_3D) + Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); + Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); + Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Real const invdtdx = 1.0_rt / (dt*dx[2]); + Real const invdtdz = 1.0_rt / (dt*dx[0]); + Real const invvol = 1.0_rt / (dx[0]*dx[2]); +#elif defined(WARPX_DIM_1D_Z) + Real const invdtdz = 1.0_rt / (dt*dx[0]); + Real const invvol = 1.0_rt / (dx[2]); +#endif + +#if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + + // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } +#endif + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long const ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); +#endif + +#if !defined(WARPX_DIM_3D) + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp_nph[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp_nph[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp_nph[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); +#endif + + // wqx, wqy wqz are particle current in each direction + Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; + } + + ParticleReal xp_nph, yp_nph, zp_nph; + GetPosition(ip, xp_nph, yp_nph, zp_nph); + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; +#else + ignore_unused(xp_n); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; +#else + ignore_unused(yp_n); +#endif + ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; + +#if !defined(WARPX_DIM_1D_Z) + amrex::Real const wqx = wq*invdtdx; +#endif +#if defined(WARPX_DIM_3D) + amrex::Real const wqy = wq*invdtdy; +#endif + amrex::Real const wqz = wq*invdtdz; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n[ip]; + amrex::Real const yp_old = yp_n[ip]; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_new, sintheta_new; + if (rp_new > 0._rt) { + costheta_new = xp_new/rp_new; + sintheta_new = yp_new/rp_new; + } else { + costheta_new = 1._rt; + sintheta_new = 0._rt; + } + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + amrex::Real costheta_old, sintheta_old; + if (rp_old > 0._rt) { + costheta_old = xp_old/rp_old; + sintheta_old = yp_old/rp_old; + } else { + costheta_old = 1._rt; + sintheta_old = 0._rt; + } + const Complex xy_new0 = Complex{costheta_new, sintheta_new}; + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + const Complex xy_old0 = Complex{costheta_old, sintheta_old}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; +#else +#if !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n[ip] - xmin)*dxi; +#endif +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n[ip] - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n[ip] - zmin)*dzi; + +#if defined(WARPX_DIM_RZ) + amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; +#elif defined(WARPX_DIM_XZ) + amrex::Real const vy = uyp_nph[ip]*gaminv; +#elif defined(WARPX_DIM_1D_Z) + amrex::Real const vx = uxp_nph[ip]*gaminv; + amrex::Real const vy = uyp_nph[ip]*gaminv; +#endif + + // Shape factor arrays + // Note that there are extra values above and below + // to possibly hold the factor for the old particle + // which can be at a different grid location. + // Keep these double to avoid bug in single precision +#if !defined(WARPX_DIM_1D_Z) + double sx_new[depos_order + 3] = {0.}; + double sx_old[depos_order + 3] = {0.}; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; +#endif + // Keep these double to avoid bug in single precision + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; + + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + Compute_shape_factor< depos_order > compute_shape_factor; + Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + +#if !defined(WARPX_DIM_1D_Z) + const int i_new = compute_shape_factor(sx_new+1, x_new); + const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); +#endif +#if defined(WARPX_DIM_3D) + const int j_new = compute_shape_factor(sy_new+1, y_new); + const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); +#endif + const int k_new = compute_shape_factor(sz_new+1, z_new); + const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); + + // computes min/max positions of current contributions +#if !defined(WARPX_DIM_1D_Z) + int dil = 1, diu = 1; + if (i_old < i_new) { dil = 0; } + if (i_old > i_new) { diu = 0; } +#endif +#if defined(WARPX_DIM_3D) + int djl = 1, dju = 1; + if (j_old < j_new) { djl = 0; } + if (j_old > j_new) { dju = 0; } +#endif + int dkl = 1, dku = 1; + if (k_old < k_new) { dkl = 0; } + if (k_old > k_new) { dku = 0; } + +#if defined(WARPX_DIM_3D) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int j=djl; j<=depos_order+2-dju; j++) { + amrex::Real sdxi = 0._rt; + for (int i=dil; i<=depos_order+1-diu; i++) { + sdxi += wqx*(sx_old[i] - sx_new[i])*( + one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) + +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); + } + } + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + amrex::Real sdyj = 0._rt; + for (int j=djl; j<=depos_order+1-dju; j++) { + sdyj += wqy*(sy_old[j] - sy_new[j])*( + one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) + +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); + } + } + } + for (int j=djl; j<=depos_order+2-dju; j++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + amrex::Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k])*( + one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) + +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real sdxi = 0._rt; + for (int i=dil; i<=depos_order+1-diu; i++) { + sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + const Complex djr_cmplx = 2._rt *sdxi*xy_mid; + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djr_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djr_cmplx.imag()); + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + Real const sdyj = wq*vy*invvol*( + one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) + +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); +#if defined(WARPX_DIM_RZ) + Complex xy_new = xy_new0; + Complex xy_mid = xy_mid0; + Complex xy_old = xy_old0; + // Throughout the following loop, xy_ takes the value e^{i m theta_} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + // The minus sign comes from the different convention with respect to Davidson et al. + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djt_cmplx.imag()); + xy_new = xy_new*xy_new0; + xy_mid = xy_mid*xy_mid0; + xy_old = xy_old*xy_old0; + } +#endif + } + } + for (int i=dil; i<=depos_order+2-diu; i++) { + Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + const Complex djz_cmplx = 2._rt * sdzk * xy_mid; + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djz_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djz_cmplx.imag()); + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } +#elif defined(WARPX_DIM_1D_Z) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real const sdxi = wq*vx*invvol*0.5_rt*(sz_old[k] + sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+k_new-1+k, 0, 0, 0), sdxi); + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real const sdyj = wq*vy*invvol*0.5_rt*(sz_old[k] + sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+k_new-1+k, 0, 0, 0), sdyj); + } + amrex::Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); + } +#endif + } + ); +#if defined(WARPX_USE_GPUCLOCK) + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + amrex::Gpu::streamSynchronize(); + *cost += *cost_real; + amrex::The_Managed_Arena()->free(cost_real); + } +#endif +} + /** * \brief Vay current deposition * ( Vay et al, 2013) @@ -941,7 +1547,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, required to have the charge of each macroparticle since \c q is a scalar. For non-ionizable species, \c ion_lev is \c null * \param[in,out] Dx_fab,Dy_fab,Dz_fab FArrayBox of Vay current density, either full array or tile - * \param[in] np_to_depose Number of particles for which current is deposited + * \param[in] np_to_deposit Number of particles for which current is deposited * \param[in] dt Time step for particle level * \param[in] relative_time Time at which to deposit D, relative to the time of the * current positions of the particles. When different than 0, @@ -957,7 +1563,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, * \param[in] load_balance_costs_update_algo Selected method for updating load balance costs */ template -void doVayDepositionShapeN (const GetParticlePosition& GetPosition, +void doVayDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::ParticleReal* const wp, const amrex::ParticleReal* const uxp, const amrex::ParticleReal* const uyp, @@ -966,7 +1572,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& Dx_fab, amrex::FArrayBox& Dy_fab, amrex::FArrayBox& Dz_fab, - long np_to_depose, + long np_to_deposit, amrex::Real dt, amrex::Real relative_time, const std::array& dx, @@ -982,14 +1588,14 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, #if defined(WARPX_DIM_RZ) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_depose, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); + np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in RZ geometry"); #endif #if defined(WARPX_DIM_1D_Z) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_depose, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); + np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in cartesian 1D geometry"); #endif @@ -1054,7 +1660,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, *cost_real = 0._rt; } #endif - amrex::ParallelFor(np_to_depose, [=] AMREX_GPU_DEVICE (long ip) + amrex::ParallelFor(np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( @@ -1068,7 +1674,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, + uzp[ip] * uzp[ip] * invcsq); // Product of particle charges and weights amrex::Real wq = q * wp[ip]; - if (do_ionization) wq *= ion_lev[ip]; + if (do_ionization) { wq *= ion_lev[ip]; } // Current particle positions (in physical units) amrex::ParticleReal xp, yp, zp; diff --git a/Source/Particles/Deposition/SharedDepositionUtils.H b/Source/Particles/Deposition/SharedDepositionUtils.H index 40407b609c0..e28835a57df 100644 --- a/Source/Particles/Deposition/SharedDepositionUtils.H +++ b/Source/Particles/Deposition/SharedDepositionUtils.H @@ -65,7 +65,7 @@ void addLocalToGlobal (const amrex::Box& bx, #if defined(AMREX_USE_HIP) || defined(AMREX_USE_CUDA) template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void depositComponent (const GetParticlePosition& GetPosition, +void depositComponent (const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, const amrex::ParticleReal * const uxp, const amrex::ParticleReal * const uyp, @@ -132,7 +132,7 @@ void depositComponent (const GetParticlePosition& GetPosition, // pcurrent is the particle current in the deposited direction #if defined(WARPX_DIM_RZ) // In RZ, wqx is actually wqr, and wqy is wqtheta - // Convert to cylinderical at the mid point + // Convert to cylindrical at the mid point const amrex::Real xpmid = xp + relative_time*vx; const amrex::Real ypmid = yp + relative_time*vy; const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index b6b74644825..573491645ea 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -37,8 +37,14 @@ struct IonizationFilterFunc int comp; int m_atomic_number; - GetParticlePosition m_get_position; + GetParticlePosition m_get_position; GetExternalEBField m_get_externalEB; + amrex::ParticleReal m_Ex_external_particle; + amrex::ParticleReal m_Ey_external_particle; + amrex::ParticleReal m_Ez_external_particle; + amrex::ParticleReal m_Bx_external_particle; + amrex::ParticleReal m_By_external_particle; + amrex::ParticleReal m_Bz_external_particle; amrex::Array4 m_ex_arr; amrex::Array4 m_ey_arr; @@ -70,6 +76,8 @@ struct IonizationFilterFunc amrex::FArrayBox const& bxfab, amrex::FArrayBox const& byfab, amrex::FArrayBox const& bzfab, + amrex::Vector& E_external_particle, + amrex::Vector& B_external_particle, const amrex::Real* AMREX_RESTRICT a_ionization_energies, const amrex::Real* AMREX_RESTRICT a_adk_prefactor, const amrex::Real* AMREX_RESTRICT a_adk_exp_prefactor, @@ -94,8 +102,12 @@ struct IonizationFilterFunc amrex::ParticleReal xp, yp, zp; m_get_position(i, xp, yp, zp); - amrex::ParticleReal ex = 0._rt, ey = 0._rt, ez = 0._rt; - amrex::ParticleReal bx = 0._rt, by = 0._rt, bz = 0._rt; + amrex::ParticleReal ex = m_Ex_external_particle; + amrex::ParticleReal ey = m_Ey_external_particle; + amrex::ParticleReal ez = m_Ez_external_particle; + amrex::ParticleReal bx = m_Bx_external_particle; + amrex::ParticleReal by = m_By_external_particle; + amrex::ParticleReal bz = m_Bz_external_particle; m_get_externalEB(i, ex, ey, ez, bx, by, bz); doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, @@ -103,6 +115,7 @@ struct IonizationFilterFunc m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); + m_get_externalEB(i, ex, ey, ez, bx, by, bz); // Compute electric field amplitude in the particle's frame of // reference (particularly important when in boosted frame). @@ -110,7 +123,7 @@ struct IonizationFilterFunc const amrex::ParticleReal uy = ptd.m_rdata[PIdx::uy][i]; const amrex::ParticleReal uz = ptd.m_rdata[PIdx::uz][i]; - const amrex::Real ga = static_cast( + const auto ga = static_cast( std::sqrt(1. + (ux*ux + uy*uy + uz*uz) * c2_inv)); const amrex::Real E = std::sqrt( - ( ux*ex + uy*ey + uz*ez ) * ( ux*ex + uy*ey + uz*ez ) * c2_inv diff --git a/Source/Particles/ElementaryProcess/Ionization.cpp b/Source/Particles/ElementaryProcess/Ionization.cpp index a4aac9d5e0f..c3681a30cad 100644 --- a/Source/Particles/ElementaryProcess/Ionization.cpp +++ b/Source/Particles/ElementaryProcess/Ionization.cpp @@ -24,25 +24,34 @@ IonizationFilterFunc::IonizationFilterFunc (const WarpXParIter& a_pti, int lev, amrex::FArrayBox const& bxfab, amrex::FArrayBox const& byfab, amrex::FArrayBox const& bzfab, + amrex::Vector& E_external_particle, + amrex::Vector& B_external_particle, const amrex::Real* const AMREX_RESTRICT a_ionization_energies, const amrex::Real* const AMREX_RESTRICT a_adk_prefactor, const amrex::Real* const AMREX_RESTRICT a_adk_exp_prefactor, const amrex::Real* const AMREX_RESTRICT a_adk_power, int a_comp, int a_atomic_number, - int a_offset) noexcept + int a_offset) noexcept: + m_ionization_energies{a_ionization_energies}, + m_adk_prefactor{a_adk_prefactor}, + m_adk_exp_prefactor{a_adk_exp_prefactor}, + m_adk_power{a_adk_power}, + comp{a_comp}, + m_atomic_number{a_atomic_number}, + m_Ex_external_particle{E_external_particle[0]}, + m_Ey_external_particle{E_external_particle[1]}, + m_Ez_external_particle{E_external_particle[2]}, + m_Bx_external_particle{B_external_particle[0]}, + m_By_external_particle{B_external_particle[1]}, + m_Bz_external_particle{B_external_particle[2]}, + m_galerkin_interpolation{WarpX::galerkin_interpolation}, + m_nox{WarpX::nox}, + m_n_rz_azimuthal_modes{WarpX::n_rz_azimuthal_modes} { - using namespace amrex::literals; - m_ionization_energies = a_ionization_energies; - m_adk_prefactor = a_adk_prefactor; - m_adk_exp_prefactor = a_adk_exp_prefactor; - m_adk_power = a_adk_power; - comp = a_comp; - m_atomic_number = a_atomic_number; - - m_get_position = GetParticlePosition(a_pti, a_offset); + m_get_position = GetParticlePosition(a_pti, a_offset); m_get_externalEB = GetExternalEBField(a_pti, a_offset); m_ex_arr = exfab.array(); @@ -69,9 +78,5 @@ IonizationFilterFunc::IonizationFilterFunc (const WarpXParIter& a_pti, int lev, const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - m_galerkin_interpolation = WarpX::galerkin_interpolation; - m_nox = WarpX::nox; - m_n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - m_lo = amrex::lbound(box); } diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H index 2abd31fb4e4..3f9ef133477 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H @@ -170,8 +170,9 @@ public: amrex::ParticleReal, pxr_p::unit_system::SI>( px, py, pz); if (gamma_photon < pxr_m::two || - chi_phot < m_bw_minimum_chi_phot) + chi_phot < m_bw_minimum_chi_phot) { return 0; + } const auto is_out = pxr_bw::evolve_optical_depth< amrex::ParticleReal, @@ -300,22 +301,22 @@ public: /** * Builds the functor to initialize the optical depth */ - BreitWheelerGetOpticalDepth build_optical_depth_functor () const; + [[nodiscard]] BreitWheelerGetOpticalDepth build_optical_depth_functor () const; /** * Builds the functor to evolve the optical depth */ - BreitWheelerEvolveOpticalDepth build_evolve_functor () const; + [[nodiscard]] BreitWheelerEvolveOpticalDepth build_evolve_functor () const; /** * Builds the functor to generate the pairs */ - BreitWheelerGeneratePairs build_pair_functor () const; + [[nodiscard]] BreitWheelerGeneratePairs build_pair_functor () const; /** * Checks if the optical tables are properly initialized */ - bool are_lookup_tables_initialized () const; + [[nodiscard]] bool are_lookup_tables_initialized () const; /** * Export lookup tables data into a raw binary Vector @@ -323,7 +324,7 @@ public: * @return the data in binary format. The Vector is empty if tables were * not previously initialized. */ - std::vector export_lookup_tables_data () const; + [[nodiscard]] std::vector export_lookup_tables_data () const; /** * Init lookup tables from raw binary data. @@ -357,9 +358,9 @@ public: * * @return default control params to generate the tables */ - PicsarBreitWheelerCtrl get_default_ctrl() const; + [[nodiscard]] PicsarBreitWheelerCtrl get_default_ctrl() const; - amrex::ParticleReal get_minimum_chi_phot() const; + [[nodiscard]] amrex::ParticleReal get_minimum_chi_phot() const; private: bool m_lookup_tables_initialized = false; diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp index 12f479d32ed..da50d41dc8f 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp @@ -30,7 +30,7 @@ using namespace std; using namespace amrex; namespace pxr_sr = picsar::multi_physics::utils::serialization; -//This file provides a wrapper aroud the breit_wheeler engine +//This file provides a wrapper around the breit_wheeler engine //provided by the PICSAR library // Factory class ============================= @@ -69,19 +69,20 @@ BreitWheelerEngine::init_lookup_tables_from_raw_data ( { auto raw_iter = raw_data.begin(); const auto size_first = pxr_sr::get_out(raw_iter); - if(size_first <= 0 || size_first >= raw_data.size() ) return false; + if(size_first <= 0 || size_first >= raw_data.size() ) { return false; } const auto raw_dndt_table = vector{ - raw_iter, raw_iter+size_first}; + raw_iter, raw_iter+static_cast(size_first)}; const auto raw_pair_prod_table = vector{ - raw_iter+size_first, raw_data.end()}; + raw_iter+static_cast(size_first), raw_data.end()}; m_dndt_table = BW_dndt_table{raw_dndt_table}; m_pair_prod_table = BW_pair_prod_table{raw_pair_prod_table}; - if (!m_dndt_table.is_init() || !m_pair_prod_table.is_init()) + if (!m_dndt_table.is_init() || !m_pair_prod_table.is_init()) { return false; + } m_bw_minimum_chi_phot = bw_minimum_chi_phot; @@ -104,8 +105,9 @@ void BreitWheelerEngine::init_builtin_tables( vector BreitWheelerEngine::export_lookup_tables_data () const { - if(!m_lookup_tables_initialized) + if(!m_lookup_tables_initialized) { return vector{}; + } const auto data_dndt = m_dndt_table.serialize(); const auto data_pair_prod = m_pair_prod_table.serialize(); @@ -114,10 +116,12 @@ vector BreitWheelerEngine::export_lookup_tables_data () const vector res{}; pxr_sr::put_in(size_first, res); - for (const auto& tmp : data_dndt) + for (const auto& tmp : data_dndt) { pxr_sr::put_in(tmp, res); - for (const auto& tmp : data_pair_prod) + } + for (const auto& tmp : data_pair_prod) { pxr_sr::put_in(tmp, res); + } return res; } diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H b/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H index 3b2093c40b7..4f26c256c76 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H @@ -36,7 +36,7 @@ /** * PICSAR uses internally some specifiers analogous to * AMREX_RESTRICT and AMREX_FORCE_INLINE. These definitions - * set the aformentioned specifiers to AMREX_RESTRICT and + * set the aforementioned specifiers to AMREX_RESTRICT and * AMREX_FORCE_INLINE. */ #define PXRMP_RESTRICT AMREX_RESTRICT diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H index 33660cde233..3ae27e411b6 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H @@ -153,8 +153,9 @@ public: const auto chi_part = QedUtils::chi_ele_pos( m_e*ux, m_e*uy, m_e*uz, ex, ey, ez, bx, by, bz); - if (chi_part < m_qs_minimum_chi_part) + if (chi_part < m_qs_minimum_chi_part) { return 0; + } const auto is_out = pxr_qs::evolve_optical_depth< amrex::ParticleReal, @@ -279,22 +280,22 @@ public: /** * Builds the functor to initialize the optical depth */ - QuantumSynchrotronGetOpticalDepth build_optical_depth_functor (); + [[nodiscard]] QuantumSynchrotronGetOpticalDepth build_optical_depth_functor (); /** * Builds the functor to evolve the optical depth */ - QuantumSynchrotronEvolveOpticalDepth build_evolve_functor (); + [[nodiscard]] QuantumSynchrotronEvolveOpticalDepth build_evolve_functor (); /** * Builds the functor to generate photons */ - QuantumSynchrotronPhotonEmission build_phot_em_functor (); + [[nodiscard]] QuantumSynchrotronPhotonEmission build_phot_em_functor (); /** * Checks if the optical tables are properly initialized */ - bool are_lookup_tables_initialized () const; + [[nodiscard]] bool are_lookup_tables_initialized () const; /** * Export lookup tables data into a raw binary Vector @@ -302,7 +303,7 @@ public: * @return the data in binary format. The Vector is empty if tables were * not previously initialized. */ - std::vector export_lookup_tables_data () const; + [[nodiscard]] std::vector export_lookup_tables_data () const; /** * Init lookup tables from raw binary data. @@ -335,9 +336,9 @@ public: * * @return default control params to generate the tables */ - PicsarQuantumSyncCtrl get_default_ctrl() const; + [[nodiscard]] PicsarQuantumSyncCtrl get_default_ctrl() const; - amrex::ParticleReal get_minimum_chi_part() const; + [[nodiscard]] amrex::ParticleReal get_minimum_chi_part() const; private: bool m_lookup_tables_initialized = false; diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp index c9bede89167..59a881814c6 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp @@ -30,7 +30,7 @@ using namespace std; using namespace amrex; namespace pxr_sr = picsar::multi_physics::utils::serialization; -//This file provides a wrapper aroud the quantum_sync engine +//This file provides a wrapper around the quantum_sync engine //provided by the PICSAR library // Factory class ============================= @@ -68,19 +68,20 @@ QuantumSynchrotronEngine::init_lookup_tables_from_raw_data ( { auto raw_iter = raw_data.begin(); const auto size_first = pxr_sr::get_out(raw_iter); - if(size_first <= 0 || size_first >= raw_data.size() ) return false; + if(size_first <= 0 || size_first >= raw_data.size() ) { return false; } const auto raw_dndt_table = vector{ - raw_iter, raw_iter+size_first}; + raw_iter, raw_iter+static_cast(size_first)}; const auto raw_phot_em_table = vector{ - raw_iter+size_first, raw_data.end()}; + raw_iter+static_cast(size_first), raw_data.end()}; m_dndt_table = QS_dndt_table{raw_dndt_table}; m_phot_em_table = QS_phot_em_table{raw_phot_em_table}; - if (!m_dndt_table.is_init() || !m_phot_em_table.is_init()) + if (!m_dndt_table.is_init() || !m_phot_em_table.is_init()) { return false; + } m_qs_minimum_chi_part = qs_minimum_chi_part; @@ -103,8 +104,9 @@ void QuantumSynchrotronEngine::init_builtin_tables( vector QuantumSynchrotronEngine::export_lookup_tables_data () const { - if(!m_lookup_tables_initialized) + if(!m_lookup_tables_initialized) { return vector{}; + } const auto data_dndt = m_dndt_table.serialize(); const auto data_phot_em = m_phot_em_table.serialize(); @@ -113,10 +115,12 @@ vector QuantumSynchrotronEngine::export_lookup_tables_data () const vector res{}; pxr_sr::put_in(size_first, res); - for (const auto& tmp : data_dndt) + for (const auto& tmp : data_dndt) { pxr_sr::put_in(tmp, res); - for (const auto& tmp : data_phot_em) + } + for (const auto& tmp : data_phot_em) { pxr_sr::put_in(tmp, res); + } return res; } diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index dbdbd0e6d53..5abc9282d4f 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -104,6 +104,8 @@ public: amrex::FArrayBox const& bxfab, amrex::FArrayBox const& byfab, amrex::FArrayBox const& bzfab, + amrex::Vector& E_external_particle, + amrex::Vector& B_external_particle, int a_offset = 0); /** @@ -130,8 +132,13 @@ public: amrex::ParticleReal xp, yp, zp; m_get_position(i_src, xp, yp, zp); - amrex::ParticleReal ex = 0._rt, ey = 0._rt, ez = 0._rt; - amrex::ParticleReal bx = 0._rt, by = 0._rt, bz = 0._rt; + amrex::ParticleReal ex = m_Ex_external_particle; + amrex::ParticleReal ey = m_Ey_external_particle; + amrex::ParticleReal ez = m_Ez_external_particle; + amrex::ParticleReal bx = m_Bx_external_particle; + amrex::ParticleReal by = m_By_external_particle; + amrex::ParticleReal bz = m_Bz_external_particle; + m_get_externalEB(i_src, ex, ey, ez, bx, by, bz); doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, @@ -168,8 +175,14 @@ private: const BreitWheelerGeneratePairs m_generate_functor; /*!< A copy of the functor to generate pairs. It contains only pointers to the lookup tables.*/ - GetParticlePosition m_get_position; + GetParticlePosition m_get_position; GetExternalEBField m_get_externalEB; + amrex::ParticleReal m_Ex_external_particle; + amrex::ParticleReal m_Ey_external_particle; + amrex::ParticleReal m_Ez_external_particle; + amrex::ParticleReal m_Bx_external_particle; + amrex::ParticleReal m_By_external_particle; + amrex::ParticleReal m_Bz_external_particle; amrex::Array4 m_ex_arr; amrex::Array4 m_ey_arr; diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp index 68475aa144b..2b380d454f4 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp @@ -26,13 +26,24 @@ PairGenerationTransformFunc (BreitWheelerGeneratePairs const generate_functor, amrex::FArrayBox const& bxfab, amrex::FArrayBox const& byfab, amrex::FArrayBox const& bzfab, - int a_offset) -: m_generate_functor(generate_functor) + amrex::Vector& E_external_particle, + amrex::Vector& B_external_particle, + int a_offset): + m_generate_functor{generate_functor}, + m_Ex_external_particle{E_external_particle[0]}, + m_Ey_external_particle{E_external_particle[1]}, + m_Ez_external_particle{E_external_particle[2]}, + m_Bx_external_particle{B_external_particle[0]}, + m_By_external_particle{B_external_particle[1]}, + m_Bz_external_particle{B_external_particle[2]}, + m_galerkin_interpolation{WarpX::galerkin_interpolation}, + m_nox{WarpX::nox}, + m_n_rz_azimuthal_modes{WarpX::n_rz_azimuthal_modes} { using namespace amrex::literals; - m_get_position = GetParticlePosition(a_pti, a_offset); + m_get_position = GetParticlePosition(a_pti, a_offset); m_get_externalEB = GetExternalEBField(a_pti, a_offset); m_ex_arr = exfab.array(); @@ -59,9 +70,5 @@ PairGenerationTransformFunc (BreitWheelerGeneratePairs const generate_functor, const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - m_galerkin_interpolation = WarpX::galerkin_interpolation; - m_nox = WarpX::nox; - m_n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - m_lo = amrex::lbound(box); } diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index 080555e6fe8..8ba5c63ad57 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -118,6 +118,8 @@ public: amrex::FArrayBox const& bxfab, amrex::FArrayBox const& byfab, amrex::FArrayBox const& bzfab, + amrex::Vector& E_external_particle, + amrex::Vector& B_external_particle, int a_offset = 0); /** @@ -141,8 +143,13 @@ public: amrex::ParticleReal xp, yp, zp; m_get_position(i_src, xp, yp, zp); - amrex::ParticleReal ex = 0._rt, ey = 0._rt, ez = 0._rt; - amrex::ParticleReal bx = 0._rt, by = 0._rt, bz = 0._rt; + amrex::ParticleReal ex = m_Ex_external_particle; + amrex::ParticleReal ey = m_Ey_external_particle; + amrex::ParticleReal ez = m_Ez_external_particle; + amrex::ParticleReal bx = m_Bx_external_particle; + amrex::ParticleReal by = m_By_external_particle; + amrex::ParticleReal bz = m_Bz_external_particle; + m_get_externalEB(i_src, ex, ey, ez, bx, by, bz); doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, @@ -178,8 +185,14 @@ private: const QuantumSynchrotronPhotonEmission m_emission_functor; /*!< A copy of the functor to generate photons. It contains only pointers to the lookup tables.*/ - GetParticlePosition m_get_position; + GetParticlePosition m_get_position; GetExternalEBField m_get_externalEB; + amrex::ParticleReal m_Ex_external_particle; + amrex::ParticleReal m_Ey_external_particle; + amrex::ParticleReal m_Ez_external_particle; + amrex::ParticleReal m_Bx_external_particle; + amrex::ParticleReal m_By_external_particle; + amrex::ParticleReal m_Bz_external_particle; amrex::Array4 m_ex_arr; amrex::Array4 m_ey_arr; diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp index 2c060a30f01..077a4659ce5 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp @@ -27,15 +27,26 @@ PhotonEmissionTransformFunc (QuantumSynchrotronGetOpticalDepth opt_depth_functor amrex::FArrayBox const& bxfab, amrex::FArrayBox const& byfab, amrex::FArrayBox const& bzfab, - int a_offset) -:m_opt_depth_functor{opt_depth_functor}, - m_opt_depth_runtime_comp{opt_depth_runtime_comp}, - m_emission_functor{emission_functor} + amrex::Vector& E_external_particle, + amrex::Vector& B_external_particle, + int a_offset): + m_opt_depth_functor{opt_depth_functor}, + m_opt_depth_runtime_comp{opt_depth_runtime_comp}, + m_emission_functor{emission_functor}, + m_Ex_external_particle{E_external_particle[0]}, + m_Ey_external_particle{E_external_particle[1]}, + m_Ez_external_particle{E_external_particle[2]}, + m_Bx_external_particle{B_external_particle[0]}, + m_By_external_particle{B_external_particle[1]}, + m_Bz_external_particle{B_external_particle[2]}, + m_galerkin_interpolation{WarpX::galerkin_interpolation}, + m_nox{WarpX::nox}, + m_n_rz_azimuthal_modes{WarpX::n_rz_azimuthal_modes} { using namespace amrex::literals; - m_get_position = GetParticlePosition(a_pti, a_offset); + m_get_position = GetParticlePosition(a_pti, a_offset); m_get_externalEB = GetExternalEBField(a_pti, a_offset); m_ex_arr = exfab.array(); @@ -62,9 +73,7 @@ PhotonEmissionTransformFunc (QuantumSynchrotronGetOpticalDepth opt_depth_functor const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - m_galerkin_interpolation = WarpX::galerkin_interpolation; - m_nox = WarpX::nox; - m_n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + m_lo = amrex::lbound(box); } diff --git a/Source/Particles/Filter/FilterFunctors.H b/Source/Particles/Filter/FilterFunctors.H index fb83df1184a..354d57e2c98 100644 --- a/Source/Particles/Filter/FilterFunctors.H +++ b/Source/Particles/Filter/FilterFunctors.H @@ -97,8 +97,8 @@ struct ParserFilter m_is_active{a_is_active}, m_function_partparser{a_filter_parser}, m_mass{a_mass}, - m_t{time}, - m_units{InputUnits::WarpX}{} + m_t{time} + {} /** * \brief return 1 if the particle is selected by the parser @@ -143,7 +143,7 @@ public: /** Store physical time on the coarsest level. */ amrex::Real m_t; /** keep track of momentum units particles will come in with **/ - InputUnits m_units; + InputUnits m_units{InputUnits::WarpX}; }; @@ -157,14 +157,14 @@ struct GeometryFilter GeometryFilter(bool a_is_active, amrex::RealBox a_domain) : m_is_active(a_is_active), m_domain(a_domain) {} /** - * \brief return 1 if the partcile is within the region described by the RealBox + * \brief return 1 if the particle is within the region described by the RealBox * \param p one particle * \return whether or not the particle is inside the region defined by m_domain */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool operator () (const SuperParticleType& p, const amrex::RandomEngine&) const noexcept { - if ( !m_is_active ) return 1; + if ( !m_is_active ) { return true; } return ! (AMREX_D_TERM( (p.pos(0) < m_domain.lo(0)) || (p.pos(0) > m_domain.hi(0) ), || (p.pos(1) < m_domain.lo(1)) || (p.pos(1) > m_domain.hi(1) ), || (p.pos(2) < m_domain.lo(2)) || (p.pos(2) > m_domain.hi(2) ))); diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index 4f5064a0e9b..dd6b7276681 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -252,7 +252,7 @@ void doGatherShapeN (const amrex::ParticleReal xp, by_arr(lo.x+l_by+iz, 0, 0, 0); } -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_XZ) // Gather field on particle Eyp from field on grid ey_arr for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ @@ -288,7 +288,47 @@ void doGatherShapeN (const amrex::ParticleReal xp, } } -#ifdef WARPX_DIM_RZ +#elif defined(WARPX_DIM_RZ) + + amrex::ParticleReal Erp = 0.; + amrex::ParticleReal Ethetap = 0.; + amrex::ParticleReal Brp = 0.; + amrex::ParticleReal Bthetap = 0.; + + // Gather field on particle Ethetap from field on grid ey_arr + for (int iz=0; iz<=depos_order; iz++){ + for (int ix=0; ix<=depos_order; ix++){ + Ethetap += sx_ey[ix]*sz_ey[iz]* + ey_arr(lo.x+j_ey+ix, lo.y+l_ey+iz, 0, 0); + } + } + // Gather field on particle Erp from field on grid ex_arr + // Gather field on particle Bzp from field on grid bz_arr + for (int iz=0; iz<=depos_order; iz++){ + for (int ix=0; ix<=depos_order-galerkin_interpolation; ix++){ + Erp += sx_ex[ix]*sz_ex[iz]* + ex_arr(lo.x+j_ex+ix, lo.y+l_ex+iz, 0, 0); + Bzp += sx_bz[ix]*sz_bz[iz]* + bz_arr(lo.x+j_bz+ix, lo.y+l_bz+iz, 0, 0); + } + } + // Gather field on particle Ezp from field on grid ez_arr + // Gather field on particle Brp from field on grid bx_arr + for (int iz=0; iz<=depos_order-galerkin_interpolation; iz++){ + for (int ix=0; ix<=depos_order; ix++){ + Ezp += sx_ez[ix]*sz_ez[iz]* + ez_arr(lo.x+j_ez+ix, lo.y+l_ez+iz, 0, 0); + Brp += sx_bx[ix]*sz_bx[iz]* + bx_arr(lo.x+j_bx+ix, lo.y+l_bx+iz, 0, 0); + } + } + // Gather field on particle Bthetap from field on grid by_arr + for (int iz=0; iz<=depos_order-galerkin_interpolation; iz++){ + for (int ix=0; ix<=depos_order-galerkin_interpolation; ix++){ + Bthetap += sx_by[ix]*sz_by[iz]* + by_arr(lo.x+j_by+ix, lo.y+l_by+iz, 0, 0); + } + } amrex::Real costheta; amrex::Real sintheta; @@ -304,28 +344,28 @@ void doGatherShapeN (const amrex::ParticleReal xp, for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { - // Gather field on particle Eyp from field on grid ey_arr + // Gather field on particle Ethetap from field on grid ey_arr for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ const amrex::Real dEy = (+ ey_arr(lo.x+j_ey+ix, lo.y+l_ey+iz, 0, 2*imode-1)*xy.real() - ey_arr(lo.x+j_ey+ix, lo.y+l_ey+iz, 0, 2*imode)*xy.imag()); - Eyp += sx_ey[ix]*sz_ey[iz]*dEy; + Ethetap += sx_ey[ix]*sz_ey[iz]*dEy; } } - // Gather field on particle Exp from field on grid ex_arr + // Gather field on particle Erp from field on grid ex_arr // Gather field on particle Bzp from field on grid bz_arr for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order-galerkin_interpolation; ix++){ const amrex::Real dEx = (+ ex_arr(lo.x+j_ex+ix, lo.y+l_ex+iz, 0, 2*imode-1)*xy.real() - ex_arr(lo.x+j_ex+ix, lo.y+l_ex+iz, 0, 2*imode)*xy.imag()); - Exp += sx_ex[ix]*sz_ex[iz]*dEx; + Erp += sx_ex[ix]*sz_ex[iz]*dEx; const amrex::Real dBz = (+ bz_arr(lo.x+j_bz+ix, lo.y+l_bz+iz, 0, 2*imode-1)*xy.real() - bz_arr(lo.x+j_bz+ix, lo.y+l_bz+iz, 0, 2*imode)*xy.imag()); Bzp += sx_bz[ix]*sz_bz[iz]*dBz; } } // Gather field on particle Ezp from field on grid ez_arr - // Gather field on particle Bxp from field on grid bx_arr + // Gather field on particle Brp from field on grid bx_arr for (int iz=0; iz<=depos_order-galerkin_interpolation; iz++){ for (int ix=0; ix<=depos_order; ix++){ const amrex::Real dEz = (+ ez_arr(lo.x+j_ez+ix, lo.y+l_ez+iz, 0, 2*imode-1)*xy.real() @@ -333,28 +373,25 @@ void doGatherShapeN (const amrex::ParticleReal xp, Ezp += sx_ez[ix]*sz_ez[iz]*dEz; const amrex::Real dBx = (+ bx_arr(lo.x+j_bx+ix, lo.y+l_bx+iz, 0, 2*imode-1)*xy.real() - bx_arr(lo.x+j_bx+ix, lo.y+l_bx+iz, 0, 2*imode)*xy.imag()); - Bxp += sx_bx[ix]*sz_bx[iz]*dBx; + Brp += sx_bx[ix]*sz_bx[iz]*dBx; } } - // Gather field on particle Byp from field on grid by_arr + // Gather field on particle Bthetap from field on grid by_arr for (int iz=0; iz<=depos_order-galerkin_interpolation; iz++){ for (int ix=0; ix<=depos_order-galerkin_interpolation; ix++){ const amrex::Real dBy = (+ by_arr(lo.x+j_by+ix, lo.y+l_by+iz, 0, 2*imode-1)*xy.real() - by_arr(lo.x+j_by+ix, lo.y+l_by+iz, 0, 2*imode)*xy.imag()); - Byp += sx_by[ix]*sz_by[iz]*dBy; + Bthetap += sx_by[ix]*sz_by[iz]*dBy; } } xy = xy*xy0; } - // Convert Exp and Eyp (which are actually Er and Etheta) to Ex and Ey - const amrex::Real Exp_save = Exp; - Exp = costheta*Exp - sintheta*Eyp; - Eyp = costheta*Eyp + sintheta*Exp_save; - const amrex::Real Bxp_save = Bxp; - Bxp = costheta*Bxp - sintheta*Byp; - Byp = costheta*Byp + sintheta*Bxp_save; -#endif + // Convert Erp and Ethetap to Ex and Ey + Exp += costheta*Erp - sintheta*Ethetap; + Eyp += costheta*Ethetap + sintheta*Erp; + Bxp += costheta*Brp - sintheta*Bthetap; + Byp += costheta*Bthetap + sintheta*Brp; #else // defined(WARPX_DIM_3D) // Gather field on particle Exp from field on grid ex_arr @@ -414,6 +451,435 @@ void doGatherShapeN (const amrex::ParticleReal xp, #endif } +/** + * \brief Energy conserving field gather for thread thread_num for the implicit scheme + * This uses the same stencil for the gather that is used for Esirkepov current deposition. + * + * \tparam depos_order Particle shape order + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half step + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param Ex_arr,Ey_arr,Ez_arr Array4 of the electric field, either full array or tile. + * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. + * \param Ex_type,Ey_type,Ez_type IndexType of the electric field + * \param Bx_type,By_type,Bz_type IndexType of the magnetic field + * \param dx 3D cell spacing + * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + */ +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherShapeNEsirkepovStencilImplicit ( + [[maybe_unused]] const amrex::ParticleReal xp_n, + [[maybe_unused]] const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + [[maybe_unused]] const amrex::ParticleReal xp_nph, + [[maybe_unused]] const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& Ex_arr, + amrex::Array4 const& Ey_arr, + amrex::Array4 const& Ez_arr, + amrex::Array4 const& Bx_arr, + amrex::Array4 const& By_arr, + amrex::Array4 const& Bz_arr, + [[maybe_unused]] const amrex::IndexType Ex_type, + [[maybe_unused]] const amrex::IndexType Ey_type, + [[maybe_unused]] const amrex::IndexType Ez_type, + [[maybe_unused]] const amrex::IndexType Bx_type, + [[maybe_unused]] const amrex::IndexType By_type, + [[maybe_unused]] const amrex::IndexType Bz_type, + const amrex::GpuArray& dx, + const amrex::GpuArray& xyzmin, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal xp_np1 = 2._prt*xp_nph - xp_n; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal yp_np1 = 2._prt*yp_nph - yp_n; +#endif + ParticleReal zp_np1 = 2._prt*zp_nph - zp_n; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n; + amrex::Real const yp_old = yp_n; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; +#else +#if !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n - xmin)*dxi; +#endif +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n - zmin)*dzi; + + // Shape factor arrays + // Note that there are extra values above and below + // to possibly hold the factor for the old particle + // which can be at a different grid location. + // Keep these double to avoid bug in single precision +#if !defined(WARPX_DIM_1D_Z) + double sx_E_new[depos_order + 3] = {0.}; + double sx_E_old[depos_order + 3] = {0.}; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double sy_E_new[depos_order + 3] = {0.}; + double sy_E_old[depos_order + 3] = {0.}; +#endif + // Keep these double to avoid bug in single precision + double sz_E_new[depos_order + 3] = {0.}; + double sz_E_old[depos_order + 3] = {0.}; + +#if defined(WARPX_DIM_3D) + double sx_B_new[depos_order + 3] = {0.}; + double sx_B_old[depos_order + 3] = {0.}; + double sy_B_new[depos_order + 3] = {0.}; + double sy_B_old[depos_order + 3] = {0.}; + double sz_B_new[depos_order + 3] = {0.}; + double sz_B_old[depos_order + 3] = {0.}; +#endif + +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + // Special shape functions are needed for By which is cell + // centered in both x and z. One lower order shape function is used. + double sx_By_new[depos_order + 2] = {0.}; + double sx_By_old[depos_order + 2] = {0.}; + double sz_By_new[depos_order + 2] = {0.}; + double sz_By_old[depos_order + 2] = {0.}; +#endif + + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + Compute_shape_factor< depos_order > compute_shape_factor; + Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + +#if !defined(WARPX_DIM_1D_Z) + const int i_E_new = compute_shape_factor(sx_E_new+1, x_new); + const int i_E_old = compute_shifted_shape_factor(sx_E_old, x_old, i_E_new); +#endif +#if defined(WARPX_DIM_3D) + const int j_E_new = compute_shape_factor(sy_E_new+1, y_new); + const int j_E_old = compute_shifted_shape_factor(sy_E_old, y_old, j_E_new); +#endif + const int k_E_new = compute_shape_factor(sz_E_new+1, z_new); + const int k_E_old = compute_shifted_shape_factor(sz_E_old, z_old, k_E_new); + +#if defined(WARPX_DIM_3D) + const int i_B_new = compute_shape_factor(sx_B_new+1, x_new - 0.5_rt); + const int i_B_old = compute_shifted_shape_factor(sx_B_old, x_old - 0.5_rt, i_B_new); + const int j_B_new = compute_shape_factor(sy_B_new+1, y_new - 0.5_rt); + const int j_B_old = compute_shifted_shape_factor(sy_B_old, y_old - 0.5_rt, j_B_new); + const int k_B_new = compute_shape_factor(sz_B_new+1, z_new - 0.5_rt); + const int k_B_old = compute_shifted_shape_factor(sz_B_old, z_old - 0.5_rt, k_B_new); +#endif + + // computes min/max positions of current contributions +#if !defined(WARPX_DIM_1D_Z) + int dil_E = 1, diu_E = 1; + if (i_E_old < i_E_new) { dil_E = 0; } + if (i_E_old > i_E_new) { diu_E = 0; } +#endif +#if defined(WARPX_DIM_3D) + int djl_E = 1, dju_E = 1; + if (j_E_old < j_E_new) { djl_E = 0; } + if (j_E_old > j_E_new) { dju_E = 0; } +#endif + int dkl_E = 1, dku_E = 1; + if (k_E_old < k_E_new) { dkl_E = 0; } + if (k_E_old > k_E_new) { dku_E = 0; } + +#if defined(WARPX_DIM_3D) + int dil_B = 1, diu_B = 1; + if (i_B_old < i_B_new) { dil_B = 0; } + if (i_B_old > i_B_new) { diu_B = 0; } + int djl_B = 1, dju_B = 1; + if (j_B_old < j_B_new) { djl_B = 0; } + if (j_B_old > j_B_new) { dju_B = 0; } + int dkl_B = 1, dku_B = 1; + if (k_B_old < k_B_new) { dkl_B = 0; } + if (k_B_old > k_B_new) { dku_B = 0; } +#endif + +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Compute_shape_factor< depos_order-1 > compute_shape_factor_By; + Compute_shifted_shape_factor< depos_order-1 > compute_shifted_shape_factor_By; + const int i_By_new = compute_shape_factor_By(sx_By_new+1, x_new - 0.5_rt); + const int i_By_old = compute_shifted_shape_factor_By(sx_By_old, x_old - 0.5_rt, i_By_new); + const int k_By_new = compute_shape_factor_By(sz_By_new+1, z_new - 0.5_rt); + const int k_By_old = compute_shifted_shape_factor_By(sz_By_old, z_old - 0.5_rt, k_By_new); + int dil_By = 1, diu_By = 1; + if (i_By_old < i_By_new) { dil_By = 0; } + if (i_By_old > i_By_new) { diu_By = 0; } + int dkl_By = 1, dku_By = 1; + if (k_By_old < k_By_new) { dkl_By = 0; } + if (k_By_old > k_By_new) { dku_By = 0; } +#endif + +#if defined(WARPX_DIM_3D) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int j=djl_E; j<=depos_order+2-dju_E; j++) { + amrex::Real sdzjk = one_third*(sy_E_new[j]*sz_E_new[k] + sy_E_old[j]*sz_E_old[k]) + +one_sixth*(sy_E_new[j]*sz_E_old[k] + sy_E_old[j]*sz_E_new[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Exp += Ex_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdxiov*sdzjk; + } + } + } + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdyik = one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k]); + amrex::Real sdyj = 0._rt; + for (int j=djl_E; j<=depos_order+1-dju_E; j++) { + sdyj += (sy_E_old[j] - sy_E_new[j]); + auto sdyjov = static_cast((y_new - y_old) == 0. ? 1. : sdyj/(y_new - y_old)); + Eyp += Ey_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdyjov*sdyik; + } + } + } + for (int j=djl_E; j<=depos_order+2-dju_E; j++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdzij = one_third*(sx_E_new[i]*sy_E_new[j] + sx_E_old[i]*sy_E_old[j]) + +one_sixth*(sx_E_new[i]*sy_E_old[j] + sx_E_old[i]*sy_E_new[j]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Ezp += Ez_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdzkov*sdzij; + } + } + } + for (int k=dkl_B; k<=depos_order+2-dku_B; k++) { + for (int j=djl_B; j<=depos_order+2-dju_B; j++) { + amrex::Real sdzjk = one_third*(sy_B_new[j]*sz_B_new[k] + sy_B_old[j]*sz_B_old[k]) + +one_sixth*(sy_B_new[j]*sz_B_old[k] + sy_B_old[j]*sz_B_new[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_B; i<=depos_order+1-diu_B; i++) { + sdxi += (sx_B_old[i] - sx_B_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Bxp += Bx_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_B_new-1+k)*sdxiov*sdzjk; + } + } + } + for (int k=dkl_B; k<=depos_order+2-dku_B; k++) { + for (int i=dil_B; i<=depos_order+2-diu_B; i++) { + amrex::Real sdyik = one_third*(sx_B_new[i]*sz_B_new[k] + sx_B_old[i]*sz_B_old[k]) + +one_sixth*(sx_B_new[i]*sz_B_old[k] + sx_B_old[i]*sz_B_new[k]); + amrex::Real sdyj = 0._rt; + for (int j=djl_B; j<=depos_order+1-dju_B; j++) { + sdyj += (sy_B_old[j] - sy_B_new[j]); + auto sdyjov = static_cast((y_new - y_old) == 0. ? 1. : sdyj/(y_new - y_old)); + Byp += By_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_B_new-1+k)*sdyjov*sdyik; + } + } + } + for (int j=djl_B; j<=depos_order+2-dju_B; j++) { + for (int i=dil_B; i<=depos_order+2-diu_B; i++) { + amrex::Real sdzij = one_third*(sx_B_new[i]*sy_B_new[j] + sx_B_old[i]*sy_B_old[j]) + +one_sixth*(sx_B_new[i]*sy_B_old[j] + sx_B_old[i]*sy_B_new[j]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_B; k<=depos_order+1-dku_B; k++) { + sdzk += (sz_B_old[k] - sz_B_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Bzp += Bz_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_E_new-1+k)*sdzkov*sdzij; + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real sdzk = 0.5_rt*(sz_E_new[k] + sz_E_old[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Exp += Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdxiov*sdzk; + Bzp += Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdxiov*sdzk; + } + } + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + Real const sdyj = ( + one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k])); + Eyp += Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdyj; + } + } + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdxi = 0.5_rt*(sx_E_new[i] + sx_E_old[i]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Ezp += Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdzkov*sdxi; + Bxp += Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdzkov*sdxi; + } + } + for (int k=dkl_By; k<=depos_order+1-dku_By; k++) { + for (int i=dil_By; i<=depos_order+1-diu_By; i++) { + Real const sdyj = ( + one_third*(sx_By_new[i]*sz_By_new[k] + sx_By_old[i]*sz_By_old[k]) + +one_sixth*(sx_By_new[i]*sz_By_old[k] + sx_By_old[i]*sz_By_new[k])); + Byp += By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 0)*sdyj; + } + } + +#ifdef WARPX_DIM_RZ + Complex xy_mid = xy_mid0; + + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + + // Gather field on particle Exp from field on grid ex_arr + // Gather field on particle Bzp from field on grid bz_arr + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real sdzk = 0.5_rt*(sz_E_new[k] + sz_E_old[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + const amrex::Real dEx = (+ Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + const amrex::Real dBz = (+ Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Exp += dEx*sdxiov*sdzk; + Bzp += dBz*sdxiov*sdzk; + } + } + // Gather field on particle Eyp from field on grid ey_arr + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + Real const sdyj = ( + one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k])); + const amrex::Real dEy = (+ Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Eyp += dEy*sdyj; + } + } + // Gather field on particle Ezp from field on grid ez_arr + // Gather field on particle Bxp from field on grid bx_arr + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdxi = 0.5_rt*(sx_E_new[i] + sx_E_old[i]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + const amrex::Real dEz = (+ Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + const amrex::Real dBx = (+ Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Ezp += dEz*sdzkov*sdxi; + Bxp += dBx*sdzkov*sdxi; + } + } + // Gather field on particle Byp from field on grid by_arr + for (int k=dkl_By; k<=depos_order+1-dku_By; k++) { + for (int i=dil_By; i<=depos_order+1-diu_By; i++) { + Real const sdyj = ( + one_third*(sx_By_new[i]*sz_By_new[k] + sx_By_old[i]*sz_By_old[k]) + +one_sixth*(sx_By_new[i]*sz_By_old[k] + sx_By_old[i]*sz_By_new[k])); + const amrex::Real dBy = (+ By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 2*imode-1)*xy_mid.real() + - By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 2*imode)*xy_mid.imag()); + Byp += dBy*sdyj; + } + } + xy_mid = xy_mid*xy_mid0; + } + + // Convert Exp and Eyp (which are actually Er and Etheta) to Ex and Ey + const amrex::Real Exp_save = Exp; + Exp = costheta_mid*Exp - sintheta_mid*Eyp; + Eyp = costheta_mid*Eyp + sintheta_mid*Exp_save; + const amrex::Real Bxp_save = Bxp; + Bxp = costheta_mid*Bxp - sintheta_mid*Byp; + Byp = costheta_mid*Byp + sintheta_mid*Bxp_save; + +#endif + +#elif defined(WARPX_DIM_1D_Z) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real const sdzk = 0.5_rt*(sz_E_old[k] + sz_E_new[k]); + Exp += Ex_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + Eyp += Ey_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + Bzp += Bz_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + } + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Bxp += Bx_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + Byp += By_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + Ezp += Ez_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + } +#endif +} + /** * \brief Field gather for particles * @@ -432,7 +898,7 @@ void doGatherShapeN (const amrex::ParticleReal xp, * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ template -void doGatherShapeN(const GetParticlePosition& getPosition, +void doGatherShapeN(const GetParticlePosition& getPosition, const GetExternalEBField& getExternalEB, amrex::ParticleReal * const Exp, amrex::ParticleReal * const Eyp, amrex::ParticleReal * const Ezp, amrex::ParticleReal * const Bxp, @@ -569,4 +1035,96 @@ void doGatherShapeN (const amrex::ParticleReal xp, } } + +/** + * \brief Field gather for a single particle + * + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half time level (n + half) + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param ex_arr,ey_arr,ez_arr Array4 of the electric field, either full array or tile. + * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. + * \param ex_type,ey_type,ez_type IndexType of the electric field + * \param bx_type,by_type,bz_type IndexType of the magnetic field + * \param dx_arr 3D cell spacing + * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + * \param nox order of the particle shape function + * \param galerkin_interpolation whether to use lower order in v + */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherShapeNImplicit ( + const amrex::ParticleReal xp_n, + const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + const amrex::ParticleReal xp_nph, + const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& ex_arr, + amrex::Array4 const& ey_arr, + amrex::Array4 const& ez_arr, + amrex::Array4 const& bx_arr, + amrex::Array4 const& by_arr, + amrex::Array4 const& bz_arr, + const amrex::IndexType ex_type, + const amrex::IndexType ey_type, + const amrex::IndexType ez_type, + const amrex::IndexType bx_type, + const amrex::IndexType by_type, + const amrex::IndexType bz_type, + const amrex::GpuArray& dx_arr, + const amrex::GpuArray& xyzmin_arr, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes, + const int nox, + const bool galerkin_interpolation) +{ + if (galerkin_interpolation) { + if (nox == 1) { + doGatherShapeNEsirkepovStencilImplicit<1>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeNEsirkepovStencilImplicit<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeNEsirkepovStencilImplicit<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } else { + if (nox == 1) { + doGatherShapeN<1,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeN<2,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeN<3,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } +} + #endif // FIELDGATHER_H_ diff --git a/Source/Particles/Gather/GetExternalFields.H b/Source/Particles/Gather/GetExternalFields.H index 5fd0acb8eb2..9fd534e33d9 100644 --- a/Source/Particles/Gather/GetExternalFields.H +++ b/Source/Particles/Gather/GetExternalFields.H @@ -23,11 +23,11 @@ */ struct GetExternalEBField { - enum ExternalFieldInitType { None, Constant, Parser, RepeatedPlasmaLens, Unknown }; + enum ExternalFieldInitType { None, Parser, RepeatedPlasmaLens, Unknown }; GetExternalEBField () = default; - GetExternalEBField (const WarpXParIter& a_pti, int a_offset = 0) noexcept; + GetExternalEBField (const WarpXParIter& a_pti, long a_offset = 0) noexcept; ExternalFieldInitType m_Etype; ExternalFieldInitType m_Btype; @@ -35,9 +35,6 @@ struct GetExternalEBField amrex::ParticleReal m_gamma_boost; amrex::ParticleReal m_uz_boost; - amrex::GpuArray m_Efield_value; - amrex::GpuArray m_Bfield_value; - amrex::ParserExecutor<4> m_Exfield_partparser; amrex::ParserExecutor<4> m_Eyfield_partparser; amrex::ParserExecutor<4> m_Ezfield_partparser; @@ -45,7 +42,7 @@ struct GetExternalEBField amrex::ParserExecutor<4> m_Byfield_partparser; amrex::ParserExecutor<4> m_Bzfield_partparser; - GetParticlePosition m_get_position; + GetParticlePosition m_get_position; amrex::Real m_time; amrex::ParticleReal m_repeated_plasma_lens_period; @@ -61,7 +58,7 @@ struct GetExternalEBField std::optional d_lattice_element_finder; - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool isNoOp () const { return (m_Etype == None && m_Btype == None && !d_lattice_element_finder.has_value()); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -81,7 +78,7 @@ struct GetExternalEBField field_Bx, field_By, field_Bz); } - if (m_Etype == None && m_Btype == None) return; + if (m_Etype == None && m_Btype == None) { return; } amrex::ParticleReal Ex = 0._prt; amrex::ParticleReal Ey = 0._prt; @@ -92,13 +89,7 @@ struct GetExternalEBField constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); - if (m_Etype == Constant) - { - Ex = m_Efield_value[0]; - Ey = m_Efield_value[1]; - Ez = m_Efield_value[2]; - } - else if (m_Etype == ExternalFieldInitType::Parser) + if (m_Etype == ExternalFieldInitType::Parser) { amrex::ParticleReal x, y, z; m_get_position(i, x, y, z); @@ -112,13 +103,7 @@ struct GetExternalEBField Ez = m_Ezfield_partparser((amrex::ParticleReal) x, (amrex::ParticleReal) y, (amrex::ParticleReal) z, lab_time); } - if (m_Btype == Constant) - { - Bx = m_Bfield_value[0]; - By = m_Bfield_value[1]; - Bz = m_Bfield_value[2]; - } - else if (m_Btype == ExternalFieldInitType::Parser) + if (m_Btype == ExternalFieldInitType::Parser) { amrex::ParticleReal x, y, z; m_get_position(i, x, y, z); diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp index c6d94686c06..8e3c679aa3a 100644 --- a/Source/Particles/Gather/GetExternalFields.cpp +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -13,7 +13,7 @@ using namespace amrex::literals; -GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, int a_offset) noexcept +GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset) noexcept { auto& warpx = WarpX::GetInstance(); auto& mypc = warpx.GetPartContainer(); @@ -22,7 +22,7 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, int a_offset) AcceleratorLattice const & accelerator_lattice = warpx.get_accelerator_lattice(lev); if (accelerator_lattice.m_lattice_defined) { - d_lattice_element_finder = accelerator_lattice.GetFinderDeviceInstance(a_pti, a_offset); + d_lattice_element_finder = accelerator_lattice.GetFinderDeviceInstance(a_pti, static_cast(a_offset)); } m_gamma_boost = WarpX::gamma_boost; @@ -31,24 +31,12 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, int a_offset) m_Etype = Unknown; m_Btype = Unknown; - if (mypc.m_E_ext_particle_s == "none") m_Etype = None; - if (mypc.m_B_ext_particle_s == "none") m_Btype = None; + if (mypc.m_E_ext_particle_s == "none") { m_Etype = None; } + if (mypc.m_B_ext_particle_s == "none") { m_Btype = None; } - if (mypc.m_E_ext_particle_s == "constant") - { - m_Etype = Constant; - m_Efield_value[0] = mypc.m_E_external_particle[0]; - m_Efield_value[1] = mypc.m_E_external_particle[1]; - m_Efield_value[2] = mypc.m_E_external_particle[2]; - } - - if (mypc.m_B_ext_particle_s == "constant") - { - m_Btype = Constant; - m_Bfield_value[0] = mypc.m_B_external_particle[0]; - m_Bfield_value[1] = mypc.m_B_external_particle[1]; - m_Bfield_value[2] = mypc.m_B_external_particle[2]; - } + // These lines will be removed once the user interface is redefined and the CI tests updated + if (mypc.m_E_ext_particle_s == "constant") { m_Etype = None; } + if (mypc.m_B_ext_particle_s == "constant") { m_Btype = None; } if (mypc.m_E_ext_particle_s == "parse_e_ext_particle_function" || mypc.m_B_ext_particle_s == "parse_b_ext_particle_function" || @@ -56,7 +44,7 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, int a_offset) mypc.m_B_ext_particle_s == "repeated_plasma_lens") { m_time = warpx.gett_new(a_pti.GetLevel()); - m_get_position = GetParticlePosition(a_pti, a_offset); + m_get_position = GetParticlePosition(a_pti, a_offset); } if (mypc.m_E_ext_particle_s == "parse_e_ext_particle_function") @@ -80,10 +68,10 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, int a_offset) if (mypc.m_E_ext_particle_s == "repeated_plasma_lens" || mypc.m_B_ext_particle_s == "repeated_plasma_lens") { - if (mypc.m_E_ext_particle_s == "repeated_plasma_lens") m_Etype = RepeatedPlasmaLens; - if (mypc.m_B_ext_particle_s == "repeated_plasma_lens") m_Btype = RepeatedPlasmaLens; + if (mypc.m_E_ext_particle_s == "repeated_plasma_lens") { m_Etype = RepeatedPlasmaLens; } + if (mypc.m_B_ext_particle_s == "repeated_plasma_lens") { m_Btype = RepeatedPlasmaLens; } m_dt = warpx.getdt(a_pti.GetLevel()); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_ux = attribs[PIdx::ux].dataPtr() + a_offset; m_uy = attribs[PIdx::uy].dataPtr() + a_offset; m_uz = attribs[PIdx::uz].dataPtr() + a_offset; diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 8605868a87e..10a10cfe190 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -41,7 +41,7 @@ struct ScaleFields { using namespace amrex::literals; - if (!m_do_scale) return; + if (!m_do_scale) { return; } // Scale the fields of particles about to cross the injection plane. // This only approximates what should be happening. The particles diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index e125bab2ec5..e6fa308431c 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -10,6 +10,7 @@ #define WARPX_LaserParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Laser/LaserProfiles.H" #include "WarpXParticleContainer.H" @@ -42,25 +43,29 @@ class LaserParticleContainer { public: LaserParticleContainer (amrex::AmrCore* amr_core, int ispecies, const std::string& name); - virtual ~LaserParticleContainer () {} + ~LaserParticleContainer () override = default; - virtual void InitData () final; + LaserParticleContainer ( LaserParticleContainer const &) = delete; + LaserParticleContainer& operator= ( LaserParticleContainer const & ) = delete; + LaserParticleContainer ( LaserParticleContainer&& ) = default; + LaserParticleContainer& operator= ( LaserParticleContainer&& ) = default; + + void InitData () final; /** * \brief Method to initialize runtime attributes. Does nothing for LaserParticleContainer. */ - virtual void DefaultInitializeRuntimeAttributes ( + void DefaultInitializeRuntimeAttributes ( amrex::ParticleTile, NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& /*pinned_tile*/, const int /*n_external_attr_real*/, - const int /*n_external_attr_int*/, - const amrex::RandomEngine& /*engine*/) override final {} + const int /*n_external_attr_int*/) final {} - virtual void ReadHeader (std::istream& is) final; + void ReadHeader (std::istream& is) final; - virtual void WriteHeader (std::ostream& os) const final; + void WriteHeader (std::ostream& os) const final; - virtual void Evolve (int lev, + void Evolve (int lev, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, @@ -69,9 +74,9 @@ public: const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false) final; + bool skip_deposition=false, PushType push_type=PushType::Explicit) final; - virtual void PushP (int lev, amrex::Real dt, + void PushP (int lev, amrex::Real dt, const amrex::MultiFab& , const amrex::MultiFab& , const amrex::MultiFab& , @@ -79,7 +84,7 @@ public: const amrex::MultiFab& , const amrex::MultiFab& ) final; - virtual void PostRestart () final; + void PostRestart () final; void calculate_laser_plane_coordinates (const WarpXParIter& pti, int np, amrex::Real * AMREX_RESTRICT pplane_Xp, diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index ff11d1a30e9..fd4e9397253 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -9,6 +9,7 @@ #include "LaserParticleContainer.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Laser/LaserProfiles.H" #include "Particles/LaserParticleContainer.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -46,7 +47,6 @@ #include #include #include -#include #include #include @@ -280,7 +280,7 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, void LaserParticleContainer::ContinuousInjection (const RealBox& injection_box) { - if (!m_enabled) return; + if (!m_enabled) { return; } // Input parameter injection_box contains small box where injection // should occur. @@ -322,7 +322,7 @@ LaserParticleContainer::ContinuousInjection (const RealBox& injection_box) void LaserParticleContainer::UpdateAntennaPosition (const amrex::Real dt) { - if (!m_enabled) return; + if (!m_enabled) { return; } const int dir = WarpX::moving_window_dir; if (do_continuous_injection and (WarpX::gamma_boost > 1)){ @@ -351,7 +351,7 @@ LaserParticleContainer::UpdateAntennaPosition (const amrex::Real dt) void LaserParticleContainer::InitData () { - if (!m_enabled) return; + if (!m_enabled) { return; } // Call InitData on max level to inject one laser particle per // finest cell. @@ -368,7 +368,7 @@ LaserParticleContainer::InitData () void LaserParticleContainer::InitData (int lev) { - if (!m_enabled) return; + if (!m_enabled) { return; } // spacing of laser particles in the laser plane. // has to be done after geometry is set up. @@ -487,7 +487,8 @@ LaserParticleContainer::InitData (int lev) const DistributionMapping plane_dm {plane_ba, nprocs}; const Vector& procmap = plane_dm.ProcessorMap(); - for (int i = 0, n = plane_ba.size(); i < n; ++i) + const auto plane_ba_size = static_cast(plane_ba.size()); + for (int i = 0; i < plane_ba_size; ++i) { if (procmap[i] == myproc) { @@ -518,15 +519,16 @@ LaserParticleContainer::InitData (int lev) particle_w.push_back(-m_weight); #else // Particles are laid out in radial spokes - const int n_spokes = (WarpX::n_rz_azimuthal_modes - 1)*m_min_particles_per_mode; + const auto n_spokes = + static_cast((WarpX::n_rz_azimuthal_modes - 1)*m_min_particles_per_mode); for (int spoke = 0 ; spoke < n_spokes ; spoke++) { - const Real phase = 2.*MathConst::pi*spoke/n_spokes; + const Real phase = 2._rt*MathConst::pi*spoke/n_spokes; for (int k = 0; k<2; ++k) { particle_x.push_back(pos[0]*std::cos(phase)); particle_y.push_back(pos[0]*std::sin(phase)); particle_z.push_back(pos[2]); } - const Real r_weight = m_weight*2.*MathConst::pi*pos[0]/n_spokes; + const Real r_weight = m_weight*2._rt*MathConst::pi*pos[0]/n_spokes; particle_w.push_back( r_weight); particle_w.push_back(-r_weight); } @@ -535,12 +537,12 @@ LaserParticleContainer::InitData (int lev) } } } - const int np = particle_z.size(); + const auto np = static_cast(particle_z.size()); amrex::Vector particle_ux(np, 0.0); amrex::Vector particle_uy(np, 0.0); amrex::Vector particle_uz(np, 0.0); - if (Verbose()) amrex::Print() << Utils::TextMsg::Info("Adding laser particles"); + if (Verbose()) { amrex::Print() << Utils::TextMsg::Info("Adding laser particles"); } amrex::Vector> attr; attr.push_back(particle_w); amrex::Vector> attr_int; @@ -560,12 +562,12 @@ LaserParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, - Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition) + Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition, PushType push_type) { WARPX_PROFILE("LaserParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("LaserParticleContainer::Evolve::ParticlePush", blp_pp); - if (!m_enabled) return; + if (!m_enabled) { return; } Real t_lab = t; if (WarpX::gamma_boost > 1) { @@ -637,18 +639,18 @@ LaserParticleContainer::Evolve (int lev, // WARPX_PROFILE_VAR_START(blp_pp); // Find the coordinates of the particles in the emission plane - calculate_laser_plane_coordinates(pti, np, + calculate_laser_plane_coordinates(pti, static_cast(np), plane_Xp.dataPtr(), plane_Yp.dataPtr()); // Calculate the laser amplitude to be emitted, // at the position of the emission plane m_up_laser_profile->fill_amplitude( - np, plane_Xp.dataPtr(), plane_Yp.dataPtr(), + static_cast(np), plane_Xp.dataPtr(), plane_Yp.dataPtr(), t_lab, amplitude_E.dataPtr()); // Calculate the corresponding momentum and position for the particles - update_laser_particle(pti, np, uxp.dataPtr(), uyp.dataPtr(), + update_laser_particle(pti, static_cast(np), uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), wp.dataPtr(), amplitude_E.dataPtr(), dt); WARPX_PROFILE_VAR_STOP(blp_pp); @@ -663,14 +665,14 @@ LaserParticleContainer::Evolve (int lev, // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, - lev, lev, dt, relative_time); + lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, - lev, lev-1, dt, relative_time); + lev, lev-1, dt, relative_time, push_type); } } @@ -700,7 +702,7 @@ LaserParticleContainer::Evolve (int lev, void LaserParticleContainer::PostRestart () { - if (!m_enabled) return; + if (!m_enabled) { return; } Real Sx, Sy; const int lev = finestLevel(); @@ -788,7 +790,7 @@ LaserParticleContainer::calculate_laser_plane_coordinates (const WarpXParIter& p Real * AMREX_RESTRICT const pplane_Xp, Real * AMREX_RESTRICT const pplane_Yp) { - const auto GetPosition = GetParticlePosition(pti); + const auto GetPosition = GetParticlePosition(pti); #if (AMREX_SPACEDIM >= 2) const Real tmp_u_X_0 = m_u_X[0]; @@ -850,8 +852,8 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, Real const * AMREX_RESTRICT const amplitude, const Real dt) { - const auto GetPosition = GetParticlePosition(pti); - auto SetPosition = SetParticlePosition(pti); + const auto GetPosition = GetParticlePosition(pti); + auto SetPosition = SetParticlePosition(pti); const Real tmp_p_X_0 = m_p_X[0]; const Real tmp_p_X_1 = m_p_X[1]; @@ -868,7 +870,7 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, np, [=] AMREX_GPU_DEVICE (int i) { // Calculate the velocity according to the amplitude of E - const Real sign_charge = (pwp[i]>0) ? 1 : -1; + const Real sign_charge = (pwp[i]>0) ? -1 : 1; const Real v_over_c = sign_charge * tmp_mobility * amplitude[i]; AMREX_ALWAYS_ASSERT_WITH_MESSAGE(amrex::Math::abs(v_over_c) < amrex::Real(1.), "Error: calculated laser particle velocity greater than c." diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index fecd39137d7..285b0a9777c 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -14,6 +14,7 @@ #include "MultiParticleContainer_fwd.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Collision/CollisionHandler.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper_fwd.H" @@ -68,22 +69,22 @@ public: MultiParticleContainer (amrex::AmrCore* amr_core); - ~MultiParticleContainer() {} + ~MultiParticleContainer() = default; - WarpXParticleContainer& + MultiParticleContainer (MultiParticleContainer const &) = delete; + MultiParticleContainer& operator= (MultiParticleContainer const & ) = delete; + MultiParticleContainer(MultiParticleContainer&& ) = default; + MultiParticleContainer& operator=(MultiParticleContainer&& ) = default; + + [[nodiscard]] WarpXParticleContainer& GetParticleContainer (int index) const {return *allcontainers[index];} - WarpXParticleContainer* + [[nodiscard]] WarpXParticleContainer* GetParticleContainerPtr (int index) const {return allcontainers[index].get();} - WarpXParticleContainer& + [[nodiscard]] WarpXParticleContainer& GetParticleContainerFromName (const std::string& name) const; -#ifdef WARPX_USE_OPENPMD - std::unique_ptr& GetUniqueContainer(int index) { - return allcontainers[index]; - } -#endif std::array meanParticleVelocity(int index) { return allcontainers[index]->meanParticleVelocity(); } @@ -107,18 +108,19 @@ public: amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false); + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, + PushType push_type=PushType::Explicit); /// /// This pushes the particle positions by one half time step for all the species in the - /// MultiParticleContainer. It is used to desynchronize the particles after initializaton + /// MultiParticleContainer. It is used to desynchronize the particles after initialization /// or when restarting from a checkpoint. /// void PushX (amrex::Real dt); /// /// This pushes the particle momenta by dt for all the species in the - /// MultiParticleContainer. It is used to desynchronize the particles after initializaton + /// MultiParticleContainer. It is used to desynchronize the particles after initialization /// or when restarting from a checkpoint. It is also used to synchronize particles at the /// the end of the run. This is the electromagnetic version. /// @@ -193,9 +195,9 @@ public: /** This function computes the box outside which Schwinger process is disabled. The box is * defined by m_qed_schwinger_xmin/xmax/ymin/ymax/zmin/zmax and the warpx level 0 geometry - * object (to make the link between Real and int quatities). + * object (to make the link between Real and int quantities). */ - amrex::Box ComputeSchwingerGlobalBox () const; + [[nodiscard]] amrex::Box ComputeSchwingerGlobalBox () const; #endif void Restart (const std::string& dir); @@ -225,18 +227,18 @@ public: * * @param[in] lev the index of the refinement level. */ - amrex::Vector GetZeroParticlesInGrid(int lev) const; + [[nodiscard]] amrex::Vector GetZeroParticlesInGrid(int lev) const; - amrex::Vector NumberOfParticlesInGrid(int lev) const; + [[nodiscard]] amrex::Vector NumberOfParticlesInGrid(int lev) const; void Increment (amrex::MultiFab& mf, int lev); void SetParticleBoxArray (int lev, amrex::BoxArray& new_ba); void SetParticleDistributionMap (int lev, amrex::DistributionMapping& new_dm); - int nSpecies () const {return species_names.size();} - int nLasers () const {return lasers_names.size();} - int nContainers () const {return allcontainers.size();} + [[nodiscard]] int nSpecies () const {return static_cast(species_names.size());} + [[nodiscard]] int nLasers () const {return static_cast(lasers_names.size());} + [[nodiscard]] int nContainers () const {return static_cast(allcontainers.size());} /** Whether back-transformed diagnostics need to be performed for any plasma species. * @@ -250,16 +252,18 @@ public: */ void SetDoBackTransformedParticles (std::string species_name, bool do_back_transformed_particles); - int nSpeciesDepositOnMainGrid () const { + [[nodiscard]] int nSpeciesDepositOnMainGrid () const + { bool const onMainGrid = true; auto const & v = m_deposit_on_main_grid; - return std::count( v.begin(), v.end(), onMainGrid ); + return static_cast(std::count( v.begin(), v.end(), onMainGrid )); } - int nSpeciesGatherFromMainGrid() const { + [[nodiscard]] int nSpeciesGatherFromMainGrid() const + { bool const fromMainGrid = true; auto const & v = m_gather_from_main_grid; - return std::count( v.begin(), v.end(), fromMainGrid ); + return static_cast(std::count( v.begin(), v.end(), fromMainGrid )); } // Inject particles during the simulation (for particles entering the @@ -273,16 +277,17 @@ public: */ void UpdateAntennaPosition(amrex::Real dt) const; - int doContinuousInjection() const; + [[nodiscard]] int doContinuousInjection() const; // Inject particles from a surface during the simulation void ContinuousFluxInjection(amrex::Real t, amrex::Real dt) const; - std::vector GetSpeciesNames() const { return species_names; } + [[nodiscard]] std::vector GetSpeciesNames() const { return species_names; } - std::vector GetLasersNames() const { return lasers_names; } + [[nodiscard]] std::vector GetLasersNames() const { return lasers_names; } - std::vector GetSpeciesAndLasersNames() const { + [[nodiscard]] std::vector GetSpeciesAndLasersNames() const + { std::vector tmp = species_names; tmp.insert(tmp.end(), lasers_names.begin(), lasers_names.end()); return tmp; @@ -294,9 +299,6 @@ public: std::string m_B_ext_particle_s = "none"; std::string m_E_ext_particle_s = "none"; - // External fields added to particle fields. - amrex::Vector m_B_external_particle; - amrex::Vector m_E_external_particle; // Parser for B_external on the particle std::unique_ptr m_Bx_particle_parser; std::unique_ptr m_By_particle_parser; @@ -329,7 +331,10 @@ public: const amrex::MultiFab& Bz); #endif - int getSpeciesID (std::string product_str) const; + [[nodiscard]] int getSpeciesID (std::string product_str) const; + + amrex::Vector>::iterator begin() {return allcontainers.begin();} + amrex::Vector>::iterator end() {return allcontainers.end();} protected: @@ -376,6 +381,7 @@ protected: std::vector species_types; template + [[nodiscard]] amrex::MFItInfo getMFItInfo (const WarpXParticleContainer& pc_src, Args const&... pc_dsts) const noexcept { @@ -423,12 +429,12 @@ protected: /** * Returns the number of species having Quantum Synchrotron process enabled */ - int NSpeciesQuantumSync() const { return m_nspecies_quantum_sync;} + [[nodiscard]] int NSpeciesQuantumSync() const { return m_nspecies_quantum_sync;} /** * Returns the number of species having Breit Wheeler process enabled */ - int NSpeciesBreitWheeler() const { return m_nspecies_breit_wheeler;} + [[nodiscard]] int NSpeciesBreitWheeler() const { return m_nspecies_breit_wheeler;} /** * Initializes the Quantum Synchrotron engine @@ -453,7 +459,7 @@ protected: void BreitWheelerGenerateTable(); /** Whether or not to activate Schwinger process */ - bool m_do_qed_schwinger = 0; + bool m_do_qed_schwinger = false; /** Name of Schwinger electron product species */ std::string m_qed_schwinger_ele_product_name; /** Name of Schwinger positron product species */ diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index b9802fa448b..a31f426a0e4 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -133,14 +133,6 @@ MultiParticleContainer::ReadParameters () { const ParmParse pp_particles("particles"); - // allocating and initializing default values of external fields for particles - m_E_external_particle.resize(3); - m_B_external_particle.resize(3); - // initialize E and B fields to 0.0 - for (int idim = 0; idim < 3; ++idim) { - m_E_external_particle[idim] = 0.0; - m_B_external_particle[idim] = 0.0; - } // default values of E_external_particle and B_external_particle // are used to set the E and B field when "constant" or "parser" // is not explicitly used in the input @@ -154,19 +146,6 @@ MultiParticleContainer::ReadParameters () m_E_ext_particle_s.end(), m_E_ext_particle_s.begin(), ::tolower); - // if the input string for B_external on particles is "constant" - // then the values for the external B on particles must - // be provided in the input file. - if (m_B_ext_particle_s == "constant") - utils::parser::getArrWithParser( - pp_particles, "B_external_particle", m_B_external_particle); - - // if the input string for E_external on particles is "constant" - // then the values for the external E on particles must - // be provided in the input file. - if (m_E_ext_particle_s == "constant") - utils::parser::getArrWithParser( - pp_particles, "E_external_particle", m_E_external_particle); // if the input string for B_ext_particle_s is // "parse_b_ext_particle_function" then the mathematical expression @@ -294,7 +273,7 @@ MultiParticleContainer::ReadParameters () it != species_names.end(), "species '" + name + "' in particles.deposit_on_main_grid must be part of particles.species_names"); - const int i = std::distance(species_names.begin(), it); + const auto i = static_cast(std::distance(species_names.begin(), it)); m_deposit_on_main_grid[i] = true; } @@ -307,7 +286,7 @@ MultiParticleContainer::ReadParameters () it != species_names.end(), "species '" + name + "' in particles.gather_from_main_grid must be part of particles.species_names"); - const int i = std::distance(species_names.begin(), it); + const auto i = static_cast(std::distance(species_names.begin(), it)); m_gather_from_main_grid.at(i) = true; } @@ -323,7 +302,7 @@ MultiParticleContainer::ReadParameters () it != species_names.end(), "species '" + name + "' in particles.rigid_injected_species must be part of particles.species_names"); - const int i = std::distance(species_names.begin(), it); + const auto i = static_cast(std::distance(species_names.begin(), it)); species_types[i] = PCTypes::RigidInjected; } } @@ -337,7 +316,7 @@ MultiParticleContainer::ReadParameters () it != species_names.end(), "species '" + name + "' in particles.photon_species must be part of particles.species_names"); - const int i = std::distance(species_names.begin(), it); + const auto i = static_cast(std::distance(species_names.begin(), it)); species_types[i] = PCTypes::Photon; } } @@ -366,7 +345,7 @@ MultiParticleContainer::ReadParameters () it != lasers_names.end(), "laser '" + name + "' in lasers.deposit_on_main_grid must be part of lasers.lasers_names"); - const int i = std::distance(lasers_names.begin(), it); + const auto i = static_cast(std::distance(lasers_names.begin(), it)); m_laser_deposit_on_main_grid[i] = true; } @@ -413,7 +392,7 @@ MultiParticleContainer::GetParticleContainerFromName (const std::string& name) c WARPX_ALWAYS_ASSERT_WITH_MESSAGE( it != species_names.end(), "unknown species name"); - const int i = std::distance(species_names.begin(), it); + const auto i = static_cast(std::distance(species_names.begin(), it)); return *allcontainers[i]; } @@ -476,21 +455,22 @@ MultiParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { if (! skip_deposition) { jx.setVal(0.0); jy.setVal(0.0); jz.setVal(0.0); - if (cjx) cjx->setVal(0.0); - if (cjy) cjy->setVal(0.0); - if (cjz) cjz->setVal(0.0); - if (rho) rho->setVal(0.0); - if (crho) crho->setVal(0.0); + if (cjx) { cjx->setVal(0.0); } + if (cjy) { cjy->setVal(0.0); } + if (cjz) { cjz->setVal(0.0); } + if (rho) { rho->setVal(0.0); } + if (crho) { crho->setVal(0.0); } } for (auto& pc : allcontainers) { pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, - rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition); + rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition, push_type); } } @@ -528,8 +508,9 @@ MultiParticleContainer::GetZeroChargeDensity (const int lev) const bool is_PSATD_RZ = false; #endif - if( !is_PSATD_RZ ) + if( !is_PSATD_RZ ) { nba.surroundingNodes(); + } auto zero_rho = std::make_unique(nba, dmap, WarpX::ncomps, ng_rho); zero_rho->setVal(amrex::Real(0.0)); @@ -576,7 +557,7 @@ MultiParticleContainer::DepositCharge ( } // Push the particles in time, if needed - if (relative_time != 0.) PushX(relative_time); + if (relative_time != 0.) { PushX(relative_time); } bool const local = true; bool const reset = false; @@ -585,13 +566,13 @@ MultiParticleContainer::DepositCharge ( // Call the deposition kernel for each species for (auto& pc : allcontainers) { - if (pc->do_not_deposit) continue; + if (pc->do_not_deposit) { continue; } pc->DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, interpolate_across_levels); } // Push the particles back in time - if (relative_time != 0.) PushX(-relative_time); + if (relative_time != 0.) { PushX(-relative_time); } #ifdef WARPX_DIM_RZ for (int lev = 0; lev < rho.size(); ++lev) @@ -607,7 +588,7 @@ MultiParticleContainer::GetChargeDensity (int lev, bool local) std::unique_ptr rho = GetZeroChargeDensity(lev); for (auto& container : allcontainers) { - if (container->do_not_deposit) continue; + if (container->do_not_deposit) { continue; } const std::unique_ptr rhoi = container->GetChargeDensity(lev, true); MultiFab::Add(*rho, *rhoi, 0, 0, rho->nComp(), rho->nGrowVect()); } @@ -671,7 +652,7 @@ Vector MultiParticleContainer::GetZeroParticlesInGrid (const int lev) const { const WarpX& warpx = WarpX::GetInstance(); - const int num_boxes = warpx.boxArray(lev).size(); + const auto num_boxes = static_cast(warpx.boxArray(lev).size()); Vector r(num_boxes, 0); return r; } @@ -694,7 +675,7 @@ MultiParticleContainer::NumberOfParticlesInGrid (int lev) const r[j] += ri[j]; } } - ParallelDescriptor::ReduceLongSum(r.data(),r.size()); + ParallelDescriptor::ReduceLongSum(r.data(),static_cast(r.size())); return r; } } @@ -731,7 +712,7 @@ MultiParticleContainer::SetParticleDistributionMap (int lev, DistributionMapping void MultiParticleContainer::ContinuousInjection (const RealBox& injection_box) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ pc->ContinuousInjection(injection_box); } @@ -741,7 +722,7 @@ MultiParticleContainer::ContinuousInjection (const RealBox& injection_box) const void MultiParticleContainer::UpdateAntennaPosition (const amrex::Real dt) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ pc->UpdateAntennaPosition(dt); } @@ -752,7 +733,7 @@ int MultiParticleContainer::doContinuousInjection () const { int warpx_do_continuous_injection = 0; - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ warpx_do_continuous_injection = 1; } @@ -767,7 +748,7 @@ MultiParticleContainer::doContinuousInjection () const void MultiParticleContainer::ContinuousFluxInjection (amrex::Real t, amrex::Real dt) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ pc->ContinuousFluxInjection(t, dt); } } @@ -811,10 +792,10 @@ MultiParticleContainer::mapSpeciesProduct () #ifdef WARPX_QED if (m_do_qed_schwinger) { - m_qed_schwinger_ele_product = - getSpeciesID(m_qed_schwinger_ele_product_name); - m_qed_schwinger_pos_product = - getSpeciesID(m_qed_schwinger_pos_product_name); + m_qed_schwinger_ele_product = + getSpeciesID(m_qed_schwinger_ele_product_name); + m_qed_schwinger_pos_product = + getSpeciesID(m_qed_schwinger_pos_product_name); } #endif } @@ -826,13 +807,13 @@ MultiParticleContainer::getSpeciesID (std::string product_str) const { auto species_and_lasers_names = GetSpeciesAndLasersNames(); int i_product = 0; - bool found = 0; + bool found = false; // Loop over species for (int i=0; i < static_cast(species_and_lasers_names.size()); i++){ // If species name matches, store its ID // into i_product if (species_and_lasers_names[i] == product_str){ - found = 1; + found = true; i_product = i; } } @@ -853,12 +834,12 @@ MultiParticleContainer::SetDoBackTransformedParticles (const bool do_back_transf void MultiParticleContainer::SetDoBackTransformedParticles (std::string species_name, const bool do_back_transformed_particles) { auto species_names_list = GetSpeciesNames(); - bool found = 0; + bool found = false; // Loop over species for (int i = 0; i < static_cast(species_names.size()); ++i) { // If species name matches, set back-transformed particles parameters if (species_names_list[i] == species_name) { - found = 1; + found = true; auto& pc = allcontainers[i]; pc->SetDoBackTransformedParticles(do_back_transformed_particles); } @@ -891,7 +872,7 @@ MultiParticleContainer::doFieldIonization (int lev, auto& pc_product = allcontainers[pc_source->ionization_product]; const SmartCopyFactory copy_factory(*pc_source, *pc_product); - auto phys_pc_ptr = static_cast(pc_source.get()); + auto *phys_pc_ptr = static_cast(pc_source.get()); auto Copy = copy_factory.getSmartCopy(); auto Transform = IonizationTransformFunc(); @@ -910,7 +891,7 @@ MultiParticleContainer::doFieldIonization (int lev, { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); auto& src_tile = pc_source ->ParticlesAt(lev, pti); auto& dst_tile = pc_product->ParticlesAt(lev, pti); @@ -920,7 +901,7 @@ MultiParticleContainer::doFieldIonization (int lev, Bx[pti], By[pti], Bz[pti]); const auto np_dst = dst_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + const auto num_added = filterCopyTransformParticles<1>(*pc_product, dst_tile, src_tile, np_dst, Filter, Copy, Transform); setNewParticleIDs(dst_tile, np_dst, num_added); @@ -928,7 +909,7 @@ MultiParticleContainer::doFieldIonization (int lev, if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[pti.index()], wt); } } @@ -997,11 +978,13 @@ void MultiParticleContainer::InitQED () } } - if(m_nspecies_quantum_sync != 0) + if(m_nspecies_quantum_sync != 0) { InitQuantumSync(); + } - if(m_nspecies_breit_wheeler !=0) + if(m_nspecies_breit_wheeler !=0) { InitBreitWheeler(); + } } @@ -1086,8 +1069,9 @@ void MultiParticleContainer::InitBreitWheeler () // considered for pair production. If a photon has chi < chi_min, // the optical depth is not evolved and photon generation is ignored amrex::Real bw_minimum_chi_part; - if(!utils::parser::queryWithParser(pp_qed_bw, "chi_min", bw_minimum_chi_part)) + if(!utils::parser::queryWithParser(pp_qed_bw, "chi_min", bw_minimum_chi_part)) { WARPX_ABORT_WITH_MESSAGE("qed_bw.chi_min should be provided!"); + } pp_qed_bw.query("lookup_table_mode", lookup_table_mode); if(lookup_table_mode.empty()){ @@ -1349,8 +1333,8 @@ MultiParticleContainer::doQEDSchwinger () * geom.CellSize(2); #endif - // Get the temporal step - const auto dt = warpx.getdt(level_0); + // Get the temporal step + const auto dt = warpx.getdt(level_0); auto& pc_product_ele = allcontainers[m_qed_schwinger_ele_product]; @@ -1370,8 +1354,8 @@ MultiParticleContainer::doQEDSchwinger () #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(Ex, TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { + for (MFIter mfi(Ex, TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { // Make the box cell centered to avoid creating particles twice on the tile edges amrex::Box box = enclosedCells(mfi.nodaltilebox()); @@ -1404,7 +1388,7 @@ MultiParticleContainer::doQEDSchwinger () const auto Transform = SchwingerTransformFunc{m_qed_schwinger_y_size, PIdx::w}; - const auto num_added = filterCreateTransformFromFAB<1>( dst_ele_tile, + const auto num_added = filterCreateTransformFromFAB<1>( *pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, box, fieldsEB, np_ele_dst, np_pos_dst,Filter, CreateEle, CreatePos, Transform); @@ -1509,7 +1493,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, // in pc_product_ele and positrons in pc_product_pos for (auto& pc_source : allcontainers){ - if(!pc_source->has_breit_wheeler()) continue; + if(!pc_source->has_breit_wheeler()) { continue; } // Get product species auto& pc_product_ele = @@ -1519,7 +1503,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, const SmartCopyFactory copy_factory_ele(*pc_source, *pc_product_ele); const SmartCopyFactory copy_factory_pos(*pc_source, *pc_product_pos); - auto phys_pc_ptr = static_cast(pc_source.get()); + auto *phys_pc_ptr = static_cast(pc_source.get()); const auto Filter = phys_pc_ptr->getPairGenerationFilterFunc(); const auto CopyEle = copy_factory_ele.getSmartCopy(); @@ -1542,12 +1526,14 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); auto Transform = PairGenerationTransformFunc(pair_gen_functor, pti, lev, Ex.nGrowVect(), Ex[pti], Ey[pti], Ez[pti], - Bx[pti], By[pti], Bz[pti]); + Bx[pti], By[pti], Bz[pti], + phys_pc_ptr->m_E_external_particle, + phys_pc_ptr->m_B_external_particle); auto& src_tile = pc_source->ParticlesAt(lev, pti); auto& dst_ele_tile = pc_product_ele->ParticlesAt(lev, pti); @@ -1555,7 +1541,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, const auto np_dst_ele = dst_ele_tile.numParticles(); const auto np_dst_pos = dst_pos_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>( + const auto num_added = filterCopyTransformParticles<1>(*pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, src_tile, np_dst_ele, np_dst_pos, Filter, CopyEle, CopyPos, Transform); @@ -1566,7 +1552,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[pti.index()], wt); } } @@ -1597,7 +1583,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, allcontainers[pc_source->m_qed_quantum_sync_phot_product]; const SmartCopyFactory copy_factory_phot(*pc_source, *pc_product_phot); - auto phys_pc_ptr = + auto *phys_pc_ptr = static_cast(pc_source.get()); const auto Filter = phys_pc_ptr->getPhotonEmissionFilterFunc(); @@ -1617,7 +1603,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); auto Transform = PhotonEmissionTransformFunc( m_shr_p_qs_engine->build_optical_depth_functor(), @@ -1625,7 +1611,9 @@ void MultiParticleContainer::doQedQuantumSync (int lev, m_shr_p_qs_engine->build_phot_em_functor(), pti, lev, Ex.nGrowVect(), Ex[pti], Ey[pti], Ez[pti], - Bx[pti], By[pti], Bz[pti]); + Bx[pti], By[pti], Bz[pti], + phys_pc_ptr->m_E_external_particle, + phys_pc_ptr->m_B_external_particle); auto& src_tile = pc_source->ParticlesAt(lev, pti); auto& dst_tile = pc_product_phot->ParticlesAt(lev, pti); @@ -1633,7 +1621,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, const auto np_dst = dst_tile.numParticles(); const auto num_added = - filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + filterCopyTransformParticles<1>(*pc_product_phot, dst_tile, src_tile, np_dst, Filter, CopyPhot, Transform); setNewParticleIDs(dst_tile, np_dst, num_added); @@ -1645,7 +1633,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[pti.index()], wt); } } diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index e74dd8a6c7e..3be0886425d 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -73,7 +73,7 @@ public: } /** Destructor for NamedComponentParticleContainer */ - virtual ~NamedComponentParticleContainer() = default; + ~NamedComponentParticleContainer() override = default; /** Construct a NamedComponentParticleContainer from a regular * amrex::ParticleContainer, and additional name-to-index maps @@ -102,9 +102,9 @@ public: NamedComponentParticleContainer& operator= ( const NamedComponentParticleContainer & ) = delete; /** Move constructor for NamedComponentParticleContainer */ - NamedComponentParticleContainer ( NamedComponentParticleContainer && ) = default; + NamedComponentParticleContainer ( NamedComponentParticleContainer && ) noexcept = default; /** Move operator for NamedComponentParticleContainer */ - NamedComponentParticleContainer& operator= ( NamedComponentParticleContainer && ) = default; + NamedComponentParticleContainer& operator= ( NamedComponentParticleContainer && ) noexcept = default; /** Create an empty particle container * @@ -169,13 +169,13 @@ public: } /** Return the name-to-index map for the compile-time and runtime-time real components */ - std::map getParticleComps () const noexcept { return particle_comps;} + [[nodiscard]] std::map getParticleComps () const noexcept { return particle_comps;} /** Return the name-to-index map for the compile-time and runtime-time integer components */ - std::map getParticleiComps () const noexcept { return particle_icomps;} + [[nodiscard]] std::map getParticleiComps () const noexcept { return particle_icomps;} /** Return the name-to-index map for the runtime-time real components */ - std::map getParticleRuntimeComps () const noexcept { return particle_runtime_comps;} + [[nodiscard]] std::map getParticleRuntimeComps () const noexcept { return particle_runtime_comps;} /** Return the name-to-index map for the runtime-time integer components */ - std::map getParticleRuntimeiComps () const noexcept { return particle_runtime_icomps;} + [[nodiscard]] std::map getParticleRuntimeiComps () const noexcept { return particle_runtime_icomps;} protected: std::map particle_comps; diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index 662e1240f8b..bbec1f54e01 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -140,10 +140,10 @@ namespace ApplyParticleBoundaries { uy = ur*std::sin(y) + ut*std::cos(y); } #else - if (change_sign_ux) ux = -ux; - if (change_sign_uy) uy = -uy; + if (change_sign_ux) { ux = -ux; } + if (change_sign_uy) { uy = -uy; } #endif - if (change_sign_uz) uz = -uz; + if (change_sign_uz) { uz = -uz; } } } diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index 7099ef4d691..1e9748b2ff5 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -23,7 +23,7 @@ class WARPX_EXPORT ParticleBoundaryBuffer public: ParticleBoundaryBuffer (); - ~ParticleBoundaryBuffer() {} + ~ParticleBoundaryBuffer() = default; /** Copy constructor for ParticleBoundaryBuffer */ ParticleBoundaryBuffer ( const ParticleBoundaryBuffer &) = delete; @@ -35,7 +35,7 @@ public: /** Move operator for NamedComponentParticleContainer */ ParticleBoundaryBuffer& operator= ( ParticleBoundaryBuffer && ) = default; - int numSpecies() const { return getSpeciesNames().size(); } + int numSpecies() const { return static_cast(getSpeciesNames().size()); } const std::vector& getSpeciesNames() const; diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 70c9459df5e..54c4396379d 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -51,12 +51,15 @@ struct CopyAndTimestamp { int src_i, int dst_i) const noexcept { dst.m_aos[dst_i] = src.m_aos[src_i]; - for (int j = 0; j < SrcData::NAR; ++j) + for (int j = 0; j < SrcData::NAR; ++j) { dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; - for (int j = 0; j < src.m_num_runtime_real; ++j) + } + for (int j = 0; j < src.m_num_runtime_real; ++j) { dst.m_runtime_rdata[j][dst_i] = src.m_runtime_rdata[j][src_i]; - for (int j = 0; j < src.m_num_runtime_int; ++j) + } + for (int j = 0; j < src.m_num_runtime_int; ++j) { dst.m_runtime_idata[j][dst_i] = src.m_runtime_idata[j][src_i]; + } dst.m_runtime_idata[m_index][dst_i] = m_step; } }; @@ -115,7 +118,7 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () #endif // Set the flag whether the boundary is active or any species for (int i = 0; i < numBoundaries(); ++i) { - if (m_do_boundary_buffer[i][ispecies]) m_do_any_boundary[i] = 1; + if (m_do_boundary_buffer[i][ispecies]) { m_do_any_boundary[i] = 1; } } } @@ -146,10 +149,10 @@ void ParticleBoundaryBuffer::printNumParticles () const { { for (int iside = 0; iside < 2; ++iside) { - auto& buffer = m_particle_containers[2*idim+iside]; + const auto& buffer = m_particle_containers[2*idim+iside]; for (int i = 0; i < numSpecies(); ++i) { - const int np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; + const auto np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; amrex::Print() << Utils::TextMsg::Info( "Species " + getSpeciesNames()[i] + " has " + std::to_string(np) + " particles in the boundary buffer " @@ -162,7 +165,7 @@ void ParticleBoundaryBuffer::printNumParticles () const { auto& buffer = m_particle_containers[2*AMREX_SPACEDIM]; for (int i = 0; i < numSpecies(); ++i) { - const int np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; + const auto np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; amrex::Print() << Utils::TextMsg::Info( "Species " + getSpeciesNames()[i] + " has " + std::to_string(np) + " particles in the EB boundary buffer" @@ -210,7 +213,7 @@ void ParticleBoundaryBuffer::clearParticles (int const i) { for (int ispecies = 0; ispecies < numSpecies(); ++ispecies) { auto& species_buffer = buffer[ispecies]; - if (species_buffer.isDefined()) species_buffer.clearParticles(); + if (species_buffer.isDefined()) { species_buffer.clearParticles(); } } } @@ -227,13 +230,13 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (geom.isPeriodic(idim)) continue; + if (geom.isPeriodic(idim)) { continue; } for (int iside = 0; iside < 2; ++iside) { auto& buffer = m_particle_containers[2*idim+iside]; for (int i = 0; i < numSpecies(); ++i) { - if (!m_do_boundary_buffer[2*idim+iside][i]) continue; + if (!m_do_boundary_buffer[2*idim+iside][i]) { continue; } const WarpXParticleContainer& pc = mypc.GetParticleContainer(i); if (!buffer[i].isDefined()) { @@ -244,16 +247,19 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, for (int lev = 0; lev < pc.numLevels(); ++lev) { const auto& plevel = pc.GetParticles(lev); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif for(PIter pti(pc, lev); pti.isValid(); ++pti) { auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - if(plevel.find(index) == plevel.end()) continue; + if(plevel.find(index) == plevel.end()) { continue; } auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile( lev, pti.index(), pti.LocalTileIndex()); const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); - if (np == 0) continue; + if (np == 0) { continue; } auto predicate = IsOutsideDomainBoundary{plo, phi, idim, iside}; @@ -307,18 +313,21 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, { const auto& plevel = pc.GetParticles(lev); auto dxi = warpx_instance.Geom(lev).InvCellSizeArray(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif for(PIter pti(pc, lev); pti.isValid(); ++pti) { auto phiarr = (*distance_to_eb[lev])[pti].array(); // signed distance function auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); if(plevel.find(index) == plevel.end()) continue; - const auto getPosition = GetParticlePosition(pti); + const auto getPosition = GetParticlePosition(pti); auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile(lev, pti.index(), pti.LocalTileIndex()); const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); - if (np == 0) continue; + if (np == 0) { continue; } using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; auto predicate = [=] AMREX_GPU_HOST_DEVICE (const SrcData& /*src*/, const int ip) @@ -370,8 +379,12 @@ int ParticleBoundaryBuffer::getNumParticlesInContainer( auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); - if (buffer[index].isDefined()) return buffer[index].TotalNumberOfParticles(false, local); - else return 0; + if (buffer[index].isDefined()){ + return static_cast(buffer[index].TotalNumberOfParticles(false, local)); + } + else{ + return 0; + } } PinnedMemoryParticleContainer & diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 09ff5cce6f0..870fc82bd0f 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -8,6 +8,12 @@ #ifndef DEFAULTINITIALIZATION_H_ #define DEFAULTINITIALIZATION_H_ +#include +#ifdef WARPX_QED +# include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" +# include "Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H" +#endif + #include #include @@ -81,4 +87,197 @@ int initializeIntValue (const InitializationPolicy policy) noexcept } } +namespace ParticleCreation { + + /** + * \brief Default initialize runtime attributes in a tile. This routine does not initialize the + * first n_external_attr_real real attributes and the first n_external_attr_int integer + * attributes, which have been in principle externally set elsewhere. + * + * @tparam[in] The type of the particle tile to operate on (e.g. could use different allocators) + * @param[inout] ptile the tile in which attributes are initialized + * @param[in] n_external_attr_real The number of real attributes that have been externally set. + * These are NOT initialized by this function. + * @param[in] n_external_attr_int The number of integer attributes that have been externally set. + * These are NOT initialized by this function. + * @param[in] user_real_attribs The names of the real components for this particle tile + * @param[in] user_int_attribs The names of the int components for this particle tile + * @param[in] particle_comps map between particle component index and component name for real comps + * @param[in] particle_icomps map between particle component index and component name for int comps + * @param[in] user_real_attrib_parser the parser functions used to initialize the user real components + * @param[in] user_int_attrib_parser the parser functions used to initialize the user int components + * @param[in] do_qed_comps whether to initialize the qed components (these are usually handled by + * SmartCopy, but NOT when adding particles in AddNParticles) + * @param[in] p_bw_engine the engine to use for setting the breit-wheeler component for QED + * @param[in] p_qs_engine the engine to use for setting the quantum synchrotron component for QED + * @param[in] ionization_initial_level the ionization level particles created should start at + * @param[in] start the index to start initializing particles + * @param[in] stop the index to stop initializing particles + */ +template +void DefaultInitializeRuntimeAttributes (PTile& ptile, + const int n_external_attr_real, + const int n_external_attr_int, + const std::vector& user_real_attribs, + const std::vector& user_int_attribs, + const std::map& particle_comps, + const std::map& particle_icomps, + const std::vector& user_real_attrib_parser, + const std::vector& user_int_attrib_parser, +#ifdef WARPX_QED + const bool do_qed_comps, + BreitWheelerEngine* p_bw_engine, + QuantumSynchrotronEngine* p_qs_engine, +#endif + const int ionization_initial_level, + int start, int stop) +{ + using namespace amrex::literals; + + // Preparing data needed for user defined attributes + const auto n_user_real_attribs = static_cast(user_real_attribs.size()); + const auto n_user_int_attribs = static_cast(user_int_attribs.size()); + const auto get_position = GetParticlePosition(ptile); + const auto soa = ptile.getParticleTileData(); + const amrex::ParticleReal* AMREX_RESTRICT ux = soa.m_rdata[PIdx::ux]; + const amrex::ParticleReal* AMREX_RESTRICT uy = soa.m_rdata[PIdx::uy]; + const amrex::ParticleReal* AMREX_RESTRICT uz = soa.m_rdata[PIdx::uz]; + constexpr int lev = 0; + const amrex::Real t = WarpX::GetInstance().gett_new(lev); + + // Initialize the last NumRuntimeRealComps() - n_external_attr_real runtime real attributes + for (int j = PIdx::nattribs + n_external_attr_real; j < ptile.NumRealComps() ; ++j) + { + auto attr_ptr = ptile.GetStructOfArrays().GetRealData(j).data(); +#ifdef WARPX_QED + // Current runtime comp is quantum synchrotron optical depth + if (particle_comps.find("opticalDepthQSR") != particle_comps.end() && + particle_comps.at("opticalDepthQSR") == j) + { + if (!do_qed_comps) { continue; } + const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = + p_qs_engine->build_optical_depth_functor(); + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelForRNG(stop - start, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { + int ip = i + start; + attr_ptr[ip] = quantum_sync_get_opt(engine); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = quantum_sync_get_opt(amrex::RandomEngine{}); + } + } + } + + // Current runtime comp is Breit-Wheeler optical depth + if (particle_comps.find("opticalDepthBW") != particle_comps.end() && + particle_comps.at("opticalDepthBW") == j) + { + if (!do_qed_comps) { continue; } + const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = + p_bw_engine->build_optical_depth_functor();; + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelForRNG(stop - start, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { + int ip = i + start; + attr_ptr[ip] = breit_wheeler_get_opt(engine); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = breit_wheeler_get_opt(amrex::RandomEngine{}); + } + } + } +#endif + + for (int ia = 0; ia < n_user_real_attribs; ++ia) + { + // Current runtime comp is ia-th user defined attribute + if (particle_comps.find(user_real_attribs[ia]) != particle_comps.end() && + particle_comps.at(user_real_attribs[ia]) == j) + { + const amrex::ParserExecutor<7> user_real_attrib_parserexec = + user_real_attrib_parser[ia]->compile<7>(); + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = user_real_attrib_parserexec(xp, yp, zp, + ux[ip], uy[ip], uz[ip], t); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = user_real_attrib_parserexec(xp, yp, zp, + ux[ip], uy[ip], uz[ip], t); + } + } + } + } + } + + // Initialize the last NumRuntimeIntComps() - n_external_attr_int runtime int attributes + for (int j = n_external_attr_int; j < ptile.NumIntComps() ; ++j) + { + auto attr_ptr = ptile.GetStructOfArrays().GetIntData(j).data(); + + // Current runtime comp is ionization level + if (particle_icomps.find("ionizationLevel") != particle_icomps.end() && + particle_icomps.at("ionizationLevel") == j) + { + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + attr_ptr[ip] = ionization_initial_level; + }); + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = ionization_initial_level; + } + } + } + + for (int ia = 0; ia < n_user_int_attribs; ++ia) + { + // Current runtime comp is ia-th user defined attribute + if (particle_icomps.find(user_int_attribs[ia]) != particle_icomps.end() && + particle_icomps.at(user_int_attribs[ia]) == j) + { + const amrex::ParserExecutor<7> user_int_attrib_parserexec = + user_int_attrib_parser[ia]->compile<7>(); + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = static_cast( + user_int_attrib_parserexec(xp, yp, zp, ux[ip], uy[ip], uz[ip], t)); + }); + } else { + for (int ip = start; ip < stop; ++ip) { + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = static_cast( + user_int_attrib_parserexec(xp, yp, zp, ux[ip], uy[ip], uz[ip], t)); + } + } + } + } + } +} + +} + #endif diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 7a85f59318f..095ebbf6a85 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -8,6 +8,8 @@ #ifndef FILTER_COPY_TRANSFORM_H_ #define FILTER_COPY_TRANSFORM_H_ +#include "Particles/ParticleCreation/DefaultInitialization.H" + #include #include @@ -47,23 +49,26 @@ * * \return num_added the number of particles that were written to dst. */ -template ::value, int> foo = 0> -Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Index dst_index, +Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, + Index* mask, Index dst_index, CopyFunc&& copy, TransFunc&& transform) noexcept { using namespace amrex; const auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); const Index num_added = N * total; - dst.resize(std::max(dst_index + num_added, dst.numParticles())); + auto old_np = dst.size(); + auto new_np = std::max(dst_index + num_added, dst.numParticles()); + dst.resize(new_np); - const auto p_offsets = offsets.dataPtr(); + auto *const p_offsets = offsets.dataPtr(); const auto src_data = src.getParticleTileData(); const auto dst_data = dst.getParticleTileData(); @@ -80,6 +85,21 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst, + 0, 0, + pc.getUserRealAttribs(), pc.getUserIntAttribs(), + pc.getParticleComps(), pc.getParticleiComps(), + pc.getUserRealAttribParser(), + pc.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc.get_breit_wheeler_engine_ptr(), + pc.get_quantum_sync_engine_ptr(), +#endif + pc.getIonizationInitialLevel(), + old_np, new_np); + Gpu::synchronize(); return num_added; } @@ -121,19 +141,19 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind * * \return num_added the number of particles that were written to dst. */ -template -Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, +Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, Index dst_index, PredFunc&& filter, CopyFunc&& copy, TransFunc&& transform) noexcept { using namespace amrex; const auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector mask(np); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); const auto src_data = src.getParticleTileData(); amrex::ParallelForRNG(np, @@ -142,9 +162,9 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, p_mask[i] = filter(src_data, i, engine); }); - return filterCopyTransformParticles(dst, src, mask.dataPtr(), dst_index, - std::forward(copy), - std::forward(transform)); + return filterCopyTransformParticles(pc, dst, src, mask.dataPtr(), dst_index, + std::forward(copy), + std::forward(transform)); } /** @@ -188,10 +208,10 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, * * \return num_added the number of particles that were written to dst. */ -template ::value, int> foo = 0> -Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, Index* mask, +Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, SrcTile& src, Index* mask, Index dst1_index, Index dst2_index, CopyFunc1&& copy1, CopyFunc2&& copy2, TransFunc&& transform) noexcept @@ -199,15 +219,20 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, using namespace amrex; auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); const Index num_added = N * total; - dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); - dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); + auto old_np1 = dst1.size(); + auto new_np1 = std::max(dst1_index + num_added, dst1.numParticles()); + dst1.resize(new_np1); + + auto old_np2 = dst2.size(); + auto new_np2 = std::max(dst2_index + num_added, dst2.numParticles()); + dst2.resize(new_np2); - auto p_offsets = offsets.dataPtr(); + auto *p_offsets = offsets.dataPtr(); const auto src_data = src.getParticleTileData(); const auto dst1_data = dst1.getParticleTileData(); @@ -230,6 +255,35 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, + 0, 0, + pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), + pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.getUserRealAttribParser(), + pc1.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc1.get_breit_wheeler_engine_ptr(), + pc1.get_quantum_sync_engine_ptr(), +#endif + pc1.getIonizationInitialLevel(), + old_np1, new_np1); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, + 0, 0, + pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), + pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.getUserRealAttribParser(), + pc2.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc2.get_breit_wheeler_engine_ptr(), + pc2.get_quantum_sync_engine_ptr(), +#endif + pc2.getIonizationInitialLevel(), + old_np2, new_np2); + Gpu::synchronize(); return num_added; } @@ -276,9 +330,9 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, * * \return num_added the number of particles that were written to dst. */ -template -Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, +Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, SrcTile& src, Index dst1_index, Index dst2_index, PredFunc&& filter, CopyFunc1&& copy1, CopyFunc2&& copy2, TransFunc&& transform) noexcept @@ -286,11 +340,11 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, using namespace amrex; auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector mask(np); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); const auto src_data = src.getParticleTileData(); amrex::ParallelForRNG(np, @@ -299,7 +353,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, p_mask[i] = filter(src_data, i, engine); }); - return filterCopyTransformParticles(dst1, dst2, src, mask.dataPtr(), + return filterCopyTransformParticles(pc1, pc2, dst1, dst2, src, mask.dataPtr(), dst1_index, dst2_index, std::forward(copy1), std::forward(copy2), diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 38dc1d6daf2..2a4c9fccad0 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -8,6 +8,7 @@ #ifndef FILTER_CREATE_TRANSFORM_FROM_FAB_H_ #define FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#include "Particles/ParticleCreation/DefaultInitialization.H" #include "WarpX.H" #include @@ -42,19 +43,20 @@ * * \return num_added the number of particles that were written to dst1 and dst2. */ -template ::value, int> foo = 0> -Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::Box box, - const FAB *src_FAB, const Index* mask, - const Index dst1_index, const Index dst2_index, - CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc&& transform) noexcept +Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, + DstTile& dst1, DstTile& dst2, const amrex::Box box, + const FAB *src_FAB, const Index* mask, + const Index dst1_index, const Index dst2_index, + CreateFunc1&& create1, CreateFunc2&& create2, + TransFunc&& transform) noexcept { using namespace amrex; const auto ncells = box.volume(); - if (ncells == 0) return 0; + if (ncells == 0) { return 0; } auto & warpx = WarpX::GetInstance(); const int level_0 = 0; @@ -83,10 +85,15 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B Gpu::DeviceVector offsets(ncells); auto total = amrex::Scan::ExclusiveSum(ncells, mask, offsets.data()); const Index num_added = N*total; - dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); - dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); + auto old_np1 = dst1.size(); + auto new_np1 = std::max(dst1_index + num_added, dst1.numParticles()); + dst1.resize(new_np1); + + auto old_np2 = dst2.size(); + auto new_np2 = std::max(dst2_index + num_added, dst2.numParticles()); + dst2.resize(new_np2); - auto p_offsets = offsets.dataPtr(); + auto *p_offsets = offsets.dataPtr(); const auto dst1_data = dst1.getParticleTileData(); const auto dst2_data = dst2.getParticleTileData(); @@ -130,6 +137,35 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, + 0, 0, + pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), + pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.getUserRealAttribParser(), + pc1.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CreateFunc functor + pc1.get_breit_wheeler_engine_ptr(), + pc1.get_quantum_sync_engine_ptr(), +#endif + pc1.getIonizationInitialLevel(), + old_np1, new_np1); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, + 0, 0, + pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), + pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.getUserRealAttribParser(), + pc2.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CreateFunc functor + pc2.get_breit_wheeler_engine_ptr(), + pc2.get_quantum_sync_engine_ptr(), +#endif + pc2.getIonizationInitialLevel(), + old_np2, new_np2); + Gpu::synchronize(); return num_added; } @@ -166,10 +202,10 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B * * \return num_added the number of particles that were written to dst1 and dst2. */ -template -Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::Box box, +Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, const amrex::Box box, const FABs& src_FABs, const Index dst1_index, const Index dst2_index, FilterFunc&& filter, CreateFunc1&& create1, CreateFunc2&& create2, @@ -184,11 +220,11 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B auto arrNumPartCreation = NumPartCreation.array(); const auto ncells = box.volume(); - if (ncells == 0) return 0; + if (ncells == 0) { return 0; } Gpu::DeviceVector mask(ncells); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); // for loop over all cells in the box. We apply the filter function to each cell // and store the result in arrNumPartCreation. If the result is strictly greater than @@ -201,7 +237,7 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B p_mask[mask_position] = (arrNumPartCreation(i,j,k) > 0); }); - return filterCreateTransformFromFAB(dst1, dst2, box, &NumPartCreation, + return filterCreateTransformFromFAB(pc1, pc2, dst1, dst2, box, &NumPartCreation, mask.dataPtr(), dst1_index, dst2_index, std::forward(create1), std::forward(create2), diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index 4b29d90e065..2c04baa18bb 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -52,16 +52,20 @@ struct SmartCopy dst.m_aos[i_dst] = src.m_aos[i_src]; // initialize the real components - for (int j = 0; j < DstData::NAR; ++j) + for (int j = 0; j < DstData::NAR; ++j) { dst.m_rdata[j][i_dst] = initializeRealValue(m_policy_real[j], engine); - for (int j = 0; j < dst.m_num_runtime_real; ++j) + } + for (int j = 0; j < dst.m_num_runtime_real; ++j) { dst.m_runtime_rdata[j][i_dst] = initializeRealValue(m_policy_real[j+DstData::NAR], engine); + } // initialize the int components - for (int j = 0; j < DstData::NAI; ++j) + for (int j = 0; j < DstData::NAI; ++j) { dst.m_idata[j][i_dst] = initializeIntValue(m_policy_int[j]); - for (int j = 0; j < dst.m_num_runtime_int; ++j) + } + for (int j = 0; j < dst.m_num_runtime_int; ++j) { dst.m_runtime_idata[j][i_dst] = initializeIntValue(m_policy_int[j+DstData::NAI]); + } // copy the shared real components for (int j = 0; j < m_num_copy_real; ++j) @@ -134,23 +138,19 @@ class SmartCopyFactory SmartCopyTag m_tag_int; PolicyVec m_policy_real; PolicyVec m_policy_int; - bool m_defined; + bool m_defined = false; public: template - SmartCopyFactory (const SrcPC& src, const DstPC& dst) noexcept - : m_defined(false) - { - m_tag_real = getSmartCopyTag(src.getParticleComps(), dst.getParticleComps()); - m_tag_int = getSmartCopyTag(src.getParticleiComps(), dst.getParticleiComps()); - - m_policy_real = getPolicies(dst.getParticleComps()); - m_policy_int = getPolicies(dst.getParticleiComps()); - - m_defined = true; - } - - SmartCopy getSmartCopy () const noexcept + SmartCopyFactory (const SrcPC& src, const DstPC& dst) noexcept : + m_tag_real{getSmartCopyTag(src.getParticleComps(), dst.getParticleComps())}, + m_tag_int{getSmartCopyTag(src.getParticleiComps(), dst.getParticleiComps())}, + m_policy_real{getPolicies(dst.getParticleComps())}, + m_policy_int{getPolicies(dst.getParticleiComps())}, + m_defined{true} + {} + + [[nodiscard]] SmartCopy getSmartCopy () const noexcept { AMREX_ASSERT(m_defined); return SmartCopy{m_tag_real.size(), @@ -163,7 +163,7 @@ public: m_policy_int.dataPtr()}; } - bool isDefined () const noexcept { return m_defined; } + [[nodiscard]] bool isDefined () const noexcept { return m_defined; } }; #endif diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index 875e69cab04..67d7767a5d3 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -62,17 +62,21 @@ struct SmartCreate prt.m_aos[i_prt].cpu() = cpu; prt.m_aos[i_prt].id() = id; - // initialize the real components - for (int j = 0; j < PartData::NAR; ++j) - prt.m_rdata[j][i_prt] = initializeRealValue(m_policy_real[j], engine); - for (int j = 0; j < prt.m_num_runtime_real; ++j) - prt.m_runtime_rdata[j][i_prt] = initializeRealValue(m_policy_real[j+PartData::NAR], engine); + // initialize the real components + for (int j = 0; j < PartData::NAR; ++j) { + prt.m_rdata[j][i_prt] = initializeRealValue(m_policy_real[j], engine); + } + for (int j = 0; j < prt.m_num_runtime_real; ++j) { + prt.m_runtime_rdata[j][i_prt] = initializeRealValue(m_policy_real[j+PartData::NAR], engine); + } - // initialize the int components - for (int j = 0; j < PartData::NAI; ++j) - prt.m_idata[j][i_prt] = initializeIntValue(m_policy_int[j]); - for (int j = 0; j < prt.m_num_runtime_int; ++j) - prt.m_runtime_idata[j][i_prt] = initializeIntValue(m_policy_int[j+PartData::NAI]); + // initialize the int components + for (int j = 0; j < PartData::NAI; ++j) { + prt.m_idata[j][i_prt] = initializeIntValue(m_policy_int[j]); + } + for (int j = 0; j < prt.m_num_runtime_int; ++j) { + prt.m_runtime_idata[j][i_prt] = initializeIntValue(m_policy_int[j+PartData::NAI]); + } } }; @@ -87,26 +91,24 @@ class SmartCreateFactory { PolicyVec m_policy_real; PolicyVec m_policy_int; - bool m_defined; + bool m_defined{false}; public: template - SmartCreateFactory (const PartTileData& part) noexcept - : m_defined(false) - { - m_policy_real = getPolicies(part.getParticleComps()); - m_policy_int = getPolicies(part.getParticleiComps()); - m_defined = true; - } + SmartCreateFactory (const PartTileData& part) noexcept: + m_policy_real{getPolicies(part.getParticleComps())}, + m_policy_int{getPolicies(part.getParticleiComps())}, + m_defined{true} + {} - SmartCreate getSmartCreate () const noexcept + [[nodiscard]] SmartCreate getSmartCreate () const noexcept { AMREX_ASSERT(m_defined); return SmartCreate{m_policy_real.dataPtr(), m_policy_int.dataPtr()}; } - bool isDefined () const noexcept { return m_defined; } + [[nodiscard]] bool isDefined () const noexcept { return m_defined; } }; #endif //SMART_CREATE_H_ diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index ee1dc321cef..732a12bb729 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -31,7 +31,7 @@ struct SmartCopyTag amrex::Gpu::DeviceVector src_comps; amrex::Gpu::DeviceVector dst_comps; - int size () const noexcept { return common_names.size(); } + [[nodiscard]] int size () const noexcept { return static_cast(common_names.size()); } }; PolicyVec getPolicies (const NameMap& names) noexcept; diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index 0f7b6e0e185..34afac53482 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -9,6 +9,7 @@ #define WARPX_PhotonParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Gather/ScaleFields.H" #include "PhysicalParticleContainer.H" @@ -25,7 +26,7 @@ /** * Photon particles have no mass, they deposit no charge, and see specific QED * effects. For these reasons, they are stored in the separate particle - * container PhotonParticleContainer, that inherts from + * container PhotonParticleContainer, that inherits from * PhysicalParticleContainer. The particle pusher and current deposition, in * particular, are overriden in this container. */ @@ -36,11 +37,16 @@ public: PhotonParticleContainer (amrex::AmrCore* amr_core, int ispecies, const std::string& name); - virtual ~PhotonParticleContainer () {} + ~PhotonParticleContainer () override = default; - virtual void InitData() override; + PhotonParticleContainer ( PhotonParticleContainer const &) = delete; + PhotonParticleContainer& operator= ( PhotonParticleContainer const & ) = delete; + PhotonParticleContainer ( PhotonParticleContainer&& ) = default; + PhotonParticleContainer& operator= ( PhotonParticleContainer&& ) = default; - virtual void Evolve (int lev, + void InitData() override; + + void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, @@ -64,9 +70,10 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; - virtual void PushPX(WarpXParIter& pti, + void PushPX(WarpXParIter& pti, amrex::FArrayBox const * exfab, amrex::FArrayBox const * eyfab, amrex::FArrayBox const * ezfab, @@ -81,7 +88,7 @@ public: DtType a_dt_type) override; // Do nothing - virtual void PushP (int /*lev*/, + void PushP (int /*lev*/, amrex::Real /*dt*/, const amrex::MultiFab& /*Ex*/, const amrex::MultiFab& /*Ey*/, @@ -92,19 +99,19 @@ public: // DepositCharge should do nothing for photons - virtual void DepositCharge (WarpXParIter& /*pti*/, + void DepositCharge (WarpXParIter& /*pti*/, RealVector const & /*wp*/, const int * const /*ion_lev*/, amrex::MultiFab* /*rho*/, int /*icomp*/, const long /*offset*/, - const long /*np_to_depose*/, + const long /*np_to_deposit*/, int /*thread_num*/, int /*lev*/, int /*depos_lev*/) override {} // DepositCurrent should do nothing for photons - virtual void DepositCurrent (WarpXParIter& /*pti*/, + void DepositCurrent (WarpXParIter& /*pti*/, RealVector const & /*wp*/, RealVector const & /*uxp*/, RealVector const & /*uyp*/, @@ -114,12 +121,13 @@ public: amrex::MultiFab * const /*jy*/, amrex::MultiFab * const /*jz*/, long const /*offset*/, - long const /*np_to_depose*/, + long const /*np_to_deposit*/, int const /*thread_num*/, int const /*lev*/, int const /*depos_lev*/, amrex::Real const /*dt*/, - amrex::Real const /*relative_time*/) override {} + amrex::Real const /*relative_time*/, + PushType /*push_type*/) override {} }; #endif // #ifndef WARPX_PhotonParticleContainer_H_ diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 9b7da18bc5e..aa9f04be224 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -129,11 +129,18 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, auto copyAttribs = CopyParticleAttribs(pti, tmp_particle_data, offset); const int do_copy = (m_do_back_transformed_particles && (a_dt_type!=DtType::SecondHalf) ); - const auto GetPosition = GetParticlePosition(pti, offset); - auto SetPosition = SetParticlePosition(pti, offset); + const auto GetPosition = GetParticlePosition(pti, offset); + auto SetPosition = SetParticlePosition(pti, offset); const auto getExternalEB = GetExternalEBField(pti, offset); + const amrex::ParticleReal Ex_external_particle = m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; + // Lower corner of tile box physical domain (take into account Galilean shift) const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); @@ -178,12 +185,16 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, np_to_push, [=] AMREX_GPU_DEVICE (long i, auto exteb_control, auto qed_control) { - if (do_copy) copyAttribs(i); + if (do_copy) { copyAttribs(i); } ParticleReal x, y, z; GetPosition(i, x, y, z); - amrex::ParticleReal Exp=0, Eyp=0, Ezp=0; - amrex::ParticleReal Bxp=0, Byp=0, Bzp=0; + amrex::ParticleReal Exp = Ex_external_particle; + amrex::ParticleReal Eyp = Ey_external_particle; + amrex::ParticleReal Ezp = Ez_external_particle; + amrex::ParticleReal Bxp = Bx_external_particle; + amrex::ParticleReal Byp = By_external_particle; + amrex::ParticleReal Bzp = Bz_external_particle; if(!t_do_not_gather){ // first gather E and B to the particle positions @@ -194,17 +205,17 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, nox, galerkin_interpolation); } - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; // workaround for nvcc + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; // workaround for nvcc if constexpr (exteb_control == has_exteb) { getExternalEB(i, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } #ifdef WARPX_QED - [[maybe_unused]] auto& evolve_opt_tmp = evolve_opt; - [[maybe_unused]] auto p_optical_depth_BW_tmp = p_optical_depth_BW; - [[maybe_unused]] auto ux_tmp = ux; // for nvhpc - [[maybe_unused]] auto uy_tmp = uy; - [[maybe_unused]] auto uz_tmp = uz; + [[maybe_unused]] const auto& evolve_opt_tmp = evolve_opt; + [[maybe_unused]] auto *p_optical_depth_BW_tmp = p_optical_depth_BW; + [[maybe_unused]] auto *ux_tmp = ux; // for nvhpc + [[maybe_unused]] auto *uy_tmp = uy; + [[maybe_unused]] auto *uz_tmp = uz; [[maybe_unused]] auto dt_tmp = dt; if constexpr (qed_control == has_qed) { evolve_opt(ux[i], uy[i], uz[i], Exp, Eyp, Ezp, Bxp, Byp, Bzp, @@ -229,10 +240,11 @@ PhotonParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { - // This does gather, push and depose. - // Push and depose have been re-written for photons + // This does gather, push and deposit. + // Push and deposit have been re-written for photons PhysicalParticleContainer::Evolve (lev, Ex, Ey, Ez, Bx, By, Bz, @@ -241,6 +253,6 @@ PhotonParticleContainer::Evolve (int lev, rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, - t, dt, a_dt_type, skip_deposition); + t, dt, a_dt_type, skip_deposition, push_type); } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 3e3ae069214..1ec3473e81b 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -11,6 +11,7 @@ #define WARPX_PhysicalParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Initialization/PlasmaInjector.H" #include "Particles/ElementaryProcess/Ionization.H" #ifdef WARPX_QED @@ -56,20 +57,25 @@ public: * the run if one of them is specified. */ void BackwardCompatibility (); - virtual ~PhysicalParticleContainer () {} + ~PhysicalParticleContainer () override = default; - virtual void InitData () override; + PhysicalParticleContainer (PhysicalParticleContainer const &) = delete; + PhysicalParticleContainer& operator= (PhysicalParticleContainer const & ) = delete; + PhysicalParticleContainer(PhysicalParticleContainer&& ) = default; + PhysicalParticleContainer& operator=(PhysicalParticleContainer&& ) = default; - virtual void ReadHeader (std::istream& is) override; + void InitData () override; - virtual void WriteHeader (std::ostream& os) const override; + void ReadHeader (std::istream& is) override; - virtual void InitIonizationModule () override; + void WriteHeader (std::ostream& os) const override; + + void InitIonizationModule () override; /* - * \brief Returns a pointer to the plasma injector. + * \brief Returns a pointer to the i'th plasma injector. */ - virtual PlasmaInjector* GetPlasmaInjector () override; + PlasmaInjector* GetPlasmaInjector (int i) override; /** * \brief Evolve is the central function PhysicalParticleContainer that @@ -100,12 +106,13 @@ public: * \param dt time step by which particles are advanced * \param a_dt_type type of time step (used for sub-cycling) * \param skip_deposition Skip the charge and current deposition. + * \param push_type Type of particle push, explicit or implicit. Defaults to explicit * * Evolve iterates over particle iterator (each box) and performs filtering, * field gather, particle push and current deposition for all particles * in the box. */ - virtual void Evolve (int lev, + void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, @@ -129,7 +136,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false ) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; virtual void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, @@ -145,7 +153,21 @@ public: amrex::Real dt, ScaleFields scaleFields, DtType a_dt_type=DtType::Full); - virtual void PushP (int lev, amrex::Real dt, + void ImplicitPushXP (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + amrex::IntVect ngEB, int /*e_is_nodal*/, + long offset, + long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type=DtType::Full); + + void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, @@ -162,7 +184,7 @@ public: amrex::iMultiFab const* current_masks, amrex::iMultiFab const* gather_masks ); - virtual void PostRestart () final {} + void PostRestart () final {} void SplitParticles (int lev); @@ -184,35 +206,40 @@ public: * number of particles per cell (in the cells of `part_realbox`). * The new particles are only created inside the intersection of `part_realbox` * with the local grid for the current proc. + * @param[in] the PlasmaInjector instance holding the input parameters * @param[in] lev the index of the refinement level * @param[in] part_realbox the box in which new particles should be created * (this box should correspond to an integer number of cells in each direction, * but its boundaries need not be aligned with the actual cells of the simulation) */ - void AddPlasma (int lev, amrex::RealBox part_realbox = amrex::RealBox()); + void AddPlasma (PlasmaInjector const& plasma_injector, int lev, amrex::RealBox part_realbox = amrex::RealBox()); /** * Create new macroparticles for this species, with a fixed * number of particles per cell in a plane. + * @param[in] the PlasmaInjector instance holding the input parameters * @param[in] dt time step size, used to partially advance the particles */ - void AddPlasmaFlux (amrex::Real dt); + void AddPlasmaFlux (PlasmaInjector const& plasma_injector, amrex::Real dt); void MapParticletoBoostedFrame (amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, amrex::Real t_lab = 0.); void AddGaussianBeam ( + PlasmaInjector const& plasma_injector, amrex::Real x_m, amrex::Real y_m, amrex::Real z_m, amrex::Real x_rms, amrex::Real y_rms, amrex::Real z_rms, amrex::Real x_cut, amrex::Real y_cut, amrex::Real z_cut, amrex::Real q_tot, long npart, int do_symmetrize, int symmetrization_order); /** Load a particle beam from an external file + * @param[in] the PlasmaInjector instance holding the input parameters * @param[in] q_tot total charge of the particle species to be initialized * @param[in] z_shift optional shift to the z position of particles (useful for boosted frame runs) */ - void AddPlasmaFromFile (amrex::ParticleReal q_tot, + void AddPlasmaFromFile (PlasmaInjector & plasma_injector, + amrex::ParticleReal q_tot, amrex::ParticleReal z_shift); void CheckAndAddParticle ( @@ -240,13 +267,12 @@ public: * These are NOT initialized by this function. * @param[in] engine the random engine, used in initialization of QED optical depths */ - virtual void DefaultInitializeRuntimeAttributes ( + void DefaultInitializeRuntimeAttributes ( amrex::ParticleTile, NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& pinned_tile, int n_external_attr_real, - int n_external_attr_int, - const amrex::RandomEngine& engine) override final; + int n_external_attr_int) final; /** * \brief Apply NCI Godfrey filter to all components of E and B before gather @@ -276,12 +302,12 @@ public: * \param Bx Field array before filtering (not modified) * \param By Field array before filtering (not modified) * \param Bz Field array before filtering (not modified) - * \param exfab pointer to the Ex field (modified) - * \param eyfab pointer to the Ey field (modified) - * \param ezfab pointer to the Ez field (modified) - * \param bxfab pointer to the Bx field (modified) - * \param byfab pointer to the By field (modified) - * \param bzfab pointer to the Bz field (modified) + * \param ex_ptr pointer to the Ex field (modified) + * \param ey_ptr pointer to the Ey field (modified) + * \param ez_ptr pointer to the Ez field (modified) + * \param bx_ptr pointer to the Bx field (modified) + * \param by_ptr pointer to the By field (modified) + * \param bz_ptr pointer to the Bz field (modified) * * The NCI Godfrey filter is applied on Ex, the result is stored in filtered_Ex * and the pointer exfab is modified (before this function is called, it points to Ex @@ -297,9 +323,9 @@ public: const amrex::FArrayBox& Ex, const amrex::FArrayBox& Ey, const amrex::FArrayBox& Ez, const amrex::FArrayBox& Bx, const amrex::FArrayBox& By, const amrex::FArrayBox& Bz, - amrex::FArrayBox const * & exfab, amrex::FArrayBox const * & eyfab, - amrex::FArrayBox const * & ezfab, amrex::FArrayBox const * & bxfab, - amrex::FArrayBox const * & byfab, amrex::FArrayBox const * & bzfab); + amrex::FArrayBox const * & ex_ptr, amrex::FArrayBox const * & ey_ptr, + amrex::FArrayBox const * & ez_ptr, amrex::FArrayBox const * & bx_ptr, + amrex::FArrayBox const * & by_ptr, amrex::FArrayBox const * & bz_ptr); /** * \brief This function determines if resampling should be done for the current species, and @@ -307,7 +333,7 @@ public: * * @param[in] timestep the current timestep. */ - void resample (int timestep, bool verbose=true) override final; + void resample (int timestep, bool verbose=true) final; #ifdef WARPX_QED //Functions decleared in WarpXParticleContainer.H @@ -343,14 +369,38 @@ public: (std::shared_ptr ptr) override; //__________ + BreitWheelerEngine* get_breit_wheeler_engine_ptr () const override { + return m_shr_p_bw_engine.get(); + } + + QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const override { + return m_shr_p_qs_engine.get(); + } + PhotonEmissionFilterFunc getPhotonEmissionFilterFunc (); PairGenerationFilterFunc getPairGenerationFilterFunc (); #endif + std::vector getUserIntAttribs () const override { + return m_user_int_attribs; + } + + std::vector getUserRealAttribs () const override { + return m_user_real_attribs; + } + + amrex::Vector< amrex::Parser* > getUserIntAttribParser () const override { + return GetVecOfPtrs(m_user_int_attrib_parser); + } + + amrex::Vector< amrex::Parser* > getUserRealAttribParser () const override { + return GetVecOfPtrs(m_user_real_attrib_parser); + } + protected: std::string species_name; - std::unique_ptr plasma_injector; + std::vector> plasma_injectors; // When true, adjust the transverse particle positions accounting // for the difference between the Lorentz transformed time of the @@ -398,9 +448,9 @@ protected: /* Vector of user-defined real attributes for species, species_name */ std::vector m_user_real_attribs; /* Vector of user-defined parser for initializing user-defined integer attributes */ - std::vector< std::unique_ptr > m_user_int_attrib_parser; + amrex::Vector< std::unique_ptr > m_user_int_attrib_parser; /* Vector of user-defined parser for initializing user-defined real attributes */ - std::vector< std::unique_ptr > m_user_real_attrib_parser; + amrex::Vector< std::unique_ptr > m_user_real_attrib_parser; }; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 1358011b9b0..cfd0074d616 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -21,6 +21,7 @@ #endif #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" +#include "Particles/ParticleCreation/DefaultInitialization.H" #include "Particles/Pusher/CopyParticleAttribs.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/PushSelector.H" @@ -83,7 +84,6 @@ #include #include #include -#include #include #include #include @@ -144,12 +144,21 @@ namespace ParticleReal x, y, z; AMREX_GPU_HOST_DEVICE - PDim3(const PDim3&) = default; + PDim3(const amrex::XDim3& a): + x{a.x}, y{a.y}, z{a.z} + {} + AMREX_GPU_HOST_DEVICE - PDim3(const amrex::XDim3& a) : x(a.x), y(a.y), z(a.z) {} + ~PDim3() = default; AMREX_GPU_HOST_DEVICE - PDim3& operator=(const PDim3&) = default; + PDim3(PDim3 const &) = default; + AMREX_GPU_HOST_DEVICE + PDim3& operator=(PDim3 const &) = default; + AMREX_GPU_HOST_DEVICE + PDim3(PDim3&&) = default; + AMREX_GPU_HOST_DEVICE + PDim3& operator=(PDim3&&) = default; }; AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -236,13 +245,81 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp { BackwardCompatibility(); - plasma_injector = std::make_unique(species_id, species_name, amr_core->Geom(0)); - physical_species = plasma_injector->getPhysicalSpecies(); - charge = plasma_injector->getCharge(); - mass = plasma_injector->getMass(); - const ParmParse pp_species_name(species_name); + std::string injection_style = "none"; + pp_species_name.query("injection_style", injection_style); + if (injection_style != "none") { + // The base plasma injector, whose input parameters have no source prefix. + // Only created if needed + plasma_injectors.push_back(std::make_unique(species_id, species_name, amr_core->Geom(0))); + } + + std::vector injection_sources; + pp_species_name.queryarr("injection_sources", injection_sources); + for (auto &source_name : injection_sources) { + plasma_injectors.push_back(std::make_unique(species_id, species_name, amr_core->Geom(0), + source_name)); + } + + // Setup the charge and mass. There are multiple ways that they can be specified, so checks are needed to + // ensure that a value is specified and warnings given if multiple values are specified. + // The ordering is that species.charge and species.mass take precedence over all other values. + // Next is charge and mass determined from species_type. + // Last is charge and mass from the plasma injector setup + bool charge_from_source = false; + bool mass_from_source = false; + for (auto const& plasma_injector : plasma_injectors) { + // For now, use the last value for charge and mass that is found. + // A check could be added for consistency of multiple values, but it'll probably never be needed + charge_from_source |= plasma_injector->queryCharge(charge); + mass_from_source |= plasma_injector->queryMass(mass); + } + + std::string physical_species_s; + const bool species_is_specified = pp_species_name.query("species_type", physical_species_s); + if (species_is_specified) { + const auto physical_species_from_string = species::from_string( physical_species_s ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(physical_species_from_string, + physical_species_s + " does not exist!"); + physical_species = physical_species_from_string.value(); + charge = species::get_charge( physical_species ); + mass = species::get_mass( physical_species ); + } + + // parse charge and mass (overriding values above) + const bool charge_is_specified = utils::parser::queryWithParser(pp_species_name, "charge", charge); + const bool mass_is_specified = utils::parser::queryWithParser(pp_species_name, "mass", mass); + + if (charge_is_specified && species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".charge' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".charge' will take precedence.\n"); + } + if (mass_is_specified && species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".mass' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".mass' will take precedence.\n"); + } + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + charge_from_source || + charge_is_specified || + species_is_specified, + "Need to specify at least one of species_type or charge for species '" + + species_name + "'." + ); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + mass_from_source || + mass_is_specified || + species_is_specified, + "Need to specify at least one of species_type or mass for species '" + + species_name + "'." + ); + pp_species_name.query("boost_adjust_transverse_positions", boost_adjust_transverse_positions); pp_species_name.query("do_backward_propagation", do_backward_propagation); pp_species_name.query("random_theta", m_rz_random_theta); @@ -267,7 +344,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp_species_name.query("do_field_ionization", do_field_ionization); pp_species_name.query("do_resampling", do_resampling); - if (do_resampling) m_resampler = Resampling(species_name); + if (do_resampling) { m_resampler = Resampling(species_name); } //check if Radiation Reaction is enabled and do consistency checks pp_species_name.query("do_classical_radiation_reaction", do_classical_radiation_reaction); @@ -289,12 +366,14 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp #ifdef WARPX_QED pp_species_name.query("do_qed_quantum_sync", m_do_qed_quantum_sync); - if (m_do_qed_quantum_sync) + if (m_do_qed_quantum_sync) { AddRealComp("opticalDepthQSR"); + } pp_species_name.query("do_qed_breit_wheeler", m_do_qed_breit_wheeler); - if (m_do_qed_breit_wheeler) + if (m_do_qed_breit_wheeler) { AddRealComp("opticalDepthBW"); + } if(m_do_qed_quantum_sync){ pp_species_name.get("qed_quantum_sync_phot_product_species", @@ -304,7 +383,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp // User-defined integer attributes pp_species_name.queryarr("addIntegerAttributes", m_user_int_attribs); - const int n_user_int_attribs = m_user_int_attribs.size(); + const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); std::vector< std::string > str_int_attrib_function; str_int_attrib_function.resize(n_user_int_attribs); m_user_int_attrib_parser.resize(n_user_int_attribs); @@ -319,7 +398,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp // User-defined real attributes pp_species_name.queryarr("addRealAttributes", m_user_real_attribs); - const int n_user_real_attribs = m_user_real_attribs.size(); + const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); std::vector< std::string > str_real_attrib_function; str_real_attrib_function.resize(n_user_real_attribs); m_user_real_attrib_parser.resize(n_user_real_attribs); @@ -366,7 +445,6 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) : WarpXParticleContainer(amr_core, 0) { - plasma_injector = std::make_unique(); } void @@ -440,6 +518,7 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame ( void PhysicalParticleContainer::AddGaussianBeam ( + PlasmaInjector const& plasma_injector, const Real x_m, const Real y_m, const Real z_m, const Real x_rms, const Real y_rms, const Real z_rms, const Real x_cut, const Real y_cut, const Real z_cut, @@ -455,7 +534,6 @@ PhysicalParticleContainer::AddGaussianBeam ( Gpu::HostVector particle_uy; Gpu::HostVector particle_uz; Gpu::HostVector particle_w; - int np = 0; if (ParallelDescriptor::IOProcessor()) { // If do_symmetrize, create either 4x or 8x fewer particles, and @@ -481,11 +559,11 @@ PhysicalParticleContainer::AddGaussianBeam ( constexpr Real y = 0._prt; const Real z = amrex::RandomNormal(z_m, z_rms); #endif - if (plasma_injector->insideBounds(x, y, z) && + if (plasma_injector.insideBounds(x, y, z) && std::abs( x - x_m ) <= x_cut * x_rms && std::abs( y - y_m ) <= y_cut * y_rms && std::abs( z - z_m ) <= z_cut * z_rms ) { - XDim3 u = plasma_injector->getMomentum(x, y, z); + XDim3 u = plasma_injector.getMomentum(x, y, z); u.x *= PhysConst::c; u.y *= PhysConst::c; u.z *= PhysConst::c; @@ -551,7 +629,7 @@ PhysicalParticleContainer::AddGaussianBeam ( } } // Add the temporary CPU vectors to the particle structure - np = particle_z.size(); + auto const np = static_cast(particle_z.size()); amrex::Vector xp(particle_x.data(), particle_x.data() + np); amrex::Vector yp(particle_y.data(), particle_y.data() + np); amrex::Vector zp(particle_z.data(), particle_z.data() + np); @@ -570,7 +648,8 @@ PhysicalParticleContainer::AddGaussianBeam ( } void -PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, +PhysicalParticleContainer::AddPlasmaFromFile(PlasmaInjector & plasma_injector, + ParticleReal q_tot, ParticleReal z_shift) { // Declare temporary vectors on the CPU @@ -585,10 +664,8 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, #ifdef WARPX_USE_OPENPMD //TODO: Make changes for read/write in multiple MPI ranks if (ParallelDescriptor::IOProcessor()) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(plasma_injector, - "AddPlasmaFromFile: plasma injector not initialized.\n"); // take ownership of the series and close it when done - auto series = std::move(plasma_injector->m_openpmd_input_series); + auto series = std::move(plasma_injector.m_openpmd_input_series); // assumption asserts: see PlasmaInjector openPMD::Iteration it = series->iterations.begin()->second; @@ -606,30 +683,30 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, #if !defined(WARPX_DIM_1D_Z) // 2D, 3D, and RZ const std::shared_ptr ptr_x = ps["position"]["x"].loadChunk(); const std::shared_ptr ptr_offset_x = ps["positionOffset"]["x"].loadChunk(); - double const position_unit_x = ps["position"]["x"].unitSI(); - double const position_offset_unit_x = ps["positionOffset"]["x"].unitSI(); + auto const position_unit_x = static_cast(ps["position"]["x"].unitSI()); + auto const position_offset_unit_x = static_cast(ps["positionOffset"]["x"].unitSI()); #endif #if !(defined(WARPX_DIM_XZ) || defined(WARPX_DIM_1D_Z)) const std::shared_ptr ptr_y = ps["position"]["y"].loadChunk(); const std::shared_ptr ptr_offset_y = ps["positionOffset"]["y"].loadChunk(); - double const position_unit_y = ps["position"]["y"].unitSI(); - double const position_offset_unit_y = ps["positionOffset"]["y"].unitSI(); + auto const position_unit_y = static_cast(ps["position"]["y"].unitSI()); + auto const position_offset_unit_y = static_cast(ps["positionOffset"]["y"].unitSI()); #endif const std::shared_ptr ptr_z = ps["position"]["z"].loadChunk(); const std::shared_ptr ptr_offset_z = ps["positionOffset"]["z"].loadChunk(); - double const position_unit_z = ps["position"]["z"].unitSI(); - double const position_offset_unit_z = ps["positionOffset"]["z"].unitSI(); + auto const position_unit_z = static_cast(ps["position"]["z"].unitSI()); + auto const position_offset_unit_z = static_cast(ps["positionOffset"]["z"].unitSI()); const std::shared_ptr ptr_ux = ps["momentum"]["x"].loadChunk(); - double const momentum_unit_x = ps["momentum"]["x"].unitSI(); + auto const momentum_unit_x = static_cast(ps["momentum"]["x"].unitSI()); const std::shared_ptr ptr_uz = ps["momentum"]["z"].loadChunk(); - double const momentum_unit_z = ps["momentum"]["z"].unitSI(); + auto const momentum_unit_z = static_cast(ps["momentum"]["z"].unitSI()); const std::shared_ptr ptr_w = ps["weighting"][openPMD::RecordComponent::SCALAR].loadChunk(); - double const w_unit = ps["weighting"][openPMD::RecordComponent::SCALAR].unitSI(); + auto const w_unit = static_cast(ps["weighting"][openPMD::RecordComponent::SCALAR].unitSI()); std::shared_ptr ptr_uy = nullptr; - double momentum_unit_y = 1.0; + auto momentum_unit_y = 1.0_prt; if (ps["momentum"].contains("y")) { ptr_uy = ps["momentum"]["y"].loadChunk(); - momentum_unit_y = ps["momentum"]["y"].unitSI(); + momentum_unit_y = static_cast(ps["momentum"]["y"].unitSI()); } series->flush(); // shared_ptr data can be read now @@ -656,7 +733,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, #endif ParticleReal const z = ptr_z.get()[i]*position_unit_z + ptr_offset_z.get()[i]*position_offset_unit_z + z_shift; - if (plasma_injector->insideBounds(x, y, z)) { + if (plasma_injector.insideBounds(x, y, z)) { ParticleReal const ux = ptr_ux.get()[i]*momentum_unit_x/mass; ParticleReal const uz = ptr_uz.get()[i]*momentum_unit_z/mass; ParticleReal uy = 0.0_prt; @@ -666,7 +743,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, CheckAndAddParticle(x, y, z, ux, uy, uz, weight, particle_x, particle_y, particle_z, particle_ux, particle_uy, particle_uz, - particle_w, t_lab); + particle_w, static_cast(t_lab)); } } auto const np = particle_z.size(); @@ -676,7 +753,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, ablastr::warn_manager::WarnPriority::high); } } // IO Processor - auto const np = particle_z.size(); + auto const np = static_cast(particle_z.size()); amrex::Vector xp(particle_x.data(), particle_x.data() + np); amrex::Vector yp(particle_y.data(), particle_y.data() + np); amrex::Vector zp(particle_z.data(), particle_z.data() + np); @@ -694,7 +771,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, 1, attr, 0, attr_int, 1); #endif // WARPX_USE_OPENPMD - ignore_unused(q_tot, z_shift); + ignore_unused(plasma_injector, q_tot, z_shift); } void @@ -703,110 +780,21 @@ PhysicalParticleContainer::DefaultInitializeRuntimeAttributes ( NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& pinned_tile, const int n_external_attr_real, - const int n_external_attr_int, - const amrex::RandomEngine& engine) + const int n_external_attr_int) { - using namespace amrex::literals; - - const int np = pinned_tile.numParticles(); - - // Preparing data needed for user defined attributes - const int n_user_real_attribs = m_user_real_attribs.size(); - const int n_user_int_attribs = m_user_int_attribs.size(); - const auto get_position = GetParticlePosition(pinned_tile); - const auto soa = pinned_tile.getParticleTileData(); - const amrex::ParticleReal* AMREX_RESTRICT ux = soa.m_rdata[PIdx::ux]; - const amrex::ParticleReal* AMREX_RESTRICT uy = soa.m_rdata[PIdx::uy]; - const amrex::ParticleReal* AMREX_RESTRICT uz = soa.m_rdata[PIdx::uz]; - constexpr int lev = 0; - const amrex::Real t = WarpX::GetInstance().gett_new(lev); - -#ifndef WARPX_QED - amrex::ignore_unused(engine); -#endif - - // Initialize the last NumRuntimeRealComps() - n_external_attr_real runtime real attributes - for (int j = PIdx::nattribs + n_external_attr_real; j < NumRealComps() ; ++j) - { - amrex::Vector attr_temp(np, 0.0_prt); + ParticleCreation::DefaultInitializeRuntimeAttributes(pinned_tile, + n_external_attr_real, n_external_attr_int, + m_user_real_attribs, m_user_int_attribs, + particle_comps, particle_icomps, + amrex::GetVecOfPtrs(m_user_real_attrib_parser), + amrex::GetVecOfPtrs(m_user_int_attrib_parser), #ifdef WARPX_QED - // Current runtime comp is quantum synchrotron optical depth - if (particle_comps.find("opticalDepthQSR") != particle_comps.end() && - particle_comps["opticalDepthQSR"] == j) - { - const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = - m_shr_p_qs_engine->build_optical_depth_functor();; - for (int i = 0; i < np; ++i) { - attr_temp[i] = quantum_sync_get_opt(engine); - } - } - - // Current runtime comp is Breit-Wheeler optical depth - if (particle_comps.find("opticalDepthBW") != particle_comps.end() && - particle_comps["opticalDepthBW"] == j) - { - const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = - m_shr_p_bw_engine->build_optical_depth_functor();; - for (int i = 0; i < np; ++i) { - attr_temp[i] = breit_wheeler_get_opt(engine); - } - } + true, + m_shr_p_bw_engine.get(), + m_shr_p_qs_engine.get(), #endif - - for (int ia = 0; ia < n_user_real_attribs; ++ia) - { - // Current runtime comp is ia-th user defined attribute - if (particle_comps.find(m_user_real_attribs[ia]) != particle_comps.end() && - particle_comps[m_user_real_attribs[ia]] == j) - { - amrex::ParticleReal xp, yp, zp; - const amrex::ParserExecutor<7> user_real_attrib_parserexec = - m_user_real_attrib_parser[ia]->compile<7>(); - for (int i = 0; i < np; ++i) { - get_position(i, xp, yp, zp); - attr_temp[i] = user_real_attrib_parserexec(xp, yp, zp, - ux[i], uy[i], uz[i], t); - } - } - } - - pinned_tile.push_back_real(j, attr_temp.data(), attr_temp.data() + np); - } - - // Initialize the last NumRuntimeIntComps() - n_external_attr_int runtime int attributes - for (int j = n_external_attr_int; j < NumIntComps() ; ++j) - { - amrex::Vector attr_temp(np, 0); - - // Current runtime comp is ionization level - if (particle_icomps.find("ionizationLevel") != particle_icomps.end() && - particle_icomps["ionizationLevel"] == j) - { - for (int i = 0; i < np; ++i) { - attr_temp[i] = ionization_initial_level; - } - } - - for (int ia = 0; ia < n_user_int_attribs; ++ia) - { - // Current runtime comp is ia-th user defined attribute - if (particle_icomps.find(m_user_int_attribs[ia]) != particle_icomps.end() && - particle_icomps[m_user_int_attribs[ia]] == j) - { - amrex::ParticleReal xp, yp, zp; - const amrex::ParserExecutor<7> user_int_attrib_parserexec = - m_user_int_attrib_parser[ia]->compile<7>(); - for (int i = 0; i < np; ++i) { - get_position(i, xp, yp, zp); - attr_temp[i] = static_cast( - user_int_attrib_parserexec(xp, yp, zp, ux[i], uy[i], uz[i], t)); - } - } - } - - pinned_tile.push_back_int(j, attr_temp.data(), attr_temp.data() + np); - } - + ionization_initial_level, + 0,pinned_tile.numParticles()); } @@ -841,95 +829,95 @@ PhysicalParticleContainer::AddParticles (int lev) { WARPX_PROFILE("PhysicalParticleContainer::AddParticles()"); - if (plasma_injector->add_single_particle) { - if (WarpX::gamma_boost > 1.) { - MapParticletoBoostedFrame(plasma_injector->single_particle_pos[0], - plasma_injector->single_particle_pos[1], - plasma_injector->single_particle_pos[2], - plasma_injector->single_particle_u[0], - plasma_injector->single_particle_u[1], - plasma_injector->single_particle_u[2]); + for (auto const& plasma_injector : plasma_injectors) { + + if (plasma_injector->add_single_particle) { + if (WarpX::gamma_boost > 1.) { + MapParticletoBoostedFrame(plasma_injector->single_particle_pos[0], + plasma_injector->single_particle_pos[1], + plasma_injector->single_particle_pos[2], + plasma_injector->single_particle_u[0], + plasma_injector->single_particle_u[1], + plasma_injector->single_particle_u[2]); + } + amrex::Vector xp = {plasma_injector->single_particle_pos[0]}; + amrex::Vector yp = {plasma_injector->single_particle_pos[1]}; + amrex::Vector zp = {plasma_injector->single_particle_pos[2]}; + amrex::Vector uxp = {plasma_injector->single_particle_u[0]}; + amrex::Vector uyp = {plasma_injector->single_particle_u[1]}; + amrex::Vector uzp = {plasma_injector->single_particle_u[2]}; + amrex::Vector> attr = {{plasma_injector->single_particle_weight}}; + amrex::Vector> attr_int; + AddNParticles(lev, 1, xp, yp, zp, uxp, uyp, uzp, + 1, attr, 0, attr_int, 0); + return; } - amrex::Vector xp = {plasma_injector->single_particle_pos[0]}; - amrex::Vector yp = {plasma_injector->single_particle_pos[1]}; - amrex::Vector zp = {plasma_injector->single_particle_pos[2]}; - amrex::Vector uxp = {plasma_injector->single_particle_u[0]}; - amrex::Vector uyp = {plasma_injector->single_particle_u[1]}; - amrex::Vector uzp = {plasma_injector->single_particle_u[2]}; - amrex::Vector> attr = {{plasma_injector->single_particle_weight}}; - amrex::Vector> attr_int; - AddNParticles(lev, 1, xp, yp, zp, uxp, uyp, uzp, - 1, attr, 0, attr_int, 0); - return; - } - if (plasma_injector->add_multiple_particles) { - if (WarpX::gamma_boost > 1.) { - for (int i=0 ; i < plasma_injector->multiple_particles_pos_x.size() ; i++) { - MapParticletoBoostedFrame(plasma_injector->multiple_particles_pos_x[i], - plasma_injector->multiple_particles_pos_y[i], - plasma_injector->multiple_particles_pos_z[i], - plasma_injector->multiple_particles_ux[i], - plasma_injector->multiple_particles_uy[i], - plasma_injector->multiple_particles_uz[i]); + if (plasma_injector->add_multiple_particles) { + if (WarpX::gamma_boost > 1.) { + for (int i=0 ; i < plasma_injector->multiple_particles_pos_x.size() ; i++) { + MapParticletoBoostedFrame(plasma_injector->multiple_particles_pos_x[i], + plasma_injector->multiple_particles_pos_y[i], + plasma_injector->multiple_particles_pos_z[i], + plasma_injector->multiple_particles_ux[i], + plasma_injector->multiple_particles_uy[i], + plasma_injector->multiple_particles_uz[i]); + } } + amrex::Vector> attr; + attr.push_back(plasma_injector->multiple_particles_weight); + amrex::Vector> attr_int; + AddNParticles(lev, static_cast(plasma_injector->multiple_particles_pos_x.size()), + plasma_injector->multiple_particles_pos_x, + plasma_injector->multiple_particles_pos_y, + plasma_injector->multiple_particles_pos_z, + plasma_injector->multiple_particles_ux, + plasma_injector->multiple_particles_uy, + plasma_injector->multiple_particles_uz, + 1, attr, 0, attr_int, 0); } - amrex::Vector> attr; - attr.push_back(plasma_injector->multiple_particles_weight); - amrex::Vector> attr_int; - AddNParticles(lev, plasma_injector->multiple_particles_pos_x.size(), - plasma_injector->multiple_particles_pos_x, - plasma_injector->multiple_particles_pos_y, - plasma_injector->multiple_particles_pos_z, - plasma_injector->multiple_particles_ux, - plasma_injector->multiple_particles_uy, - plasma_injector->multiple_particles_uz, - 1, attr, 0, attr_int, 0); - return; - } - if (plasma_injector->gaussian_beam) { - AddGaussianBeam(plasma_injector->x_m, - plasma_injector->y_m, - plasma_injector->z_m, - plasma_injector->x_rms, - plasma_injector->y_rms, - plasma_injector->z_rms, - plasma_injector->x_cut, - plasma_injector->y_cut, - plasma_injector->z_cut, - plasma_injector->q_tot, - plasma_injector->npart, - plasma_injector->do_symmetrize, - plasma_injector->symmetrization_order); - - - return; - } + if (plasma_injector->gaussian_beam) { + AddGaussianBeam(*plasma_injector, + plasma_injector->x_m, + plasma_injector->y_m, + plasma_injector->z_m, + plasma_injector->x_rms, + plasma_injector->y_rms, + plasma_injector->z_rms, + plasma_injector->x_cut, + plasma_injector->y_cut, + plasma_injector->z_cut, + plasma_injector->q_tot, + plasma_injector->npart, + plasma_injector->do_symmetrize, + plasma_injector->symmetrization_order); + } - if (plasma_injector->external_file) { - AddPlasmaFromFile(plasma_injector->q_tot, - plasma_injector->z_shift); - return; - } + if (plasma_injector->external_file) { + AddPlasmaFromFile(*plasma_injector, + plasma_injector->q_tot, + plasma_injector->z_shift); + } - if ( plasma_injector->doInjection() ) { - AddPlasma( lev ); + if ( plasma_injector->doInjection() ) { + AddPlasma(*plasma_injector, lev); + } } } void -PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) +PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int lev, RealBox part_realbox) { WARPX_PROFILE("PhysicalParticleContainer::AddPlasma()"); // If no part_realbox is provided, initialize particles in the whole domain const Geometry& geom = Geom(lev); - if (!part_realbox.ok()) part_realbox = geom.ProbDomain(); + if (!part_realbox.ok()) { part_realbox = geom.ProbDomain(); } - const int num_ppc = plasma_injector->num_particles_per_cell; + const int num_ppc = plasma_injector.num_particles_per_cell; #ifdef WARPX_DIM_RZ - Real rmax = std::min(plasma_injector->xmax, part_realbox.hi(0)); + Real rmax = std::min(plasma_injector.xmax, part_realbox.hi(0)); #endif const auto dx = geom.CellSizeArray(); @@ -956,18 +944,18 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) fine_injection_box.coarsen(rrfac); } - InjectorPosition* inj_pos = plasma_injector->getInjectorPosition(); - InjectorDensity* inj_rho = plasma_injector->getInjectorDensity(); - InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentumDevice(); + InjectorPosition* inj_pos = plasma_injector.getInjectorPosition(); + InjectorDensity* inj_rho = plasma_injector.getInjectorDensity(); + InjectorMomentum* inj_mom = plasma_injector.getInjectorMomentumDevice(); const Real gamma_boost = WarpX::gamma_boost; const Real beta_boost = WarpX::beta_boost; const Real t = WarpX::GetInstance().gett_new(lev); - const Real density_min = plasma_injector->density_min; - const Real density_max = plasma_injector->density_max; + const Real density_min = plasma_injector.density_min; + const Real density_max = plasma_injector.density_max; #ifdef WARPX_DIM_RZ const int nmodes = WarpX::n_rz_azimuthal_modes; - bool radially_weighted = plasma_injector->radially_weighted; + bool radially_weighted = plasma_injector.radially_weighted; #endif MFItInfo info; @@ -984,7 +972,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); const Box& tile_box = mfi.tilebox(); const RealBox tile_realbox = WarpX::getRealBox(tile_box, lev); @@ -1033,7 +1021,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // count the number of particles that each cell in overlap_box could add Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); - auto pcounts = counts.data(); + auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { @@ -1061,12 +1049,15 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const auto zlim = GpuArray{lo.z,(lo.z+hi.z)/2._rt,hi.z}; const auto checker = [&](){ - for (const auto& x : xlim) - for (const auto& y : ylim) - for (const auto& z : zlim) + for (const auto& x : xlim) { + for (const auto& y : ylim) { + for (const auto& z : zlim) { if (inj_pos->insideBounds(x,y,z) and (inj_rho->getDensity(x,y,z) > 0) ) { return 1; } + } + } + } return 0; }; const int flag_pcount = checker(); @@ -1120,8 +1111,8 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) pa[ia] = soa.GetRealData(ia).data() + old_size; } // user-defined integer and real attributes - const int n_user_int_attribs = m_user_int_attribs.size(); - const int n_user_real_attribs = m_user_real_attribs.size(); + const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); + const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); amrex::Gpu::PinnedVector pa_user_int_pinned(n_user_int_attribs); amrex::Gpu::PinnedVector pa_user_real_pinned(n_user_real_attribs); amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); @@ -1174,12 +1165,14 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // has to be initialized const bool loc_has_quantum_sync = has_quantum_sync(); const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) + if (loc_has_quantum_sync) { p_optical_depth_QSR = soa.GetRealData( particle_comps["opticalDepthQSR"]).data() + old_size; - if(loc_has_breit_wheeler) + } + if(loc_has_breit_wheeler) { p_optical_depth_BW = soa.GetRealData( particle_comps["opticalDepthBW"]).data() + old_size; + } //If needed, get the appropriate functors from the engines QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; @@ -1201,7 +1194,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the // next redistribute. - const auto poffset = offset.data(); + auto *const poffset = offset.data(); #ifdef WARPX_DIM_RZ const bool rz_random_theta = m_rz_random_theta; #endif @@ -1212,7 +1205,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const auto index = overlap_box.index(iv); #ifdef WARPX_DIM_RZ Real theta_offset = 0._rt; - if (rz_random_theta) theta_offset = amrex::Random(engine) * 2._rt * MathConst::pi; + if (rz_random_theta) { theta_offset = amrex::Random(engine) * 2._rt * MathConst::pi; } #endif Real scale_fac = 0.0_rt; @@ -1432,7 +1425,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -1441,16 +1434,16 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) } void -PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) +PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, amrex::Real dt) { WARPX_PROFILE("PhysicalParticleContainer::AddPlasmaFlux()"); const Geometry& geom = Geom(0); const amrex::RealBox& part_realbox = geom.ProbDomain(); - const amrex::Real num_ppc_real = plasma_injector->num_particles_per_cell_real; + const amrex::Real num_ppc_real = plasma_injector.num_particles_per_cell_real; #ifdef WARPX_DIM_RZ - Real rmax = std::min(plasma_injector->xmax, geom.ProbDomain().hi(0)); + Real rmax = std::min(plasma_injector.xmax, geom.ProbDomain().hi(0)); #endif const auto dx = geom.CellSizeArray(); @@ -1459,20 +1452,20 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) Real scale_fac = 0._rt; // Scale particle weight by the area of the emitting surface, within one cell #if defined(WARPX_DIM_3D) - scale_fac = dx[0]*dx[1]*dx[2]/dx[plasma_injector->flux_normal_axis]/num_ppc_real; + scale_fac = dx[0]*dx[1]*dx[2]/dx[plasma_injector.flux_normal_axis]/num_ppc_real; #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) scale_fac = dx[0]*dx[1]/num_ppc_real; // When emission is in the r direction, the emitting surface is a cylinder. // The factor 2*pi*r is added later below. - if (plasma_injector->flux_normal_axis == 0) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 0) { scale_fac /= dx[0]; } // When emission is in the z direction, the emitting surface is an annulus // The factor 2*pi*r is added later below. - if (plasma_injector->flux_normal_axis == 2) scale_fac /= dx[1]; + if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[1]; } // When emission is in the theta direction (flux_normal_axis == 1), // the emitting surface is a rectangle, within the plane of the simulation #elif defined(WARPX_DIM_1D_Z) scale_fac = dx[0]/num_ppc_real; - if (plasma_injector->flux_normal_axis == 2) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[0]; } #endif amrex::LayoutData* cost = WarpX::getCosts(0); @@ -1499,16 +1492,16 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) fine_injection_box.coarsen(rrfac); } - InjectorPosition* flux_pos = plasma_injector->getInjectorFluxPosition(); - InjectorFlux* inj_flux = plasma_injector->getInjectorFlux(); - InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentumDevice(); + InjectorPosition* flux_pos = plasma_injector.getInjectorFluxPosition(); + InjectorFlux* inj_flux = plasma_injector.getInjectorFlux(); + InjectorMomentum* inj_mom = plasma_injector.getInjectorMomentumDevice(); constexpr int level_zero = 0; const amrex::Real t = WarpX::GetInstance().gett_new(level_zero); #ifdef WARPX_DIM_RZ const int nmodes = WarpX::n_rz_azimuthal_modes; const bool rz_random_theta = m_rz_random_theta; - bool radially_weighted = plasma_injector->radially_weighted; + bool radially_weighted = plasma_injector.radially_weighted; #endif MFItInfo info; @@ -1525,7 +1518,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); const Box& tile_box = mfi.tilebox(); const RealBox tile_realbox = WarpX::getRealBox(tile_box, 0); @@ -1539,30 +1532,30 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) for (int dir=0; dirflux_normal_axis) { + if (dir == plasma_injector.flux_normal_axis) { #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (2*dir == plasma_injector->flux_normal_axis) { + if (2*dir == plasma_injector.flux_normal_axis) { // The above formula captures the following cases: // - flux_normal_axis=0 (emission along x/r) and dir=0 // - flux_normal_axis=2 (emission along z) and dir=1 #elif defined(WARPX_DIM_1D_Z) - if ( (dir==0) && (plasma_injector->flux_normal_axis==2) ) { + if ( (dir==0) && (plasma_injector.flux_normal_axis==2) ) { #endif - if (plasma_injector->flux_direction > 0) { - if (plasma_injector->surface_flux_pos < tile_realbox.lo(dir) || - plasma_injector->surface_flux_pos >= tile_realbox.hi(dir)) { + if (plasma_injector.flux_direction > 0) { + if (plasma_injector.surface_flux_pos < tile_realbox.lo(dir) || + plasma_injector.surface_flux_pos >= tile_realbox.hi(dir)) { no_overlap = true; break; } } else { - if (plasma_injector->surface_flux_pos <= tile_realbox.lo(dir) || - plasma_injector->surface_flux_pos > tile_realbox.hi(dir)) { + if (plasma_injector.surface_flux_pos <= tile_realbox.lo(dir) || + plasma_injector.surface_flux_pos > tile_realbox.hi(dir)) { no_overlap = true; break; } } - overlap_realbox.setLo( dir, plasma_injector->surface_flux_pos ); - overlap_realbox.setHi( dir, plasma_injector->surface_flux_pos ); + overlap_realbox.setLo( dir, plasma_injector.surface_flux_pos ); + overlap_realbox.setHi( dir, plasma_injector.surface_flux_pos ); overlap_box.setSmall( dir, 0 ); overlap_box.setBig( dir, 0 ); shifted[dir] = @@ -1605,7 +1598,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) // count the number of particles that each cell in overlap_box could add Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); - auto pcounts = counts.data(); + auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { @@ -1670,8 +1663,8 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) } // user-defined integer and real attributes - const int n_user_int_attribs = m_user_int_attribs.size(); - const int n_user_real_attribs = m_user_real_attribs.size(); + const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); + const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); amrex::Gpu::PinnedVector pa_user_int_pinned(n_user_int_attribs); amrex::Gpu::PinnedVector pa_user_real_pinned(n_user_real_attribs); amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); @@ -1724,12 +1717,14 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) // has to be initialized const bool loc_has_quantum_sync = has_quantum_sync(); const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) + if (loc_has_quantum_sync) { p_optical_depth_QSR = soa.GetRealData( particle_comps["opticalDepthQSR"]).data() + old_size; - if(loc_has_breit_wheeler) + } + if(loc_has_breit_wheeler) { p_optical_depth_BW = soa.GetRealData( particle_comps["opticalDepthBW"]).data() + old_size; + } //If needed, get the appropriate functors from the engines QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; @@ -1747,14 +1742,14 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) const bool loc_do_field_ionization = do_field_ionization; const int loc_ionization_initial_level = ionization_initial_level; #ifdef WARPX_DIM_RZ - int const loc_flux_normal_axis = plasma_injector->flux_normal_axis; + int const loc_flux_normal_axis = plasma_injector.flux_normal_axis; #endif // Loop over all new particles and inject them (creates too many // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the // next redistribute. - const auto poffset = offset.data(); + auto *const poffset = offset.data(); amrex::ParallelForRNG(overlap_box, [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept { @@ -1833,7 +1828,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) // Rotate the momentum // This because, when the flux direction is e.g. "r" // the `inj_mom` objects generates a v*Gaussian distribution - // along the Cartesian "x" directionm by default. This + // along the Cartesian "x" direction by default. This // needs to be rotated along "r". Real ur = pu.x; Real ut = pu.y; @@ -1923,7 +1918,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -1964,7 +1959,8 @@ PhysicalParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition) + Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { WARPX_PROFILE("PhysicalParticleContainer::Evolve()"); @@ -1987,8 +1983,9 @@ PhysicalParticleContainer::Evolve (int lev, const auto t_lev = pti.GetLevel(); const auto index = pti.GetPairIndex(); tmp_particle_data.resize(finestLevel()+1); - for (int i = 0; i < TmpIdx::nattribs; ++i) + for (int i = 0; i < TmpIdx::nattribs; ++i) { tmp_particle_data[t_lev][index][i].resize(np); + } } } @@ -2011,7 +2008,7 @@ PhysicalParticleContainer::Evolve (int lev, { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); const Box& box = pti.validbox(); @@ -2092,10 +2089,17 @@ PhysicalParticleContainer::Evolve (int lev, WARPX_PROFILE_VAR_START(blp_fg); const auto np_to_push = np_gather; const auto gather_lev = lev; - PushPX(pti, exfab, eyfab, ezfab, - bxfab, byfab, bzfab, - Ex.nGrowVect(), e_is_nodal, - 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + if (push_type == PushType::Explicit) { + PushPX(pti, exfab, eyfab, ezfab, + bxfab, byfab, bzfab, + Ex.nGrowVect(), e_is_nodal, + 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + } else if (push_type == PushType::Implicit) { + ImplicitPushXP(pti, exfab, eyfab, ezfab, + bxfab, byfab, bzfab, + Ex.nGrowVect(), e_is_nodal, + 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + } if (np_gather < np) { @@ -2126,11 +2130,19 @@ PhysicalParticleContainer::Evolve (int lev, // Field gather and push for particles in gather buffers e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); - PushPX(pti, cexfab, ceyfab, cezfab, - cbxfab, cbyfab, cbzfab, - cEx->nGrowVect(), e_is_nodal, - nfine_gather, np-nfine_gather, - lev, lev-1, dt, ScaleFields(false), a_dt_type); + if (push_type == PushType::Explicit) { + PushPX(pti, cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrowVect(), e_is_nodal, + nfine_gather, np-nfine_gather, + lev, lev-1, dt, ScaleFields(false), a_dt_type); + } else if (push_type == PushType::Implicit) { + ImplicitPushXP(pti, cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrowVect(), e_is_nodal, + nfine_gather, np-nfine_gather, + lev, lev-1, dt, ScaleFields(false), a_dt_type); + } } WARPX_PROFILE_VAR_STOP(blp_fg); @@ -2138,8 +2150,8 @@ PhysicalParticleContainer::Evolve (int lev, // Current Deposition if (!skip_deposition) { - // Deposit at t_{n+1/2} - const amrex::Real relative_time = -0.5_rt * dt; + // Deposit at t_{n+1/2} with explicit push + const amrex::Real relative_time = (push_type == PushType::Explicit ? -0.5_rt * dt : 0.0_rt); const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; @@ -2147,14 +2159,14 @@ PhysicalParticleContainer::Evolve (int lev, // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, - lev, lev, dt, relative_time); + lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, - lev, lev-1, dt, relative_time); + lev, lev-1, dt, relative_time, push_type); } } // end of "if electrostatic_solver_id == ElectrostaticSolverAlgo::None" } // end of "if do_not_push" @@ -2180,7 +2192,7 @@ PhysicalParticleContainer::Evolve (int lev, if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[pti.index()], wt); } } @@ -2299,9 +2311,9 @@ PhysicalParticleContainer::SplitParticles (int lev) // Loop over particle interator for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - const auto GetPosition = GetParticlePosition(pti); + const auto GetPosition = GetParticlePosition(pti); - const amrex::Vector ppc_nd = plasma_injector->num_particles_per_cell_each_dim; + const amrex::Vector ppc_nd = plasma_injectors[0]->num_particles_per_cell_each_dim; const std::array& dx = WarpX::CellSize(lev); amrex::Vector split_offset = {dx[0]/2._rt, dx[1]/2._rt, @@ -2460,7 +2472,8 @@ PhysicalParticleContainer::SplitParticles (int lev) 0, attr_int, 1, NoSplitParticleID); // Copy particles from tmp to current particle container - addParticles(pctmp_split,1); + constexpr bool local_flag = true; + addParticles(pctmp_split,local_flag); // Clear tmp container pctmp_split.clearParticles(); } @@ -2472,7 +2485,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, { WARPX_PROFILE("PhysicalParticleContainer::PushP()"); - if (do_not_push) return; + if (do_not_push) { return; } const std::array& dx = WarpX::CellSize(std::max(lev,0)); @@ -2495,10 +2508,17 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const FArrayBox& byfab = By[pti]; const FArrayBox& bzfab = Bz[pti]; - const auto getPosition = GetParticlePosition(pti); + const auto getPosition = GetParticlePosition(pti); const auto getExternalEB = GetExternalEBField(pti); + const amrex::ParticleReal Ex_external_particle = m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; + const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const Dim3 lo = lbound(box); @@ -2554,8 +2574,12 @@ PhysicalParticleContainer::PushP (int lev, Real dt, amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); - amrex::ParticleReal Exp = 0._rt, Eyp = 0._rt, Ezp = 0._rt; - amrex::ParticleReal Bxp = 0._rt, Byp = 0._rt, Bzp = 0._rt; + amrex::ParticleReal Exp = Ex_external_particle; + amrex::ParticleReal Eyp = Ey_external_particle; + amrex::ParticleReal Ezp = Ez_external_particle; + amrex::ParticleReal Bxp = Bx_external_particle; + amrex::ParticleReal Byp = By_external_particle; + amrex::ParticleReal Bzp = Bz_external_particle; if (!t_do_not_gather){ // first gather E and B to the particle positions @@ -2567,7 +2591,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, } // Externally applied E and B-field in Cartesian co-ordinates - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } @@ -2610,9 +2634,11 @@ PhysicalParticleContainer::PushP (int lev, Real dt, void PhysicalParticleContainer::ContinuousInjection (const RealBox& injection_box) { - // Inject plasma on level 0. Paticles will be redistributed. + // Inject plasma on level 0. Particles will be redistributed. const int lev=0; - AddPlasma(lev, injection_box); + for (auto const& plasma_injector : plasma_injectors) { + AddPlasma(*plasma_injector, lev, injection_box); + } } /* \brief Inject a flux of particles during the simulation @@ -2620,13 +2646,15 @@ PhysicalParticleContainer::ContinuousInjection (const RealBox& injection_box) void PhysicalParticleContainer::ContinuousFluxInjection (amrex::Real t, amrex::Real dt) { - if (plasma_injector->doFluxInjection()){ - // Check the optional parameters for start and stop of injection - if ( ((plasma_injector->flux_tmin<0) || (t>=plasma_injector->flux_tmin)) && - ((plasma_injector->flux_tmax<0) || (t< plasma_injector->flux_tmax)) ){ + for (auto const& plasma_injector : plasma_injectors) { + if (plasma_injector->doFluxInjection()){ + // Check the optional parameters for start and stop of injection + if ( ((plasma_injector->flux_tmin<0) || (t>=plasma_injector->flux_tmin)) && + ((plasma_injector->flux_tmax<0) || (t< plasma_injector->flux_tmax)) ){ - AddPlasmaFlux(dt); + AddPlasmaFlux(*plasma_injector, dt); + } } } } @@ -2653,7 +2681,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, (gather_lev==(lev )), "Gather buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_push == 0) return; + if (np_to_push == 0) { return; } // Get cell size on gather_lev const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); @@ -2671,11 +2699,18 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, // Add guard cells to the box. box.grow(ngEB); - const auto getPosition = GetParticlePosition(pti, offset); - auto setPosition = SetParticlePosition(pti, offset); + const auto getPosition = GetParticlePosition(pti, offset); + auto setPosition = SetParticlePosition(pti, offset); const auto getExternalEB = GetExternalEBField(pti, offset); + const amrex::ParticleReal Ex_external_particle = m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; + // Lower corner of tile box physical domain (take into account Galilean shift) const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); @@ -2745,7 +2780,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, #ifdef WARPX_QED const auto do_sync = m_do_qed_quantum_sync; amrex::Real t_chi_max = 0.0; - if (do_sync) t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); + if (do_sync) { t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); } QuantumSynchrotronEvolveOpticalDepth evolve_opt; amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR = nullptr; @@ -2790,8 +2825,12 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, z_old[ip] = zp; } - amrex::ParticleReal Exp = 0._rt, Eyp = 0._rt, Ezp = 0._rt; - amrex::ParticleReal Bxp = 0._rt, Byp = 0._rt, Bzp = 0._rt; + amrex::ParticleReal Exp = Ex_external_particle; + amrex::ParticleReal Eyp = Ey_external_particle; + amrex::ParticleReal Ezp = Ez_external_particle; + amrex::ParticleReal Bxp = Bx_external_particle; + amrex::ParticleReal Byp = By_external_particle; + amrex::ParticleReal Bzp = Bz_external_particle; if(!t_do_not_gather){ // first gather E and B to the particle positions @@ -2802,7 +2841,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, nox, galerkin_interpolation); } - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } @@ -2813,34 +2852,48 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, if (!do_sync) #endif { - doParticlePush<0>(getPosition, setPosition, copyAttribs, ip, - ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, - m, q, pusher_algo, do_crr, do_copy, + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, #ifdef WARPX_QED - t_chi_max, + t_chi_max, #endif - dt); + dt); + + UpdatePosition(xp, yp, zp, ux[ip], uy[ip], uz[ip], dt); + setPosition(ip, xp, yp, zp); } #ifdef WARPX_QED else { if constexpr (qed_control == has_qed) { - doParticlePush<1>(getPosition, setPosition, copyAttribs, ip, - ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, - m, q, pusher_algo, do_crr, do_copy, - t_chi_max, - dt); + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + + UpdatePosition(xp, yp, zp, ux[ip], uy[ip], uz[ip], dt); + setPosition(ip, xp, yp, zp); } } #endif #ifdef WARPX_QED [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; - [[maybe_unused]] auto foo_podq = p_optical_depth_QSR; - [[maybe_unused]] auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc if constexpr (qed_control == has_qed) { if (local_has_quantum_sync) { evolve_opt(ux[ip], uy[ip], uz[ip], @@ -2854,10 +2907,265 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, }); } +/* \brief Perform the implicit particle push operation in one fused kernel + * The main difference from PushPX is the order of operations: + * - push position by 1/2 dt + * - gather fields + * - push velocity by dt + * - average old and new velocity to get time centered value + * The routines ends with both position and velocity at the half time level. + */ +void +PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + amrex::IntVect ngEB, int /*e_is_nodal*/, + long offset, + long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE((gather_lev==(lev-1)) || + (gather_lev==(lev )), + "Gather buffers only work for lev-1"); + // If no particles, do not do anything + if (np_to_push == 0) { return; } + + // Get cell size on gather_lev + const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + + // Get box from which field is gathered. + // If not gathering from the finest level, the box is coarsened. + Box box; + if (lev == gather_lev) { + box = pti.tilebox(); + } else { + const IntVect& ref_ratio = WarpX::RefRatio(gather_lev); + box = amrex::coarsen(pti.tilebox(),ref_ratio); + } + + // Add guard cells to the box. + box.grow(ngEB); + + auto setPosition = SetParticlePosition(pti, offset); + + const auto getExternalEB = GetExternalEBField(pti, offset); + + const amrex::ParticleReal Ex_external_particle = m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; + + // Lower corner of tile box physical domain (take into account Galilean shift) + const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + + const Dim3 lo = lbound(box); + + bool galerkin_interpolation = WarpX::galerkin_interpolation; + int nox = WarpX::nox; + int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + amrex::Array4 const& ex_arr = exfab->array(); + amrex::Array4 const& ey_arr = eyfab->array(); + amrex::Array4 const& ez_arr = ezfab->array(); + amrex::Array4 const& bx_arr = bxfab->array(); + amrex::Array4 const& by_arr = byfab->array(); + amrex::Array4 const& bz_arr = bzfab->array(); + + amrex::IndexType const ex_type = exfab->box().ixType(); + amrex::IndexType const ey_type = eyfab->box().ixType(); + amrex::IndexType const ez_type = ezfab->box().ixType(); + amrex::IndexType const bx_type = bxfab->box().ixType(); + amrex::IndexType const by_type = byfab->box().ixType(); + amrex::IndexType const bz_type = bzfab->box().ixType(); + + auto& attribs = pti.GetAttribs(); + ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr() + offset; + ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr() + offset; + ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr() + offset; + +#if (AMREX_SPACEDIM >= 2) + ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + int do_copy = (m_do_back_transformed_particles && (a_dt_type!=DtType::SecondHalf) ); + CopyParticleAttribs copyAttribs; + if (do_copy) { + copyAttribs = CopyParticleAttribs(pti, tmp_particle_data, offset); + } + + int* AMREX_RESTRICT ion_lev = nullptr; + if (do_field_ionization) { + ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr() + offset; + } + + // Loop over the particles and update their momentum + const amrex::ParticleReal q = this->charge; + const amrex::ParticleReal m = this-> mass; + + const auto pusher_algo = WarpX::particle_pusher_algo; + const auto do_crr = do_classical_radiation_reaction; +#ifdef WARPX_QED + const auto do_sync = m_do_qed_quantum_sync; + amrex::Real t_chi_max = 0.0; + if (do_sync) { t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); } + + QuantumSynchrotronEvolveOpticalDepth evolve_opt; + amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR = nullptr; + const bool local_has_quantum_sync = has_quantum_sync(); + if (local_has_quantum_sync) { + evolve_opt = m_shr_p_qs_engine->build_evolve_functor(); + p_optical_depth_QSR = pti.GetAttribs(particle_comps["opticalDepthQSR"]).dataPtr() + offset; + } +#endif + + const auto t_do_not_gather = do_not_gather; + + enum exteb_flags : int { no_exteb, has_exteb }; + enum qed_flags : int { no_qed, has_qed }; + + int exteb_runtime_flag = getExternalEB.isNoOp() ? no_exteb : has_exteb; +#ifdef WARPX_QED + int qed_runtime_flag = (local_has_quantum_sync || do_sync) ? has_qed : no_qed; +#else + int qed_runtime_flag = no_qed; +#endif + + // Using this version of ParallelFor with compile time options + // improves performance when qed or external EB are not used by reducing + // register pressure. + amrex::ParallelFor(TypeList, + CompileTimeOptions>{}, + {exteb_runtime_flag, qed_runtime_flag}, + np_to_push, [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, + auto qed_control) + { + // Position advance starts from the position at the start of the step + // but uses the most recent velocity. +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal xp = x_n[ip]; + amrex::ParticleReal xp_n = x_n[ip]; +#else + amrex::ParticleReal xp = 0._rt; + amrex::ParticleReal xp_n = 0._rt; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal yp = y_n[ip]; + amrex::ParticleReal yp_n = y_n[ip]; +#else + amrex::ParticleReal yp = 0._rt; + amrex::ParticleReal yp_n = 0._rt; +#endif + amrex::ParticleReal zp = z_n[ip]; + amrex::ParticleReal zp_n = z_n[ip]; + + UpdatePositionImplicit(xp, yp, zp, ux_n[ip], uy_n[ip], uz_n[ip], ux[ip], uy[ip], uz[ip], 0.5_rt*dt); + setPosition(ip, xp, yp, zp); + + amrex::ParticleReal Exp = Ex_external_particle; + amrex::ParticleReal Eyp = Ey_external_particle; + amrex::ParticleReal Ezp = Ez_external_particle; + amrex::ParticleReal Bxp = Bx_external_particle; + amrex::ParticleReal Byp = By_external_particle; + amrex::ParticleReal Bzp = Bz_external_particle; + + if(!t_do_not_gather){ + // first gather E and B to the particle positions + doGatherShapeNImplicit(xp_n, yp_n, zp_n, xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + nox, galerkin_interpolation); + } + + // Externally applied E and B-field in Cartesian co-ordinates + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; + if constexpr (exteb_control == has_exteb) { + getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + } + + scaleFields(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + // The momentum push starts with the velocity at the start of the step + ux[ip] = ux_n[ip]; + uy[ip] = uy_n[ip]; + uz[ip] = uz_n[ip]; + +#ifdef WARPX_QED + if (!do_sync) +#endif + { + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, +#ifdef WARPX_QED + t_chi_max, +#endif + dt); + } +#ifdef WARPX_QED + else { + if constexpr (qed_control == has_qed) { + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + } + } +#endif + +#ifdef WARPX_QED + [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + if constexpr (qed_control == has_qed) { + if (local_has_quantum_sync) { + evolve_opt(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp,Bxp, Byp, Bzp, + dt, p_optical_depth_QSR[ip]); + } + } +#else + amrex::ignore_unused(qed_control); +#endif + + // Take average to get the time centered value + ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); + uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); + uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + + }); +} + void PhysicalParticleContainer::InitIonizationModule () { - if (!do_field_ionization) return; + if (!do_field_ionization) { return; } const ParmParse pp_species_name(species_name); if (charge != PhysConst::q_e){ ablastr::warn_manager::WMRecordWarning("Species", @@ -2936,6 +3244,7 @@ PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti, WARPX_PROFILE("PhysicalParticleContainer::getIonizationFunc()"); return {pti, lev, ngEB, Ex, Ey, Ez, Bx, By, Bz, + m_E_external_particle, m_B_external_particle, ionization_energies.dataPtr(), adk_prefactor.dataPtr(), adk_exp_prefactor.dataPtr(), @@ -2944,9 +3253,13 @@ PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti, ion_atomic_number}; } -PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector () +PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector (int i) { - return plasma_injector.get(); + if (i < 0 || i >= static_cast(plasma_injectors.size())) { + return nullptr; + } else { + return plasma_injectors[i].get(); + } } void PhysicalParticleContainer::resample (const int timestep, const bool verbose) diff --git a/Source/Particles/Pusher/CopyParticleAttribs.H b/Source/Particles/Pusher/CopyParticleAttribs.H index 697fe1e0b38..4f961fe94aa 100644 --- a/Source/Particles/Pusher/CopyParticleAttribs.H +++ b/Source/Particles/Pusher/CopyParticleAttribs.H @@ -22,7 +22,7 @@ struct CopyParticleAttribs { using TmpParticles = WarpXParticleContainer::TmpParticles; - GetParticlePosition m_get_position; + GetParticlePosition m_get_position; const amrex::ParticleReal* AMREX_RESTRICT uxp = nullptr; const amrex::ParticleReal* AMREX_RESTRICT uyp = nullptr; @@ -47,11 +47,11 @@ struct CopyParticleAttribs * always start at the particle with index 0. */ CopyParticleAttribs (const WarpXParIter& a_pti, TmpParticles& tmp_particle_data, - int a_offset = 0) noexcept + long a_offset = 0) noexcept { - if (tmp_particle_data.empty()) return; + if (tmp_particle_data.empty()) { return; } - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); uxp = attribs[PIdx::ux].dataPtr() + a_offset; uyp = attribs[PIdx::uy].dataPtr() + a_offset; @@ -66,14 +66,14 @@ struct CopyParticleAttribs uypold = tmp_particle_data[lev].at(index)[TmpIdx::uyold].dataPtr() + a_offset; uzpold = tmp_particle_data[lev].at(index)[TmpIdx::uzold].dataPtr() + a_offset; - m_get_position = GetParticlePosition(a_pti, a_offset); + m_get_position = GetParticlePosition(a_pti, a_offset); } /** \brief copy the position and momentum of particle i to the * temporary data holder */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator() (const int i) const noexcept + void operator() (const long i) const noexcept { AMREX_ASSERT(uxp != nullptr); AMREX_ASSERT(uyp != nullptr); diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H index d8cce61756d..e4477a2a60d 100644 --- a/Source/Particles/Pusher/GetAndSetPosition.H +++ b/Source/Particles/Pusher/GetAndSetPosition.H @@ -9,6 +9,7 @@ #define WARPX_PARTICLES_PUSHER_GETANDSETPOSITION_H_ #include "Particles/WarpXParticleContainer.H" +#include "Particles/NamedComponentParticleContainer.H" #include #include @@ -18,7 +19,11 @@ /** \brief Extract the cartesian position coordinates of the particle * p and store them in the variables `x`, `y`, `z` - * This version operates on a SuperParticle */ + * This version operates on a SuperParticle + * + * \tparam T_PIdx particle index enumerator + */ +template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, amrex::ParticleReal& x, @@ -26,7 +31,7 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, amrex::ParticleReal& z) noexcept { #ifdef WARPX_DIM_RZ - const amrex::ParticleReal theta = p.rdata(PIdx::theta); + const amrex::ParticleReal theta = p.rdata(T_PIdx::theta); const amrex::ParticleReal r = p.pos(0); x = r*std::cos(theta); y = r*std::sin(theta); @@ -48,7 +53,10 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, /** \brief Functor that can be used to extract the positions of the macroparticles * inside a ParallelFor kernel + * + * \tparam T_PIdx particle index enumerator */ +template struct GetParticlePosition { using PType = WarpXParticleContainer::ParticleType; @@ -74,13 +82,13 @@ struct GetParticlePosition * \param a_offset offset to apply to the particle indices */ template - GetParticlePosition (const ptiType& a_pti, int a_offset = 0) noexcept + GetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { const auto& aos = a_pti.GetArrayOfStructs(); m_structs = aos().dataPtr() + a_offset; #if defined(WARPX_DIM_RZ) const auto& soa = a_pti.GetStructOfArrays(); - m_theta = soa.GetRealData(PIdx::theta).dataPtr() + a_offset; + m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } @@ -88,7 +96,7 @@ struct GetParticlePosition * located at index `i + a_offset` and store them in the variables * `x`, `y`, `z` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator() (const int i, RType& x, RType& y, RType& z) const noexcept + void operator() (const long i, RType& x, RType& y, RType& z) const noexcept { const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ @@ -117,7 +125,7 @@ struct GetParticlePosition * This is only different for RZ since this returns (r, theta, z) * in that case. */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void AsStored (const int i, RType& x, RType& y, RType& z) const noexcept + void AsStored (const long i, RType& x, RType& y, RType& z) const noexcept { const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ @@ -143,9 +151,11 @@ struct GetParticlePosition /** \brief Functor that can be used to modify the positions of the macroparticles, * inside a ParallelFor kernel. * + * \tparam T_PIdx particle index enumerator * \param a_pti iterator to the tile being modified * \param a_offset offset to apply to the particle indices */ +template struct SetParticlePosition { using PType = WarpXParticleContainer::ParticleType; @@ -157,20 +167,20 @@ struct SetParticlePosition #endif template - SetParticlePosition (const ptiType& a_pti, int a_offset = 0) noexcept + SetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { auto& aos = a_pti.GetArrayOfStructs(); m_structs = aos().dataPtr() + a_offset; #if defined(WARPX_DIM_RZ) auto& soa = a_pti.GetStructOfArrays(); - m_theta = soa.GetRealData(PIdx::theta).dataPtr() + a_offset; + m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } /** \brief Set the position of the particle at index `i + a_offset` * to `x`, `y`, `z` */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator() (const int i, RType x, RType y, RType z) const noexcept + void operator() (const long i, RType x, RType y, RType z) const noexcept { #if defined(WARPX_DIM_XZ) amrex::ignore_unused(y); @@ -199,7 +209,7 @@ struct SetParticlePosition * This is only different for RZ since the input should * be (r, theta, z) in that case. */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void AsStored (const int i, RType x, RType y, RType z) const noexcept + void AsStored (const long i, RType x, RType y, RType z) const noexcept { #if defined(WARPX_DIM_XZ) amrex::ignore_unused(y); diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H index b1e59029958..2a6ac8a7d6c 100644 --- a/Source/Particles/Pusher/PushSelector.H +++ b/Source/Particles/Pusher/PushSelector.H @@ -12,7 +12,6 @@ #include "Particles/Pusher/UpdateMomentumBorisWithRadiationReaction.H" #include "Particles/Pusher/UpdateMomentumHigueraCary.H" #include "Particles/Pusher/UpdateMomentumVay.H" -#include "Particles/Pusher/UpdatePosition.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -21,56 +20,45 @@ #include /** - * \brief Push position and momentum for a single particle + * \brief Push momentum for a single particle * * \tparam do_sync Whether to include quantum synchrotron radiation (QSR) - * \param GetPosition A functor for returning the particle position. - * \param SetPosition A functor for setting the particle position. - * \param copyAttribs A functor for storing the old u and x - * \param i The index of the particle to work on * \param ux, uy, uz Particle momentum * \param Ex, Ey, Ez Electric field on particles. * \param Bx, By, Bz Magnetic field on particles. - * \param ion_lev Ionization level of this particle (0 if ioniziation not on) + * \param ion_lev Ionization level of this particle (0 if ionization not on) * \param m Mass of this species. * \param a_q Charge of this species. * \param pusher_algo 0: Boris, 1: Vay, 2: HigueraCary * \param do_crr Whether to do the classical radiation reaction - * \param do_copy Whether to copy the old x and u for the BTD * \param t_chi_max Cutoff chi for QSR * \param dt Time step size */ template AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void doParticlePush(const GetParticlePosition& GetPosition, - const SetParticlePosition& SetPosition, - const CopyParticleAttribs& copyAttribs, - const long i, - amrex::ParticleReal& ux, - amrex::ParticleReal& uy, - amrex::ParticleReal& uz, - const amrex::ParticleReal Ex, - const amrex::ParticleReal Ey, - const amrex::ParticleReal Ez, - const amrex::ParticleReal Bx, - const amrex::ParticleReal By, - const amrex::ParticleReal Bz, - const int ion_lev, - const amrex::ParticleReal m, - const amrex::ParticleReal a_q, - const int pusher_algo, - const int do_crr, - const int do_copy, +void doParticleMomentumPush(amrex::ParticleReal& ux, + amrex::ParticleReal& uy, + amrex::ParticleReal& uz, + const amrex::ParticleReal Ex, + const amrex::ParticleReal Ey, + const amrex::ParticleReal Ez, + const amrex::ParticleReal Bx, + const amrex::ParticleReal By, + const amrex::ParticleReal Bz, + const int ion_lev, + const amrex::ParticleReal m, + const amrex::ParticleReal a_q, + const int pusher_algo, + const int do_crr, #ifdef WARPX_QED - const amrex::Real t_chi_max, + const amrex::Real t_chi_max, #endif - const amrex::Real dt) + const amrex::Real dt) { amrex::ParticleReal qp = a_q; if (ion_lev) { qp *= ion_lev; } - if (do_copy) copyAttribs(i); if (do_crr) { #ifdef WARPX_QED amrex::ignore_unused(t_chi_max); @@ -88,10 +76,6 @@ void doParticlePush(const GetParticlePosition& GetPosition, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); } - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else #endif { @@ -99,35 +83,19 @@ void doParticlePush(const GetParticlePosition& GetPosition, UpdateMomentumBorisWithRadiationReaction(ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } } else if (pusher_algo == ParticlePusherAlgo::Boris) { UpdateMomentumBoris( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else if (pusher_algo == ParticlePusherAlgo::Vay) { UpdateMomentumVay( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else if (pusher_algo == ParticlePusherAlgo::HigueraCary) { UpdateMomentumHigueraCary( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } //else { // amrex::Abort("Unknown particle pusher"); // } diff --git a/Source/Particles/Pusher/UpdateMomentumHigueraCary.H b/Source/Particles/Pusher/UpdateMomentumHigueraCary.H index 359fc34d508..8da3a8cfb2c 100644 --- a/Source/Particles/Pusher/UpdateMomentumHigueraCary.H +++ b/Source/Particles/Pusher/UpdateMomentumHigueraCary.H @@ -17,49 +17,49 @@ * \brief Push the particle's positions over one timestep, * given the value of its momenta `ux`, `uy`, `uz` */ - +template AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdateMomentumHigueraCary( - amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, - const amrex::ParticleReal Ex, const amrex::ParticleReal Ey, const amrex::ParticleReal Ez, - const amrex::ParticleReal Bx, const amrex::ParticleReal By, const amrex::ParticleReal Bz, - const amrex::ParticleReal q, const amrex::ParticleReal m, const amrex::Real dt ) + T& ux, T& uy, T& uz, + const T Ex, const T Ey, const T Ez, + const T Bx, const T By, const T Bz, + const T q, const T m, const amrex::Real dt ) { using namespace amrex::literals; // Constants - const amrex::ParticleReal qmt = 0.5_prt*q*dt/m; - constexpr amrex::ParticleReal invclight = 1._prt/PhysConst::c; - constexpr amrex::ParticleReal invclightsq = 1._prt/(PhysConst::c*PhysConst::c); + const T qmt = 0.5_prt*q*dt/m; + constexpr T invclight = 1._prt/PhysConst::c; + constexpr T invclightsq = 1._prt/(PhysConst::c*PhysConst::c); // Compute u_minus - const amrex::ParticleReal umx = ux + qmt*Ex; - const amrex::ParticleReal umy = uy + qmt*Ey; - const amrex::ParticleReal umz = uz + qmt*Ez; + const T umx = ux + qmt*Ex; + const T umy = uy + qmt*Ey; + const T umz = uz + qmt*Ez; // Compute gamma squared of u_minus - amrex::ParticleReal gamma = 1._prt + (umx*umx + umy*umy + umz*umz)*invclightsq; + T gamma = 1._prt + (umx*umx + umy*umy + umz*umz)*invclightsq; // Compute beta and betam squared - const amrex::ParticleReal betax = qmt*Bx; - const amrex::ParticleReal betay = qmt*By; - const amrex::ParticleReal betaz = qmt*Bz; - const amrex::ParticleReal betam = betax*betax + betay*betay + betaz*betaz; + const T betax = qmt*Bx; + const T betay = qmt*By; + const T betaz = qmt*Bz; + const T betam = betax*betax + betay*betay + betaz*betaz; // Compute sigma - const amrex::ParticleReal sigma = gamma - betam; + const T sigma = gamma - betam; // Get u* - const amrex::ParticleReal ust = (umx*betax + umy*betay + umz*betaz)*invclight; - // Get new gamma inversed + const T ust = (umx*betax + umy*betay + umz*betaz)*invclight; + // Get new gamma inverse gamma = 1._prt/std::sqrt(0.5_prt*(sigma + std::sqrt(sigma*sigma + 4._prt*(betam + ust*ust)) )); // Compute t - const amrex::ParticleReal tx = gamma*betax; - const amrex::ParticleReal ty = gamma*betay; - const amrex::ParticleReal tz = gamma*betaz; + const T tx = gamma*betax; + const T ty = gamma*betay; + const T tz = gamma*betaz; // Compute s - const amrex::ParticleReal s = 1._prt/(1._prt+(tx*tx + ty*ty + tz*tz)); + const T s = 1._prt/(1._prt+(tx*tx + ty*ty + tz*tz)); // Compute um dot t - const amrex::ParticleReal umt = umx*tx + umy*ty + umz*tz; + const T umt = umx*tx + umy*ty + umz*tz; // Compute u_plus - const amrex::ParticleReal upx = s*( umx + umt*tx + umy*tz - umz*ty ); - const amrex::ParticleReal upy = s*( umy + umt*ty + umz*tx - umx*tz ); - const amrex::ParticleReal upz = s*( umz + umt*tz + umx*ty - umy*tx ); + const T upx = s*( umx + umt*tx + umy*tz - umz*ty ); + const T upy = s*( umy + umt*ty + umz*tx - umx*tz ); + const T upz = s*( umz + umt*tz + umx*ty - umy*tx ); // Get new u ux = upx + qmt*Ex + upy*tz - upz*ty; uy = upy + qmt*Ey + upz*tx - upx*tz; diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 4bce5c10211..4743c79c00f 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -16,7 +16,10 @@ /** \brief Push the particle's positions over one timestep, - * given the value of its momenta `ux`, `uy`, `uz` */ + * given the value of its momenta `ux`, `uy`, `uz`. + * This uses the standard leapfrog algorithm + * x^{n+1} - x^{n} = dt*u^{n+1/2}/gamma^{n+1/2} + */ AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, @@ -42,4 +45,43 @@ void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::Parti z += uz * inv_gamma * dt; } +/** \brief Push the particle's positions over one timestep, + * given the value of its momenta `ux`, `uy`, `uz`. + * The implicit version is the Crank-Nicolson scheme, + * x^{n+1} - x^{n} = dt*(u^{n+1} + u^{n})/(gamma^{n+1} + gamma^{n}) + * See Eqs. 15 and 17 in Chen, JCP 407 (2020) 109228. + */ +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, + const amrex::ParticleReal ux_n, const amrex::ParticleReal uy_n, const amrex::ParticleReal uz_n, + const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, + const amrex::Real dt ) +{ + using namespace amrex::literals; + + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The ux,uy,uz are the velocities at time level n+1/2 + const amrex::ParticleReal ux_np1 = 2._prt*ux - ux_n; + const amrex::ParticleReal uy_np1 = 2._prt*uy - uy_n; + const amrex::ParticleReal uz_np1 = 2._prt*uz - uz_n; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (ux_n*ux_n + uy_n*uy_n + uz_n*uz_n)*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (ux_np1*ux_np1 + uy_np1*uy_np1 + uz_np1*uz_np1)*inv_c2); + const amrex::ParticleReal inv_gamma = 2.0_prt/(gamma_n + gamma_np1); + + // Update positions over one time step +#if (AMREX_SPACEDIM >= 2) + x += ux * inv_gamma * dt; +#else + amrex::ignore_unused(x); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) // RZ pushes particles in 3D + y += uy * inv_gamma * dt; +#else + amrex::ignore_unused(y); +#endif + z += uz * inv_gamma * dt; +} + #endif // WARPX_PARTICLES_PUSHER_UPDATEPOSITION_H_ diff --git a/Source/Particles/Resampling/LevelingThinning.H b/Source/Particles/Resampling/LevelingThinning.H index d7db1e7021c..fa05525e270 100644 --- a/Source/Particles/Resampling/LevelingThinning.H +++ b/Source/Particles/Resampling/LevelingThinning.H @@ -45,7 +45,7 @@ public: * @param[in] lev the index of the refinement level. * @param[in] pc a pointer to the particle container. */ - void operator() (WarpXParIter& pti, int lev, WarpXParticleContainer* pc) const override final; + void operator() (WarpXParIter& pti, int lev, WarpXParticleContainer* pc) const final; private: amrex::Real m_target_ratio = amrex::Real(1.5); diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 6877529053c..680e33ebe6a 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -70,9 +70,9 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, // algorithm. auto bins = ParticleUtils::findParticlesInEachCell(lev, pti, ptile); - const int n_cells = bins.numBins(); - const auto indices = bins.permutationPtr(); - const auto cell_offsets = bins.offsetsPtr(); + const auto n_cells = static_cast(bins.numBins()); + auto *const indices = bins.permutationPtr(); + auto *const cell_offsets = bins.offsetsPtr(); const amrex::Real target_ratio = m_target_ratio; const int min_ppc = m_min_ppc; @@ -83,14 +83,15 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, { // The particles that are in the cell `i_cell` are // given by the `indices[cell_start:cell_stop]` - const auto cell_start = cell_offsets[i_cell]; + const auto cell_start = static_cast(cell_offsets[i_cell]); const auto cell_stop = static_cast(cell_offsets[i_cell+1]); const int cell_numparts = cell_stop - cell_start; // do nothing for cells with less particles than min_ppc // (this intentionally includes skipping empty cells, too) - if (cell_numparts < min_ppc) + if (cell_numparts < min_ppc) { return; + } amrex::Real average_weight = 0._rt; // First loop over cell particles to compute average particle weight in the cell diff --git a/Source/Particles/Resampling/Resampling.H b/Source/Particles/Resampling/Resampling.H index 820b14f5dc5..35439e905df 100644 --- a/Source/Particles/Resampling/Resampling.H +++ b/Source/Particles/Resampling/Resampling.H @@ -30,6 +30,15 @@ struct ResamplingAlgorithm * \brief Virtual destructor of the abstract ResamplingAlgorithm class */ virtual ~ResamplingAlgorithm () = default; + + /** Default constructor */ + ResamplingAlgorithm () = default; + + + ResamplingAlgorithm ( ResamplingAlgorithm const &) = default; + ResamplingAlgorithm& operator= ( ResamplingAlgorithm const & ) = default; + ResamplingAlgorithm ( ResamplingAlgorithm&& ) = default; + ResamplingAlgorithm& operator= ( ResamplingAlgorithm&& ) = default; }; /** diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index 0f6f20f5f71..bc20420ea6e 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -9,6 +9,7 @@ #define WARPX_RigidInjectedParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Gather/ScaleFields.H" #include "PhysicalParticleContainer.H" @@ -48,13 +49,19 @@ public: RigidInjectedParticleContainer (amrex::AmrCore* amr_core, int ispecies, const std::string& name); - virtual ~RigidInjectedParticleContainer () {} + ~RigidInjectedParticleContainer () override = default; - virtual void InitData() override; + RigidInjectedParticleContainer ( RigidInjectedParticleContainer const &) = delete; + RigidInjectedParticleContainer& operator= ( RigidInjectedParticleContainer const & ) = delete; + RigidInjectedParticleContainer ( RigidInjectedParticleContainer&& ) = default; + RigidInjectedParticleContainer& operator= ( RigidInjectedParticleContainer&& ) = default; + + + void InitData() override; virtual void RemapParticles(); - virtual void Evolve (int lev, + void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, @@ -78,9 +85,10 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false ) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; - virtual void PushPX (WarpXParIter& pti, + void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, amrex::FArrayBox const * eyfab, amrex::FArrayBox const * ezfab, @@ -94,7 +102,7 @@ public: amrex::Real dt, ScaleFields scaleFields, DtType a_dt_type=DtType::Full) override; - virtual void PushP (int lev, amrex::Real dt, + void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, @@ -102,9 +110,9 @@ public: const amrex::MultiFab& By, const amrex::MultiFab& Bz) override; - virtual void ReadHeader (std::istream& is) override; + void ReadHeader (std::istream& is) override; - virtual void WriteHeader (std::ostream& os) const override; + void WriteHeader (std::ostream& os) const override; private: @@ -116,7 +124,7 @@ private: amrex::Vector zinject_plane_levels; - // Temporary quantites + // Temporary quantities amrex::ParticleReal zinject_plane_lev; amrex::ParticleReal zinject_plane_lev_previous; bool done_injecting_lev; diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 28364cb6ef7..d50851ffb1b 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -117,12 +117,12 @@ RigidInjectedParticleContainer::RemapParticles() for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { const auto& attribs = pti.GetAttribs(); - const auto uxp = attribs[PIdx::ux].dataPtr(); - const auto uyp = attribs[PIdx::uy].dataPtr(); - const auto uzp = attribs[PIdx::uz].dataPtr(); + const auto *const uxp = attribs[PIdx::ux].dataPtr(); + const auto *const uyp = attribs[PIdx::uy].dataPtr(); + const auto *const uzp = attribs[PIdx::uz].dataPtr(); - const auto GetPosition = GetParticlePosition(pti); - auto SetPosition = SetParticlePosition(pti); + const auto GetPosition = GetParticlePosition(pti); + auto SetPosition = SetParticlePosition(pti); // Loop over particles const long np = pti.numParticles(); @@ -181,8 +181,8 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, amrex::Gpu::DeviceVector optical_depth_save; #endif - const auto GetPosition = GetParticlePosition(pti, offset); - auto SetPosition = SetParticlePosition(pti, offset); + const auto GetPosition = GetParticlePosition(pti, offset); + auto SetPosition = SetParticlePosition(pti, offset); amrex::ParticleReal* const AMREX_RESTRICT ux = uxp.dataPtr() + offset; amrex::ParticleReal* const AMREX_RESTRICT uy = uyp.dataPtr() + offset; @@ -299,7 +299,8 @@ RigidInjectedParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { // Update location of injection plane in the boosted frame @@ -307,7 +308,7 @@ RigidInjectedParticleContainer::Evolve (int lev, zinject_plane_levels[lev] -= dt*WarpX::beta_boost*PhysConst::c; zinject_plane_lev = zinject_plane_levels[lev]; - // Set the done injecting flag whan the inject plane moves out of the + // Set the done injecting flag when the inject plane moves out of the // simulation domain. // It is much easier to do this check, rather than checking if all of the // particles have crossed the inject plane. @@ -324,7 +325,7 @@ RigidInjectedParticleContainer::Evolve (int lev, rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, - t, dt, a_dt_type, skip_deposition); + t, dt, a_dt_type, skip_deposition, push_type); } void @@ -334,7 +335,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, { WARPX_PROFILE("RigidInjectedParticleContainer::PushP"); - if (do_not_push) return; + if (do_not_push) { return; } const std::array& dx = WarpX::CellSize(std::max(lev,0)); @@ -357,10 +358,17 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const FArrayBox& byfab = By[pti]; const FArrayBox& bzfab = Bz[pti]; - const auto getPosition = GetParticlePosition(pti); + const auto getPosition = GetParticlePosition(pti); const auto getExternalEB = GetExternalEBField(pti); + const amrex::ParticleReal Ex_external_particle = m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; + const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const Dim3 lo = lbound(box); @@ -426,8 +434,12 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); - amrex::ParticleReal Exp = 0._prt, Eyp = 0._prt, Ezp = 0._prt; - amrex::ParticleReal Bxp = 0._prt, Byp = 0._prt, Bzp = 0._prt; + amrex::ParticleReal Exp = Ex_external_particle; + amrex::ParticleReal Eyp = Ey_external_particle; + amrex::ParticleReal Ezp = Ez_external_particle; + amrex::ParticleReal Bxp = Bx_external_particle; + amrex::ParticleReal Byp = By_external_particle; + amrex::ParticleReal Bzp = Bz_external_particle; // first gather E and B to the particle positions doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, @@ -436,7 +448,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 60e54b5d59b..a0b4ed63a30 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -89,8 +89,14 @@ struct Compute_shifted_shape_factor const T x_old, const int i_new) const { - if constexpr (depos_order == 1){ - const auto i = static_cast(x_old); + if constexpr (depos_order == 0){ + const auto i = static_cast(std::floor(x_old + T(0.5))); + const int i_shift = i - i_new; + sx[1+i_shift] = T(1.0); + return i; + } + else if constexpr (depos_order == 1){ + const auto i = static_cast(std::floor(x_old)); const int i_shift = i - i_new; const T xint = x_old - T(i); sx[1+i_shift] = T(1.0) - xint; diff --git a/Source/Particles/Sorting/CMakeLists.txt b/Source/Particles/Sorting/CMakeLists.txt index d3c378e43b8..60a156f4649 100644 --- a/Source/Particles/Sorting/CMakeLists.txt +++ b/Source/Particles/Sorting/CMakeLists.txt @@ -3,5 +3,6 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE Partition.cpp + SortingUtils.cpp ) endforeach() diff --git a/Source/Particles/Sorting/Make.package b/Source/Particles/Sorting/Make.package index 16efc02b89e..e6ad1604dc7 100644 --- a/Source/Particles/Sorting/Make.package +++ b/Source/Particles/Sorting/Make.package @@ -1,2 +1,4 @@ CEXE_sources += Partition.cpp +CEXE_sources += SortingUtils.cpp + VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Sorting diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 18bebed185f..58511cfd5e7 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -76,7 +76,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // - Find the indices that reorder particles so that the last particles // are in the larger buffer fillWithConsecutiveIntegers( pid ); - auto const sep = stablePartition( pid.begin(), pid.end(), inexflag ); + auto *const sep = stablePartition( pid.begin(), pid.end(), inexflag ); // At the end of this step, `pid` contains the indices that should be used to // reorder the particles, and `sep` is the position in the array that // separates the particles that deposit/gather on the fine patch (first part) @@ -110,7 +110,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // the smaller buffer, by looking up the mask. Store the answer in `inexflag`. amrex::ParallelFor( np - n_fine, fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, n_fine) ); - auto const sep2 = stablePartition( sep, pid.end(), inexflag ); + auto *const sep2 = stablePartition( sep, pid.end(), inexflag ); if (bmasks == gather_masks) { nfine_gather = iteratorDistance(pid.begin(), sep2); diff --git a/Source/Particles/Sorting/SortingUtils.H b/Source/Particles/Sorting/SortingUtils.H index 0bec92d6efc..ac2c63e88f8 100644 --- a/Source/Particles/Sorting/SortingUtils.H +++ b/Source/Particles/Sorting/SortingUtils.H @@ -19,18 +19,7 @@ * * \param[inout] v Vector of integers, to be filled by this routine */ -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) -{ -#ifdef AMREX_USE_GPU - // On GPU: Use amrex - auto data = v.data(); - auto N = v.size(); - AMREX_FOR_1D( N, i, {data[i] = i;}); -#else - // On CPU: Use std library - std::iota( v.begin(), v.end(), 0L ); -#endif -} +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); /** \brief Find the indices that would reorder the elements of `predicate` * so that the elements with non-zero value precede the other elements @@ -95,13 +84,13 @@ class fillBufferFlag public: fillBufferFlag( WarpXParIter const& pti, amrex::iMultiFab const* bmasks, amrex::Gpu::DeviceVector& inexflag, - amrex::Geometry const& geom ) { - + amrex::Geometry const& geom ): // Extract simple structure that can be used directly on the GPU - m_particles = pti.GetArrayOfStructs().data(); - m_buffer_mask = (*bmasks)[pti].array(); - m_inexflag_ptr = inexflag.dataPtr(); - m_domain = geom.Domain(); + m_domain{geom.Domain()}, + m_inexflag_ptr{inexflag.dataPtr()}, + m_particles{pti.GetArrayOfStructs().data()}, + m_buffer_mask{(*bmasks)[pti].array()} + { for (int idim=0; idim m_prob_lo; - amrex::GpuArray m_inv_cell_size; amrex::Box m_domain; int* m_inexflag_ptr; WarpXParticleContainer::ParticleType const* m_particles; amrex::Array4 m_buffer_mask; + amrex::GpuArray m_prob_lo; + amrex::GpuArray m_inv_cell_size; }; /** \brief Functor that fills the elements of the particle array `inexflag` @@ -154,14 +143,14 @@ class fillBufferFlagRemainingParticles amrex::Geometry const& geom, amrex::Gpu::DeviceVector const& particle_indices, long const start_index ) : - m_start_index(start_index) { - + m_domain{geom.Domain()}, // Extract simple structure that can be used directly on the GPU - m_particles = pti.GetArrayOfStructs().data(); - m_buffer_mask = (*bmasks)[pti].array(); - m_inexflag_ptr = inexflag.dataPtr(); - m_indices_ptr = particle_indices.dataPtr(); - m_domain = geom.Domain(); + m_inexflag_ptr{inexflag.dataPtr()}, + m_particles{pti.GetArrayOfStructs().data()}, + m_buffer_mask{(*bmasks)[pti].array()}, + m_start_index{start_index}, + m_indices_ptr{particle_indices.dataPtr()} + { for (int idim=0; idim const& src, amrex::Gpu::DeviceVector& dst, - amrex::Gpu::DeviceVector const& indices ) { - // Extract simple structure that can be used directly on the GPU - m_src_ptr = src.dataPtr(); - m_dst_ptr = dst.dataPtr(); - m_indices_ptr = indices.dataPtr(); - } + amrex::Gpu::DeviceVector const& indices ): + // Extract simple structure that can be used directly on the GPU + m_src_ptr{src.dataPtr()}, + m_dst_ptr{dst.dataPtr()}, + m_indices_ptr{indices.dataPtr()} + {} AMREX_GPU_DEVICE AMREX_FORCE_INLINE void operator()( const long ip ) const { diff --git a/Source/Particles/Sorting/SortingUtils.cpp b/Source/Particles/Sorting/SortingUtils.cpp new file mode 100644 index 00000000000..699119e8e18 --- /dev/null +++ b/Source/Particles/Sorting/SortingUtils.cpp @@ -0,0 +1,22 @@ +/* Copyright 2019-2020 Andrew Myers, Maxence Thevenet, Remi Lehe + * Weiqun Zhang + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "SortingUtils.H" + +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) +{ +#ifdef AMREX_USE_GPU + // On GPU: Use amrex + auto data = v.data(); + auto N = v.size(); + AMREX_FOR_1D( N, i, {data[i] = i;}); +#else + // On CPU: Use std library + std::iota( v.begin(), v.end(), 0L ); +#endif +} diff --git a/Source/Particles/SpeciesPhysicalProperties.cpp b/Source/Particles/SpeciesPhysicalProperties.cpp index 118db98f2c2..20ca5ed48b9 100644 --- a/Source/Particles/SpeciesPhysicalProperties.cpp +++ b/Source/Particles/SpeciesPhysicalProperties.cpp @@ -126,8 +126,9 @@ namespace { constexpr auto quiet_NaN = std::numeric_limits::quiet_NaN(); - // The atomic mass data below is from this NIST page + // The atomic mass data below is from these NIST pages // https://physics.nist.gov/cgi-bin/Compositions/stand_alone.pl?ele=&ascii=ascii2&isotype=some + // https://physics.nist.gov/cgi-bin/cuu/Value?malu const std::map species_to_properties { @@ -177,7 +178,7 @@ namespace { amrex::Real(4.00260325413) * PhysConst::m_u, amrex::Real(2) * PhysConst::q_e}}, {PhysicalSpecies::alpha, Properties{ - amrex::Real(4.00260325413) * PhysConst::m_u - amrex::Real(2) * PhysConst::m_e, + amrex::Real(4.001506179127) * PhysConst::m_u, amrex::Real(2) * PhysConst::q_e}}, {PhysicalSpecies::lithium, Properties{ amrex::Real(6.967) * PhysConst::m_u, diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index c6da328264b..e791ac25d22 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -13,6 +13,7 @@ #include "WarpXParticleContainer_fwd.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Initialization/PlasmaInjector.H" #include "Particles/ParticleBoundaries.H" #include "SpeciesPhysicalProperties.H" @@ -57,23 +58,28 @@ public: WarpXParIter (ContainerType& pc, int level, amrex::MFItInfo& info); - const std::array& GetAttribs () const { + [[nodiscard]] const std::array& GetAttribs () const + { return GetStructOfArrays().GetRealData(); } - std::array& GetAttribs () { + [[nodiscard]] std::array& GetAttribs () + { return GetStructOfArrays().GetRealData(); } - const RealVector& GetAttribs (int comp) const { + [[nodiscard]] const RealVector& GetAttribs (int comp) const + { return GetStructOfArrays().GetRealData(comp); } - RealVector& GetAttribs (int comp) { + [[nodiscard]] RealVector& GetAttribs (int comp) + { return GetStructOfArrays().GetRealData(comp); } - IntVector& GetiAttribs (int comp) { + [[nodiscard]] IntVector& GetiAttribs (int comp) + { return GetStructOfArrays().GetIntData(comp); } }; @@ -115,7 +121,13 @@ public: using DiagnosticParticles = amrex::Vector, DiagnosticParticleData> >; WarpXParticleContainer (amrex::AmrCore* amr_core, int ispecies); - virtual ~WarpXParticleContainer() {} + ~WarpXParticleContainer() override = default; + + // Move and copy operations + WarpXParticleContainer(const WarpXParticleContainer&) = delete; + WarpXParticleContainer& operator=(const WarpXParticleContainer&) = delete; + WarpXParticleContainer(WarpXParticleContainer&&) = default; + WarpXParticleContainer& operator=(WarpXParticleContainer&&) = default; virtual void InitData () = 0; @@ -125,7 +137,7 @@ public: * \brief Virtual function that returns a pointer to the plasma injector, * for derived classes that define one (PhysicalParticleContainer). */ - virtual PlasmaInjector* GetPlasmaInjector () { return nullptr; } + virtual PlasmaInjector* GetPlasmaInjector (const int /*i*/) { return nullptr; } /** * Evolve is the central WarpXParticleContainer function that advances @@ -140,7 +152,8 @@ public: amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false) = 0; + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, + PushType push_type=PushType::Explicit) = 0; virtual void PostRestart () = 0; @@ -155,12 +168,11 @@ public: NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& pinned_tile, int n_external_attr_real, - int n_external_attr_int, - const amrex::RandomEngine& engine) = 0; + int n_external_attr_int) = 0; /// /// This pushes the particle positions by one half time step. - /// It is used to desynchronize the particles after initializaton + /// It is used to desynchronize the particles after initialization /// or when restarting from a checkpoint. /// void PushX ( amrex::Real dt); @@ -219,7 +231,7 @@ public: amrex::MultiFab* rho, int icomp, long offset, - long np_to_depose, + long np_to_deposit, int thread_num, int lev, int depos_lev); @@ -234,12 +246,13 @@ public: amrex::MultiFab* jy, amrex::MultiFab* jz, long offset, - long np_to_depose, + long np_to_deposit, int thread_num, int lev, int depos_lev, amrex::Real dt, - amrex::Real relative_time); + amrex::Real relative_time, + PushType push_type); // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and @@ -297,7 +310,7 @@ public: * @param[in] id if different than -1, this id will be assigned to the particles (used for * particle tagging in some routines, e.g. SplitParticle) */ - void AddNParticles (int lev, int n, + void AddNParticles (int lev, long n, amrex::Vector const & x, amrex::Vector const & y, amrex::Vector const & z, @@ -329,6 +342,10 @@ public: int self_fields_max_iters = 200; int self_fields_verbosity = 2; + // External fields added to particle fields. + amrex::Vector m_B_external_particle; + amrex::Vector m_E_external_particle; + //! Current injection position amrex::Real m_current_injection_position; @@ -387,7 +404,20 @@ public: * they do not exist (or if they were defined by default, i.e., * without runtime component). */ - void defineAllParticleTiles () noexcept; + void defineAllParticleTiles () noexcept; + + virtual std::vector getUserIntAttribs () const { return std::vector{}; } + + virtual std::vector getUserRealAttribs () const { return std::vector{}; } + + virtual amrex::Vector< amrex::Parser* > getUserIntAttribParser () const { return amrex::Vector{}; } + + virtual amrex::Vector< amrex::Parser* > getUserRealAttribParser () const { return amrex::Vector{}; } + +#ifdef WARPX_QED + virtual BreitWheelerEngine* get_breit_wheeler_engine_ptr () const {return nullptr;} + virtual QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const {return nullptr;} +#endif protected: int species_id; @@ -458,12 +488,14 @@ public: using TmpParticles = amrex::Vector >; TmpParticles getTmpParticleData () const noexcept {return tmp_particle_data;} + + int getIonizationInitialLevel () const noexcept {return ionization_initial_level;} + protected: TmpParticles tmp_particle_data; private: - virtual void particlePostLocate(ParticleType& p, const amrex::ParticleLocData& pld, - int lev) override; + void particlePostLocate(ParticleType& p, const amrex::ParticleLocData& pld, int lev) override; }; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index d2da12888ca..f1b19dd8407 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -20,6 +20,7 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "Utils/Parser/ParserUtils.H" #include "WarpX.H" #include @@ -58,7 +59,6 @@ #include #include #include -#include #include @@ -90,6 +90,17 @@ WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies) SetParticleSize(); ReadParameters(); + // Reading the external fields needs to be here since ReadParameters + // is static but the m_E_external_particle and B are not + const ParmParse pp_particles("particles"); + + // allocating and initializing default values of external fields for particles + m_E_external_particle.resize(3, 0.); + m_B_external_particle.resize(3, 0.); + + utils::parser::queryArrWithParser(pp_particles, "E_external_particle", m_E_external_particle); + utils::parser::queryArrWithParser(pp_particles, "B_external_particle", m_B_external_particle); + // Initialize temporary local arrays for charge/current deposition #ifdef AMREX_USE_OMP int num_threads = 1; @@ -144,7 +155,7 @@ WarpXParticleContainer::AllocData () } void -WarpXParticleContainer::AddNParticles (int /*lev*/, int n, +WarpXParticleContainer::AddNParticles (int /*lev*/, long n, amrex::Vector const & x, amrex::Vector const & y, amrex::Vector const & z, @@ -164,13 +175,13 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, int n, WARPX_ALWAYS_ASSERT_WITH_MESSAGE(nattr_int <= NumIntComps(), "Too many integer attributes specified"); - int ibegin = 0; - int iend = n; + long ibegin = 0; + long iend = n; if (!uniqueparticles) { const int myproc = amrex::ParallelDescriptor::MyProc(); const int nprocs = amrex::ParallelDescriptor::NProcs(); - const int navg = n/nprocs; - const int nleft = n - navg * nprocs; + const auto navg = n/nprocs; + const auto nleft = n - navg * nprocs; if (myproc < nleft) { ibegin = myproc*(navg+1); iend = ibegin + navg+1; @@ -196,7 +207,7 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, int n, amrex::Vector theta(np); #endif - for (int i = ibegin; i < iend; ++i) + for (auto i = ibegin; i < iend; ++i) { ParticleType p; if (id==-1) @@ -268,9 +279,9 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, int n, pinned_tile.push_back_int(j, attr_int[j].data() + ibegin, attr_int[j].data() + iend); } + pinned_tile.resize(np); // Default initialize the other real and integer runtime attributes - DefaultInitializeRuntimeAttributes(pinned_tile, nattr_real - 1, nattr_int, - amrex::RandomEngine{}); + DefaultInitializeRuntimeAttributes(pinned_tile, nattr_real - 1, nattr_int); auto old_np = particle_tile.numParticles(); auto new_np = old_np + pinned_tile.numParticles(); @@ -284,22 +295,22 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, int n, } /* \brief Current Deposition for thread thread_num - * \param pti : Particle iterator - * \param wp : Array of particle weights - * \param uxp uyp uzp : Array of particle momenta - * \param ion_lev : Pointer to array of particle ionization level. This is - required to have the charge of each macroparticle - since q is a scalar. For non-ionizable species, - ion_lev is a null pointer. - * \param jx jy jz : Full array of current density - * \param offset : Index of first particle for which current is deposited - * \param np_to_depose: Number of particles for which current is deposited. - Particles [offset,offset+np_tp_depose] deposit current - * \param thread_num : Thread number (if tiling) - * \param lev : Level of box that contains particles - * \param depos_lev : Level on which particles deposit (if buffers are used) - * \param dt : Time step for particle level - * \param relative_time: Time at which to deposit J, relative to the time of the + * \param pti Particle iterator + * \param wp Array of particle weights + * \param uxp uyp uzp Array of particle momenta + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param jx jy jz Full array of current density + * \param offset Index of first particle for which current is deposited + * \param np_to_deposit Number of particles for which current is deposited. + Particles [offset,offset+np_to_deposit] deposit current + * \param thread_num Thread number (if tiling) + * \param lev Level of box that contains particles + * \param depos_lev Level on which particles deposit (if buffers are used) + * \param dt Time step for particle level + * \param relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. @@ -310,19 +321,19 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, RealVector const & uyp, RealVector const & uzp, int const * const ion_lev, amrex::MultiFab * const jx, amrex::MultiFab * const jy, amrex::MultiFab * const jz, - long const offset, long const np_to_depose, + long const offset, long const np_to_deposit, int const thread_num, const int lev, int const depos_lev, - amrex::Real const dt, amrex::Real const relative_time) + amrex::Real const dt, amrex::Real const relative_time, PushType push_type) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || (depos_lev==(lev )), "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_depose == 0) return; + if (np_to_deposit == 0) { return; } // If user decides not to deposit - if (do_not_deposit) return; + if (do_not_deposit) { return; } // Number of guard cells for local deposition of J const WarpX& warpx = WarpX::GetInstance(); @@ -424,7 +435,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, Array4 const& jz_arr = local_jz[thread_num].array(); #endif - const auto GetPosition = GetParticlePosition(pti, offset); + const auto GetPosition = GetParticlePosition(pti, offset); // Lower corner of tile box physical domain // Note that this includes guard cells since it is after tilebox.ngrow @@ -459,7 +470,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, { auto& ptile = ParticlesAt(lev, pti); auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto *pstruct_ptr = aos().dataPtr(); const int ntiles = numTilesInBox(box, true, bin_size); @@ -491,6 +502,9 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WARPX_PROFILE_VAR_STOP(blp_get_max_tilesize); // Now pick current deposition algorithm + if (push_type == PushType::Implicit) { + amrex::Abort("Cannot do shared memory deposition with implicit algorithm"); + } if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { WARPX_ABORT_WITH_MESSAGE("Cannot do shared memory deposition with Esirkepov algorithm"); } @@ -501,25 +515,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WARPX_PROFILE_VAR_START(direct_current_dep_kernel); if (WarpX::nox == 1){ doDepositionSharedShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } WARPX_PROFILE_VAR_STOP(direct_current_dep_kernel); } @@ -527,73 +541,158 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // If not doing shared memory deposition, call normal kernels else { if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { - if (WarpX::nox == 1){ - doEsirkepovDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 2){ - doEsirkepovDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 3){ - doEsirkepovDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + if (push_type == PushType::Explicit) { + if (WarpX::nox == 1){ + doEsirkepovDepositionShapeN<1>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doEsirkepovDepositionShapeN<2>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doEsirkepovDepositionShapeN<3>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } + } else if (push_type == PushType::Implicit) { +#if (AMREX_SPACEDIM >= 2) + auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); + const ParticleReal* xp_n_data = xp_n.dataPtr() + offset; +#else + const ParticleReal* xp_n_data = nullptr; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + auto& yp_n = pti.GetAttribs(particle_comps["y_n"]); + const ParticleReal* yp_n_data = yp_n.dataPtr() + offset; +#else + const ParticleReal* yp_n_data = nullptr; +#endif + auto& zp_n = pti.GetAttribs(particle_comps["z_n"]); + const ParticleReal* zp_n_data = zp_n.dataPtr() + offset; + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doChargeConservingDepositionShapeNImplicit<1>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doChargeConservingDepositionShapeNImplicit<2>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doChargeConservingDepositionShapeNImplicit<3>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { + if (push_type == PushType::Implicit) { + WARPX_ABORT_WITH_MESSAGE("The Vay algorithm cannot be used with implicit algorithm."); + } if (WarpX::nox == 1){ doVayDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 2){ doVayDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 3){ doVayDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } else { // Direct deposition - if (WarpX::nox == 1){ - doDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 2){ - doDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 3){ - doDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + if (push_type == PushType::Explicit) { + if (WarpX::nox == 1){ + doDepositionShapeN<1>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doDepositionShapeN<2>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doDepositionShapeN<3>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } + } else if (push_type == PushType::Implicit) { + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doDepositionShapeNImplicit<1>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doDepositionShapeNImplicit<2>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doDepositionShapeNImplicit<3>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } } } } @@ -615,7 +714,7 @@ WarpXParticleContainer::DepositCurrent ( const amrex::Real dt, const amrex::Real relative_time) { // Loop over the refinement levels - int const finest_level = J.size() - 1; + auto const finest_level = static_cast(J.size() - 1); for (int lev = 0; lev <= finest_level; ++lev) { // Loop over particle tiles and deposit current on each level @@ -642,7 +741,7 @@ WarpXParticleContainer::DepositCurrent ( DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, J[lev][0].get(), J[lev][1].get(), J[lev][2].get(), - 0, np, thread_num, lev, lev, dt, relative_time); + 0, np, thread_num, lev, lev, dt, relative_time, PushType::Explicit); } #ifdef AMREX_USE_OMP } @@ -651,28 +750,28 @@ WarpXParticleContainer::DepositCurrent ( } /* \brief Charge Deposition for thread thread_num - * \param pti : Particle iterator - * \param wp : Array of particle weights - * \param ion_lev : Pointer to array of particle ionization level. This is - required to have the charge of each macroparticle - since q is a scalar. For non-ionizable species, - ion_lev is a null pointer. - * \param rho : Full array of charge density - * \param icomp : Component of rho into which charge is deposited. - 0: old value (before particle push). - 1: new value (after particle push). - * \param offset : Index of first particle for which charge is deposited - * \param np_to_depose: Number of particles for which charge is deposited. - Particles [offset,offset+np_tp_depose) deposit charge - * \param thread_num : Thread number (if tiling) - * \param lev : Level of box that contains particles - * \param depos_lev : Level on which particles deposit (if buffers are used) + * \param pti Particle iterator + * \param wp Array of particle weights + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param rho Full array of charge density + * \param icomp Component of rho into which charge is deposited. + 0: old value (before particle push). + 1: new value (after particle push). + * \param offset Index of first particle for which charge is deposited + * \param np_to_deposit Number of particles for which charge is deposited. + Particles [offset,offset+np_to_deposit) deposit charge + * \param thread_num Thread number (if tiling) + * \param lev Level of box that contains particles + * \param depos_lev Level on which particles deposit (if buffers are used) */ void WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const int * const ion_lev, amrex::MultiFab* rho, const int icomp, - const long offset, const long np_to_depose, + const long offset, const long np_to_deposit, const int thread_num, const int lev, const int depos_lev) { if (WarpX::do_shared_mem_charge_deposition) @@ -682,7 +781,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_depose == 0) return; + if (np_to_deposit == 0) { return; } // Number of guard cells for local deposition of rho const WarpX& warpx = WarpX::GetInstance(); @@ -784,7 +883,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, auto& ptile = ParticlesAt(lev, pti); auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto *pstruct_ptr = aos().dataPtr(); Box box = pti.validbox(); box.grow(ng_rho); @@ -813,19 +912,19 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, auto& ptile = ParticlesAt(lev, pti); auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto *pstruct_ptr = aos().dataPtr(); Box box = pti.validbox(); box.grow(ng_rho); const amrex::IntVect bin_size = WarpX::shared_tilesize; - const auto offsets_ptr = bins.offsetsPtr(); - auto tbox_ptr = tboxes.dataPtr(); - auto permutation = bins.permutationPtr(); + auto *const offsets_ptr = bins.offsetsPtr(); + auto *tbox_ptr = tboxes.dataPtr(); + auto *permutation = bins.permutationPtr(); amrex::ParallelFor(bins.numBins(), [=] AMREX_GPU_DEVICE (int ibin) { - const int bin_start = offsets_ptr[ibin]; - const int bin_stop = offsets_ptr[ibin+1]; + const auto bin_start = offsets_ptr[ibin]; + const auto bin_stop = offsets_ptr[ibin+1]; if (bin_start < bin_stop) { auto p = pstruct_ptr[permutation[bin_start]]; Box tbx; @@ -846,7 +945,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; - const auto boxes_ptr = tboxes.dataPtr(); + auto *const boxes_ptr = tboxes.dataPtr(); reduce_op.eval(tboxes.size(), reduce_data, [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple { @@ -870,26 +969,26 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, amrex::LayoutData* costs = WarpX::getCosts(lev); amrex::Real* cost = costs ? &((*costs)[pti.index()]) : nullptr; - const auto GetPosition = GetParticlePosition(pti, offset); + const auto GetPosition = GetParticlePosition(pti, offset); const Geometry& geom = Geom(lev); Box box = pti.validbox(); box.grow(ng_rho); if (WarpX::nox == 1){ doChargeDepositionSharedShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_depose, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 2){ doChargeDepositionSharedShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_depose, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 3){ doChargeDepositionSharedShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_depose, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); @@ -943,13 +1042,13 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noz); ablastr::particles::deposit_charge( - pti, wp, this->charge, ion_lev, - rho, local_rho[thread_num], - WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, - ng_rho, depos_lev, ref_ratio, - offset, np_to_depose, - icomp, nc, - cost, WarpX::load_balance_costs_update_algo, WarpX::do_device_synchronize + pti, wp, this->charge, ion_lev, + rho, local_rho[thread_num], + WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, + ng_rho, depos_lev, ref_ratio, + offset, np_to_deposit, + icomp, nc, + cost, WarpX::load_balance_costs_update_algo, WarpX::do_device_synchronize ); } } @@ -964,7 +1063,7 @@ WarpXParticleContainer::DepositCharge (amrex::Vector(rho.size() - 1); for (int lev = 0; lev <= finest_level; ++lev) { DepositCharge ( @@ -1003,7 +1102,7 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, { // Reset the rho array if reset is True int const nc = WarpX::ncomps; - if (reset) rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); + if (reset) { rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); } // Loop over particle tiles and deposit charge on each level #ifdef AMREX_USE_OMP @@ -1070,8 +1169,9 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) #else const bool is_PSATD_RZ = false; #endif - if( !is_PSATD_RZ ) + if( !is_PSATD_RZ ) { nba.surroundingNodes(); + } // Number of guard cells for local deposition of rho const WarpX& warpx = WarpX::GetInstance(); @@ -1096,7 +1196,7 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { for (int lev = 0; lev <= nLevels; ++lev) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - const auto wp = pti.GetAttribs(PIdx::w).data(); + auto *const wp = pti.GetAttribs(PIdx::w).data(); reduce_op.eval(pti.numParticles(), reduce_data, [=] AMREX_GPU_DEVICE (int ip) @@ -1106,7 +1206,7 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { total_charge = get<0>(reduce_data.value()); - if (!local) ParallelDescriptor::ReduceRealSum(total_charge); + if (!local) { ParallelDescriptor::ReduceRealSum(total_charge); } total_charge *= this->charge; return total_charge; } @@ -1219,7 +1319,7 @@ amrex::ParticleReal WarpXParticleContainer::maxParticleVelocity(bool local) { } } - if (!local) ParallelAllReduce::Max(max_v, ParallelDescriptor::Communicator()); + if (!local) { ParallelAllReduce::Max(max_v, ParallelDescriptor::Communicator()); } return max_v; } @@ -1237,7 +1337,7 @@ WarpXParticleContainer::PushX (int lev, amrex::Real dt) { WARPX_PROFILE("WarpXParticleContainer::PushX()"); - if (do_not_push) return; + if (do_not_push) { return; } amrex::LayoutData* costs = WarpX::getCosts(lev); @@ -1252,14 +1352,14 @@ WarpXParticleContainer::PushX (int lev, amrex::Real dt) { amrex::Gpu::synchronize(); } - Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); // // Particle Push // - const auto GetPosition = GetParticlePosition(pti); - auto SetPosition = SetParticlePosition(pti); + const auto GetPosition = GetParticlePosition(pti); + auto SetPosition = SetParticlePosition(pti); // - momenta are stored as a struct of array, in `attribs` auto& attribs = pti.GetAttribs(); @@ -1280,7 +1380,7 @@ WarpXParticleContainer::PushX (int lev, amrex::Real dt) if (costs && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*costs)[pti.index()], wt); } } @@ -1312,7 +1412,7 @@ WarpXParticleContainer::particlePostLocate(ParticleType& p, const ParticleLocData& pld, const int lev) { - if (not do_splitting) return; + if (not do_splitting) { return; } // Tag particle if goes to higher level. // It will be split later in the loop @@ -1334,7 +1434,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ WARPX_PROFILE("WarpXParticleContainer::ApplyBoundaryConditions()"); // Periodic boundaries are handled in AMReX code - if (m_boundary_conditions.CheckAll(ParticleBoundaryType::Periodic)) return; + if (m_boundary_conditions.CheckAll(ParticleBoundaryType::Periodic)) { return; } auto boundary_conditions = m_boundary_conditions.data; @@ -1345,8 +1445,8 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ #endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - auto GetPosition = GetParticlePosition(pti); - auto SetPosition = SetParticlePosition(pti); + auto GetPosition = GetParticlePosition(pti); + auto SetPosition = SetParticlePosition(pti); #ifndef WARPX_DIM_1D_Z const Real xmin = Geom(lev).ProbLo(0); const Real xmax = Geom(lev).ProbHi(0); @@ -1373,7 +1473,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ ParticleType& p = pp[i]; // skip particles that are already flagged for removal - if (p.id() < 0) return; + if (p.id() < 0) { return; } ParticleReal x, y, z; GetPosition.AsStored(i, x, y, z); diff --git a/Source/Python/CMakeLists.txt b/Source/Python/CMakeLists.txt index 4b4be199c31..17a75301306 100644 --- a/Source/Python/CMakeLists.txt +++ b/Source/Python/CMakeLists.txt @@ -7,10 +7,7 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE # callback hooks - WarpX_py.cpp - - # legacy C wrapper APIs - WarpXWrappers.cpp + callbacks.cpp ) if(WarpX_PYTHON) target_sources(pyWarpX_${SD} diff --git a/Source/Python/Make.package b/Source/Python/Make.package index 9f421456b40..bf7121e3c77 100644 --- a/Source/Python/Make.package +++ b/Source/Python/Make.package @@ -1,4 +1,3 @@ -CEXE_sources += WarpXWrappers.cpp -CEXE_sources += WarpX_py.cpp +CEXE_sources += callbacks.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Python diff --git a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp index d048b4d9c8f..600d56a62c9 100644 --- a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp +++ b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp @@ -15,9 +15,4 @@ void init_PinnedMemoryParticleContainer (py::module& m) PinnedMemoryParticleContainer, amrex::ParticleContainer<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> > pmpc (m, "PinnedMemoryParticleContainer"); - pmpc - .def("num_int_comps", - [](PinnedMemoryParticleContainer& pc) { return pc.NumIntComps(); } - ) - ; } diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index b5d3b16269c..1473a750941 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -85,7 +85,6 @@ void init_WarpXParticleContainer (py::module& m) py::arg("nattr_int"), py::arg("attr_int"), py::arg("uniqueparticles"), py::arg("id")=-1 ) - .def("num_real_comps", &WarpXParticleContainer::NumRealComps) .def("get_comp_index", [](WarpXParticleContainer& pc, std::string comp_name) { @@ -102,6 +101,10 @@ void init_WarpXParticleContainer (py::module& m) &WarpXParticleContainer::TotalNumberOfParticles, py::arg("valid_particles_only"), py::arg("local") ) + .def("sum_particle_charge", + &WarpXParticleContainer::sumParticleCharge, + py::arg("local") + ) .def("deposit_charge", [](WarpXParticleContainer& pc, amrex::MultiFab* rho, const int lev) @@ -115,5 +118,12 @@ void init_WarpXParticleContainer (py::module& m) }, py::arg("rho"), py::arg("lev") ) + .def("get_charge_density", + [](WarpXParticleContainer& pc, int lev, bool local) + { + return pc.GetChargeDensity(lev, local); + }, + py::arg("lev"), py::arg("local") + ) ; } diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 3429f33f5b9..e42b8268fe1 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -25,7 +25,10 @@ #endif // use PSATD ifdef #include #include +#include #include +#include +#include #include #include #include @@ -83,7 +86,15 @@ void init_WarpX (py::module& m) "Evolve the simulation the specified number of steps" ) - // from AmrCore->AmrMesh + // from amrex::AmrCore / amrex::AmrMesh + .def_property_readonly("max_level", + [](WarpX const & wx){ return wx.maxLevel(); }, + "The maximum mesh-refinement level for the simulation." + ) + .def_property_readonly("finest_level", + [](WarpX const & wx){ return wx.finestLevel(); }, + "The currently finest level of mesh-refinement used. This is always less or equal to max_level." + ) .def("Geom", //[](WarpX const & wx, int const lev) { return wx.Geom(lev); }, py::overload_cast< int >(&WarpX::Geom, py::const_), @@ -109,7 +120,15 @@ void init_WarpX (py::module& m) }, py::arg("multifab_name"), py::return_value_policy::reference_internal, - "Return MultiFabs by name, e.g., 'Efield_aux[x][l=0]', 'Efield_cp[x][l=0]', ..." + R"doc(Return MultiFabs by name, e.g., ``\"Efield_aux[x][level=0]\"``, ``\"Efield_cp[x][level=0]\"``, ... + +The physical fields in WarpX have the following naming: + +- ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level +- ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. +- ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher).)doc" ) .def("multi_particle_container", [](WarpX& wx){ return &wx.GetPartContainer(); }, @@ -137,22 +156,26 @@ void init_WarpX (py::module& m) // Expose functions to get the current simulation step and time .def("getistep", [](WarpX const & wx, int lev){ return wx.getistep(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current step on mesh-refinement level ``lev``." ) .def("gett_new", [](WarpX const & wx, int lev){ return wx.gett_new(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current physical time on mesh-refinement level ``lev``." ) .def("getdt", [](WarpX const & wx, int lev){ return wx.getdt(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current physical time step size on mesh-refinement level ``lev``." ) .def("set_potential_on_eb", [](WarpX& wx, std::string potential) { wx.m_poisson_boundary_handler.setPotentialEB(potential); }, - py::arg("potential") + py::arg("potential"), + "Sets the EB potential string and updates the function parser." ) ; diff --git a/Source/Python/WarpXWrappers.H b/Source/Python/WarpXWrappers.H deleted file mode 100644 index d2d57445880..00000000000 --- a/Source/Python/WarpXWrappers.H +++ /dev/null @@ -1,80 +0,0 @@ -/* Copyright 2019 Andrew Myers, David Grote, Maxence Thevenet - * Remi Lehe, Weiqun Zhang - * - * This file is part of WarpX. - * - * This file is a legacy file and will be removed soon. - * Please do NOT add new bindings here! Please see the other files - * in this directory for the new pybind11-based bindings! - * - * License: BSD-3-Clause-LBNL - */ -#ifndef WARPX_WRAPPERS_H_ -#define WARPX_WRAPPERS_H_ - -#include "Particles/WarpXParticleContainer.H" -#include "Evolve/WarpXDtType.H" -#include -#include - -#ifdef AMREX_USE_MPI -# include -#endif - -#ifdef __cplusplus -extern "C" { -#endif - - int warpx_Real_size(); - int warpx_ParticleReal_size(); - - int warpx_nSpecies(); - - bool warpx_use_fdtd_nci_corr(); - - int warpx_galerkin_interpolation(); - - void amrex_init_with_inited_mpi (int argc, char* argv[], MPI_Comm mpicomm); - - typedef void(*WARPX_CALLBACK_PY_FUNC_0)(); - - void warpx_ConvertLabParamsToBoost(); - - void warpx_ReadBCParams(); - - void warpx_CheckGriddingForRZSpectral(); - - amrex::Real warpx_getCellSize(int dir, int lev); - - amrex::Real warpx_sumParticleCharge(const char* char_species_name, bool local); - - void warpx_ComputeDt (); - void warpx_MoveWindow (int step, bool move_j); - - void warpx_EvolveE (amrex::Real dt); - void warpx_EvolveB (amrex::Real dt, DtType a_dt_type); - void warpx_FillBoundaryE (); - void warpx_FillBoundaryB (); - void warpx_SyncCurrent ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer); - void warpx_UpdateAuxilaryData (); - void warpx_PushParticlesandDepose (amrex::Real cur_time); - - void warpx_setistep (int lev, int ii); - void warpx_sett_new (int lev, amrex::Real time); - amrex::Real warpx_getdt (int lev); - - int warpx_maxStep (); - amrex::Real warpx_stopTime (); - - int warpx_finestLevel (); - - void mypc_Redistribute (); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/Source/Python/WarpXWrappers.cpp b/Source/Python/WarpXWrappers.cpp deleted file mode 100644 index 44820dd1f53..00000000000 --- a/Source/Python/WarpXWrappers.cpp +++ /dev/null @@ -1,172 +0,0 @@ -/* Copyright 2019 Andrew Myers, Axel Huebl, David Grote - * Luca Fedeli, Maxence Thevenet, Remi Lehe - * Weiqun Zhang - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#include "BoundaryConditions/PML.H" -#include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" -#include "Initialization/WarpXAMReXInit.H" -#include "Particles/MultiParticleContainer.H" -#include "Particles/ParticleBoundaryBuffer.H" -#include "Particles/WarpXParticleContainer.H" -#include "Utils/WarpXProfilerWrapper.H" -#include "Utils/WarpXUtil.H" -#include "WarpX.H" -#include "WarpXWrappers.H" -#include "WarpX_py.H" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - - int warpx_Real_size() - { - return (int)sizeof(amrex::Real); - } - - int warpx_ParticleReal_size() - { - return (int)sizeof(amrex::ParticleReal); - } - - int warpx_nSpecies() - { - const auto & mypc = WarpX::GetInstance().GetPartContainer(); - return mypc.nSpecies(); - } - - bool warpx_use_fdtd_nci_corr() - { - return WarpX::use_fdtd_nci_corr; - } - - int warpx_galerkin_interpolation() - { - return WarpX::galerkin_interpolation; - } - - void amrex_init_with_inited_mpi (int argc, char* argv[], MPI_Comm /* mpicomm */) - { - warpx::initialization::amrex_init(argc, argv, true); - } - - void warpx_ConvertLabParamsToBoost() - { - ConvertLabParamsToBoost(); - } - - void warpx_ReadBCParams() - { - ReadBCParams(); - } - - void warpx_CheckGriddingForRZSpectral() - { - CheckGriddingForRZSpectral(); - } - - amrex::Real warpx_getCellSize(int dir, int lev) { - const std::array& dx = WarpX::CellSize(lev); - return dx[dir]; - } - - amrex::Real warpx_sumParticleCharge(const char* char_species_name, const bool local) - { - auto & mypc = WarpX::GetInstance().GetPartContainer(); - const std::string species_name(char_species_name); - auto & myspc = mypc.GetParticleContainerFromName(species_name); - return myspc.sumParticleCharge(local); - } - - void warpx_ComputeDt () { - WarpX& warpx = WarpX::GetInstance(); - warpx.ComputeDt(); - } - void warpx_MoveWindow (int step,bool move_j) { - WarpX& warpx = WarpX::GetInstance(); - warpx.MoveWindow(step, move_j); - } - - void warpx_EvolveE (amrex::Real dt) { - WarpX& warpx = WarpX::GetInstance(); - warpx.EvolveE(dt); - } - void warpx_EvolveB (amrex::Real dt, DtType a_dt_type) { - WarpX& warpx = WarpX::GetInstance(); - warpx.EvolveB(dt, a_dt_type); - } - void warpx_FillBoundaryE () { - WarpX& warpx = WarpX::GetInstance(); - warpx.FillBoundaryE(warpx.getngEB()); - } - void warpx_FillBoundaryB () { - WarpX& warpx = WarpX::GetInstance(); - warpx.FillBoundaryB(warpx.getngEB()); - } - void warpx_SyncCurrent ( - const amrex::Vector,3>>& J_fp, - const amrex::Vector,3>>& J_cp, - const amrex::Vector,3>>& J_buffer) { - WarpX& warpx = WarpX::GetInstance(); - warpx.SyncCurrent(J_fp, J_cp, J_buffer); - } - void warpx_UpdateAuxilaryData () { - WarpX& warpx = WarpX::GetInstance(); - warpx.UpdateAuxilaryData(); - } - void warpx_PushParticlesandDepose (amrex::Real cur_time) { - WarpX& warpx = WarpX::GetInstance(); - warpx.PushParticlesandDepose(cur_time); - } - - void warpx_setistep (int lev, int ii) { - WarpX& warpx = WarpX::GetInstance(); - warpx.setistep(lev, ii); - } - void warpx_sett_new (int lev, amrex::Real time) { - WarpX& warpx = WarpX::GetInstance(); - warpx.sett_new(lev, time); - } - amrex::Real warpx_getdt (int lev) { - const WarpX& warpx = WarpX::GetInstance(); - return warpx.getdt(lev); - } - - int warpx_maxStep () { - const WarpX& warpx = WarpX::GetInstance(); - return warpx.maxStep(); - } - amrex::Real warpx_stopTime () { - const WarpX& warpx = WarpX::GetInstance(); - return warpx.stopTime(); - } - - int warpx_finestLevel () { - const WarpX& warpx = WarpX::GetInstance(); - return warpx.finestLevel(); - } - - void mypc_Redistribute () { - auto & mypc = WarpX::GetInstance().GetPartContainer(); - mypc.Redistribute(); - } diff --git a/Source/Python/WarpX_py.cpp b/Source/Python/WarpX_py.cpp deleted file mode 100644 index 49328d0a4a7..00000000000 --- a/Source/Python/WarpX_py.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* Copyright 2019-2022 The WarpX Community - * - * This file is part of WarpX. - * - * Authors: David Grote, Maxence Thevenet, Weiqun Zhang, Roelof Groenewald - * - * License: BSD-3-Clause-LBNL - */ -#include "WarpX_py.H" - -std::map< std::string, std::function > warpx_callback_py_map; - -void InstallPythonCallback ( std::string name, std::function callback ) -{ - warpx_callback_py_map[name] = callback; -} - -bool IsPythonCallbackInstalled ( std::string name ) -{ - return (warpx_callback_py_map.count(name) == 1u); -} - -// Execute Python callbacks of the type given by the input string -void ExecutePythonCallback ( std::string name ) -{ - if ( IsPythonCallbackInstalled(name) ) { - WARPX_PROFILE("warpx_py_"+name); - warpx_callback_py_map[name](); - } -} - -void ClearPythonCallback ( std::string name ) -{ - warpx_callback_py_map.erase(name); -} diff --git a/Source/Python/WarpX_py.H b/Source/Python/callbacks.H similarity index 89% rename from Source/Python/WarpX_py.H rename to Source/Python/callbacks.H index a18f9a7f1ce..f0c98caf636 100644 --- a/Source/Python/WarpX_py.H +++ b/Source/Python/callbacks.H @@ -2,16 +2,17 @@ * * This file is part of WarpX. * - * Authors: David Grote, Maxence Thevenet, Weiqun Zhang, Roelof Groenewald + * Authors: David Grote, Maxence Thevenet, Weiqun Zhang, Roelof Groenewald, Axel Huebl * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_PY_H_ -#define WARPX_PY_H_ +#ifndef WARPX_PY_CALLBACKS_H_ +#define WARPX_PY_CALLBACKS_H_ #include "Utils/export.H" #include "Utils/WarpXProfilerWrapper.H" +#include #include #include @@ -45,4 +46,4 @@ void ExecutePythonCallback ( std::string name ); */ void ClearPythonCallback ( std::string name ); -#endif +#endif // WARPX_PY_CALLBACKS_H_ diff --git a/Source/Python/callbacks.cpp b/Source/Python/callbacks.cpp new file mode 100644 index 00000000000..930c88e65d1 --- /dev/null +++ b/Source/Python/callbacks.cpp @@ -0,0 +1,53 @@ +/* Copyright 2019-2022 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: David Grote, Maxence Thevenet, Weiqun Zhang, Roelof Groenewald, Axel Huebl + * + * License: BSD-3-Clause-LBNL + */ +#include "callbacks.H" + +#include +#include +#include + + +std::map< std::string, std::function > warpx_callback_py_map; + +void InstallPythonCallback ( std::string name, std::function callback ) +{ + warpx_callback_py_map[name] = callback; +} + +bool IsPythonCallbackInstalled ( std::string name ) +{ + return (warpx_callback_py_map.count(name) == 1u); +} + +// Execute Python callbacks of the type given by the input string +void ExecutePythonCallback ( std::string name ) +{ + if ( IsPythonCallbackInstalled(name) ) { + WARPX_PROFILE("warpx_py_" + name); + try { + warpx_callback_py_map[name](); + } catch (std::exception &e) { + std::cerr << "Python callback '" << name << "' failed!" << std::endl; + std::cerr << e.what() << std::endl; + std::exit(3); // note: NOT amrex::Abort(), to avoid hangs with MPI + + // future note: + // if we want to rethrow/raise exceptions from Python callbacks through here (C++) and + // back the managing Python interpreter, we first need to discard and clear + // out the Python error in py::error_already_set. Otherwise, MPI-runs will hang + // (and Python will be in continued error state). + // https://pybind11.readthedocs.io/en/stable/advanced/exceptions.html#handling-unraisable-exceptions + } + } +} + +void ClearPythonCallback ( std::string name ) +{ + warpx_callback_py_map.erase(name); +} diff --git a/Source/Python/pyWarpX.cpp b/Source/Python/pyWarpX.cpp index 64db77153bd..26f4c77502d 100644 --- a/Source/Python/pyWarpX.cpp +++ b/Source/Python/pyWarpX.cpp @@ -4,7 +4,7 @@ * License: BSD-3-Clause-LBNL */ #include "pyWarpX.H" -#include "WarpX_py.H" +#include "callbacks.H" #include // todo: move this out to Python/WarpX.cpp #include // todo: move to its own Python/Utils.cpp diff --git a/Source/Utils/CMakeLists.txt b/Source/Utils/CMakeLists.txt index 9ded83a0c29..3d804fe9cde 100644 --- a/Source/Utils/CMakeLists.txt +++ b/Source/Utils/CMakeLists.txt @@ -4,12 +4,12 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE Interpolate.cpp ParticleUtils.cpp + SpeciesUtils.cpp RelativeCellPosition.cpp WarpXAlgorithmSelection.cpp WarpXMovingWindow.cpp WarpXTagging.cpp WarpXUtil.cpp - WarpXrocfftUtil.cpp WarpXVersion.cpp ) endforeach() diff --git a/Source/Utils/Make.package b/Source/Utils/Make.package index eb318f8d23c..dd7e61ff4fa 100644 --- a/Source/Utils/Make.package +++ b/Source/Utils/Make.package @@ -1,13 +1,13 @@ CEXE_sources += WarpXMovingWindow.cpp CEXE_sources += WarpXTagging.cpp CEXE_sources += WarpXUtil.cpp -CEXE_sources += WarpXrocfftUtil.cpp CEXE_sources += WarpXVersion.cpp CEXE_sources += WarpXAlgorithmSelection.cpp CEXE_sources += Interpolate.cpp CEXE_sources += IntervalsParser.cpp CEXE_sources += RelativeCellPosition.cpp CEXE_sources += ParticleUtils.cpp +CEXE_sources += SpeciesUtils.cpp include $(WARPX_HOME)/Source/Utils/Algorithms/Make.package include $(WARPX_HOME)/Source/Utils/Logo/Make.package diff --git a/Source/Utils/Parser/IntervalsParser.H b/Source/Utils/Parser/IntervalsParser.H index 7c4e861ca15..3e0533aaf77 100644 --- a/Source/Utils/Parser/IntervalsParser.H +++ b/Source/Utils/Parser/IntervalsParser.H @@ -42,7 +42,7 @@ namespace utils::parser * * @param[in] n the input integer */ - bool contains (int n) const; + [[nodiscard]] bool contains (int n) const; /** * \brief A method that returns the smallest integer strictly greater than n such that @@ -50,7 +50,7 @@ namespace utils::parser * * @param[in] n the input integer */ - int nextContains (int n) const; + [[nodiscard]] int nextContains (int n) const; /** * \brief A method that returns the greatest integer strictly smaller than n such that @@ -58,31 +58,31 @@ namespace utils::parser * * @param[in] n the input integer */ - int previousContains (int n) const; + [[nodiscard]] int previousContains (int n) const; /** * \brief A method that returns the slice period. * */ - int getPeriod () const; + [[nodiscard]] int getPeriod () const; /** * \brief A method that returns the slice start. * */ - int getStart () const; + [[nodiscard]] int getStart () const; /** * \brief A method that returns the slice stop. * */ - int getStop () const; + [[nodiscard]] int getStop () const; /** * @brief A method that returns the number of integers contained by the slice. * */ - int numContained() const; + [[nodiscard]] int numContained() const; private: bool m_isBTD = false; @@ -122,7 +122,7 @@ namespace utils::parser * * @param[in] n the input integer */ - bool contains (int n) const; + [[nodiscard]] bool contains (int n) const; /** * \brief A method that returns the smallest integer strictly greater than n such that @@ -130,7 +130,7 @@ namespace utils::parser * * @param[in] n the input integer */ - int nextContains (int n) const; + [[nodiscard]] int nextContains (int n) const; /** * \brief A method that returns the greatest integer strictly smaller than n such that @@ -138,7 +138,7 @@ namespace utils::parser * * @param[in] n the input integer */ - int previousContains (int n) const; + [[nodiscard]] int previousContains (int n) const; /** * \brief A method that returns the greatest integer smaller than or equal to n such that @@ -146,7 +146,7 @@ namespace utils::parser * * @param[in] n the input integer */ - int previousContainsInclusive (int n) const; + [[nodiscard]] int previousContainsInclusive (int n) const; /** * \brief A method the local period (in timesteps) of the IntervalsParser at timestep n. @@ -154,13 +154,13 @@ namespace utils::parser * * @param[in] n the input integer */ - int localPeriod (int n) const; + [[nodiscard]] int localPeriod (int n) const; /** * \brief A method that returns true if any of the slices contained by the IntervalsParser * has a strictly positive period. */ - bool isActivated () const; + [[nodiscard]] bool isActivated () const; private: std::vector m_slices; @@ -193,26 +193,26 @@ namespace utils::parser /** * @brief Return the total number of unique labframe snapshots */ - int NumSnapshots () const; + [[nodiscard]] int NumSnapshots () const; /** * @brief Return the iteration number stored at index i_buffer * * @param i_buffer buffer or iteration index, between 0 and NumSnapshots */ - int GetBTDIteration(int i_buffer) const; + [[nodiscard]] int GetBTDIteration(int i_buffer) const; /** * @brief Return the final BTD iteration * */ - int GetFinalIteration() const; + [[nodiscard]] int GetFinalIteration() const; /** * \brief A method that returns true if any of the slices contained by the IntervalsParser * has a strictly positive period. */ - bool isActivated () const; + [[nodiscard]] bool isActivated () const; private: std::vector m_btd_iterations; diff --git a/Source/Utils/Parser/IntervalsParser.cpp b/Source/Utils/Parser/IntervalsParser.cpp index 35d13143ddb..6564003abdf 100644 --- a/Source/Utils/Parser/IntervalsParser.cpp +++ b/Source/Utils/Parser/IntervalsParser.cpp @@ -16,9 +16,9 @@ #include -utils::parser::SliceParser::SliceParser (const std::string& instr, const bool isBTD) +utils::parser::SliceParser::SliceParser (const std::string& instr, const bool isBTD): + m_isBTD{isBTD} { - m_isBTD = isBTD; // split string and trim whitespaces auto insplit = ablastr::utils::text::split_string>( instr, m_separator, true); @@ -92,7 +92,7 @@ utils::parser::IntervalsParser::IntervalsParser ( const std::vector& instr_vec) { std::string inconcatenated; - for (const auto& instr_element : instr_vec) inconcatenated +=instr_element; + for (const auto& instr_element : instr_vec) { inconcatenated +=instr_element; } auto insplit = ablastr::utils::text::split_string>( inconcatenated, m_separator); @@ -102,7 +102,7 @@ utils::parser::IntervalsParser::IntervalsParser ( const SliceParser temp_slice(inslc); m_slices.push_back(temp_slice); if ((temp_slice.getPeriod() > 0) && - (temp_slice.getStop() >= temp_slice.getStart())) m_activated = true; + (temp_slice.getStop() >= temp_slice.getStart())) { m_activated = true; } } } @@ -155,7 +155,7 @@ utils::parser::BTDIntervalsParser::BTDIntervalsParser ( const std::vector& instr_vec) { std::string inconcatenated; - for (const auto& instr_element : instr_vec) inconcatenated +=instr_element; + for (const auto& instr_element : instr_vec) { inconcatenated +=instr_element; } auto const insplit = ablastr::utils::text::split_string>( inconcatenated, std::string(1,m_separator)); @@ -205,7 +205,7 @@ utils::parser::BTDIntervalsParser::BTDIntervalsParser ( } else { - btd_iter_ind = m_btd_iterations.size() - 1; + btd_iter_ind = static_cast(m_btd_iterations.size() - 1); while (start < m_btd_iterations.at(btd_iter_ind) and btd_iter_ind>0) { btd_iter_ind--; @@ -256,7 +256,7 @@ utils::parser::BTDIntervalsParser::BTDIntervalsParser ( int utils::parser::BTDIntervalsParser::NumSnapshots () const { - return m_btd_iterations.size(); + return static_cast(m_btd_iterations.size()); } diff --git a/Source/Utils/Parser/ParserUtils.H b/Source/Utils/Parser/ParserUtils.H index 870c84e241b..dbc4845d1bf 100644 --- a/Source/Utils/Parser/ParserUtils.H +++ b/Source/Utils/Parser/ParserUtils.H @@ -63,9 +63,28 @@ namespace utils::parser * \param query_string ParmParse.query will look for this string * \param stored_string variable in which the string to parse is stored */ + void Store_parserString( + amrex::ParmParse const& pp, + std::string const& query_string, + std::string& stored_string); + + /** + * \brief Parse a string (typically a mathematical expression) from the + * input file and store it into a variable. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * get(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] pp used to read the query_string `pp.=string` + * \param[in] group name of the optional group + * \param[in] query_string ParmParse.query will look for this string + * \param[out] stored_string variable in which the string to parse is stored + */ void Store_parserString( const amrex::ParmParse &pp, - std::string query_string, + std::string const& group, + std::string const& query_string, std::string& stored_string); @@ -128,9 +147,10 @@ namespace utils::parser auto parser = makeParser(str_val, {}); - if (std::is_same::value) { + if constexpr (std::is_same::value) { - val = safeCastToInt(std::round(parser.compileHost<0>()()), str); + val = safeCastToInt( + static_cast(std::round(parser.compileHost<0>()())), str); } else { val = static_cast(parser.compileHost<0>()()); @@ -154,8 +174,9 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if (std::is_same::value) { - val[i] = safeCastToInt(std::round(parser.compileHost<0>()()), str); + if constexpr (std::is_same::value) { + val[i] = safeCastToInt( + static_cast(std::round(parser.compileHost<0>()())), str); } else { val[i] = static_cast(parser.compileHost<0>()()); @@ -196,8 +217,9 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if (std::is_same::value) { - val[i] = safeCastToInt(std::round(parser.compileHost<0>()()), str); + if constexpr (std::is_same::value) { + val[i] = safeCastToInt( + static_cast(std::round(parser.compileHost<0>()())), str); } else { val[i] = static_cast(parser.compileHost<0>()()); @@ -228,8 +250,9 @@ namespace utils::parser Store_parserString(a_pp, str, str_val); auto parser = makeParser(str_val, {}); - if (std::is_same::value) { - val = safeCastToInt(std::round(parser.compileHost<0>()()), str); + if constexpr (std::is_same::value) { + val = safeCastToInt( + static_cast(std::round(parser.compileHost<0>()())), str); } else { val = static_cast(parser.compileHost<0>()()); @@ -247,8 +270,9 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if (std::is_same::value) { - val[i] = safeCastToInt(std::round(parser.compileHost<0>()()), str); + if constexpr (std::is_same::value) { + val[i] = safeCastToInt( + static_cast(std::round(parser.compileHost<0>()())), str); } else { val[i] = static_cast(parser.compileHost<0>()()); @@ -284,8 +308,9 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if (std::is_same::value) { - val[i] = safeCastToInt(std::round(parser.compileHost<0>()()), str); + if constexpr (std::is_same::value) { + val[i] = safeCastToInt( + static_cast(std::round(parser.compileHost<0>()())), str); } else { val[i] = static_cast(parser.compileHost<0>()()); @@ -293,6 +318,208 @@ namespace utils::parser } } + + /** Similar to amrex::ParmParse::query, but also supports math expressions for the value. + * + * amrex::ParmParse::query reads a name and a value from the input file. This function does the + * same, and applies the amrex::Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * queryWithParser(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored, either a scalar or vector + */ + template + int queryWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, T& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return queryWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return queryWithParser(a_pp, grp_str.c_str(), val); + } + } + + + template + int queryArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return queryArrWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return queryArrWithParser(a_pp, grp_str.c_str(), val); + } + } + + + /** Similar to amrex::ParmParse::query, but also supports math expressions for the value. + * + * amrex::ParmParse::query reads a name and a value from the input file. This function does the + * same, and applies the amrex::Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * queryArrWithParser(pp, "group", "name", val, start_ix, num_val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored, either a scalar or vector + * \param[in] start_ix start index in the list of inputs values (optional with arrays, default is + * amrex::ParmParse::FIRST for starting with the first input value) + * \param[in] num_val number of input values to use (optional with arrays, default is + * amrex::ParmParse::LAST for reading until the last input value) + */ + template + int queryArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val, + const int start_ix, const int num_val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return queryArrWithParser(a_pp, str, val, start_ix, num_val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return queryArrWithParser(a_pp, grp_str.c_str(), val, start_ix, num_val); + } + } + + /** Wraps around amrex::ParmParse::query, but also supports an option group name + * + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * get(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + */ + int query (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val); + + /** Similar to amrex::ParmParse::get, but also supports math expressions for the value. + * + * amrex::ParmParse::get reads a name and a value from the input file. This function does the + * same, and applies the Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * getWithParser(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + */ + template + void getWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, T& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + getWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + getWithParser(a_pp, grp_str.c_str(), val); + } + } + + template + void getArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + getArrWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + getArrWithParser(a_pp, grp_str.c_str(), val); + } + } + + + /** Similar to amrex::ParmParse::get, but also supports math expressions for the value. + * + * amrex::ParmParse::get reads a name and a value from the input file. This function does the + * same, and applies the Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * getArrWithParser(pp, "group", "name", val, start_ix, num_val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + * \param[in] start_ix start index in the list of inputs values (optional with arrays, default is + * amrex::ParmParse::FIRST for starting with the first input value) + * \param[in] num_val number of input values to use (optional with arrays, default is + * amrex::ParmParse::LAST for reading until the last input value) + */ + template + void getArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val, + const int start_ix, const int num_val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + getArrWithParser(a_pp, str, val, start_ix, num_val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + getArrWithParser(a_pp, grp_str.c_str(), val, start_ix, num_val); + } + } + + /** Wraps around amrex::ParmParse::get, but also supports an option group name + * + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * get(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + */ + void get (amrex::ParmParse const& a_pp, std::string const& group, char const * str, std::string& val); + } #endif // WARPX_UTILS_PARSER_PARSERUTILS_H_ diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index 0add95df1ae..c2da9577947 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -19,8 +19,8 @@ #include void utils::parser::Store_parserString( - const amrex::ParmParse& pp, - std::string query_string, + amrex::ParmParse const& pp, + std::string const& query_string, std::string& stored_string) { std::vector f; @@ -32,12 +32,60 @@ void utils::parser::Store_parserString( f.clear(); } +void utils::parser::Store_parserString( + amrex::ParmParse const& a_pp, + std::string const& group, + std::string const& query_string, + std::string& stored_string) +{ + const bool is_specified_without_group = a_pp.contains(query_string.c_str()); + const std::string grp_str = group + "." + query_string; + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + utils::parser::Store_parserString(a_pp, query_string, stored_string); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + utils::parser::Store_parserString(a_pp, grp_str, stored_string); + } +} + +int utils::parser::query (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val) +{ + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return a_pp.query(str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return a_pp.query(grp_str.c_str(), val); + } +} + +void utils::parser::get (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val) +{ + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + a_pp.get(str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + a_pp.get(grp_str.c_str(), val); + } +} namespace { template< typename int_type > AMREX_FORCE_INLINE int_type safeCastTo(const amrex::Real x, const std::string& real_name) { - int_type result = int_type(0); + auto result = int_type(0); bool error_detected = false; std::string assert_msg; // (2.0*(numeric_limits::max()/2+1)) converts numeric_limits::max()+1 to a real ensuring accuracy to all digits @@ -87,7 +135,7 @@ amrex::Parser utils::parser::makeParser ( parser.registerVariables(varnames); std::set symbols = parser.symbols(); - for (auto const& v : varnames) symbols.erase(v); + for (auto const& v : varnames) { symbols.erase(v); } // User can provide inputs under this name, through which expressions // can be provided for arbitrary variables. PICMI inputs are aware of @@ -133,9 +181,9 @@ amrex::Parser utils::parser::makeParser ( const auto constant = warpx_constants.find(*it); if (constant != warpx_constants.end()) { - parser.setConstant(*it, constant->second); - it = symbols.erase(it); - continue; + parser.setConstant(*it, constant->second); + it = symbols.erase(it); + continue; } ++it; diff --git a/Source/Utils/ParticleUtils.H b/Source/Utils/ParticleUtils.H index 50dc12a722c..b04176d4d83 100644 --- a/Source/Utils/ParticleUtils.H +++ b/Source/Utils/ParticleUtils.H @@ -99,8 +99,8 @@ namespace ParticleUtils { // precompute repeatedly used quantities constexpr auto c2 = PhysConst::c * PhysConst::c; const auto V2 = (Vx*Vx + Vy*Vy + Vz*Vz); - const auto gamma_V = 1.0_prt / sqrt(1.0_prt - V2 / c2); - const auto gamma_u = sqrt(1.0_prt + (ux*ux + uy*uy + uz*uz) / c2); + const auto gamma_V = 1.0_prt / std::sqrt(1.0_prt - V2 / c2); + const auto gamma_u = std::sqrt(1.0_prt + (ux*ux + uy*uy + uz*uz) / c2); // copy velocity vector values const auto vx = ux; @@ -181,8 +181,8 @@ namespace ParticleUtils { */ AMREX_GPU_HOST_DEVICE AMREX_INLINE bool containsInclusive (amrex::RealBox const& tilebox, amrex::XDim3 const point) { - const auto xlo = tilebox.lo(); - const auto xhi = tilebox.hi(); + const auto *const xlo = tilebox.lo(); + const auto *const xhi = tilebox.hi(); return AMREX_D_TERM((xlo[0] <= point.x) && (point.x <= xhi[0]), && (xlo[1] <= point.y) && (point.y <= xhi[1]), && (xlo[2] <= point.z) && (point.z <= xhi[2])); diff --git a/Source/Utils/RelativeCellPosition.cpp b/Source/Utils/RelativeCellPosition.cpp index 1a108b54b56..d30da1841cb 100644 --- a/Source/Utils/RelativeCellPosition.cpp +++ b/Source/Utils/RelativeCellPosition.cpp @@ -22,8 +22,9 @@ utils::getRelativeCellPosition(amrex::MultiFab const& mf) // WarpX::grid_type==GridType::Collocated means: all indices/directions on CellIndex::NODE for (int d = 0; d < AMREX_SPACEDIM; d++) { - if (idx_type.cellCentered(d)) + if (idx_type.cellCentered(d)) { relative_position.at(d) = 0.5; + } } return relative_position; diff --git a/Source/Utils/SpeciesUtils.H b/Source/Utils/SpeciesUtils.H new file mode 100644 index 00000000000..3f5d42910c4 --- /dev/null +++ b/Source/Utils/SpeciesUtils.H @@ -0,0 +1,42 @@ +/* Copyright 2023 RemiLehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_SPECIES_UTILS_H_ +#define WARPX_SPECIES_UTILS_H_ + +#include +#include "Initialization/InjectorDensity.H" +#include "Initialization/InjectorMomentum.H" +#include "Particles/SpeciesPhysicalProperties.H" + +namespace SpeciesUtils { + + void StringParseAbortMessage(const std::string& var, + const std::string& name); + + void extractSpeciesProperties ( std::string const& species_name, + std::string const& injection_style, amrex::Real& charge, amrex::Real& mass, + PhysicalSpecies& physical_species); + + void parseDensity (std::string const& species_name, std::string const& source_name, + std::unique_ptr& h_inj_rho, + std::unique_ptr& density_parser); + + void parseMomentum (std::string const& species_name, std::string const& source_name, const std::string& style, + std::unique_ptr& h_inj_mom, + std::unique_ptr& ux_parser, + std::unique_ptr& uy_parser, + std::unique_ptr& uz_parser, + std::unique_ptr& ux_th_parser, + std::unique_ptr& uy_th_parser, + std::unique_ptr& uz_th_parser, + std::unique_ptr& h_mom_temp, + std::unique_ptr& h_mom_vel, + int flux_normal_axis=0, int flux_direction=0); + +} + +#endif // WARPX_SPECIES_UTILS_H_ diff --git a/Source/Utils/SpeciesUtils.cpp b/Source/Utils/SpeciesUtils.cpp new file mode 100644 index 00000000000..f8515592a07 --- /dev/null +++ b/Source/Utils/SpeciesUtils.cpp @@ -0,0 +1,286 @@ +/* Copyright 2023 Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "SpeciesUtils.H" +#include +#include "Utils/TextMsg.H" +#include "Utils/Parser/ParserUtils.H" + +namespace SpeciesUtils { + + void StringParseAbortMessage(const std::string& var, + const std::string& name) { + std::stringstream stringstream; + std::string string; + stringstream << var << " string '" << name << "' not recognized."; + string = stringstream.str(); + WARPX_ABORT_WITH_MESSAGE(string); + } + + void extractSpeciesProperties (std::string const& species_name, + std::string const& injection_style, amrex::Real& charge, amrex::Real& mass, + PhysicalSpecies& physical_species ) + { + const amrex::ParmParse pp_species_name(species_name); + std::string physical_species_s; + const bool species_is_specified = pp_species_name.query("species_type", physical_species_s); + if (species_is_specified){ + const auto physical_species_from_string = species::from_string( physical_species_s ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(physical_species_from_string, + physical_species_s + " does not exist!"); + physical_species = physical_species_from_string.value(); + charge = species::get_charge( physical_species ); + mass = species::get_mass( physical_species ); + } + + // parse charge and mass + const bool charge_is_specified = + utils::parser::queryWithParser(pp_species_name, "charge", charge); + const bool mass_is_specified = + utils::parser::queryWithParser(pp_species_name, "mass", mass); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + charge_is_specified || + species_is_specified || + (injection_style == "external_file"), + "Need to specify at least one of species_type or charge for species '" + + species_name + "'." + ); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + mass_is_specified || + species_is_specified || + (injection_style == "external_file"), + "Need to specify at least one of species_type or mass for species '" + + species_name + "'." + ); + + if ( charge_is_specified && species_is_specified ){ + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".charge' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".charge' will take precedence.\n"); + + } + + if ( mass_is_specified && species_is_specified ){ + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".mass' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".mass' will take precedence.\n"); + } + } + + // Depending on injection type at runtime, initialize inj_rho + // so that inj_rho->getDensity calls + // InjectorPosition[Constant or Predefined or etc.].getDensity. + void parseDensity (std::string const& species_name, std::string const& source_name, + std::unique_ptr& h_inj_rho, + std::unique_ptr& density_parser) + { + amrex::ParmParse pp_species(species_name); + + // parse density information + std::string rho_prof_s; + utils::parser::get(pp_species, source_name, "profile", rho_prof_s); + std::transform(rho_prof_s.begin(), rho_prof_s.end(), + rho_prof_s.begin(), ::tolower); + if (rho_prof_s == "constant") { + amrex::Real density; + utils::parser::getWithParser(pp_species, source_name, "density", density); + // Construct InjectorDensity with InjectorDensityConstant. + h_inj_rho.reset(new InjectorDensity((InjectorDensityConstant*)nullptr, density)); + } else if (rho_prof_s == "predefined") { + // Construct InjectorDensity with InjectorDensityPredefined. + h_inj_rho.reset(new InjectorDensity((InjectorDensityPredefined*)nullptr,species_name)); + } else if (rho_prof_s == "parse_density_function") { + std::string str_density_function; + utils::parser::Store_parserString(pp_species, source_name, "density_function(x,y,z)", str_density_function); + // Construct InjectorDensity with InjectorDensityParser. + density_parser = std::make_unique( + utils::parser::makeParser(str_density_function,{"x","y","z"})); + h_inj_rho.reset(new InjectorDensity((InjectorDensityParser*)nullptr, + density_parser->compile<3>())); + } else { + StringParseAbortMessage("Density profile type", rho_prof_s); + } + } + + // Depending on injection type at runtime, initialize inj_mom + // so that inj_mom->getMomentum calls + // InjectorMomentum[Constant or Gaussian or etc.].getMomentum. + void parseMomentum (std::string const& species_name, std::string const& source_name, const std::string& style, + std::unique_ptr& h_inj_mom, + std::unique_ptr& ux_parser, + std::unique_ptr& uy_parser, + std::unique_ptr& uz_parser, + std::unique_ptr& ux_th_parser, + std::unique_ptr& uy_th_parser, + std::unique_ptr& uz_th_parser, + std::unique_ptr& h_mom_temp, + std::unique_ptr& h_mom_vel, + int flux_normal_axis, int flux_direction) + { + using namespace amrex::literals; + + amrex::ParmParse pp_species(species_name); + + // parse momentum information + std::string mom_dist_s; + utils::parser::get(pp_species, source_name, "momentum_distribution_type", mom_dist_s); + std::transform(mom_dist_s.begin(), + mom_dist_s.end(), + mom_dist_s.begin(), + ::tolower); + if (mom_dist_s == "at_rest") { + constexpr amrex::Real ux = 0._rt; + constexpr amrex::Real uy = 0._rt; + constexpr amrex::Real uz = 0._rt; + // Construct InjectorMomentum with InjectorMomentumConstant. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); + } else if (mom_dist_s == "constant") { + amrex::Real ux = 0._rt; + amrex::Real uy = 0._rt; + amrex::Real uz = 0._rt; + utils::parser::queryWithParser(pp_species, source_name, "ux", ux); + utils::parser::queryWithParser(pp_species, source_name, "uy", uy); + utils::parser::queryWithParser(pp_species, source_name, "uz", uz); + // Construct InjectorMomentum with InjectorMomentumConstant. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); + } else if (mom_dist_s == "gaussian") { + amrex::Real ux_m = 0._rt; + amrex::Real uy_m = 0._rt; + amrex::Real uz_m = 0._rt; + amrex::Real ux_th = 0._rt; + amrex::Real uy_th = 0._rt; + amrex::Real uz_th = 0._rt; + utils::parser::queryWithParser(pp_species, source_name, "ux_m", ux_m); + utils::parser::queryWithParser(pp_species, source_name, "uy_m", uy_m); + utils::parser::queryWithParser(pp_species, source_name, "uz_m", uz_m); + utils::parser::queryWithParser(pp_species, source_name, "ux_th", ux_th); + utils::parser::queryWithParser(pp_species, source_name, "uy_th", uy_th); + utils::parser::queryWithParser(pp_species, source_name, "uz_th", uz_th); + // Construct InjectorMomentum with InjectorMomentumGaussian. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussian*)nullptr, + ux_m, uy_m, uz_m, ux_th, uy_th, uz_th)); + } else if (mom_dist_s == "gaussianflux") { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(style == "nfluxpercell", + "Error: gaussianflux can only be used with injection_style = NFluxPerCell"); + amrex::Real ux_m = 0._rt; + amrex::Real uy_m = 0._rt; + amrex::Real uz_m = 0._rt; + amrex::Real ux_th = 0._rt; + amrex::Real uy_th = 0._rt; + amrex::Real uz_th = 0._rt; + utils::parser::queryWithParser(pp_species, source_name, "ux_m", ux_m); + utils::parser::queryWithParser(pp_species, source_name, "uy_m", uy_m); + utils::parser::queryWithParser(pp_species, source_name, "uz_m", uz_m); + utils::parser::queryWithParser(pp_species, source_name, "ux_th", ux_th); + utils::parser::queryWithParser(pp_species, source_name, "uy_th", uy_th); + utils::parser::queryWithParser(pp_species, source_name, "uz_th", uz_th); + // Construct InjectorMomentum with InjectorMomentumGaussianFlux. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussianFlux*)nullptr, + ux_m, uy_m, uz_m, ux_th, uy_th, uz_th, + flux_normal_axis, flux_direction)); + } else if (mom_dist_s == "uniform") { + amrex::Real ux_min = 0._rt; + amrex::Real uy_min = 0._rt; + amrex::Real uz_min = 0._rt; + amrex::Real ux_max = 0._rt; + amrex::Real uy_max = 0._rt; + amrex::Real uz_max = 0._rt; + utils::parser::queryWithParser(pp_species, source_name, "ux_min", ux_min); + utils::parser::queryWithParser(pp_species, source_name, "uy_min", uy_min); + utils::parser::queryWithParser(pp_species, source_name, "uz_min", uz_min); + utils::parser::queryWithParser(pp_species, source_name, "ux_max", ux_max); + utils::parser::queryWithParser(pp_species, source_name, "uy_max", uy_max); + utils::parser::queryWithParser(pp_species, source_name, "uz_max", uz_max); + // Construct InjectorMomentum with InjectorMomentumUniform. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumUniform*)nullptr, + ux_min, uy_min, uz_min, ux_max, uy_max, uz_max)); + } else if (mom_dist_s == "maxwell_boltzmann"){ + h_mom_temp = std::make_unique(pp_species, source_name); + const GetTemperature getTemp(*h_mom_temp); + h_mom_vel = std::make_unique(pp_species, source_name); + const GetVelocity getVel(*h_mom_vel); + // Construct InjectorMomentum with InjectorMomentumBoltzmann. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumBoltzmann*)nullptr, getTemp, getVel)); + } else if (mom_dist_s == "maxwell_juttner"){ + h_mom_temp = std::make_unique(pp_species, source_name); + const GetTemperature getTemp(*h_mom_temp); + h_mom_vel = std::make_unique(pp_species, source_name); + const GetVelocity getVel(*h_mom_vel); + // Construct InjectorMomentum with InjectorMomentumJuttner. + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumJuttner*)nullptr, getTemp, getVel)); + } else if (mom_dist_s == "radial_expansion") { + amrex::Real u_over_r = 0._rt; + utils::parser::queryWithParser(pp_species, source_name, "u_over_r", u_over_r); + // Construct InjectorMomentum with InjectorMomentumRadialExpansion. + h_inj_mom.reset(new InjectorMomentum + ((InjectorMomentumRadialExpansion*)nullptr, u_over_r)); + } else if (mom_dist_s == "parse_momentum_function") { + std::string str_momentum_function_ux; + std::string str_momentum_function_uy; + std::string str_momentum_function_uz; + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_ux(x,y,z)", str_momentum_function_ux); + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uy(x,y,z)", str_momentum_function_uy); + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uz(x,y,z)", str_momentum_function_uz); + // Construct InjectorMomentum with InjectorMomentumParser. + ux_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_ux, {"x","y","z"})); + uy_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_uy, {"x","y","z"})); + uz_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_uz, {"x","y","z"})); + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumParser*)nullptr, + ux_parser->compile<3>(), + uy_parser->compile<3>(), + uz_parser->compile<3>())); + } else if (mom_dist_s == "gaussian_parse_momentum_function") { + std::string str_momentum_function_ux_m; + std::string str_momentum_function_uy_m; + std::string str_momentum_function_uz_m; + std::string str_momentum_function_ux_th; + std::string str_momentum_function_uy_th; + std::string str_momentum_function_uz_th; + utils::parser::Store_parserString(pp_species, source_name, + "momentum_function_ux_m(x,y,z)", str_momentum_function_ux_m); + utils::parser::Store_parserString(pp_species, source_name, + "momentum_function_uy_m(x,y,z)", str_momentum_function_uy_m); + utils::parser::Store_parserString(pp_species, source_name, + "momentum_function_uz_m(x,y,z)", str_momentum_function_uz_m); + utils::parser::Store_parserString(pp_species, source_name, + "momentum_function_ux_th(x,y,z)", str_momentum_function_ux_th); + utils::parser::Store_parserString(pp_species, source_name, + "momentum_function_uy_th(x,y,z)", str_momentum_function_uy_th); + utils::parser::Store_parserString(pp_species, source_name, + "momentum_function_uz_th(x,y,z)", str_momentum_function_uz_th); + // Construct InjectorMomentum with InjectorMomentumParser. + ux_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_ux_m, {"x","y","z"})); + uy_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_uy_m, {"x","y","z"})); + uz_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_uz_m, {"x","y","z"})); + ux_th_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_ux_th, {"x","y","z"})); + uy_th_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_uy_th, {"x","y","z"})); + uz_th_parser = std::make_unique( + utils::parser::makeParser(str_momentum_function_uz_th, {"x","y","z"})); + h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussianParser*)nullptr, + ux_parser->compile<3>(), + uy_parser->compile<3>(), + uz_parser->compile<3>(), + ux_th_parser->compile<3>(), + uy_th_parser->compile<3>(), + uz_th_parser->compile<3>())); + } else { + StringParseAbortMessage("Momentum distribution type", mom_dist_s); + } + } + +} diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 24d6edccdbf..87db0ae9b9b 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -23,6 +23,17 @@ struct MediumForEM { }; }; +/** + * \brief struct to select the overall evolve scheme + */ +struct EvolveScheme { + enum { + Explicit = 0, + ImplicitPicard = 1, + SemiImplicitPicard = 2 + }; +}; + /** * \brief struct to select algorithm for macroscopic Maxwell solver LaxWendroff (semi-implicit) represents sigma*E = sigma*0.5*(E^(n) + E^(n+1)) @@ -74,23 +85,23 @@ struct ParticlePusherAlgo { struct CurrentDepositionAlgo { enum { - Esirkepov = 0, - Direct = 1, - Vay = 2 + Esirkepov = 0, + Direct = 1, + Vay = 2 }; }; struct ChargeDepositionAlgo { // Only the Standard algorithm is implemented enum { - Standard = 0 + Standard = 0 }; }; struct GatheringAlgo { enum { - EnergyConserving = 0, - MomentumConserving + EnergyConserving = 0, + MomentumConserving }; }; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index d9cd5c138a8..75c488134e0 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -20,9 +20,16 @@ #include #include -// Define dictionary with correspondance between user-input strings, +// Define dictionary with correspondence between user-input strings, // and corresponding integer for use inside the code +const std::map evolve_scheme_to_int = { + {"explicit", EvolveScheme::Explicit }, + {"implicit_picard", EvolveScheme::ImplicitPicard }, + {"semi_implicit_picard", EvolveScheme::SemiImplicitPicard }, + {"default", EvolveScheme::Explicit } +}; + const std::map grid_to_int = { {"collocated", GridType::Collocated}, {"staggered", GridType::Staggered}, @@ -147,7 +154,9 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ // Pick the right dictionary std::map algo_to_int; - if (0 == std::strcmp(pp_search_key, "maxwell_solver")) { + if (0 == std::strcmp(pp_search_key, "evolve_scheme")) { + algo_to_int = evolve_scheme_to_int; + } else if (0 == std::strcmp(pp_search_key, "maxwell_solver")) { algo_to_int = electromagnetic_solver_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "grid_type")) { algo_to_int = grid_to_int; @@ -158,8 +167,10 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ } else if (0 == std::strcmp(pp_search_key, "current_deposition")) { algo_to_int = current_deposition_algo_to_int; if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD || - WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) + WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC || + WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::None) { algo_to_int["default"] = CurrentDepositionAlgo::Direct; + } } else if (0 == std::strcmp(pp_search_key, "charge_deposition")) { algo_to_int = charge_deposition_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "field_gathering")) { diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index f968af81c13..fdcb7cb8da4 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -12,7 +12,10 @@ #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) # include "BoundaryConditions/PML_RZ.H" #endif +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" @@ -79,7 +82,8 @@ WarpX::UpdateInjectionPosition (const amrex::Real a_dt) current_injection_position[dir] = pc.m_current_injection_position; #endif - PlasmaInjector* plasma_injector = pc.GetPlasmaInjector(); + // This only uses the base plasma injector + PlasmaInjector* plasma_injector = pc.GetPlasmaInjector(0); amrex::Real v_shift = 0._rt; if (plasma_injector != nullptr) @@ -140,7 +144,7 @@ WarpX::MoveWindow (const int step, bool move_j) if (step == end_moving_window_step) { amrex::Print() << Utils::TextMsg::Info("Stopping moving window"); } - if (!moving_window_active(step)) return 0; + if (!moving_window_active(step)) { return 0; } // Update the continuous position of the moving window, // and of the plasma injection @@ -161,7 +165,7 @@ WarpX::MoveWindow (const int step, bool move_j) const amrex::Real* cdx = geom[0].CellSize(); const int num_shift_base = static_cast((moving_window_x - current_lo[dir]) / cdx[dir]); - if (num_shift_base == 0) return 0; + if (num_shift_base == 0) { return 0; } // update the problem domain. Note the we only do this on the base level because // amrex::Geometry objects share the same, static RealBox. @@ -178,20 +182,20 @@ WarpX::MoveWindow (const int step, bool move_j) // slice box is modified only if slice diagnostics is initialized in input // if ( slice_plot_int > 0 ) { - amrex::Real new_slice_lo[AMREX_SPACEDIM]; - amrex::Real new_slice_hi[AMREX_SPACEDIM]; - const amrex::Real* current_slice_lo = slice_realbox.lo(); - const amrex::Real* current_slice_hi = slice_realbox.hi(); - for ( int i = 0; i < AMREX_SPACEDIM; i++) { - new_slice_lo[i] = current_slice_lo[i]; - new_slice_hi[i] = current_slice_hi[i]; - } - const int num_shift_base_slice = static_cast ((moving_window_x - - current_slice_lo[dir]) / cdx[dir]); - new_slice_lo[dir] = current_slice_lo[dir] + num_shift_base_slice*cdx[dir]; - new_slice_hi[dir] = current_slice_hi[dir] + num_shift_base_slice*cdx[dir]; - slice_realbox.setLo(new_slice_lo); - slice_realbox.setHi(new_slice_hi); + amrex::Real new_slice_lo[AMREX_SPACEDIM]; + amrex::Real new_slice_hi[AMREX_SPACEDIM]; + const amrex::Real* current_slice_lo = slice_realbox.lo(); + const amrex::Real* current_slice_hi = slice_realbox.hi(); + for ( int i = 0; i < AMREX_SPACEDIM; i++) { + new_slice_lo[i] = current_slice_lo[i]; + new_slice_hi[i] = current_slice_hi[i]; + } + const int num_shift_base_slice = static_cast ((moving_window_x - + current_slice_lo[dir]) / cdx[dir]); + new_slice_lo[dir] = current_slice_lo[dir] + num_shift_base_slice*cdx[dir]; + new_slice_hi[dir] = current_slice_hi[dir] + num_shift_base_slice*cdx[dir]; + slice_realbox.setLo(new_slice_lo); + slice_realbox.setHi(new_slice_hi); } int num_shift = num_shift_base; @@ -215,27 +219,29 @@ WarpX::MoveWindow (const int step, bool move_j) amrex::ParserExecutor<3> Efield_parser; bool use_Bparser = false; bool use_Eparser = false; - if (B_ext_grid_s == "parse_b_ext_grid_function") { + if (m_p_ext_field_params->B_ext_grid_type == + ExternalFieldType::parse_ext_grid_function) { use_Bparser = true; - if (dim == 0) Bfield_parser = Bxfield_parser->compile<3>(); - if (dim == 1) Bfield_parser = Byfield_parser->compile<3>(); - if (dim == 2) Bfield_parser = Bzfield_parser->compile<3>(); + if (dim == 0) { Bfield_parser = m_p_ext_field_params->Bxfield_parser->compile<3>(); } + if (dim == 1) { Bfield_parser = m_p_ext_field_params->Byfield_parser->compile<3>(); } + if (dim == 2) { Bfield_parser = m_p_ext_field_params->Bzfield_parser->compile<3>(); } } - if (E_ext_grid_s == "parse_e_ext_grid_function") { + if (m_p_ext_field_params->E_ext_grid_type == + ExternalFieldType::parse_ext_grid_function) { use_Eparser = true; - if (dim == 0) Efield_parser = Exfield_parser->compile<3>(); - if (dim == 1) Efield_parser = Eyfield_parser->compile<3>(); - if (dim == 2) Efield_parser = Ezfield_parser->compile<3>(); + if (dim == 0) { Efield_parser = m_p_ext_field_params->Exfield_parser->compile<3>(); } + if (dim == 1) { Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); } + if (dim == 2) { Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); } } shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); if (fft_do_time_averaging) { shiftMF(*Bfield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params-> E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); @@ -257,16 +263,16 @@ WarpX::MoveWindow (const int step, bool move_j) if (lev > 0) { // coarse grid shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); if (fft_do_time_averaging) { shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); @@ -357,6 +363,18 @@ WarpX::MoveWindow (const int step, bool move_j) } } } + + // Shift values of N, NU for each fluid species + if (do_fluid_species) { + const int n_fluid_species = myfl->nSpecies(); + for (int i=0; iGetFluidContainer(i); + shiftMF( *fl.N[lev], geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *fl.NU[lev][0], geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *fl.NU[lev][1], geom[lev], num_shift, dir, lev, do_update_cost ); + shiftMF( *fl.NU[lev][2], geom[lev], num_shift, dir, lev, do_update_cost ); + } + } } // Loop over species (particles and lasers) @@ -410,6 +428,30 @@ WarpX::MoveWindow (const int step, bool move_j) } } + // Continuously inject fluid species in new cells (by default only on level 0) + const int lev = 0; + // Find box in which to initialize new fluid cells + amrex::Box injection_box = geom[lev].Domain(); + injection_box.surroundingNodes(); // get nodal box + // Restrict box in the direction of the moving window, to only include the new cells + if (moving_window_v > 0._rt) + { + injection_box.setSmall( dir, injection_box.bigEnd(dir) - num_shift_base + 1 ); + } + else if (moving_window_v < 0._rt) + { + injection_box.setBig( dir, injection_box.smallEnd(dir) + num_shift_base - 1 ); + } + // Loop over fluid species, and fill the values of the new cells + if (do_fluid_species) { + const int n_fluid_species = myfl->nSpecies(); + const amrex::Real cur_time = t_new[0]; + for (int i=0; iGetFluidContainer(i); + fl.InitData( lev, injection_box, cur_time ); + } + } + return num_shift_base; } @@ -486,7 +528,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, { amrex::Gpu::synchronize(); } - amrex::Real wt = amrex::second(); + auto wt = static_cast(amrex::second()); auto const& dstfab = mf.array(mfi); auto const& srcfab = tmpmf.array(mfi); @@ -551,7 +593,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); - wt = amrex::second() - wt; + wt = static_cast(amrex::second()) - wt; amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } @@ -562,14 +604,15 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, // guard region both radially and longitudinally. These are the PML cells in the overlapping // longitudinal region. FillBoundary normally does not update these cells. // This update is needed so that the cells at the end of the FABs are updated appropriately - // with the data shifted from the nieghboring FAB. Without this update, the RZ PML becomes + // with the data shifted from the neighboring FAB. Without this update, the RZ PML becomes // unstable with the moving grid. // This code creates a temporary MultiFab using a BoxList where the radial size of all of // its boxes is increased so that the radial guard cells are included in the boxes valid domain. // The temporary MultiFab is setup to refer to the data of the original Multifab (this can // be done since the shape of the data is all the same, just the indexing is different). amrex::BoxList bl; - for (int i = 0, N=ba.size(); i < N; ++i) { + const auto ba_size = static_cast(ba.size()); + for (int i = 0; i < ba_size; ++i) { bl.push_back(amrex::grow(ba[i], 0, mf.nGrowVect()[0])); } const amrex::BoxArray rba(std::move(bl)); diff --git a/Source/Utils/WarpXTagging.cpp b/Source/Utils/WarpXTagging.cpp index da810244088..192fc9e9152 100644 --- a/Source/Utils/WarpXTagging.cpp +++ b/Source/Utils/WarpXTagging.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -30,9 +31,10 @@ WarpX::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) const auto problo = Geom(lev).ProbLoArray(); const auto dx = Geom(lev).CellSizeArray(); + amrex::ParserExecutor<3> ref_parser; + if (ref_patch_parser) { ref_parser = ref_patch_parser->compile<3>(); } const auto ftlo = fine_tag_lo; const auto fthi = fine_tag_hi; - #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -45,7 +47,21 @@ WarpX::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) const RealVect pos {AMREX_D_DECL((i+0.5_rt)*dx[0]+problo[0], (j+0.5_rt)*dx[1]+problo[1], (k+0.5_rt)*dx[2]+problo[2])}; - if (pos > ftlo && pos < fthi) { + bool tag_val = false; + if (ref_parser) { +#if defined (WARPX_DIM_3D) + tag_val = (ref_parser(pos[0], pos[1], pos[2]) == 1); +#elif defined (WARPX_DIM_XZ) || defined (WARPX_DIM_RZ) + amrex::Real unused = 0.0; + tag_val = (ref_parser(pos[0], unused, pos[1]) == 1); +#elif defined (WARPX_DIM_1D_Z) + amrex::Real unused = 0.0; + tag_val = (ref_parser(unused, unused, pos[0]) == 1); +#endif + } else { + tag_val = (pos > ftlo && pos < fthi); + } + if ( tag_val == 1) { fab(i,j,k) = TagBox::SET; } }); diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index c65603f4d02..5b917a81fdf 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -146,7 +146,7 @@ void ConvertLabParamsToBoost() ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction); - if (gamma_boost <= 1.) return; + if (gamma_boost <= 1.) { return; } Vector prob_lo(AMREX_SPACEDIM); Vector prob_hi(AMREX_SPACEDIM); @@ -289,11 +289,17 @@ void CheckDims () #endif const ParmParse pp_geometry("geometry"); std::string dims; - pp_geometry.get("dims", dims); std::string dims_error = "The selected WarpX executable was built as '"; dims_error.append(dims_compiled).append("'-dimensional, but the "); - dims_error.append("inputs file declares 'geometry.dims = ").append(dims).append("'.\n"); - dims_error.append("Please re-compile with a different WarpX_DIMS option or select the right executable name."); + if (pp_geometry.contains("dims")) { + pp_geometry.get("dims", dims); + dims_error.append("inputs file declares 'geometry.dims = ").append(dims).append("'.\n"); + dims_error.append("Please re-compile with a different WarpX_DIMS option or select the right executable name."); + } else { + dims = "Not specified"; + dims_error.append("inputs file does not declare 'geometry.dims'. Please add 'geometry.dims = "); + dims_error.append(dims_compiled).append("' to inputs file."); + } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(dims == dims_compiled, dims_error); } @@ -307,8 +313,9 @@ void CheckGriddingForRZSpectral () const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); // only check for PSATD in RZ - if (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { return; + } int max_level; Vector n_cell(AMREX_SPACEDIM, -1); @@ -413,10 +420,12 @@ void ReadBCParams () const ParmParse pp_boundary("boundary"); pp_boundary.queryarr("field_lo", field_BC_lo, 0, AMREX_SPACEDIM); pp_boundary.queryarr("field_hi", field_BC_hi, 0, AMREX_SPACEDIM); - if (pp_boundary.queryarr("particle_lo", particle_BC_lo, 0, AMREX_SPACEDIM)) + if (pp_boundary.queryarr("particle_lo", particle_BC_lo, 0, AMREX_SPACEDIM)) { particle_boundary_specified = true; - if (pp_boundary.queryarr("particle_hi", particle_BC_hi, 0, AMREX_SPACEDIM)) + } + if (pp_boundary.queryarr("particle_hi", particle_BC_hi, 0, AMREX_SPACEDIM)) { particle_boundary_specified = true; + } AMREX_ALWAYS_ASSERT(field_BC_lo.size() == AMREX_SPACEDIM); AMREX_ALWAYS_ASSERT(field_BC_hi.size() == AMREX_SPACEDIM); AMREX_ALWAYS_ASSERT(particle_BC_lo.size() == AMREX_SPACEDIM); diff --git a/Source/Utils/WarpXVersion.cpp b/Source/Utils/WarpXVersion.cpp index 43978a4795a..41abfebb38c 100644 --- a/Source/Utils/WarpXVersion.cpp +++ b/Source/Utils/WarpXVersion.cpp @@ -17,10 +17,11 @@ WarpX::Version () #ifdef WARPX_GIT_VERSION version = std::string(WARPX_GIT_VERSION); #endif - if( version.empty() ) + if( version.empty() ) { return {"Unknown"}; - else + } else { return version; + } } std::string @@ -30,8 +31,9 @@ WarpX::PicsarVersion () #ifdef PICSAR_GIT_VERSION version = std::string(PICSAR_GIT_VERSION); #endif - if( version.empty() ) + if( version.empty() ) { return {"Unknown"}; - else + } else { return version; + } } diff --git a/Source/Utils/WarpX_Complex.H b/Source/Utils/WarpX_Complex.H index 048ee29940f..7a1f1669286 100644 --- a/Source/Utils/WarpX_Complex.H +++ b/Source/Utils/WarpX_Complex.H @@ -9,7 +9,7 @@ #define WARPX_COMPLEX_H_ #ifdef WARPX_USE_PSATD -# include "FieldSolver/SpectralSolver/AnyFFT.H" +# include #endif #include @@ -22,7 +22,7 @@ using Complex = amrex::GpuComplex; #ifdef WARPX_USE_PSATD -static_assert(sizeof(Complex) == sizeof(AnyFFT::Complex), +static_assert(sizeof(Complex) == sizeof(ablastr::math::anyfft::Complex), "The complex type in WarpX and the FFT library do not match."); #endif diff --git a/Source/Utils/WarpXrocfftUtil.H b/Source/Utils/WarpXrocfftUtil.H deleted file mode 100644 index 85332e6de3b..00000000000 --- a/Source/Utils/WarpXrocfftUtil.H +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright 2023 Luca Fedeli - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ - -#ifndef WARPX_ROCFFT_UTIL_H_ -#define WARPX_ROCFFT_UTIL_H_ - -namespace utils::rocfft -{ - /** This function is a wrapper around rocff_setup(). - * It is a no-op in case rocfft is not used. - */ - void setup(); - - /** This function is a wrapper around rocff_cleanup(). - * It is a no-op in case rocfft is not used. - */ - void cleanup(); -} - -#endif //WARPX_ROCFFT_UTILS_H_ diff --git a/Source/Utils/WarpXrocfftUtil.cpp b/Source/Utils/WarpXrocfftUtil.cpp deleted file mode 100644 index cc12d691449..00000000000 --- a/Source/Utils/WarpXrocfftUtil.cpp +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2023 Luca Fedeli - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ - -#include "WarpXrocfftUtil.H" - -#include - -#if defined(AMREX_USE_HIP) && defined(WARPX_USE_PSATD) -# if __has_include() // ROCm 5.3+ -# include -# else -# include -# endif -#endif - -void -utils::rocfft::setup() -{ -#if defined(AMREX_USE_HIP) && defined(WARPX_USE_PSATD) - rocfft_setup(); -#endif -} - -void -utils::rocfft::cleanup() -{ -#if defined(AMREX_USE_HIP) && defined(WARPX_USE_PSATD) - rocfft_cleanup(); -#endif -} diff --git a/Source/Utils/check_interp_points_and_weights.py b/Source/Utils/check_interp_points_and_weights.py index a1d17c8dd3d..8bf2cf08490 100644 --- a/Source/Utils/check_interp_points_and_weights.py +++ b/Source/Utils/check_interp_points_and_weights.py @@ -100,8 +100,8 @@ def refinement_points_and_weights( ii, sc, sf, cr ): # Input coarsening ratio cr = int( input( "\n Select coarsening ratio (cr=1,2,4): cr=" ) ) if ( cr!=1 and cr!=2 and cr!=4 ): - print() - sys.exit( 'coarsening ratio cr={} is not valid'.format( cr ) ) + print() + sys.exit( 'coarsening ratio cr={} is not valid'.format( cr ) ) # Loop over possible staggering of coarse and fine grid (cell-centered or nodal) for sc in [0,1]: diff --git a/Source/WarpX.H b/Source/WarpX.H index 7466bd92496..7a7c5ee89c1 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -20,9 +20,13 @@ #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties_fwd.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel_fwd.H" #include "Filter/NCIGodfreyFilter_fwd.H" +#include "Initialization/ExternalField_fwd.H" #include "Particles/ParticleBoundaryBuffer_fwd.H" #include "Particles/MultiParticleContainer_fwd.H" #include "Particles/WarpXParticleContainer_fwd.H" +#include "Fluids/MultiFluidContainer_fwd.H" +#include "Fluids/WarpXFluidContainer_fwd.H" + #ifdef WARPX_USE_PSATD # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ_fwd.H" @@ -31,12 +35,13 @@ # include "FieldSolver/SpectralSolver/SpectralSolver_fwd.H" # endif #endif +#include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "Filter/BilinearFilter.H" #include "Parallelization/GuardCellManager.H" -#include "AcceleratorLattice/AcceleratorLattice.H" #include "Utils/Parser/IntervalsParser.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/export.H" @@ -95,7 +100,7 @@ public: static void Finalize(); /** Destructor */ - ~WarpX (); + ~WarpX () override; /** Copy constructor */ WarpX ( WarpX const &) = delete; @@ -110,13 +115,20 @@ public: static std::string Version (); //!< Version of WarpX executable static std::string PicsarVersion (); //!< Version of PICSAR dependency - int Verbose () const { return verbose; } + [[nodiscard]] int Verbose () const { return verbose; } void InitData (); void Evolve (int numsteps = -1); + void EvolveImplicitPicardInit (int lev); + void SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev); + void FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev); + void FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Efield_fp, + amrex::Vector, 3 > >& Efield_n); + MultiParticleContainer& GetPartContainer () { return *mypc; } + MultiFluidContainer& GetFluidContainer () { return *myfl; } MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} @@ -128,36 +140,18 @@ public: amrex::Real external_field=0.0, bool useparser = false, amrex::ParserExecutor<3> const& field_parser={}); - //! Author of an input file / simulation setup - static std::string authors; - - //! Initial electric field on the grid - static amrex::Vector E_external_grid; - //! Initial magnetic field on the grid - static amrex::Vector B_external_grid; - - //! Initialization type for external magnetic field on the grid - static std::string B_ext_grid_s; - //! Initialization type for external electric field on the grid - static std::string E_ext_grid_s; - - //! Whether to apply the effect of an externally-defined electric field - static bool add_external_E_field; - //! Whether to apply the effect of an externally-defined magnetic field - static bool add_external_B_field; - - //! User-defined parser to initialize x-component of the magnetic field on the grid - std::unique_ptr Bxfield_parser; - //! User-defined parser to initialize y-component of the magnetic field on the grid - std::unique_ptr Byfield_parser; - //! User-defined parser to initialize z-component of the magnetic field on the grid - std::unique_ptr Bzfield_parser; - //! User-defined parser to initialize x-component of the electric field on the grid - std::unique_ptr Exfield_parser; - //! User-defined parser to initialize y-component of the electric field on the grid - std::unique_ptr Eyfield_parser; - //! User-defined parser to initialize z-component of the electric field on the grid - std::unique_ptr Ezfield_parser; + /** + * \brief + * If an authors' string is specified in the inputfile, this method returns that string. + * Otherwise, it returns an empty string. + */ + [[nodiscard]] std::string GetAuthors () const { return m_authors; } + + /** Maximum level up to which the externally defined electric and magnetic fields are initialized. + * The default value is set to the max levels in the simulation. + * if lev > maxlevel_extEMfield_init, the fields on those levels will have a default value of 0 + */ + int maxlevel_extEMfield_init; // Algorithms //! Integer that corresponds to the current deposition algorithm (Esirkepov, direct, Vay) @@ -170,6 +164,14 @@ public: static short particle_pusher_algo; //! Integer that corresponds to the type of Maxwell solver (Yee, CKC, PSATD, ECT) static short electromagnetic_solver_id; + //! Integer that corresponds to the evolve scheme (explicit, implicit_picard, semi_implicit_picard) + static short evolve_scheme; + //! The maximum number of Picard iterations to do each time step + static int max_picard_iterations; + //! The tolerance for the Picard iteration convergence + static amrex::Real picard_iteration_tolerance; + //! Flags whether the Picard iterations are required to converge + static bool require_picard_convergence; /** Records a number corresponding to the load balance cost update strategy * being used (0, 1, 2 corresponding to timers, heuristic, or gpuclock). */ @@ -433,13 +435,32 @@ public: const std::string& name, std::optional initial_value); + /** + * \brief + * Allocate the MultiFab so that is like the specified MultiFab (same ba and dm) + * and optionally initialize it. This also adds the MultiFab + * to the map of MultiFabs (used to ease the access to MultiFabs from the Python + * interface + * + * \param mf[out] The MultiFab unique pointer to be allocated + * \param mf_model[in] The MultiFab to model + * \param name[in] The name of the MultiFab to use in the map + * \param initial_value[in] The optional initial value + */ + static void AllocInitMultiFabFromModel ( + std::unique_ptr& mf, + amrex::MultiFab& mf_model, + int level, + const std::string& name, + std::optional initial_value = {}); + // Maps of all of the MultiFabs and iMultiFabs used (this can include MFs from other classes) // This is a convenience for the Python interface, allowing all MultiFabs // to be easily referenced from Python. static std::map multifab_map; static std::map imultifab_map; - std::array + [[nodiscard]] std::array get_array_Bfield_aux (const int lev) const { return { Bfield_aux[lev][0].get(), @@ -447,7 +468,8 @@ public: Bfield_aux[lev][2].get() }; } - std::array + + [[nodiscard]] std::array get_array_Efield_aux (const int lev) const { return { Efield_aux[lev][0].get(), @@ -456,28 +478,28 @@ public: }; } - amrex::MultiFab * get_pointer_Efield_aux (int lev, int direction) const { return Efield_aux[lev][direction].get(); } - amrex::MultiFab * get_pointer_Bfield_aux (int lev, int direction) const { return Bfield_aux[lev][direction].get(); } - - amrex::MultiFab * get_pointer_Efield_fp (int lev, int direction) const { return Efield_fp[lev][direction].get(); } - amrex::MultiFab * get_pointer_Bfield_fp (int lev, int direction) const { return Bfield_fp[lev][direction].get(); } - amrex::MultiFab * get_pointer_current_fp (int lev, int direction) const { return current_fp[lev][direction].get(); } - amrex::MultiFab * get_pointer_current_fp_nodal (int lev, int direction) const { return current_fp_nodal[lev][direction].get(); } - amrex::MultiFab * get_pointer_rho_fp (int lev) const { return rho_fp[lev].get(); } - amrex::MultiFab * get_pointer_F_fp (int lev) const { return F_fp[lev].get(); } - amrex::MultiFab * get_pointer_G_fp (int lev) const { return G_fp[lev].get(); } - amrex::MultiFab * get_pointer_phi_fp (int lev) const { return phi_fp[lev].get(); } - amrex::MultiFab * get_pointer_vector_potential_fp (int lev, int direction) const { return vector_potential_fp_nodal[lev][direction].get(); } - - amrex::MultiFab * get_pointer_Efield_cp (int lev, int direction) const { return Efield_cp[lev][direction].get(); } - amrex::MultiFab * get_pointer_Bfield_cp (int lev, int direction) const { return Bfield_cp[lev][direction].get(); } - amrex::MultiFab * get_pointer_current_cp (int lev, int direction) const { return current_cp[lev][direction].get(); } - amrex::MultiFab * get_pointer_rho_cp (int lev) const { return rho_cp[lev].get(); } - amrex::MultiFab * get_pointer_F_cp (int lev) const { return F_cp[lev].get(); } - amrex::MultiFab * get_pointer_G_cp (int lev) const { return G_cp[lev].get(); } - - amrex::MultiFab * get_pointer_edge_lengths (int lev, int direction) const { return m_edge_lengths[lev][direction].get(); } - amrex::MultiFab * get_pointer_face_areas (int lev, int direction) const { return m_face_areas[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_Efield_aux (int lev, int direction) const { return Efield_aux[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_aux (int lev, int direction) const { return Bfield_aux[lev][direction].get(); } + + [[nodiscard]] amrex::MultiFab * get_pointer_Efield_fp (int lev, int direction) const { return Efield_fp[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_fp (int lev, int direction) const { return Bfield_fp[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_current_fp (int lev, int direction) const { return current_fp[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_current_fp_nodal (int lev, int direction) const { return current_fp_nodal[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_rho_fp (int lev) const { return rho_fp[lev].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_F_fp (int lev) const { return F_fp[lev].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_G_fp (int lev) const { return G_fp[lev].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_phi_fp (int lev) const { return phi_fp[lev].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_vector_potential_fp (int lev, int direction) const { return vector_potential_fp_nodal[lev][direction].get(); } + + [[nodiscard]] amrex::MultiFab * get_pointer_Efield_cp (int lev, int direction) const { return Efield_cp[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_cp (int lev, int direction) const { return Bfield_cp[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_current_cp (int lev, int direction) const { return current_cp[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_rho_cp (int lev) const { return rho_cp[lev].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_F_cp (int lev) const { return F_cp[lev].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_G_cp (int lev) const { return G_cp[lev].get(); } + + [[nodiscard]] amrex::MultiFab * get_pointer_edge_lengths (int lev, int direction) const { return m_edge_lengths[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab * get_pointer_face_areas (int lev, int direction) const { return m_face_areas[lev][direction].get(); } const amrex::MultiFab& getEfield (int lev, int direction) {return *Efield_aux[lev][direction];} const amrex::MultiFab& getBfield (int lev, int direction) {return *Bfield_aux[lev][direction];} @@ -502,14 +524,18 @@ public: const amrex::MultiFab& getEfield_avg_cp (int lev, int direction) {return *Efield_avg_cp[lev][direction];} const amrex::MultiFab& getBfield_avg_cp (int lev, int direction) {return *Bfield_avg_cp[lev][direction];} - bool DoPML () const {return do_pml;} + const amrex::MultiFab& getedgelengths (int lev, int direction) {return *m_edge_lengths[lev][direction];} + const amrex::MultiFab& getfaceareas (int lev, int direction) {return *m_face_areas[lev][direction];} + + [[nodiscard]] bool DoPML () const {return do_pml;} + [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) const PML_RZ* getPMLRZ() {return pml_rz[0].get();} #endif /** get low-high-low-high-... vector for each direction indicating if mother grid PMLs are enabled */ - std::vector getPMLdirections() const; + [[nodiscard]] std::vector getPMLdirections() const; static amrex::LayoutData* getCosts (int lev); @@ -533,7 +559,7 @@ public: amrex::Vector mirror_z_width; amrex::Vector mirror_z_npoints; - /// object with all reduced diagnotics, similar to MultiParticleContainer for species. + /// object with all reduced diagnostics, similar to MultiParticleContainer for species. std::unique_ptr reduced_diags; void applyMirrors(amrex::Real time); @@ -644,7 +670,7 @@ public: /** \brief returns the load balance interval */ - utils::parser::IntervalsParser get_load_balance_intervals () const + [[nodiscard]] utils::parser::IntervalsParser get_load_balance_intervals () const { return load_balance_intervals; } @@ -699,6 +725,11 @@ public: void ApplyEfieldBoundary (int lev, PatchType patch_type); void ApplyBfieldBoundary (int lev, PatchType patch_type, DtType dt_type); +#ifdef WARPX_DIM_RZ + // Applies the boundary conditions that are specific to the axis when in RZ. + void ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev); +#endif + /** * \brief When the Ohm's law solver is used, the electron pressure values * on PEC boundaries are set to enforce a zero derivative Neumann condition, @@ -721,21 +752,6 @@ public: void CopyJPML (); bool isAnyBoundaryPML(); - /** - * \brief Synchronize the nodal points of the PML MultiFabs - */ - void NodalSyncPML (); - - /** - * \brief Synchronize the nodal points of the PML MultiFabs for given MR level - */ - void NodalSyncPML (int lev); - - /** - * \brief Synchronize the nodal points of the PML MultiFabs for given MR level and patch - */ - void NodalSyncPML (int lev, PatchType patch_type); - PML* GetPML (int lev); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) PML_RZ* GetPML_RZ (int lev); @@ -757,8 +773,10 @@ public: void doQEDEvents (int lev); #endif - void PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false); - void PushParticlesandDepose ( amrex::Real cur_time, bool skip_current=false); + void PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false, + PushType push_type=PushType::Explicit); + void PushParticlesandDeposit (amrex::Real cur_time, bool skip_current=false, + PushType push_type=PushType::Explicit); // This function does aux(lev) = fp(lev) + I(aux(lev-1)-cp(lev)). // Caller must make sure fp and cp have ghost cells filled. @@ -826,25 +844,25 @@ public: const amrex::Vector>& charge_cp, const amrex::Vector>& charge_buffer); - amrex::Vector getnsubsteps () const {return nsubsteps;} - int getnsubsteps (int lev) const {return nsubsteps[lev];} - amrex::Vector getistep () const {return istep;} - int getistep (int lev) const {return istep[lev];} + [[nodiscard]] amrex::Vector getnsubsteps () const {return nsubsteps;} + [[nodiscard]] int getnsubsteps (int lev) const {return nsubsteps[lev];} + [[nodiscard]] amrex::Vector getistep () const {return istep;} + [[nodiscard]] int getistep (int lev) const {return istep[lev];} void setistep (int lev, int ii) {istep[lev] = ii;} - amrex::Vector gett_old () const {return t_old;} - amrex::Real gett_old (int lev) const {return t_old[lev];} - amrex::Vector gett_new () const {return t_new;} - amrex::Real gett_new (int lev) const {return t_new[lev];} + [[nodiscard]] amrex::Vector gett_old () const {return t_old;} + [[nodiscard]] amrex::Real gett_old (int lev) const {return t_old[lev];} + [[nodiscard]] amrex::Vector gett_new () const {return t_new;} + [[nodiscard]] amrex::Real gett_new (int lev) const {return t_new[lev];} void sett_new (int lev, amrex::Real time) {t_new[lev] = time;} - amrex::Vector getdt () const {return dt;} - amrex::Real getdt (int lev) const {return dt.at(lev);} - int getdo_moving_window() const {return do_moving_window;} - amrex::Real getmoving_window_x() const {return moving_window_x;} - bool getis_synchronized() const {return is_synchronized;} + [[nodiscard]] amrex::Vector getdt () const {return dt;} + [[nodiscard]] amrex::Real getdt (int lev) const {return dt.at(lev);} + [[nodiscard]] int getdo_moving_window() const {return do_moving_window;} + [[nodiscard]] amrex::Real getmoving_window_x() const {return moving_window_x;} + [[nodiscard]] bool getis_synchronized() const {return is_synchronized;} - int maxStep () const {return max_step;} + [[nodiscard]] int maxStep () const {return max_step;} void updateMaxStep (const int new_max_step) {max_step = new_max_step;} - amrex::Real stopTime () const {return stop_time;} + [[nodiscard]] amrex::Real stopTime () const {return stop_time;} void updateStopTime (const amrex::Real new_stop_time) {stop_time = new_stop_time;} void AverageAndPackFields( amrex::Vector& varnames, @@ -918,12 +936,12 @@ public: void ComputeDivE(amrex::MultiFab& divE, int lev); - amrex::IntVect getngEB() const { return guard_cells.ng_alloc_EB; } - amrex::IntVect getngF() const { return guard_cells.ng_alloc_F; } - amrex::IntVect getngUpdateAux() const { return guard_cells.ng_UpdateAux; } - amrex::IntVect get_ng_depos_J() const {return guard_cells.ng_depos_J;} - amrex::IntVect get_ng_depos_rho() const {return guard_cells.ng_depos_rho;} - amrex::IntVect get_ng_fieldgather () const {return guard_cells.ng_FieldGather;} + [[nodiscard]] amrex::IntVect getngEB() const { return guard_cells.ng_alloc_EB; } + [[nodiscard]] amrex::IntVect getngF() const { return guard_cells.ng_alloc_F; } + [[nodiscard]] amrex::IntVect getngUpdateAux() const { return guard_cells.ng_UpdateAux; } + [[nodiscard]] amrex::IntVect get_ng_depos_J() const {return guard_cells.ng_depos_J;} + [[nodiscard]] amrex::IntVect get_ng_depos_rho() const {return guard_cells.ng_depos_rho;} + [[nodiscard]] amrex::IntVect get_ng_fieldgather () const {return guard_cells.ng_FieldGather;} /** Coarsest-level Domain Decomposition * @@ -932,7 +950,7 @@ public: * * @return the number of MPI processes per dimension if specified, otherwise a 0-vector */ - amrex::IntVect get_numprocs() const {return numprocs;} + [[nodiscard]] amrex::IntVect get_numprocs() const {return numprocs;} ElectrostaticSolver::PoissonBoundaryHandler m_poisson_boundary_handler; void ComputeSpaceChargeField (bool reset_fields); @@ -1047,11 +1065,114 @@ public: // This needs to be public for CUDA. //! Tagging cells for refinement - virtual void ErrorEst (int lev, amrex::TagBoxArray& tags, amrex::Real time, int /*ngrow*/) final; + void ErrorEst (int lev, amrex::TagBoxArray& tags, amrex::Real time, int /*ngrow*/) final; // Return the accelerator lattice instance defined at the given refinement level const AcceleratorLattice& get_accelerator_lattice (int lev) {return *(m_accelerator_lattice[lev]);} + // for cuda + void BuildBufferMasksInBox ( amrex::Box tbx, amrex::IArrayBox &buffer_mask, + const amrex::IArrayBox &guard_mask, int ng ); +#ifdef AMREX_USE_EB + amrex::EBFArrayBoxFactory const& fieldEBFactory (int lev) const noexcept { + return static_cast(*m_field_factory[lev]); + } +#endif + + void InitEB (); + +#ifdef AMREX_USE_EB + /** + * \brief Compute the length of the mesh edges. Here the length is a value in [0, 1]. + * An edge of length 0 is fully covered. + */ + static void ComputeEdgeLengths (std::array< std::unique_ptr, 3 >& edge_lengths, + const amrex::EBFArrayBoxFactory& eb_fact); + /** + * \brief Compute the area of the mesh faces. Here the area is a value in [0, 1]. + * An edge of area 0 is fully covered. + */ + static void ComputeFaceAreas (std::array< std::unique_ptr, 3 >& face_areas, + const amrex::EBFArrayBoxFactory& eb_fact); + + /** + * \brief Scale the edges lengths by the mesh width to obtain the real lengths. + */ + static void ScaleEdges (std::array< std::unique_ptr, 3 >& edge_lengths, + const std::array& cell_size); + /** + * \brief Scale the edges areas by the mesh width to obtain the real areas. + */ + static void ScaleAreas (std::array< std::unique_ptr, 3 >& face_areas, + const std::array& cell_size); + /** + * \brief Initialize information for cell extensions. + * The flags convention for m_flag_info_face is as follows + * - 0 for unstable cells + * - 1 for stable cells which have not been intruded + * - 2 for stable cells which have been intruded + * Here we cannot know if a cell is intruded or not so we initialize all stable cells with 1 + */ + void MarkCells(); +#endif + + /** + * \brief Compute the level set function used for particle-boundary interaction. + */ + void ComputeDistanceToEB (); + /** + * \brief Auxiliary function to count the amount of faces which still need to be extended + */ + amrex::Array1D CountExtFaces(); + /** + * \brief Main function computing the cell extension. Where possible it computes one-way + * extensions and, when this is not possible, it does eight-ways extensions. + */ + void ComputeFaceExtensions(); + /** + * \brief Initialize the memory for the FaceInfoBoxes + */ + void InitBorrowing(); + /** + * \brief Shrink the vectors in the FaceInfoBoxes + */ + void ShrinkBorrowing(); + /** + * \brief Do the one-way extension + */ + void ComputeOneWayExtensions(); + /** + * \brief Do the eight-ways extension + */ + void ComputeEightWaysExtensions(); + /** + * \brief Whenever an unstable cell cannot be extended we increase its area to be the minimal for stability. + * This is the method Benkler-Chavannes-Kuster method and it is less accurate than the regular ECT but it + * still works better than staircasing. (see https://ieeexplore.ieee.org/document/1638381) + * + * @param idim Integer indicating the dimension (x->0, y->1, z->2) for which the BCK correction is done + * + */ + void ApplyBCKCorrection(int idim); + + /** + * \brief Subtract the average of the cumulative sums of the preliminary current D + * from the current J (computed from D according to the Vay deposition scheme) + */ + void PSATDSubtractCurrentPartialSumsAvg (); + +#ifdef WARPX_USE_PSATD + +# ifdef WARPX_DIM_RZ + SpectralSolverRZ& +# else + SpectralSolver& +# endif + get_spectral_solver_fp (int lev) {return *spectral_solver_fp[lev];} +#endif + + FiniteDifferenceSolver * get_pointer_fdtd_solver_fp (int lev) { return m_fdtd_solver_fp[lev].get(); } + protected: /** @@ -1068,11 +1189,11 @@ protected: * then, the E and B fields at all the levels are initialized with * user-defined values for E_external_grid and B_external_grid. * If the initialization type for B-field is set to - * "parse_B_ext_grid_function", then, the parser is used to read + * "parse_ext_grid_function", then, the parser is used to read * Bx_external_grid_function(x,y,z), By_external_grid_function(x,y,z), * and Bz_external_grid_function(x,y,z). * Similarly, if the E-field initialization type is set to - * "parse_E_ext_grid_function", then, the parser is used to read + * "parse_ext_grid_function", then, the parser is used to read * Ex_external_grid_function(x,y,z), Ey_external_grid_function(x,y,z), * and Ex_external_grid_function(x,y,z). The parser for the E and B * initialization assumes that the function has three independent @@ -1083,28 +1204,28 @@ protected: //! Use this function to override the Level 0 grids made by AMReX. //! This function is called in amrex::AmrCore::InitFromScratch. - virtual void PostProcessBaseGrids (amrex::BoxArray& ba0) const final; + void PostProcessBaseGrids (amrex::BoxArray& ba0) const final; //! Make a new level from scratch using provided BoxArray and //! DistributionMapping. Only used during initialization. Called //! by AmrCoreInitFromScratch. - virtual void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) final; + void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& new_grids, + const amrex::DistributionMapping& new_dmap) final; //! Make a new level using provided BoxArray and //! DistributionMapping and fill with interpolated coarse level //! data. Called by AmrCore::regrid. - virtual void MakeNewLevelFromCoarse (int /*lev*/, amrex::Real /*time*/, const amrex::BoxArray& /*ba*/, + void MakeNewLevelFromCoarse (int /*lev*/, amrex::Real /*time*/, const amrex::BoxArray& /*ba*/, const amrex::DistributionMapping& /*dm*/) final; //! Remake an existing level using provided BoxArray and //! DistributionMapping and fill with existing fine and coarse //! data. Called by AmrCore::regrid. - virtual void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray& ba, + void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) final; //! Delete level data. Called by AmrCore::regrid. - virtual void ClearLevel (int lev) final; + void ClearLevel (int lev) final; private: @@ -1128,11 +1249,6 @@ private: //! Complete the asynchronous broadcast of signal flags, and initiate a checkpoint if requested void HandleSignals (); - /// - /// Advance the simulation by numsteps steps, electromagnetic case. - /// - void EvolveEM(int numsteps); - void FillBoundaryB (int lev, PatchType patch_type, amrex::IntVect ng, std::optional nodal_sync = std::nullopt); void FillBoundaryE (int lev, PatchType patch_type, amrex::IntVect ng, std::optional nodal_sync = std::nullopt); void FillBoundaryF (int lev, PatchType patch_type, amrex::IntVect ng, std::optional nodal_sync = std::nullopt); @@ -1143,13 +1259,15 @@ private: void AddExternalFields (); - void OneStep_nosub (amrex::Real t); - void OneStep_sub1 (amrex::Real t); + void OneStep_nosub (amrex::Real cur_time); + void OneStep_sub1 (amrex::Real cur_time); + + void OneStep_ImplicitPicard(amrex::Real cur_time); /** * \brief Perform one PIC iteration, with the multiple J deposition per time step */ - void OneStep_multiJ (amrex::Real t); + void OneStep_multiJ (amrex::Real cur_time); void RestrictCurrentFromFineToCoarsePatch ( const amrex::Vector,3>>& J_fp, @@ -1218,10 +1336,10 @@ private: void InitFromScratch (); - void AllocLevelData (int lev, const amrex::BoxArray& new_grids, - const amrex::DistributionMapping& new_dmap); + void AllocLevelData (int lev, const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm); - amrex::DistributionMapping + [[nodiscard]] amrex::DistributionMapping GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, int lev) const; void InitFromCheckpoint (); @@ -1251,14 +1369,13 @@ private: void PerformanceHints (); void BuildBufferMasks (); -public: // for cuda - void BuildBufferMasksInBox ( amrex::Box tbx, amrex::IArrayBox &buffer_mask, - const amrex::IArrayBox &guard_mask, int ng ); -private: - const amrex::iMultiFab* getCurrentBufferMasks (int lev) const { + + [[nodiscard]] const amrex::iMultiFab* getCurrentBufferMasks (int lev) const { return current_buffer_masks[lev].get(); } - const amrex::iMultiFab* getGatherBufferMasks (int lev) const { + + [[nodiscard]] const amrex::iMultiFab* getGatherBufferMasks (int lev) const + { return gather_buffer_masks[lev].get(); } @@ -1316,6 +1433,9 @@ private: # endif #endif + //! Author of an input file / simulation setup + std::string m_authors; + amrex::Vector istep; // which step? amrex::Vector nsubsteps; // how many substeps on each level? @@ -1327,6 +1447,10 @@ private: std::unique_ptr mypc; std::unique_ptr multi_diags; + // Fluid container + bool do_fluid_species = false; + std::unique_ptr myfl; + // // Fields: First array for level, second for direction // @@ -1347,6 +1471,12 @@ private: amrex::Vector, 3 > > Efield_avg_fp; amrex::Vector, 3 > > Bfield_avg_fp; + // Implicit, fields at start of step and from the previous iteration + amrex::Vector, 3 > > Efield_n; + amrex::Vector, 3 > > Bfield_n; + amrex::Vector, 3 > > Efield_save; + amrex::Vector, 3 > > Bfield_save; + // Memory buffers for computing magnetostatic fields // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order amrex::Vector, 3 > > vector_potential_fp_nodal; @@ -1446,6 +1576,9 @@ private: #endif amrex::Real v_particle_pml; + // External fields parameters + std::unique_ptr m_p_ext_field_params; + amrex::Real moving_window_x = std::numeric_limits::max(); // Plasma injection parameters @@ -1502,7 +1635,7 @@ private: // Other runtime parameters int verbose = 1; - bool use_hybrid_QED = 0; + bool use_hybrid_QED = false; int max_step = std::numeric_limits::max(); amrex::Real stop_time = std::numeric_limits::max(); @@ -1514,7 +1647,7 @@ private: std::string restart_chkfile; /** When `true`, write the diagnostics after restart at the time of the restart. */ - bool write_diagonstics_on_restart = false; + bool write_diagnostics_on_restart = false; amrex::VisMF::Header::Version plotfile_headerversion = amrex::VisMF::Header::Version_v1; amrex::VisMF::Header::Version slice_plotfile_headerversion = amrex::VisMF::Header::Version_v1; @@ -1527,6 +1660,8 @@ private: amrex::RealVect fine_tag_lo; amrex::RealVect fine_tag_hi; + //! User-defined parser to define refinement patches + std::unique_ptr ref_patch_parser; bool is_synchronized = true; @@ -1568,100 +1703,12 @@ private: // Factory for field data amrex::Vector > > m_field_factory; - amrex::FabFactory const& fieldFactory (int lev) const noexcept { + [[nodiscard]] + amrex::FabFactory const& fieldFactory (int lev) const noexcept + { return *m_field_factory[lev]; } -#ifdef AMREX_USE_EB -public: - amrex::EBFArrayBoxFactory const& fieldEBFactory (int lev) const noexcept { - return static_cast(*m_field_factory[lev]); - } -#endif - -public: - void InitEB (); - /** - * \brief Compute the length of the mesh edges. Here the length is a value in [0, 1]. - * An edge of length 0 is fully covered. - */ - -public: -#ifdef AMREX_USE_EB - static void ComputeEdgeLengths (std::array< std::unique_ptr, 3 >& edge_lengths, - const amrex::EBFArrayBoxFactory& eb_fact); - /** - * \brief Compute the area of the mesh faces. Here the area is a value in [0, 1]. - * An edge of area 0 is fully covered. - */ - static void ComputeFaceAreas (std::array< std::unique_ptr, 3 >& face_areas, - const amrex::EBFArrayBoxFactory& eb_fact); - - /** - * \brief Scale the edges lengths by the mesh width to obtain the real lengths. - */ - static void ScaleEdges (std::array< std::unique_ptr, 3 >& edge_lengths, - const std::array& cell_size); - /** - * \brief Scale the edges areas by the mesh width to obtain the real areas. - */ - static void ScaleAreas (std::array< std::unique_ptr, 3 >& face_areas, - const std::array& cell_size); - /** - * \brief Initialize information for cell extensions. - * The flags convention for m_flag_info_face is as follows - * - 0 for unstable cells - * - 1 for stable cells which have not been intruded - * - 2 for stable cells which have been intruded - * Here we cannot know if a cell is intruded or not so we initialize all stable cells with 1 - */ - void MarkCells(); - /** - * \brief Compute the level set function used for particle-boundary interaction. - */ -#endif - void ComputeDistanceToEB (); - /** - * \brief Auxiliary function to count the amount of faces which still need to be extended - */ - amrex::Array1D CountExtFaces(); - /** - * \brief Main function computing the cell extension. Where possible it computes one-way - * extensions and, when this is not possible, it does eight-ways extensions. - */ - void ComputeFaceExtensions(); - /** - * \brief Initialize the memory for the FaceInfoBoxes - */ - void InitBorrowing(); - /** - * \brief Shrink the vectors in the FaceInfoBoxes - */ - void ShrinkBorrowing(); - /** - * \brief Do the one-way extension - */ - void ComputeOneWayExtensions(); - /** - * \brief Do the eight-ways extension - */ - void ComputeEightWaysExtensions(); - /** - * \brief Whenever an unstable cell cannot be extended we increase its area to be the minimal for stability. - * This is the method Benkler-Chavannes-Kuster method and it is less accurate than the regular ECT but it - * still works better than staircasing. (see https://ieeexplore.ieee.org/document/1638381) - * - * @param idim Integer indicating the dimension (x->0, y->1, z->2) for which the BCK correction is done - * - */ - void ApplyBCKCorrection(int idim); - /** - * \brief Subtract the average of the cumulative sums of the preliminary current D - * from the current J (computed from D according to the Vay deposition scheme) - */ - void PSATDSubtractCurrentPartialSumsAvg (); - -private: void ScrapeParticles (); void PushPSATD (); @@ -1832,20 +1879,8 @@ private: amrex::Vector> spectral_solver_cp; # endif -public: - -# ifdef WARPX_DIM_RZ - SpectralSolverRZ& -# else - SpectralSolver& -# endif - get_spectral_solver_fp (int lev) {return *spectral_solver_fp[lev];} #endif -public: - FiniteDifferenceSolver * get_pointer_fdtd_solver_fp (int lev) { return m_fdtd_solver_fp[lev].get(); } - -private: amrex::Vector> m_fdtd_solver_fp; amrex::Vector> m_fdtd_solver_cp; }; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 508790ef20f..f3008e6157d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -29,7 +29,10 @@ #endif // use PSATD ifdef #include "FieldSolver/WarpX_FDTD.H" #include "Filter/NCIGodfreyFilter.H" +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" +#include "Fluids/MultiFluidContainer.H" +#include "Fluids/WarpXFluidContainer.H" #include "Particles/ParticleBoundaryBuffer.H" #include "AcceleratorLattice/AcceleratorLattice.H" #include "Utils/TextMsg.H" @@ -83,15 +86,6 @@ using namespace amrex; -Vector WarpX::E_external_grid(3, 0.0); -Vector WarpX::B_external_grid(3, 0.0); - -std::string WarpX::authors; -std::string WarpX::B_ext_grid_s = "default"; -std::string WarpX::E_ext_grid_s = "default"; -bool WarpX::add_external_E_field = false; -bool WarpX::add_external_B_field = false; - int WarpX::do_moving_window = 0; int WarpX::start_moving_window_step = 0; int WarpX::end_moving_window_step = -1; @@ -117,6 +111,10 @@ short WarpX::charge_deposition_algo; short WarpX::field_gathering_algo; short WarpX::particle_pusher_algo; short WarpX::electromagnetic_solver_id; +short WarpX::evolve_scheme; +int WarpX::max_picard_iterations = 10; +Real WarpX::picard_iteration_tolerance = 1.e-7; +bool WarpX::require_picard_convergence = true; short WarpX::psatd_solution_type; short WarpX::J_in_time; short WarpX::rho_in_time; @@ -200,7 +198,7 @@ int WarpX::self_fields_verbosity = 2; bool WarpX::do_subcycling = false; bool WarpX::do_multi_J = false; int WarpX::do_multi_J_n_depositions; -bool WarpX::safe_guard_cells = 0; +bool WarpX::safe_guard_cells = false; std::map WarpX::multifab_map; std::map WarpX::imultifab_map; @@ -316,6 +314,11 @@ WarpX::WarpX () // Particle Boundary Buffer (i.e., scraped particles on boundary) m_particle_boundary_buffer = std::make_unique(); + // Fluid Container + if (do_fluid_species) { + myfl = std::make_unique(nlevs_max); + } + Efield_aux.resize(nlevs_max); Bfield_aux.resize(nlevs_max); @@ -511,7 +514,7 @@ WarpX::ReadParameters () const ParmParse pp;// Traditionally, max_step and stop_time do not have prefix. utils::parser::queryWithParser(pp, "max_step", max_step); utils::parser::queryWithParser(pp, "stop_time", stop_time); - pp.query("authors", authors); + pp.query("authors", m_authors); } { @@ -522,7 +525,7 @@ WarpX::ReadParameters () { const ParmParse pp_algo("algo"); - electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); + electromagnetic_solver_id = static_cast(GetAlgorithmInteger(pp_algo, "maxwell_solver")); } { @@ -541,13 +544,13 @@ WarpX::ReadParameters () if(std::string str_abort_on_warning_threshold; pp_warpx.query("abort_on_warning_threshold", str_abort_on_warning_threshold)){ std::optional abort_on_warning_threshold = std::nullopt; - if (str_abort_on_warning_threshold == "high") + if (str_abort_on_warning_threshold == "high") { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::high; - else if (str_abort_on_warning_threshold == "medium" ) + } else if (str_abort_on_warning_threshold == "medium" ) { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::medium; - else if (str_abort_on_warning_threshold == "low") + } else if (str_abort_on_warning_threshold == "low") { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::low; - else { + } else { WARPX_ABORT_WITH_MESSAGE(str_abort_on_warning_threshold +"is not a valid option for warpx.abort_on_warning_threshold (use: low, medium or high)"); } @@ -609,7 +612,7 @@ WarpX::ReadParameters () } } - pp_warpx.query("write_diagonstics_on_restart", write_diagonstics_on_restart); + pp_warpx.query("write_diagnostics_on_restart", write_diagnostics_on_restart); pp_warpx.queryarr("checkpoint_signals", signals_in); #if defined(__linux__) || defined(__APPLE__) @@ -718,22 +721,16 @@ WarpX::ReadParameters () moving_window_v *= PhysConst::c; } - pp_warpx.query("B_ext_grid_init_style", WarpX::B_ext_grid_s); - pp_warpx.query("E_ext_grid_init_style", WarpX::E_ext_grid_s); - - if (WarpX::B_ext_grid_s == "read_from_file") - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, - "External field reading is not implemented for more than one level"); - add_external_B_field = true; - } - if (WarpX::E_ext_grid_s == "read_from_file") - { + m_p_ext_field_params = std::make_unique(pp_warpx); + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file || + m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file){ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, - "External field reading is not implemented for more than one level"); - add_external_E_field = true; + "External field reading is not implemented for more than one level"); } + maxlevel_extEMfield_init = maxLevel(); + pp_warpx.query("maxlevel_extEMfield_init", maxlevel_extEMfield_init); + electrostatic_solver_id = GetAlgorithmInteger(pp_warpx, "do_electrostatic"); // if an electrostatic solver is used, set the Maxwell solver to None if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) { @@ -778,7 +775,7 @@ WarpX::ReadParameters () // Filter currently not working with FDTD solver in RZ geometry: turn OFF by default // (see https://github.com/ECP-WarpX/WarpX/issues/1943) #ifdef WARPX_DIM_RZ - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) WarpX::use_filter = false; + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { WarpX::use_filter = false; } #endif // Read filter and fill IntVect filter_npass_each_dir with @@ -830,7 +827,7 @@ WarpX::ReadParameters () pp_warpx.query("do_single_precision_comms", do_single_precision_comms); #ifdef AMREX_USE_FLOAT if (do_single_precision_comms) { - do_single_precision_comms = 0; + do_single_precision_comms = false; ablastr::warn_manager::WMRecordWarning( "comms", "Overwrote warpx.do_single_precision_comms to be 0, since WarpX was built in single precision.", @@ -850,8 +847,9 @@ WarpX::ReadParameters () const bool shared_tilesize_is_specified = utils::parser::queryArrWithParser(pp_warpx, "shared_tilesize", vect_shared_tilesize, 0, AMREX_SPACEDIM); if (shared_tilesize_is_specified){ - for (int i=0; i(quantum_xi * PhysConst::c * PhysConst::c); } - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - !( - ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML && - WarpX::field_boundary_lo[idim] == FieldBoundaryType::Absorbing_SilverMueller ) || - ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML && - WarpX::field_boundary_hi[idim] == FieldBoundaryType::Absorbing_SilverMueller ) - ), - "PML and Silver-Mueller boundary conditions cannot be activated at the same time."); + const auto at_least_one_boundary_is_pml = + (std::any_of(WarpX::field_boundary_lo.begin(), WarpX::field_boundary_lo.end(), + [](const auto& cc){return cc == FieldBoundaryType::PML;}) + || + std::any_of(WarpX::field_boundary_hi.begin(), WarpX::field_boundary_hi.end(), + [](const auto& cc){return cc == FieldBoundaryType::PML;}) + ); + const auto at_least_one_boundary_is_silver_mueller = + (std::any_of(WarpX::field_boundary_lo.begin(), WarpX::field_boundary_lo.end(), + [](const auto& cc){return cc == FieldBoundaryType::Absorbing_SilverMueller;}) + || + std::any_of(WarpX::field_boundary_hi.begin(), WarpX::field_boundary_hi.end(), + [](const auto& cc){return cc == FieldBoundaryType::Absorbing_SilverMueller;}) + ); - if (WarpX::field_boundary_lo[idim] == FieldBoundaryType::Absorbing_SilverMueller || - WarpX::field_boundary_hi[idim] == FieldBoundaryType::Absorbing_SilverMueller) - { - // SilverMueller is implemented for Yee - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee, - "The Silver-Mueller boundary condition can only be used with the Yee solver."); - } - } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !(at_least_one_boundary_is_pml && at_least_one_boundary_is_silver_mueller), + "PML and Silver-Mueller boundary conditions cannot be activated at the same time."); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (!at_least_one_boundary_is_silver_mueller) || + (electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee), + "The Silver-Mueller boundary condition can only be used with the Yee solver."); utils::parser::queryWithParser(pp_warpx, "pml_ncell", pml_ncell); utils::parser::queryWithParser(pp_warpx, "pml_delta", pml_delta); @@ -919,12 +921,12 @@ WarpX::ReadParameters () // If WarpX::do_dive_cleaning = true, set also WarpX::do_pml_dive_cleaning = true // (possibly overwritten by users in the input file, see query below) - if (do_dive_cleaning) do_pml_dive_cleaning = true; + if (do_dive_cleaning) { do_pml_dive_cleaning = true; } // If WarpX::do_divb_cleaning = true, set also WarpX::do_pml_divb_cleaning = true // (possibly overwritten by users in the input file, see query below) // TODO Implement div(B) cleaning in PML with FDTD and remove second if condition - if (do_divb_cleaning && electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) do_pml_divb_cleaning = true; + if (do_divb_cleaning && electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { do_pml_divb_cleaning = true; } #endif // Query input parameters to use div(E) and div(B) cleaning in PMLs @@ -993,20 +995,39 @@ WarpX::ReadParameters () if (maxLevel() > 0) { Vector lo, hi; - utils::parser::getArrWithParser(pp_warpx, "fine_tag_lo", lo); - utils::parser::getArrWithParser(pp_warpx, "fine_tag_hi", hi); - fine_tag_lo = RealVect{lo}; - fine_tag_hi = RealVect{hi}; + bool fine_tag_lo_specified = utils::parser::queryArrWithParser(pp_warpx, "fine_tag_lo", lo); + bool fine_tag_hi_specified = utils::parser::queryArrWithParser(pp_warpx, "fine_tag_hi", hi); + std::string ref_patch_function; + bool parser_specified = pp_warpx.query("ref_patch_function(x,y,z)",ref_patch_function); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( ((fine_tag_lo_specified && fine_tag_hi_specified) || + parser_specified ), + "For max_level > 0, you need to either set\ + warpx.fine_tag_lo and warpx.fine_tag_hi\ + or warpx.ref_patch_function(x,y,z)"); + + if ( (fine_tag_lo_specified && fine_tag_hi_specified) && parser_specified) { + ablastr::warn_manager::WMRecordWarning("Refined patch", "Both fine_tag_lo,fine_tag_hi\ + and ref_patch_function(x,y,z) are provided. Note that fine_tag_lo/fine_tag_hi will\ + override the ref_patch_function(x,y,z) for defining the refinement patches"); + } + if (fine_tag_lo_specified && fine_tag_hi_specified) { + fine_tag_lo = RealVect{lo}; + fine_tag_hi = RealVect{hi}; + } else { + utils::parser::Store_parserString(pp_warpx, "ref_patch_function(x,y,z)", ref_patch_function); + ref_patch_parser = std::make_unique( + utils::parser::makeParser(ref_patch_function,{"x","y","z"})); + } } pp_warpx.query("do_dynamic_scheduling", do_dynamic_scheduling); // Integer that corresponds to the type of grid used in the simulation // (collocated, staggered, hybrid) - grid_type = GetAlgorithmInteger(pp_warpx, "grid_type"); + grid_type = static_cast(GetAlgorithmInteger(pp_warpx, "grid_type")); // Use same shape factors in all directions, for gathering - if (grid_type == GridType::Collocated) galerkin_interpolation = false; + if (grid_type == GridType::Collocated) { galerkin_interpolation = false; } #ifdef WARPX_DIM_RZ // Only needs to be set with WARPX_DIM_RZ, otherwise defaults to 1 @@ -1015,6 +1036,25 @@ WarpX::ReadParameters () "The number of azimuthal modes (n_rz_azimuthal_modes) must be at least 1"); #endif + // Check whether fluid species will be used + { + const ParmParse pp_fluids("fluids"); + std::vector fluid_species_names = {}; + pp_fluids.queryarr("species_names", fluid_species_names); + do_fluid_species = !fluid_species_names.empty(); + if (do_fluid_species) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level <= 1, + "Fluid species cannot currently be used with mesh refinement."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + electrostatic_solver_id != ElectrostaticSolverAlgo::Relativistic, + "Fluid species cannot currently be used with the relativistic electrostatic solver."); +#ifdef WARPX_DIM_RZ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( n_rz_azimuthal_modes <= 1, + "Fluid species cannot be used with more than 1 azimuthal mode."); +#endif + } + } + // Set default parameters with hybrid grid (parsed later below) if (grid_type == GridType::Hybrid) { @@ -1100,11 +1140,13 @@ WarpX::ReadParameters () } #endif - // note: current_deposition must be set after maxwell_solver is already determined, + // note: current_deposition must be set after maxwell_solver (electromagnetic_solver_id) or + // do_electrostatic (electrostatic_solver_id) are already determined, // because its default depends on the solver selection - current_deposition_algo = GetAlgorithmInteger(pp_algo, "current_deposition"); - charge_deposition_algo = GetAlgorithmInteger(pp_algo, "charge_deposition"); - particle_pusher_algo = GetAlgorithmInteger(pp_algo, "particle_pusher"); + current_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "current_deposition")); + charge_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "charge_deposition")); + particle_pusher_algo = static_cast(GetAlgorithmInteger(pp_algo, "particle_pusher")); + evolve_scheme = static_cast(GetAlgorithmInteger(pp_algo, "evolve_scheme")); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo != CurrentDepositionAlgo::Esirkepov || @@ -1136,7 +1178,7 @@ WarpX::ReadParameters () // Query algo.field_gathering from input, set field_gathering_algo to // "default" if not found (default defined in Utils/WarpXAlgorithmSelection.cpp) - field_gathering_algo = GetAlgorithmInteger(pp_algo, "field_gathering"); + field_gathering_algo = static_cast(GetAlgorithmInteger(pp_algo, "field_gathering")); // Set default field gathering algorithm for hybrid grids (momentum-conserving) std::string tmp_algo; @@ -1166,13 +1208,63 @@ WarpX::ReadParameters () } // Use same shape factors in all directions, for gathering - if (field_gathering_algo == GatheringAlgo::MomentumConserving) galerkin_interpolation = false; + if (field_gathering_algo == GatheringAlgo::MomentumConserving) { galerkin_interpolation = false; } + + // With the PSATD solver, momentum-conserving field gathering + // combined with mesh refinement does not seem to work correctly + // TODO Needs debugging + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD && + field_gathering_algo == GatheringAlgo::MomentumConserving && + maxLevel() > 0) + { + WARPX_ABORT_WITH_MESSAGE( + "With the PSATD solver, momentum-conserving field gathering" + " combined with mesh refinement is currently not implemented"); + } em_solver_medium = GetAlgorithmInteger(pp_algo, "em_solver_medium"); if (em_solver_medium == MediumForEM::Macroscopic ) { macroscopic_solver_algo = GetAlgorithmInteger(pp_algo,"macroscopic_sigma_method"); } + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + utils::parser::queryWithParser(pp_algo, "max_picard_iterations", max_picard_iterations); + utils::parser::queryWithParser(pp_algo, "picard_iteration_tolerance", picard_iteration_tolerance); + utils::parser::queryWithParser(pp_algo, "require_picard_convergence", require_picard_convergence); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + current_deposition_algo == CurrentDepositionAlgo::Esirkepov || + current_deposition_algo == CurrentDepositionAlgo::Direct, + "Only Esirkepov or Direct current deposition supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || + electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC, + "Only the Yee EM solver is supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + particle_pusher_algo == ParticlePusherAlgo::Boris || + particle_pusher_algo == ParticlePusherAlgo::HigueraCary, + "Only the Boris and Higuera particle pushers are supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + field_gathering_algo != GatheringAlgo::MomentumConserving, + "With implicit and semi-implicit schemes, the momentum conserving field gather is not supported as it would not conserve energy"); + + if (current_deposition_algo == CurrentDepositionAlgo::Direct) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !galerkin_interpolation, + "With implicit and semi-implicit schemes and direct deposition, the Galerkin field gathering must be turned off in order to conserve energy"); + } + + if (current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + galerkin_interpolation, + "With implicit and semi-implicit schemes and Esirkepov deposition, the Galerkin field gathering must be turned on in order to conserve energy"); + } + } + // Load balancing parameters std::vector load_balance_intervals_string_vec = {"0"}; pp_algo.queryarr("load_balance_intervals", load_balance_intervals_string_vec); @@ -1180,11 +1272,12 @@ WarpX::ReadParameters () load_balance_intervals_string_vec); pp_algo.query("load_balance_with_sfc", load_balance_with_sfc); // Knapsack factor only used with non-SFC strategy - if (!load_balance_with_sfc) + if (!load_balance_with_sfc) { pp_algo.query("load_balance_knapsack_factor", load_balance_knapsack_factor); + } utils::parser::queryWithParser(pp_algo, "load_balance_efficiency_ratio_threshold", load_balance_efficiency_ratio_threshold); - load_balance_costs_update_algo = GetAlgorithmInteger(pp_algo, "load_balance_costs_update"); + load_balance_costs_update_algo = static_cast(GetAlgorithmInteger(pp_algo, "load_balance_costs_update")); if (WarpX::load_balance_costs_update_algo==LoadBalanceCostsUpdateAlgo::Heuristic) { utils::parser::queryWithParser( pp_algo, "costs_heuristic_cells_wt", costs_heuristic_cells_wt); @@ -1265,8 +1358,9 @@ WarpX::ReadParameters () pp_warpx, "sort_bin_size", vect_sort_bin_size, 0, AMREX_SPACEDIM); if (sort_bin_size_is_specified){ - for (int i=0; i(GetAlgorithmInteger(pp_psatd, "solution_type")); // Integers that correspond to the time dependency of J (constant, linear) // and rho (linear, quadratic) for the PSATD algorithm - J_in_time = GetAlgorithmInteger(pp_psatd, "J_in_time"); - rho_in_time = GetAlgorithmInteger(pp_psatd, "rho_in_time"); + J_in_time = static_cast(GetAlgorithmInteger(pp_psatd, "J_in_time")); + rho_in_time = static_cast(GetAlgorithmInteger(pp_psatd, "rho_in_time")); if (psatd_solution_type != PSATDSolutionType::FirstOrder || !do_multi_J) { @@ -1392,7 +1487,7 @@ WarpX::ReadParameters () // TODO Remove this default when current correction will // be implemented for the multi-J algorithm as well. - if (do_multi_J) current_correction = false; + if (do_multi_J) { current_correction = false; } pp_psatd.query("current_correction", current_correction); @@ -1469,8 +1564,8 @@ WarpX::ReadParameters () } // Scale the Galilean/comoving velocity by the speed of light - for (auto& vv : m_v_galilean) vv*= PhysConst::c; - for (auto& vv : m_v_comoving) vv*= PhysConst::c; + for (auto& vv : m_v_galilean) { vv*= PhysConst::c; } + for (auto& vv : m_v_comoving) { vv*= PhysConst::c; } const auto v_galilean_is_zero = std::all_of(m_v_galilean.begin(), m_v_galilean.end(), @@ -2035,6 +2130,11 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d AllocLevelMFs(lev, ba, dm, guard_cells.ng_alloc_EB, guard_cells.ng_alloc_J, guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F, guard_cells.ng_alloc_G, aux_is_nodal); + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + EvolveImplicitPicardInit(lev); + } + m_accelerator_lattice[lev] = std::make_unique(); m_accelerator_lattice[lev]->InitElementFinder(lev, ba, dm); @@ -2146,28 +2246,28 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // const std::array dx = CellSize(lev); - AllocInitMultiFab(Bfield_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[x]"); - AllocInitMultiFab(Bfield_fp[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[y]"); - AllocInitMultiFab(Bfield_fp[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[z]"); + AllocInitMultiFab(Bfield_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[x]", 0.0_rt); + AllocInitMultiFab(Bfield_fp[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[y]", 0.0_rt); + AllocInitMultiFab(Bfield_fp[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp[z]", 0.0_rt); - AllocInitMultiFab(Efield_fp[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[x]"); - AllocInitMultiFab(Efield_fp[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[y]"); - AllocInitMultiFab(Efield_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[z]"); + AllocInitMultiFab(Efield_fp[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[x]", 0.0_rt); + AllocInitMultiFab(Efield_fp[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[y]", 0.0_rt); + AllocInitMultiFab(Efield_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp[z]", 0.0_rt); AllocInitMultiFab(current_fp[lev][0], amrex::convert(ba, jx_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[x]", 0.0_rt); AllocInitMultiFab(current_fp[lev][1], amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[y]", 0.0_rt); AllocInitMultiFab(current_fp[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[z]", 0.0_rt); // Match external field MultiFabs to fine patch - if (add_external_B_field) { - AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[x]"); - AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[y]"); - AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[z]"); + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); + AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); + AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); } - if (add_external_E_field) { - AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[x]"); - AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[y]"); - AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[z]"); + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { + AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); + AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); + AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); } if (do_current_centering) @@ -2218,15 +2318,23 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm ); } + // Allocate extra multifabs needed for fluids + if (do_fluid_species) { + myfl->AllocateLevelMFs(lev, ba, dm); + auto & warpx = GetInstance(); + const amrex::Real cur_time = warpx.gett_new(lev); + myfl->InitData(lev, geom[lev].Domain(),cur_time); + } + if (fft_do_time_averaging) { - AllocInitMultiFab(Bfield_avg_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[x]"); - AllocInitMultiFab(Bfield_avg_fp[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[y]"); - AllocInitMultiFab(Bfield_avg_fp[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[z]"); + AllocInitMultiFab(Bfield_avg_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[x]", 0.0_rt); + AllocInitMultiFab(Bfield_avg_fp[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[y]", 0.0_rt); + AllocInitMultiFab(Bfield_avg_fp[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[z]", 0.0_rt); - AllocInitMultiFab(Efield_avg_fp[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[x]"); - AllocInitMultiFab(Efield_avg_fp[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[y]"); - AllocInitMultiFab(Efield_avg_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[z]"); + AllocInitMultiFab(Efield_avg_fp[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[x]", 0.0_rt); + AllocInitMultiFab(Efield_avg_fp[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[y]", 0.0_rt); + AllocInitMultiFab(Efield_avg_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[z]", 0.0_rt); } #ifdef AMREX_USE_EB @@ -2431,24 +2539,24 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm const std::array cdx = CellSize(lev-1); // Create the MultiFabs for B - AllocInitMultiFab(Bfield_cp[lev][0], amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[x]"); - AllocInitMultiFab(Bfield_cp[lev][1], amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[y]"); - AllocInitMultiFab(Bfield_cp[lev][2], amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[z]"); + AllocInitMultiFab(Bfield_cp[lev][0], amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[x]", 0.0_rt); + AllocInitMultiFab(Bfield_cp[lev][1], amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[y]", 0.0_rt); + AllocInitMultiFab(Bfield_cp[lev][2], amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_cp[z]", 0.0_rt); // Create the MultiFabs for E - AllocInitMultiFab(Efield_cp[lev][0], amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[x]"); - AllocInitMultiFab(Efield_cp[lev][1], amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[y]"); - AllocInitMultiFab(Efield_cp[lev][2], amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[z]"); + AllocInitMultiFab(Efield_cp[lev][0], amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[x]", 0.0_rt); + AllocInitMultiFab(Efield_cp[lev][1], amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[y]", 0.0_rt); + AllocInitMultiFab(Efield_cp[lev][2], amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_cp[z]", 0.0_rt); if (fft_do_time_averaging) { - AllocInitMultiFab(Bfield_avg_cp[lev][0], amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[x]"); - AllocInitMultiFab(Bfield_avg_cp[lev][1], amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[y]"); - AllocInitMultiFab(Bfield_avg_cp[lev][2], amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[z]"); + AllocInitMultiFab(Bfield_avg_cp[lev][0], amrex::convert(cba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[x]", 0.0_rt); + AllocInitMultiFab(Bfield_avg_cp[lev][1], amrex::convert(cba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[y]", 0.0_rt); + AllocInitMultiFab(Bfield_avg_cp[lev][2], amrex::convert(cba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_cp[z]", 0.0_rt); - AllocInitMultiFab(Efield_avg_cp[lev][0], amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[x]"); - AllocInitMultiFab(Efield_avg_cp[lev][1], amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[y]"); - AllocInitMultiFab(Efield_avg_cp[lev][2], amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[z]"); + AllocInitMultiFab(Efield_avg_cp[lev][0], amrex::convert(cba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[x]", 0.0_rt); + AllocInitMultiFab(Efield_avg_cp[lev][1], amrex::convert(cba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[y]", 0.0_rt); + AllocInitMultiFab(Efield_avg_cp[lev][2], amrex::convert(cba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_cp[z]", 0.0_rt); } // Create the MultiFabs for the current @@ -2595,7 +2703,7 @@ void WarpX::AllocLevelSpectralSolverRZ (amrex::Vector(WarpX::do_multi_J_n_depositions); + if (WarpX::do_multi_J) { solver_dt /= static_cast(WarpX::do_multi_J_n_depositions); } auto pss = std::make_unique(lev, realspace_ba, @@ -2648,7 +2756,7 @@ void WarpX::AllocLevelSpectralSolver (amrex::Vector(WarpX::do_multi_J_n_depositions); + if (WarpX::do_multi_J) { solver_dt /= static_cast(WarpX::do_multi_J_n_depositions); } auto pss = std::make_unique(lev, realspace_ba, @@ -2999,29 +3107,29 @@ amrex::Vector WarpX::getFornbergStencilCoefficients(const int n_ord // Coefficients for collocated (nodal) finite-difference approximation if (a_grid_type == GridType::Collocated) { - // First coefficient - coeffs.at(0) = m * 2. / (m+1); - // Other coefficients by recurrence - for (int n = 1; n < m; n++) - { - coeffs.at(n) = - (m-n) * 1. / (m+n+1) * coeffs.at(n-1); - } + // First coefficient + coeffs.at(0) = m * 2._rt / (m+1); + // Other coefficients by recurrence + for (int n = 1; n < m; n++) + { + coeffs.at(n) = - (m-n) * 1._rt / (m+n+1) * coeffs.at(n-1); + } } // Coefficients for staggered finite-difference approximation else { - Real prod = 1.; - for (int k = 1; k < m+1; k++) - { - prod *= (m + k) / (4. * k); - } - // First coefficient - coeffs.at(0) = 4 * m * prod * prod; - // Other coefficients by recurrence - for (int n = 1; n < m; n++) - { - coeffs.at(n) = - ((2*n-1) * (m-n)) * 1. / ((2*n+1) * (m+n)) * coeffs.at(n-1); - } + Real prod = 1.; + for (int k = 1; k < m+1; k++) + { + prod *= (m + k) / (4._rt * k); + } + // First coefficient + coeffs.at(0) = 4_rt * m * prod * prod; + // Other coefficients by recurrence + for (int n = 1; n < m; n++) + { + coeffs.at(n) = - ((2_rt*n-1) * (m-n)) * 1._rt / ((2_rt*n+1) * (m+n)) * coeffs.at(n-1); + } } return coeffs; @@ -3135,8 +3243,8 @@ bool WarpX::isAnyBoundaryPML() { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML) return true; - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML) return true; + if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML) { return true; } + if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML) { return true; } } return false; } @@ -3206,3 +3314,21 @@ WarpX::AliasInitMultiFab ( } multifab_map[name_with_suffix] = mf.get(); } + +void +WarpX::AllocInitMultiFabFromModel ( + std::unique_ptr& mf, + amrex::MultiFab& mf_model, + const int level, + const std::string& name, + std::optional initial_value) +{ + const auto name_with_suffix = TagWithLevelSuffix(name, level); + const auto tag = amrex::MFInfo().SetTag(name_with_suffix); + mf = std::make_unique(mf_model.boxArray(), mf_model.DistributionMap(), + mf_model.nComp(), mf_model.nGrowVect(), tag); + if (initial_value) { + mf->setVal(*initial_value); + } + multifab_map[name_with_suffix] = mf.get(); +} diff --git a/Source/ablastr/CMakeLists.txt b/Source/ablastr/CMakeLists.txt index 95b2ea0e24c..a41fa4ae3a0 100644 --- a/Source/ablastr/CMakeLists.txt +++ b/Source/ablastr/CMakeLists.txt @@ -1,5 +1,6 @@ -#add_subdirectory(fields) add_subdirectory(coarsen) +add_subdirectory(math) +#add_subdirectory(fields) add_subdirectory(parallelization) #add_subdirectory(particles) #add_subdirectory(profiler) diff --git a/Source/ablastr/Make.package b/Source/ablastr/Make.package index 2d848e63a44..3426020e45d 100644 --- a/Source/ablastr/Make.package +++ b/Source/ablastr/Make.package @@ -1,6 +1,7 @@ #CEXE_sources += ParticleBoundaries.cpp include $(WARPX_HOME)/Source/ablastr/coarsen/Make.package +include $(WARPX_HOME)/Source/ablastr/math/Make.package include $(WARPX_HOME)/Source/ablastr/parallelization/Make.package include $(WARPX_HOME)/Source/ablastr/particles/Make.package include $(WARPX_HOME)/Source/ablastr/utils/Make.package diff --git a/Source/ablastr/coarsen/average.H b/Source/ablastr/coarsen/average.H index 315938acb48..621b4cb54e2 100644 --- a/Source/ablastr/coarsen/average.H +++ b/Source/ablastr/coarsen/average.H @@ -72,20 +72,22 @@ namespace ablastr::coarsen::average // Compute number of points for (int l = 0; l < 3; ++l) { - if (cr[l] == 1) + if (cr[l] == 1) { np[l] = 1; // no coarsening - else + } else { np[l] = cr[l] * (1 - sf[l]) * (1 - sc[l]) // cell-centered + (2 * (cr[l] - 1) + 1) * sf[l] * sc[l]; // nodal + } } // Compute starting indices of source array (fine) for (int l = 0; l < 3; ++l) { - if (cr[l] == 1) + if (cr[l] == 1) { idx_min[l] = ic[l]; // no coarsening - else + } else { idx_min[l] = ic[l] * cr[l] * (1 - sf[l]) * (1 - sc[l]) // cell-centered + (ic[l] * cr[l] - cr[l] + 1) * sf[l] * sc[l]; // nodal + } } // Auxiliary integer variables diff --git a/Source/ablastr/coarsen/sample.H b/Source/ablastr/coarsen/sample.H index 6ef0962168a..80eac14e833 100644 --- a/Source/ablastr/coarsen/sample.H +++ b/Source/ablastr/coarsen/sample.H @@ -67,14 +67,14 @@ namespace ablastr::coarsen::sample // Compute number of points for ( int l = 0; l < 3; ++l ) { - if ( cr[l] == 1 ) np[l] = 1+amrex::Math::abs(sf[l]-sc[l]); // no coarsening - else np[l] = 2-sf[l]; + if ( cr[l] == 1 ) { np[l] = 1+amrex::Math::abs(sf[l]-sc[l]); // no coarsening + } else { np[l] = 2-sf[l]; } } // Compute starting indices of source array (fine) for ( int l = 0; l < 3; ++l ) { - if ( cr[l] == 1 ) idx_min[l] = ic[l]-sc[l]*(1-sf[l]); // no coarsening - else idx_min[l] = ic[l]*cr[l]+static_cast(cr[l]/2)*(1-sc[l])-(1-sf[l]); + if ( cr[l] == 1 ) { idx_min[l] = ic[l]-sc[l]*(1-sf[l]); // no coarsening + } else { idx_min[l] = ic[l]*cr[l]+static_cast(cr[l]/2)*(1-sc[l])-(1-sf[l]); } } // Auxiliary integer variables diff --git a/Source/ablastr/coarsen/sample.cpp b/Source/ablastr/coarsen/sample.cpp index 65ada612905..b5661037c7e 100644 --- a/Source/ablastr/coarsen/sample.cpp +++ b/Source/ablastr/coarsen/sample.cpp @@ -29,21 +29,22 @@ namespace ablastr::coarsen::sample void Loop ( amrex::MultiFab& mf_dst, - const amrex::MultiFab& mf_src, - const int dcomp, - const int scomp, - const int ncomp, - const amrex::IntVect ngrowvect, - const amrex::IntVect crse_ratio + const amrex::MultiFab& mf_src, + const int dcomp, + const int scomp, + const int ncomp, + const amrex::IntVect ngrowvect, + const amrex::IntVect crse_ratio ) { // Staggering of source fine MultiFab and destination coarse MultiFab const amrex::IntVect stag_src = mf_src.boxArray().ixType().toIntVect(); const amrex::IntVect stag_dst = mf_dst.boxArray().ixType().toIntVect(); - if ( crse_ratio > amrex::IntVect(1) ) + if ( crse_ratio > amrex::IntVect(1) ) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( ngrowvect == amrex::IntVect(0), "option of filling guard cells of destination MultiFab with coarsening not supported for this interpolation" ); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect() >= stag_dst-stag_src+ngrowvect, "source fine MultiFab does not have enough guard cells for this interpolation" ); @@ -118,9 +119,9 @@ namespace ablastr::coarsen::sample "source MultiFab converted to staggering of destination MultiFab is not coarsenable" ); ba_tmp.coarsen( crse_ratio ); - if ( ba_tmp == mf_dst.boxArray() and mf_src.DistributionMap() == mf_dst.DistributionMap() ) + if ( ba_tmp == mf_dst.boxArray() and mf_src.DistributionMap() == mf_dst.DistributionMap() ) { Loop( mf_dst, mf_src, dcomp, scomp, ncomp, ngrowvect, crse_ratio ); - else + } else { // Cannot coarsen into MultiFab with different BoxArray or DistributionMapping: // 1) create temporary MultiFab on coarsened version of source BoxArray with same DistributionMapping diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index c5824be1774..eaeadfbb2fb 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -12,21 +12,6 @@ #include #include -#include -#include -#include -#include -#include -#include -#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) -# include -#endif -#ifdef AMREX_USE_EB -# include -#endif -#include -#include -#include #include #include @@ -45,12 +30,21 @@ #include #include #include +#include #include +#include +#include #include +#include #include #include #include -#include +#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) +# include +#endif +#ifdef AMREX_USE_EB +# include +#endif #include #include @@ -81,7 +75,7 @@ namespace details AMREX_GPU_DEVICE AMREX_FORCE_INLINE void - operator() (long i, long j, long k) const noexcept + operator() (int i, int j, int k) const noexcept { amrex::mf_nodebilin_interp(i, j, k, 0, m_phi_fp_arr, 0, m_phi_cp_arr, 0, m_refratio); @@ -155,7 +149,7 @@ computePhi (amrex::Vector const & rho, rel_ref_ratio = amrex::Vector{{amrex::IntVect(AMREX_D_DECL(1, 1, 1))}}; } - int const finest_level = rho.size() - 1u; + auto const finest_level = static_cast(rho.size() - 1); // scale rho appropriately; also determine if rho is zero everywhere amrex::Real max_norm_b = 0.0; @@ -168,7 +162,7 @@ computePhi (amrex::Vector const & rho, const bool always_use_bnorm = (max_norm_b > 0); if (!always_use_bnorm) { - if (absolute_tolerance == 0.0) absolute_tolerance = amrex::Real(1e-6); + if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } ablastr::warn_manager::WMRecordWarning( "ElectrostaticSolver", "Max norm of rho is 0", @@ -196,10 +190,10 @@ computePhi (amrex::Vector const & rho, geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; int max_semicoarsening_level = 0; int semicoarsening_direction = -1; - const int min_dir = std::distance(dx_scaled.begin(), - std::min_element(dx_scaled.begin(),dx_scaled.end())); - const int max_dir = std::distance(dx_scaled.begin(), - std::max_element(dx_scaled.begin(),dx_scaled.end())); + const auto min_dir = static_cast(std::distance(dx_scaled.begin(), + std::min_element(dx_scaled.begin(),dx_scaled.end()))); + const auto max_dir = static_cast(std::distance(dx_scaled.begin(), + std::max_element(dx_scaled.begin(),dx_scaled.end()))); if (dx_scaled[max_dir] > dx_scaled[min_dir]) { semicoarsening_direction = max_dir; max_semicoarsening_level = static_cast @@ -312,9 +306,11 @@ computePhi (amrex::Vector const & rho, } // Run additional operations, such as calculation of the E field for embedded boundaries - if constexpr (!std::is_same::value) - if (post_phi_calculation.has_value()) + if constexpr (!std::is_same::value) { + if (post_phi_calculation.has_value()) { post_phi_calculation.value()(mlmg, lev); + } + } } // loop over lev(els) } diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index 3284b49c8fe..a8833b070b8 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -13,19 +13,6 @@ #include #include -#include -#include -#include -#include -#include -#include -#ifdef AMREX_USE_EB -# include -#endif -#include -#include -#include - #include #include #include @@ -43,12 +30,18 @@ #include #include #include +#include +#include +#include #include #include +#include #include #include #include -#include +#ifdef AMREX_USE_EB +# include +#endif #include #include @@ -115,7 +108,7 @@ computeVectorPotential ( amrex::Vector > co rel_ref_ratio = amrex::Vector{{amrex::IntVect(AMREX_D_DECL(1, 1, 1))}}; } - int const finest_level = curr.size() - 1u; + auto const finest_level = static_cast(curr.size()) - 1; // scale J appropriately; also determine if current is zero everywhere amrex::Real max_comp_J = 0.0; @@ -129,7 +122,7 @@ computeVectorPotential ( amrex::Vector > co const bool always_use_bnorm = (max_comp_J > 0); if (!always_use_bnorm) { - if (absolute_tolerance == 0.0) absolute_tolerance = amrex::Real(1e-6); + if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } ablastr::warn_manager::WMRecordWarning( "MagnetostaticSolver", "Max norm of J is 0", @@ -249,9 +242,11 @@ computeVectorPotential ( amrex::Vector > co curr[lev][adim]->mult(-1._rt/ablastr::constant::SI::mu0); } // Loop over adim // Run additional operations, such as calculation of the B fields for embedded boundaries - if constexpr (!std::is_same::value) - if (post_A_calculation.has_value()) + if constexpr (!std::is_same::value) { + if (post_A_calculation.has_value()) { post_A_calculation.value()(mlmg, lev); + } + } } // loop over lev(els) } } // namepace Magnetostatic diff --git a/Source/ablastr/math/CMakeLists.txt b/Source/ablastr/math/CMakeLists.txt new file mode 100644 index 00000000000..9093da83ae1 --- /dev/null +++ b/Source/ablastr/math/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(fft) diff --git a/Source/ablastr/math/Make.package b/Source/ablastr/math/Make.package new file mode 100644 index 00000000000..a0e95b11225 --- /dev/null +++ b/Source/ablastr/math/Make.package @@ -0,0 +1,3 @@ +include $(WARPX_HOME)/Source/ablastr/math/fft/Make.package + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/ablastr diff --git a/Source/FieldSolver/SpectralSolver/AnyFFT.H b/Source/ablastr/math/fft/AnyFFT.H similarity index 58% rename from Source/FieldSolver/SpectralSolver/AnyFFT.H rename to Source/ablastr/math/fft/AnyFFT.H index 79dbdf0e03d..8a4c11c7654 100644 --- a/Source/FieldSolver/SpectralSolver/AnyFFT.H +++ b/Source/ablastr/math/fft/AnyFFT.H @@ -1,73 +1,89 @@ -/* Copyright 2019-2020 +/* Copyright 2019-2023 * - * This file is part of WarpX. + * This file is part of ABLASTR. * * License: BSD-3-Clause-LBNL */ -#ifndef ANYFFT_H_ -#define ANYFFT_H_ - -#include -#include - -#if defined(AMREX_USE_CUDA) -# include -#elif defined(AMREX_USE_HIP) -# if __has_include() // ROCm 5.3+ -# include -# else -# include -# endif -#else -# include +#ifndef ABLASTR_ANYFFT_H_ +#define ABLASTR_ANYFFT_H_ + +#ifdef ABLASTR_USE_FFT +# include +# include + +# if defined(AMREX_USE_CUDA) +# include +# elif defined(AMREX_USE_HIP) +# if __has_include() // ROCm 5.3+ +# include +# else +# include +# endif +# else +# include +# endif #endif + /** * Wrapper around FFT libraries. The header file defines the API and the base types * (Complex and VendorFFTPlan), and the implementation for different FFT libraries is * done in different cpp files. This wrapper only depends on the underlying FFT library * AND on AMReX (There is no dependence on WarpX). */ -namespace AnyFFT +namespace ablastr::math::anyfft { + + /** This function is a wrapper around rocff_setup(). + * It is a no-op in case rocfft is not used. + */ + void setup(); + + /** This function is a wrapper around rocff_cleanup(). + * It is a no-op in case rocfft is not used. + */ + void cleanup(); + +#ifdef ABLASTR_USE_FFT + // First, define library-dependent types (complex, FFT plan) /** Complex type for FFT, depends on FFT library */ -#if defined(AMREX_USE_CUDA) -# ifdef AMREX_USE_FLOAT - using Complex = cuComplex; -# else - using Complex = cuDoubleComplex; -# endif -#elif defined(AMREX_USE_HIP) -# ifdef AMREX_USE_FLOAT - using Complex = float2; -# else - using Complex = double2; -# endif -#else -# ifdef AMREX_USE_FLOAT - using Complex = fftwf_complex; -# else - using Complex = fftw_complex; -# endif -#endif +# if defined(AMREX_USE_CUDA) +# ifdef AMREX_USE_FLOAT + using Complex = cuComplex; +# else + using Complex = cuDoubleComplex; +# endif +# elif defined(AMREX_USE_HIP) +# ifdef AMREX_USE_FLOAT + using Complex = float2; +# else + using Complex = double2; +# endif +# else +# ifdef AMREX_USE_FLOAT + using Complex = fftwf_complex; +# else + using Complex = fftw_complex; +# endif +# endif /** Library-dependent FFT plans type, which holds one fft plan per box * (plans are only initialized for the boxes that are owned by the local MPI rank). */ -#if defined(AMREX_USE_CUDA) - using VendorFFTPlan = cufftHandle; -#elif defined(AMREX_USE_HIP) - using VendorFFTPlan = rocfft_plan; -#else -# ifdef AMREX_USE_FLOAT - using VendorFFTPlan = fftwf_plan; -# else - using VendorFFTPlan = fftw_plan; -# endif -#endif +# if defined(AMREX_USE_CUDA) + using VendorFFTPlan = cufftHandle; +# elif defined(AMREX_USE_HIP) + using VendorFFTPlan = rocfft_plan; +# else +# ifdef AMREX_USE_FLOAT + using VendorFFTPlan = fftwf_plan; +# else + using VendorFFTPlan = fftw_plan; +# endif +# endif // Second, define library-independent API @@ -108,6 +124,9 @@ namespace AnyFFT * \param[out] fft_plan plan for which the FFT is performed */ void Execute(FFTplan& fft_plan); + +#endif + } -#endif // ANYFFT_H_ +#endif // ABLASTR_ANYFFT_H_ diff --git a/Source/ablastr/math/fft/CMakeLists.txt b/Source/ablastr/math/fft/CMakeLists.txt new file mode 100644 index 00000000000..f7d689c98e9 --- /dev/null +++ b/Source/ablastr/math/fft/CMakeLists.txt @@ -0,0 +1,14 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + if(WarpX_PSATD STREQUAL ON) + if(WarpX_COMPUTE STREQUAL CUDA) + target_sources(ablastr_${SD} PRIVATE WrapCuFFT.cpp) + elseif(WarpX_COMPUTE STREQUAL HIP) + target_sources(ablastr_${SD} PRIVATE WrapRocFFT.cpp) + else() + target_sources(ablastr_${SD} PRIVATE WrapFFTW.cpp) + endif() + else() + target_sources(ablastr_${SD} PRIVATE WrapNoFFT.cpp) + endif() +endforeach() diff --git a/Source/ablastr/math/fft/Make.package b/Source/ablastr/math/fft/Make.package new file mode 100644 index 00000000000..b04062bd9d5 --- /dev/null +++ b/Source/ablastr/math/fft/Make.package @@ -0,0 +1,13 @@ +ifeq ($(USE_PSATD),TRUE) + ifeq ($(USE_CUDA),TRUE) + CEXE_sources += WrapCuFFT.cpp + else ifeq ($(USE_HIP),TRUE) + CEXE_sources += WrapRocFFT.cpp + else + CEXE_sources += WrapFFTW.cpp + endif +else + CEXE_sources += WrapNoFFT.cpp +endif + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/ablastr/math/fft diff --git a/Source/FieldSolver/SpectralSolver/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp similarity index 84% rename from Source/FieldSolver/SpectralSolver/WrapCuFFT.cpp rename to Source/ablastr/math/fft/WrapCuFFT.cpp index 59b200a32fc..8ecea2a2486 100644 --- a/Source/FieldSolver/SpectralSolver/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -1,17 +1,21 @@ -/* Copyright 2019-2020 +/* Copyright 2019-2023 * - * This file is part of WarpX. + * This file is part of ABLASTR. * * License: BSD-3-Clause-LBNL */ #include "AnyFFT.H" -#include "Utils/TextMsg.H" +#include "ablastr/utils/TextMsg.H" -namespace AnyFFT +namespace ablastr::math::anyfft { + void setup(){/*nothing to do*/} + + void cleanup(){/*nothing to do*/} + #ifdef AMREX_USE_FLOAT cufftType VendorR2C = CUFFT_R2C; cufftType VendorC2R = CUFFT_C2R; @@ -37,7 +41,7 @@ namespace AnyFFT result = cufftPlan2d( &(fft_plan.m_plan), real_size[1], real_size[0], VendorR2C); } else { - WARPX_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); + ABLASTR_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); } } else { if (dim == 3) { @@ -47,15 +51,12 @@ namespace AnyFFT result = cufftPlan2d( &(fft_plan.m_plan), real_size[1], real_size[0], VendorC2R); } else { - WARPX_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); + ABLASTR_ABORT_WITH_MESSAGE("only dim=2 and dim=3 have been implemented"); } } - if ( result != CUFFT_SUCCESS ) { - amrex::Print() << Utils::TextMsg::Err( - "cufftplan failed! Error: " - + cufftErrorToString(result)); - } + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(result == CUFFT_SUCCESS, + "cufftplan failed! Error: " + cufftErrorToString(result)); // Store meta-data in fft_plan fft_plan.m_real_array = real_array; @@ -89,11 +90,11 @@ namespace AnyFFT result = cufftExecZ2D(fft_plan.m_plan, fft_plan.m_complex_array, fft_plan.m_real_array); #endif } else { - WARPX_ABORT_WITH_MESSAGE( - "direction must be AnyFFT::direction::R2C or AnyFFT::direction::C2R"); + ABLASTR_ABORT_WITH_MESSAGE( + "direction must be FFTplan::direction::R2C or FFTplan::direction::C2R"); } if ( result != CUFFT_SUCCESS ) { - WARPX_ABORT_WITH_MESSAGE( + ABLASTR_ABORT_WITH_MESSAGE( "forward transform using cufftExec failed ! Error: " +cufftErrorToString(result)); } diff --git a/Source/FieldSolver/SpectralSolver/WrapFFTW.cpp b/Source/ablastr/math/fft/WrapFFTW.cpp similarity index 91% rename from Source/FieldSolver/SpectralSolver/WrapFFTW.cpp rename to Source/ablastr/math/fft/WrapFFTW.cpp index df330c4f67c..6711bbface9 100644 --- a/Source/FieldSolver/SpectralSolver/WrapFFTW.cpp +++ b/Source/ablastr/math/fft/WrapFFTW.cpp @@ -1,22 +1,25 @@ -/* Copyright 2019-2021 +/* Copyright 2019-2023 * - * This file is part of WarpX. + * This file is part of ABLASTR. * * License: BSD-3-Clause-LBNL */ #include "AnyFFT.H" -#include "Utils/TextMsg.H" +#include "ablastr/utils/TextMsg.H" #include #include #include -#include - -namespace AnyFFT +namespace ablastr::math::anyfft { + + void setup(){/*nothing to do*/} + + void cleanup(){/*nothing to do*/} + #ifdef AMREX_USE_FLOAT const auto VendorCreatePlanR2C3D = fftwf_plan_dft_r2c_3d; const auto VendorCreatePlanC2R3D = fftwf_plan_dft_c2r_3d; @@ -54,7 +57,7 @@ namespace AnyFFT fft_plan.m_plan = VendorCreatePlanR2C2D( real_size[1], real_size[0], real_array, complex_array, FFTW_ESTIMATE); } else { - WARPX_ABORT_WITH_MESSAGE( + ABLASTR_ABORT_WITH_MESSAGE( "only dim=2 and dim=3 have been implemented"); } } else if (dir == direction::C2R){ @@ -65,7 +68,7 @@ namespace AnyFFT fft_plan.m_plan = VendorCreatePlanC2R2D( real_size[1], real_size[0], complex_array, real_array, FFTW_ESTIMATE); } else { - WARPX_ABORT_WITH_MESSAGE( + ABLASTR_ABORT_WITH_MESSAGE( "only dim=2 and dim=3 have been implemented. Should be easy to add dim=1."); } } diff --git a/Source/ablastr/math/fft/WrapNoFFT.cpp b/Source/ablastr/math/fft/WrapNoFFT.cpp new file mode 100644 index 00000000000..c6dc5cdf484 --- /dev/null +++ b/Source/ablastr/math/fft/WrapNoFFT.cpp @@ -0,0 +1,17 @@ +/* Copyright 2019-2023 + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ + +#include "AnyFFT.H" + +namespace ablastr::math::anyfft +{ + + void setup(){/*nothing to do*/} + + void cleanup(){/*nothing to do*/} + +} diff --git a/Source/FieldSolver/SpectralSolver/WrapRocFFT.cpp b/Source/ablastr/math/fft/WrapRocFFT.cpp similarity index 89% rename from Source/FieldSolver/SpectralSolver/WrapRocFFT.cpp rename to Source/ablastr/math/fft/WrapRocFFT.cpp index 5f3131dec47..4bd39efab57 100644 --- a/Source/FieldSolver/SpectralSolver/WrapRocFFT.cpp +++ b/Source/ablastr/math/fft/WrapRocFFT.cpp @@ -1,26 +1,34 @@ -/* Copyright 2019-2020 +/* Copyright 2019-2023 * - * This file is part of WarpX. + * This file is part of ABLASTR. * * License: BSD-3-Clause-LBNL */ #include "AnyFFT.H" -#include "Utils/TextMsg.H" +#include "ablastr/utils/TextMsg.H" -namespace AnyFFT +namespace ablastr::math::anyfft { + void setup() + { + rocfft_setup(); + } + + void cleanup() + { + rocfft_cleanup(); + } std::string rocfftErrorToString (const rocfft_status err); - namespace { - void assert_rocfft_status (std::string const& name, rocfft_status status) + namespace + { + void assert_rocfft_status (std::string const& name, rocfft_status const& status) { - if (status != rocfft_status_success) { - WARPX_ABORT_WITH_MESSAGE( - name + " failed! Error: " + rocfftErrorToString(status)); - } + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(status == rocfft_status_success, + name + " failed! Error: " + rocfftErrorToString(status)); } } @@ -91,8 +99,8 @@ namespace AnyFFT (void**)&(fft_plan.m_real_array), // out execinfo); } else { - WARPX_ABORT_WITH_MESSAGE( - "direction must be AnyFFT::direction::R2C or AnyFFT::direction::C2R"); + ABLASTR_ABORT_WITH_MESSAGE( + "direction must be FFTplan::direction::R2C or FFTplan::direction::C2R"); } assert_rocfft_status("rocfft_execute", result); diff --git a/Source/ablastr/parallelization/KernelTimer.H b/Source/ablastr/parallelization/KernelTimer.H index 0d11220daba..9e94014822b 100644 --- a/Source/ablastr/parallelization/KernelTimer.H +++ b/Source/ablastr/parallelization/KernelTimer.H @@ -57,19 +57,31 @@ public: } //! Destructor. - AMREX_GPU_DEVICE - ~KernelTimer () { #if (defined AMREX_USE_GPU) - if (m_do_timing && m_cost) { # if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) - m_wt = clock64() - m_wt; - amrex::Gpu::Atomic::Add( m_cost, amrex::Real(m_wt)); + AMREX_GPU_DEVICE + ~KernelTimer () + { + if (m_do_timing && m_cost) { + m_wt = clock64() - m_wt; + amrex::Gpu::Atomic::Add( m_cost, amrex::Real(m_wt)); + } + } # elif defined(AMREX_USE_DPCPP) - // To be updated + // To be updated + AMREX_GPU_DEVICE + ~KernelTimer () = default; # endif - } + +#else +~KernelTimer () = default; #endif //AMREX_USE_GPU - } + + + KernelTimer ( KernelTimer const &) = default; + KernelTimer& operator= ( KernelTimer const & ) = default; + KernelTimer ( KernelTimer&& ) = default; + KernelTimer& operator= ( KernelTimer&& ) = default; #if (defined AMREX_USE_GPU) private: diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index 2fcdcf3b768..da24af94dfb 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -40,8 +40,8 @@ namespace ablastr::particles * \param num_rho_deposition_guards number of ghost cells to use for rho (default: rho.nGrowVect()) * \param depos_lev the level to deposit the particles to (default: lev) * \param rel_ref_ratio mesh refinement ratio between lev and depos_lev (default: 1) - * \param offset index to start at when looping over particles to depose (default: 0) - * \param np_to_depose number of particles to depose (default: pti.numParticles()) + * \param offset index to start at when looping over particles to deposit (default: 0) + * \param np_to_deposit number of particles to deposit (default: pti.numParticles()) * \param icomp component in MultiFab to start depositing to * \param nc number of components to deposit * \param cost pointer to (load balancing) cost corresponding to box where present @@ -65,7 +65,7 @@ deposit_charge (typename T_PC::ParIterType& pti, std::optional depos_lev = std::nullopt, std::optional rel_ref_ratio = std::nullopt, long const offset = 0, - std::optional np_to_depose = std::nullopt, + std::optional np_to_deposit = std::nullopt, int const icomp = 0, int const nc = 1, amrex::Real * const AMREX_RESTRICT cost = nullptr, long const load_balance_costs_update_algo = 0, @@ -73,8 +73,9 @@ deposit_charge (typename T_PC::ParIterType& pti, { // deposition guards amrex::IntVect ng_rho = rho->nGrowVect(); - if (num_rho_deposition_guards.has_value()) + if (num_rho_deposition_guards.has_value()) { ng_rho = num_rho_deposition_guards.value(); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho <= rho->nGrowVect(), "num_rho_deposition_guards are larger than allocated!"); // particle shape @@ -82,14 +83,16 @@ deposit_charge (typename T_PC::ParIterType& pti, // used for MR when we want to deposit for a subset of the particles on the level in the // current box; with offset, we start at a later particle index - if (!np_to_depose.has_value()) - np_to_depose = pti.numParticles(); - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(np_to_depose.value() + offset <= pti.numParticles(), - "np_to_depose + offset are out-of-bounds for particle iterator"); + if (!np_to_deposit.has_value()) { + np_to_deposit = pti.numParticles(); + } + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(np_to_deposit.value() + offset <= pti.numParticles(), + "np_to_deposit + offset are out-of-bounds for particle iterator"); int const lev = pti.GetLevel(); - if (!depos_lev.has_value()) + if (!depos_lev.has_value()) { depos_lev = lev; + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev.value() == (lev-1)) || (depos_lev.value() == (lev )), "Deposition buffers only work for lev or lev-1"); @@ -100,7 +103,7 @@ deposit_charge (typename T_PC::ParIterType& pti, } // If no particles, do not do anything - if (np_to_depose == 0) return; + if (np_to_deposit == 0) { return; } // Extract deposition order and check that particles shape fits within the guard cells. // NOTE: In specific situations where the staggering of rho and the charge deposition algorithm @@ -170,7 +173,7 @@ deposit_charge (typename T_PC::ParIterType& pti, auto & rho_fab = local_rho; #endif - const auto GetPosition = GetParticlePosition(pti, offset); + const auto GetPosition = GetParticlePosition(pti, offset); // Indices of the lower bound const amrex::Dim3 lo = lbound(tilebox); @@ -179,17 +182,17 @@ deposit_charge (typename T_PC::ParIterType& pti, if (nox == 1){ doChargeDepositionShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_depose.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); } else if (nox == 2){ doChargeDepositionShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_depose.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); } else if (nox == 3){ doChargeDepositionShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_depose.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); } diff --git a/Source/ablastr/profiler/ProfilerWrapper.H b/Source/ablastr/profiler/ProfilerWrapper.H index 1dbe80c59b1..6b476d78114 100644 --- a/Source/ablastr/profiler/ProfilerWrapper.H +++ b/Source/ablastr/profiler/ProfilerWrapper.H @@ -21,8 +21,9 @@ namespace ablastr::profiler AMREX_FORCE_INLINE void device_synchronize(bool const do_device_synchronize = false) { - if (do_device_synchronize) + if (do_device_synchronize) { amrex::Gpu::synchronize(); + } } /** An object that conditionally calls device_synchronize() on destruction @@ -38,6 +39,12 @@ namespace ablastr::profiler device_synchronize(m_do_device_synchronize); } + // default move and copy operations + SynchronizeOnDestruct(const SynchronizeOnDestruct&) = default; + SynchronizeOnDestruct& operator=(const SynchronizeOnDestruct&) = default; + SynchronizeOnDestruct(SynchronizeOnDestruct&&) = default; + SynchronizeOnDestruct& operator=(SynchronizeOnDestruct&& field_data) = default; + bool m_do_device_synchronize = false; }; diff --git a/Source/ablastr/utils/Communication.H b/Source/ablastr/utils/Communication.H index 9be9fec1e58..85f58ff9f65 100644 --- a/Source/ablastr/utils/Communication.H +++ b/Source/ablastr/utils/Communication.H @@ -60,7 +60,8 @@ void ParallelAdd (amrex::MultiFab &dst, void FillBoundary (amrex::MultiFab &mf, bool do_single_precision_comms, - const amrex::Periodicity &period = amrex::Periodicity::NonPeriodic()); + const amrex::Periodicity &period = amrex::Periodicity::NonPeriodic(), + std::optional nodal_sync=std::nullopt); void FillBoundary (amrex::MultiFab &mf, amrex::IntVect ng, @@ -77,7 +78,7 @@ void FillBoundary (amrex::iMultiFab& mf, void FillBoundary(amrex::Vector const &mf, bool do_single_precision_comms, - const amrex::Periodicity &period); + const amrex::Periodicity &period, std::optional nodal_sync=std::nullopt); void SumBoundary (amrex::MultiFab &mf, diff --git a/Source/ablastr/utils/Communication.cpp b/Source/ablastr/utils/Communication.cpp index 3ae18ad8d1c..bfe34d7d875 100644 --- a/Source/ablastr/utils/Communication.cpp +++ b/Source/ablastr/utils/Communication.cpp @@ -114,18 +114,18 @@ void FillBoundary (amrex::MultiFab &mf, } } -void FillBoundary (amrex::MultiFab &mf, bool do_single_precision_comms, const amrex::Periodicity &period) +void FillBoundary (amrex::MultiFab &mf, bool do_single_precision_comms, const amrex::Periodicity &period, std::optional nodal_sync) { amrex::IntVect const ng = mf.n_grow; - FillBoundary(mf, ng, do_single_precision_comms, period); + FillBoundary(mf, ng, do_single_precision_comms, period, nodal_sync); } void FillBoundary (amrex::Vector const &mf, bool do_single_precision_comms, - const amrex::Periodicity &period) + const amrex::Periodicity &period, std::optional nodal_sync) { - for (auto x : mf) { - ablastr::utils::communication::FillBoundary(*x, do_single_precision_comms, period); + for (auto *x : mf) { + ablastr::utils::communication::FillBoundary(*x, do_single_precision_comms, period, nodal_sync); } } @@ -180,7 +180,7 @@ void OverrideSync (amrex::MultiFab &mf, { BL_PROFILE("ablastr::utils::communication::OverrideSync"); - if (mf.ixType().cellCentered()) return; + if (mf.ixType().cellCentered()) { return; } if (do_single_precision_comms) { diff --git a/Source/ablastr/utils/Serialization.H b/Source/ablastr/utils/Serialization.H index 29173a8eeec..b4dd2efa8ac 100644 --- a/Source/ablastr/utils/Serialization.H +++ b/Source/ablastr/utils/Serialization.H @@ -73,8 +73,9 @@ namespace ablastr::utils::serialization ", except vectors of std::string."); put_in(static_cast(val.size()), vec); - for (const auto &el : val) + for (const auto &el : val) { put_in(el, vec); + } } } @@ -145,8 +146,9 @@ namespace ablastr::utils::serialization const auto length = get_out(it); std::vector res(length); - for (int i = 0; i < length; ++i) + for (int i = 0; i < length; ++i) { res[i] = get_out(it); + } return res; } diff --git a/Source/ablastr/utils/SignalHandling.H b/Source/ablastr/utils/SignalHandling.H index 559fb93a080..d6d10b61f31 100644 --- a/Source/ablastr/utils/SignalHandling.H +++ b/Source/ablastr/utils/SignalHandling.H @@ -63,6 +63,9 @@ public: //! Check whether a given action has been requested, and reset the associated flag static bool TestAndResetActionRequestFlag (int action_to_test); + // Don't allow clients to incorrectly try to construct and use an instance of this type + SignalHandling () = delete; + private: //! Is any signal handling action configured in signal_conf_requests ? static bool m_any_signal_action_active; @@ -81,9 +84,6 @@ private: //! Boolean flags transmitted between CheckSignals() and //! HandleSignals() to indicate actions requested by signals static bool signal_actions_requested[SIGNAL_REQUESTS_SIZE]; - - // Don't allow clients to incorrectly try to construct and use an instance of this type - SignalHandling () = delete; }; } // namespace ablastr::utils diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp index feaf69d002f..9521e5f9c3d 100644 --- a/Source/ablastr/utils/SignalHandling.cpp +++ b/Source/ablastr/utils/SignalHandling.cpp @@ -90,7 +90,7 @@ SignalHandling::parseSignalNameToNumber (const std::string &str) std::string name_upper = sp.abbrev; std::string name_lower = name_upper; for (char &c : name_lower) { - c = std::tolower(c); + c = static_cast(std::tolower(static_cast(c))); } signals_parser.setConstant(name_upper, sp.value); @@ -146,8 +146,9 @@ SignalHandling::CheckSignals () { // Is any signal handling action configured? // If not, we can skip all handling and the MPI communication as well. - if (!m_any_signal_action_active) + if (!m_any_signal_action_active) { return; + } // We assume that signals will definitely be delivered to rank 0, // and may be delivered to other ranks as well. For coordination, @@ -171,12 +172,12 @@ SignalHandling::CheckSignals () } #if defined(AMREX_USE_MPI) - auto comm = amrex::ParallelDescriptor::Communicator(); // Due to a bug in Cray's MPICH 8.1.13 implementation (CUDA builds on Perlmutter@NERSC in 2022), // we cannot use the MPI_CXX_BOOL C++ datatype here. See WarpX PR #3029 and NERSC INC0183281 static_assert(sizeof(bool) == 1, "We communicate bools as 1 byte-sized type in MPI"); BL_MPI_REQUIRE(MPI_Ibcast(signal_actions_requested, SIGNAL_REQUESTS_SIZE, - MPI_BYTE, 0, comm,&signal_mpi_ibcast_request)); + MPI_BYTE, 0, amrex::ParallelDescriptor::Communicator(), + &signal_mpi_ibcast_request)); #endif } @@ -185,8 +186,9 @@ SignalHandling::WaitSignals () { // Is any signal handling action configured? // If not, we can skip all handling and the MPI communication as well. - if (!m_any_signal_action_active) + if (!m_any_signal_action_active) { return; + } #if defined(AMREX_USE_MPI) BL_MPI_REQUIRE(MPI_Wait(&signal_mpi_ibcast_request, MPI_STATUS_IGNORE)); diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.H b/Source/ablastr/utils/msg_logger/MsgLogger.H index 2296eceac25..01a61be5c80 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.H +++ b/Source/ablastr/utils/msg_logger/MsgLogger.H @@ -65,7 +65,7 @@ namespace ablastr::utils::msg_logger * * @return a byte vector */ - std::vector serialize() const; + [[nodiscard]] std::vector serialize() const; /** * \brief This function generates a Msg struct from a byte vector @@ -101,7 +101,7 @@ namespace ablastr::utils::msg_logger * * @return a byte vector */ - std::vector serialize() const; + [[nodiscard]] std::vector serialize() const; /** * \brief This function generates a MsgWithCounter struct from a byte vector @@ -140,7 +140,7 @@ namespace ablastr::utils::msg_logger * * @return a byte vector */ - std::vector serialize() const; + [[nodiscard]] std::vector serialize() const; /** * \brief This function generates a MsgWithCounterAndRanks struct from a byte vector @@ -203,7 +203,7 @@ namespace ablastr::utils::msg_logger * * @return a vector of the recorded messages */ - std::vector get_msgs() const; + [[nodiscard]] std::vector get_msgs() const; /** * \brief This function returns a vector containing the recorded messages @@ -211,7 +211,8 @@ namespace ablastr::utils::msg_logger * * @return a vector of the recorded messages with counters */ - std::vector get_msgs_with_counter() const; + [[nodiscard]] std::vector + get_msgs_with_counter() const; /** * \brief This collective function generates a vector containing the messages @@ -220,7 +221,7 @@ namespace ablastr::utils::msg_logger * * @return a vector of messages with counters and ranks if I/O rank, an empty vector otherwise */ - std::vector + [[nodiscard]] std::vector collective_gather_msgs_with_counter_and_ranks() const; private: @@ -231,7 +232,7 @@ namespace ablastr::utils::msg_logger * * @return a vector of messages with counters and ranks */ - std::vector + [[nodiscard]] std::vector one_rank_gather_msgs_with_counter_and_ranks() const; #ifdef AMREX_USE_MPI @@ -243,8 +244,8 @@ namespace ablastr::utils::msg_logger * @param[in] how_many_msgs the number of messages that the current rank has * @return a pair containing the ID of the "gather rank" and its number of messages */ - std::pair find_gather_rank_and_its_msgs( - int how_many_msgs) const; + [[nodiscard]] std::pair + find_gather_rank_and_its_msgs(int how_many_msgs) const; /** * \brief This function uses data gathered on the "gather rank" to generate @@ -256,7 +257,7 @@ namespace ablastr::utils::msg_logger * @param[in] gather_rank the ID of the "gather rank" * @return if gather_rank==m_rank a vector of messages with global counters and emitting rank lists, dummy data otherwise */ - std::vector + [[nodiscard]] std::vector compute_msgs_with_counter_and_ranks( const std::map& my_msg_map, const std::vector& all_data, diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.cpp b/Source/ablastr/utils/msg_logger/MsgLogger.cpp index 4ba36ee82e9..629eab8c25d 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.cpp +++ b/Source/ablastr/utils/msg_logger/MsgLogger.cpp @@ -98,25 +98,27 @@ namespace std::string abl_msg_logger::PriorityToString(const Priority& priority) { - if(priority == Priority::high) + if(priority == Priority::high) { return "high"; - else if (priority == Priority::medium) + } else if (priority == Priority::medium) { return "medium"; - else + } else { return "low"; + } } Priority abl_msg_logger::StringToPriority(const std::string& priority_string) { - if(priority_string == "high") + if(priority_string == "high") { return Priority::high; - else if (priority_string == "medium") + } else if (priority_string == "medium") { return Priority::medium; - else if (priority_string == "low") + } else if (priority_string == "low") { return Priority::low; - else + } else { ABLASTR_ABORT_WITH_MESSAGE( "Priority string '" + priority_string + "' not recognized"); + } //this silences a "non-void function does not return a value in all control paths" warning return Priority::low; @@ -223,8 +225,9 @@ std::vector Logger::get_msgs() const { auto res = std::vector{}; - for (const auto& msg_w_counter : m_messages) + for (const auto& msg_w_counter : m_messages) { res.emplace_back(msg_w_counter.first); + } return res; } @@ -233,8 +236,9 @@ std::vector Logger::get_msgs_with_counter() const { auto res = std::vector{}; - for (const auto& msg : m_messages) + for (const auto& msg : m_messages) { res.emplace_back(MsgWithCounter{msg.first, msg.second}); + } return res; } @@ -246,8 +250,9 @@ Logger::collective_gather_msgs_with_counter_and_ranks() const #ifdef AMREX_USE_MPI // Trivial case of only one rank - if (m_num_procs == 1) + if (m_num_procs == 1) { return one_rank_gather_msgs_with_counter_and_ranks(); + } // Find out who is the "gather rank" and how many messages it has const auto my_msgs = get_msgs(); @@ -256,8 +261,9 @@ Logger::collective_gather_msgs_with_counter_and_ranks() const find_gather_rank_and_its_msgs(how_many_msgs); // If the "gather rank" has zero messages there are no messages at all - if(gather_rank_how_many_msgs == 0) + if(gather_rank_how_many_msgs == 0) { return std::vector{}; + } // All the ranks receive the msgs of the "gather rank" as a byte array const auto serialized_gather_rank_msgs = @@ -347,7 +353,7 @@ Logger::compute_msgs_with_counter_and_ranks( const std::vector& displacements, const int gather_rank) const { - if(m_rank != gather_rank) return std::vector{}; + if(m_rank != gather_rank) { return std::vector{}; } std::vector msgs_with_counter_and_ranks; @@ -369,8 +375,9 @@ Logger::compute_msgs_with_counter_and_ranks( #pragma omp parallel for #endif for(int rr = 0; rr < m_num_procs; ++rr){ //for each rank - if(rr == gather_rank) // (skip gather_rank) + if(rr == gather_rank) { // (skip gather_rank) continue; + } // get counters generated by rank rr auto it = all_data.begin() + displacements[rr]; @@ -461,8 +468,9 @@ void Logger::swap_with_io_rank( if (gather_rank != m_io_rank){ if(m_rank == gather_rank){ auto package = std::vector{}; - for (const auto& el: msgs_with_counter_and_ranks) + for (const auto& el: msgs_with_counter_and_ranks) { abl_ser::put_in_vec(el.serialize(), package); + } auto package_size = static_cast(package.size()); amrex::ParallelDescriptor::Send(&package_size, 1, m_io_rank, 0); @@ -510,9 +518,10 @@ get_serialized_gather_rank_msgs( amrex::ParallelDescriptor::Bcast( &size_serialized_gather_rank_msgs, 1, gather_rank); - if (!is_gather_rank) + if (!is_gather_rank) { serialized_gather_rank_msgs.resize( size_serialized_gather_rank_msgs); + } amrex::ParallelDescriptor::Bcast( serialized_gather_rank_msgs.data(), @@ -554,9 +563,10 @@ compute_package_for_gather_rank( // Add the additional messages seen by the current rank to the package abl_ser::put_in(static_cast(msgs_to_send.size()), package); - for (const auto& el : msgs_to_send) + for (const auto& el : msgs_to_send) { abl_ser::put_in_vec( MsgWithCounter{el.first, el.second}.serialize(), package); + } return package; } diff --git a/Source/ablastr/utils/timer/Timer.H b/Source/ablastr/utils/timer/Timer.H index 6743607a0d6..efbb7d6a2bb 100644 --- a/Source/ablastr/utils/timer/Timer.H +++ b/Source/ablastr/utils/timer/Timer.H @@ -8,8 +8,6 @@ #ifndef ABLASTR_TIMER_H_ #define ABLASTR_TIMER_H_ -#include - namespace ablastr::utils::timer { @@ -25,7 +23,7 @@ namespace ablastr::utils::timer /** * \brief The constructor. */ - Timer(); + Timer() = default; /** @@ -46,7 +44,7 @@ namespace ablastr::utils::timer * * @return the duration */ - amrex::Real get_duration () noexcept; + double get_duration () noexcept; /** @@ -55,12 +53,12 @@ namespace ablastr::utils::timer * * @return the maximum duration across all the MPI ranks */ - amrex::Real get_global_duration (); + double get_global_duration (); private: - amrex::Real m_start_time /*! The start time*/; - amrex::Real m_stop_time /*! The stop time*/; + double m_start_time /*! The start time*/; + double m_stop_time /*! The stop time*/; }; } diff --git a/Source/ablastr/utils/timer/Timer.cpp b/Source/ablastr/utils/timer/Timer.cpp index 6ce44eab312..5682a07c290 100644 --- a/Source/ablastr/utils/timer/Timer.cpp +++ b/Source/ablastr/utils/timer/Timer.cpp @@ -11,8 +11,6 @@ using namespace ablastr::utils::timer; -Timer::Timer(){} - void Timer::record_start_time() noexcept { @@ -25,13 +23,13 @@ Timer::record_stop_time() noexcept m_stop_time = amrex::ParallelDescriptor::second(); } -amrex::Real +double Timer::get_duration () noexcept { return m_stop_time - m_start_time; } -amrex::Real +double Timer::get_global_duration () { auto duration = this->get_duration(); diff --git a/Source/ablastr/warn_manager/WarnManager.H b/Source/ablastr/warn_manager/WarnManager.H index 75fd642a884..737fb8fba73 100644 --- a/Source/ablastr/warn_manager/WarnManager.H +++ b/Source/ablastr/warn_manager/WarnManager.H @@ -52,12 +52,27 @@ namespace ablastr::warn_manager /** * A singleton class should not be cloneable. */ - WarnManager(WarnManager &other) = delete; + WarnManager( const WarnManager& ) = delete; + + /** + * A singleton class should not be cloneable. + */ + WarnManager( WarnManager&& ) = delete; /** * A singleton class should not be assignable. */ - void operator=(const WarnManager &) = delete; + WarnManager& operator=( const WarnManager & ) = delete; + + /** + * A singleton class should not be assignable. + */ + WarnManager& operator=( const WarnManager&& ) = delete; + + /** + * Default destructor + */ + ~WarnManager() = default; static WarnManager& GetInstance(); @@ -80,7 +95,7 @@ namespace ablastr::warn_manager * @param[in] when a string to mark when the warnings are printed out (it appears in the warning list) * @return a string containing the "local" warning list */ - std::string PrintLocalWarnings( + [[nodiscard]] std::string PrintLocalWarnings( const std::string& when) const; /** @@ -90,7 +105,7 @@ namespace ablastr::warn_manager * @param[in] when a string to mark when the warnings are printed out (it appears in the warning list) * @return a string containing the "global" warning list */ - std::string PrintGlobalWarnings( + [[nodiscard]] std::string PrintGlobalWarnings( const std::string& when) const; /** @@ -105,7 +120,7 @@ namespace ablastr::warn_manager * * @return the m_always_warn_immediately flag */ - bool GetAlwaysWarnImmediatelyFlag() const; + [[nodiscard]] bool GetAlwaysWarnImmediatelyFlag() const; /** * \brief Setter for the m_abort_on_warning_threshold flag @@ -120,7 +135,7 @@ namespace ablastr::warn_manager * * @return the m_abort_on_warning_threshold flag */ - std::optional GetAbortThreshold() const; + [[nodiscard]] std::optional GetAbortThreshold() const; /** * \brief This function reads warning messages from the inputfile. It is intended for @@ -148,7 +163,7 @@ namespace ablastr::warn_manager * @param[in] msg_with_counter a MessageWithCounter * @return a string containing the warning message */ - std::string PrintWarnMsg( + [[nodiscard]] std::string PrintWarnMsg( const ablastr::utils::msg_logger::MsgWithCounter& msg_with_counter) const; /** @@ -159,7 +174,7 @@ namespace ablastr::warn_manager * @param[in] msg_with_counter_and_ranks a MsgWithCounterAndRanks * @return a string containing the warning message */ - std::string PrintWarnMsg( + [[nodiscard]] std::string PrintWarnMsg( const ablastr::utils::msg_logger::MsgWithCounterAndRanks& msg_with_counter_and_ranks) const; /** diff --git a/Source/ablastr/warn_manager/WarnManager.cpp b/Source/ablastr/warn_manager/WarnManager.cpp index 889f2df848d..ee052ded299 100644 --- a/Source/ablastr/warn_manager/WarnManager.cpp +++ b/Source/ablastr/warn_manager/WarnManager.cpp @@ -29,15 +29,16 @@ namespace const abl_msg_logger::Priority& priority) { using namespace abl_msg_logger; - if (priority == Priority::low) + if (priority == Priority::low) { return WarnPriority::low; - else if (priority == Priority::medium) + } else if (priority == Priority::medium) { return WarnPriority::medium; - else if (priority == Priority::high) + } else if (priority == Priority::high) { return WarnPriority::high; - else + } else { ABLASTR_ABORT_WITH_MESSAGE( "Parsing Priority to WarnPriority has failed"); + } return WarnPriority::high; } @@ -59,10 +60,11 @@ void WarnManager::RecordWarning( WarnPriority priority) { auto msg_priority = abl_msg_logger::Priority::high; - if(priority == WarnPriority::low) + if(priority == WarnPriority::low) { msg_priority = abl_msg_logger::Priority::low; - else if(priority == WarnPriority::medium) + } else if(priority == WarnPriority::medium) { msg_priority = abl_msg_logger::Priority::medium; + } if(m_always_warn_immediately){ @@ -86,10 +88,11 @@ void WarnManager::RecordWarning( if(m_abort_on_warning_threshold){ auto abort_priority = abl_msg_logger::Priority::high; - if(m_abort_on_warning_threshold == WarnPriority::low) + if(m_abort_on_warning_threshold == WarnPriority::low) { abort_priority = abl_msg_logger::Priority::low; - else if(m_abort_on_warning_threshold == WarnPriority::medium) + } else if(m_abort_on_warning_threshold == WarnPriority::medium) { abort_priority = abl_msg_logger::Priority::medium; + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( msg_priority < abort_priority, @@ -130,8 +133,9 @@ std::string WarnManager::PrintGlobalWarnings(const std::string& when) const auto all_warnings = m_p_logger->collective_gather_msgs_with_counter_and_ranks(); - if(m_rank != amrex::ParallelDescriptor::IOProcessorNumber()) + if(m_rank != amrex::ParallelDescriptor::IOProcessorNumber()) { return "[see I/O rank message]"; + } std::sort(all_warnings.begin(), all_warnings.end(), [](const auto& a, const auto& b){ @@ -217,23 +221,25 @@ std::string WarnManager::PrintWarnMsg( { std::stringstream ss; ss << "* --> "; - if (msg_with_counter.msg.priority == abl_msg_logger::Priority::high) + if (msg_with_counter.msg.priority == abl_msg_logger::Priority::high) { ss << "[!!!]"; - else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::medium) + } else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::medium) { ss << "[!! ]"; - else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::low) + } else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::low) { ss << "[! ]"; - else + } else { ss << "[???]"; + } ss << " [" + msg_with_counter.msg.topic << "] "; - if(msg_with_counter.counter == 2) + if(msg_with_counter.counter == 2) { ss << "[raised twice]\n"; - else if(msg_with_counter.counter == 1) + } else if(msg_with_counter.counter == 1) { ss << "[raised once]\n"; - else + } else { ss << "[raised " << msg_with_counter.counter << " times]\n"; + } ss << MsgFormatter(msg_with_counter.msg.text, warn_line_size, warn_tab_size); @@ -248,8 +254,9 @@ std::string WarnManager::PrintWarnMsg( std::string raised_by = "@ Raised by: "; if (!msg_with_counter_and_ranks.all_ranks){ - for (const auto rr : msg_with_counter_and_ranks.ranks) + for (const auto rr : msg_with_counter_and_ranks.ranks) { raised_by += " " + std::to_string(rr); + } } else{ raised_by += "ALL\n"; @@ -296,8 +303,9 @@ WarnManager::MsgFormatter( msg, line_size-prefix_length); std::stringstream ss_out; - for (const auto& line : wrapped_text) + for (const auto& line : wrapped_text) { ss_out << prefix << line << "\n"; + } return ss_out.str(); } diff --git a/Source/main.cpp b/Source/main.cpp index cfe80534987..2a1b828c64f 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -10,8 +10,8 @@ #include "Initialization/WarpXAMReXInit.H" #include "Utils/WarpXProfilerWrapper.H" -#include "Utils/WarpXrocfftUtil.H" +#include #include #include #include @@ -25,7 +25,7 @@ int main(int argc, char* argv[]) warpx::initialization::amrex_init(argc, argv); - utils::rocfft::setup(); + ablastr::math::anyfft::setup(); { WARPX_PROFILE_VAR("main()", pmain); @@ -39,14 +39,14 @@ int main(int argc, char* argv[]) warpx.Evolve(); - //Print warning messages at the end of the simulation amrex::Print() << ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("THE END"); timer.record_stop_time(); - if (warpx.Verbose()) { + if (warpx.Verbose()) + { amrex::Print() << "Total Time : " - << timer.get_global_duration() << '\n'; + << timer.get_global_duration() << '\n'; } WARPX_PROFILE_VAR_STOP(pmain); @@ -54,7 +54,7 @@ int main(int argc, char* argv[]) WarpX::Finalize(); } - utils::rocfft::cleanup(); + ablastr::math::anyfft::cleanup(); amrex::Finalize(); diff --git a/Tools/Algorithms/psatd_pml.ipynb b/Tools/Algorithms/psatd_pml.ipynb new file mode 100644 index 00000000000..897ffa70b9e --- /dev/null +++ b/Tools/Algorithms/psatd_pml.ipynb @@ -0,0 +1,607 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import inspect\n", + "import sympy as sp\n", + "from sympy import *\n", + "from sympy.solvers.solveset import linsolve\n", + "\n", + "sp.init_session()\n", + "sp.init_printing()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Global variables:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Dimensional parameters\n", + "dd = 9\n", + "DD = 12\n", + "\n", + "# Speed of light, time, time step\n", + "c = sp.symbols(r'c', real = True, positive = True)\n", + "t = sp.symbols(r't', real = True)\n", + "tn = sp.symbols(r't_n', real = True)\n", + "dt = sp.symbols(r'\\Delta{t}', real = True, positive = True)\n", + "\n", + "# Components of k vector, omega, norm of k vector\n", + "kx = sp.symbols(r'k_x', real = True)\n", + "ky = sp.symbols(r'k_y', real = True)\n", + "kz = sp.symbols(r'k_z', real = True)\n", + "om = sp.symbols(r'omega', real = True, positive = True)\n", + "knorm = sp.sqrt(kx**2 + ky**2 + kz**2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Method:\n", + "The goal is to solve a system of second-order ordinary differential equations in time of the form $\\boldsymbol{\\ddot{X}} = M \\boldsymbol{X}$, where the $d \\times d$ matrix $M$ has eigenpairs $(\\lambda_i, \\{\\boldsymbol{v}_{i,1}, \\dots, \\boldsymbol{v}_{i,\\mu_i}\\})$, where $\\mu_i$ denotes the algebraic multiplicity of $\\lambda_i$ and $\\sum_i \\mu_i = d$.\n", + "Assuming that all eigenvalues are either zero or negative, we can write $\\lambda_i = - \\omega_i^2$, with $\\omega_i = \\sqrt{- \\lambda_i} \\geq 0$.\n", + "Then the solution of $\\boldsymbol{\\ddot{X}} = M \\boldsymbol{X}$ reads\n", + "$$\n", + "\\boldsymbol{X} = \\sum_i \\sum_j^{\\mu_i} \\left(a_{i,j} C(\\omega_i, t) + b_{i,j} S(\\omega_i, t) \\right) \\boldsymbol{v}_{i,j} \\,,\n", + "$$\n", + "where $a_{i,j}$ and $b_{i,j}$ are integration constants to be determined by the initial conditions $\\boldsymbol{X}(t_n)$ and $\\boldsymbol{\\dot{X}}(t_n)$, $C(\\omega, t) = \\cos(\\omega \\, (t - t_n))$, and\n", + "$$\n", + "S(\\omega, t) =\n", + "\\begin{cases}\n", + "(t - t_n) & \\omega = 0 \\,, \\\\\n", + "\\dfrac{\\sin(\\omega \\, (t - t_n))}{\\omega} & \\omega \\neq 0 \\,.\n", + "\\end{cases}\n", + "$$\n", + "We remark that\n", + "$$\n", + "\\begin{aligned}\n", + "& \\boldsymbol{X}(t_n) = \\sum_i \\sum_j^{\\mu_i} \\left(a_{i,j} C(\\omega_i, t_n) + b_{i,j} S(\\omega_i, t_n) \\right) \\boldsymbol{v}_{i,j} = \\sum_i \\sum_j^{\\mu_i} a_{i,j} \\boldsymbol{v}_{i,j} \\,, \\\\\n", + "& \\boldsymbol{\\dot{X}}(t_n) = \\sum_i \\sum_j^{\\mu_i} \\left(a_{i,j} \\dot{C}(\\omega_i, t_n) + b_{i,j} \\dot{S}(\\omega_i, t_n) \\right) \\boldsymbol{v}_{i,j} = \\sum_i \\sum_j^{\\mu_i} b_{i,j} \\boldsymbol{v}_{i,j} \\,.\n", + "\\end{aligned}\n", + "$$\n", + "In fact, the second time derivative of $\\boldsymbol{X}$ yields\n", + "$$\n", + "\\begin{split}\n", + "\\boldsymbol{\\ddot{X}} & = \\sum_i \\sum_j^{\\mu_i} \\left(a_{i,j} \\ddot{C}(\\omega_i, t) + b_{i,j} \\ddot{S}(\\omega_i, t) \\right) \\boldsymbol{v}_{i,j} \\\\\n", + "& = \\sum_i (-\\omega_i^2) \\sum_j^{\\mu_i} \\left(a_{i,j} C(\\omega_i, t) + b_{i,j} S(\\omega_i, t) \\right) \\boldsymbol{v}_{i,j} = M \\boldsymbol{X} \\,,\n", + "\\end{split}\n", + "$$\n", + "where we used the fact that, by definition, $M \\boldsymbol{v}_{i,j} = \\lambda_i \\boldsymbol{v}_{i,j} = - \\omega_i^2 \\boldsymbol{v}_{i,j}$ for all $j = 1, \\dots, \\mu_i$." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Auxiliary functions:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def C(omega, t):\n", + " return sp.cos(omega * (t-tn))\n", + "\n", + "def S(omega, t):\n", + " return (t-tn) if omega == 0. else sp.sin(omega * (t-tn)) / omega\n", + "\n", + "def Xt(eigenpairs, a, b, t):\n", + " \"\"\"\n", + " Compute X(t) according to the formulas above\n", + " for a given set of eigenpairs and coefficients.\n", + " \"\"\"\n", + " XX = zeros(DD, 1)\n", + " # Index used for the integration constants a_n and b_n\n", + " i = 0\n", + " # Loop over matrix eigenpairs\n", + " for ep in eigenpairs:\n", + " # ep[0] is an eigenvalue and om = sp.sqrt(-ep[0])\n", + " omega = 0. if ep[0] == 0. else om\n", + " # am is the algebraic multiplicity of the eigenvalue\n", + " am = ep[1]\n", + " # vF is the list of all eigenvectors corresponding to the eigenvalue\n", + " vX = ep[2]\n", + " # Loop over algebraic multiplicity of the eigenvalue\n", + " for j in range(am):\n", + " XX += (a[i] * C(omega, t) + b[i] * S(omega, t)) * vX[j]\n", + " i += 1\n", + " return XX\n", + "\n", + "def evolve(X, dX_dt, d2X_dt2):\n", + " \"\"\"\n", + " Solve ordinary differential equation X'' = M*X.\n", + " \"\"\"\n", + " # Set matrix for given ODE\n", + " MX = zeros(DD)\n", + " for i in range(DD):\n", + " for j in range(DD):\n", + " MX[i,j] = d2X_dt2[i].coeff(X[j], 1)\n", + " #MX /= c**2\n", + "\n", + " print()\n", + " print(r'Matrix:')\n", + " display(MX)\n", + "\n", + " # Characteristic matrix\n", + " lamda = sp.symbols(r'lamda')\n", + " Id = eye(DD)\n", + " MX_charmat = MX - lamda * Id\n", + "\n", + " # Characteristic polynomial\n", + " MX_charpoly = MX_charmat.det()\n", + " MX_charpoly = factor(MX_charpoly.as_expr())\n", + "\n", + " print(r'Characteristic polynomial:')\n", + " display(MX_charpoly)\n", + " \n", + " MX_eigenvals = sp.solve(MX_charpoly, lamda)\n", + "\n", + " # List of eigenvectors\n", + " MX_eigenvects = []\n", + "\n", + " # List of eigenpairs\n", + " MX_eigenpairs = []\n", + " \n", + " # Compute eigenvectors as null spaces\n", + " for l in MX_eigenvals:\n", + "\n", + " # M - lamda * Id\n", + " A = MX_charmat.subs(lamda, l)\n", + " A.simplify()\n", + "\n", + " print(r'Eigenvalue:')\n", + " display(l)\n", + "\n", + " print(r'Characteristic matrix:')\n", + " display(A)\n", + "\n", + " # Perform Gaussian elimination (necessary for lamda != 0)\n", + " if (l != 0.):\n", + " print(r'Gaussian elimination:')\n", + " print(r'A[0,:] += A[1,:]')\n", + " A[0,:] += A[1,:]\n", + " print(r'A[0,:] += A[2,:]')\n", + " A[0,:] += A[2,:]\n", + " print(r'Swap A[0,:] and A[11,:]')\n", + " row = A[11,:]\n", + " A[11,:] = A[0,:]\n", + " A[0,:] = row\n", + " print(r'A[3,:] += A[4,:]')\n", + " A[3,:] += A[4,:]\n", + " print(r'A[3,:] += A[5,:]')\n", + " A[3,:] += A[5,:]\n", + " print(r'Swap A[3,:] and A[10,:]')\n", + " row = A[10,:]\n", + " A[10,:] = A[3,:]\n", + " A[3,:] = row\n", + " print(r'A[0,:] += A[3,:]')\n", + " A[0,:] += A[3,:]\n", + " print(r'A[0,:] += A[9,:]')\n", + " A[0,:] += A[9,:]\n", + " print(r'Swap A[0,:] and A[9,:]')\n", + " row = A[9,:]\n", + " A[9,:] = A[0,:]\n", + " A[0,:] = row\n", + " print(r'A[6,:] += A[8,:]')\n", + " A[6,:] += A[8,:]\n", + " print(r'A[6,:] += A[7,:]')\n", + " A[6,:] += A[7,:]\n", + " print(r'Swap A[6,:] and A[8,:]')\n", + " row = A[8,:]\n", + " A[8,:] = A[6,:]\n", + " A[6,:] = row\n", + " print(r'A[6,:] += A[7,:]')\n", + " A[6,:] += A[7,:]\n", + " print(r'A[4,:] += A[5,:]')\n", + " A[4,:] += A[5,:]\n", + " A.simplify()\n", + " display(A)\n", + "\n", + " # Compute null space and store eigenvectors\n", + " v = A.nullspace()\n", + " MX_eigenvects.append(v)\n", + "\n", + " print(r'Eigenvectors:')\n", + " display(v)\n", + " \n", + " # Store eigenpairs (eigenvalue, algebraic multiplicity, eigenvectors)\n", + " MX_eigenpairs.append((l, len(v), v))\n", + " \n", + " #print(r'Eigenpairs:')\n", + " #display(MX_eigenpairs)\n", + "\n", + " # Verify that the eigenpairs satisfy the characteristic equations\n", + " for ep in MX_eigenpairs:\n", + " for j in range(ep[1]):\n", + " diff = MX * ep[2][j] - ep[0] * ep[2][j]\n", + " diff.simplify()\n", + " if diff != zeros(DD,1):\n", + " print('The charcteristic equation is not verified for some eigenpairs')\n", + " display(diff)\n", + "\n", + " # Define integration constants\n", + " a = []\n", + " b = []\n", + " for i in range(DD):\n", + " an = r'a_{:d}'.format(i+1)\n", + " bn = r'b_{:d}'.format(i+1) \n", + " a.append(sp.symbols(an))\n", + " b.append(sp.symbols(bn))\n", + "\n", + " # Set equations corresponding to initial conditions\n", + " lhs_a = Xt(MX_eigenpairs, a, b, tn) - X\n", + " lhs_b = Xt(MX_eigenpairs, a, b, t ).diff(t).subs(t, tn) - dX_dt\n", + "\n", + " # Compute integration constants from initial conditions\n", + " # (convert list of tuples to list using list comprehension)\n", + " a = list(linsolve(list(lhs_a), a))\n", + " a = [item for el in a for item in el]\n", + " b = list(linsolve(list(lhs_b), b))\n", + " b = [item for el in b for item in el]\n", + "\n", + " # Evaluate solution at t = tn + dt\n", + " X_new = Xt(MX_eigenpairs, a, b, tn+dt).expand()\n", + " for d in range(DD):\n", + " for Eij in E:\n", + " X_new[d] = X_new[d].collect(Eij)\n", + " for Bij in B:\n", + " X_new[d] = X_new[d].collect(Bij)\n", + " for Fi in F:\n", + " X_new[d] = X_new[d].collect(Fi)\n", + " for Gi in G:\n", + " X_new[d] = X_new[d].collect(Gi)\n", + "\n", + " # Check correctness by taking *second* derivative\n", + " # and comparing with initial right-hand side at time tn\n", + " X_t = Xt(MX_eigenpairs, a, b, t)\n", + " diff = X_t.diff(t).diff(t).subs(t, tn).subs(om, c*knorm).expand() - d2X_dt2\n", + " diff.simplify()\n", + " if diff != zeros(DD,1):\n", + " print('Integration in time failed')\n", + " display(diff)\n", + " \n", + " return X_t, X_new" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### First-order and second-order ODEs for $\\boldsymbol{E}$, $\\boldsymbol{B}$, $F$ and $G$:\n", + "Equations for the $\\boldsymbol{E}$ field:\n", + "$$\n", + "\\begin{alignat*}{3}\n", + "& \\frac{\\partial E_{xx}}{\\partial t} = c^2 i k_x (F_x + F_y + F_z) \\quad\n", + "&& \\frac{\\partial E_{xy}}{\\partial t} = c^2 i k_y (B_{zx} + B_{zy} + B_{zz}) \\quad\n", + "&& \\frac{\\partial E_{xz}}{\\partial t} = -c^2 i k_z (B_{yx} + B_{yy} + B_{yz}) \\\\[5pt]\n", + "& \\frac{\\partial E_{yx}}{\\partial t} = -c^2 i k_x (B_{zx} + B_{zy} + B_{zz}) \\quad\n", + "&& \\frac{\\partial E_{yy}}{\\partial t} = c^2 i k_y (F_x + F_y + F_z) \\quad\n", + "&& \\frac{\\partial E_{yz}}{\\partial t} = c^2 i k_z (B_{xx} + B_{xy} + B_{xz}) \\\\[5pt]\n", + "& \\frac{\\partial E_{zx}}{\\partial t} = c^2 i k_x (B_{yx} + B_{yy} + B_{yz}) \\quad\n", + "&& \\frac{\\partial E_{zy}}{\\partial t} = -c^2 i k_y (B_{xx} + B_{xy} + B_{xz})\\quad\n", + "&& \\frac{\\partial E_{zz}}{\\partial t} = c^2 i k_z (F_x + F_y + F_z)\n", + "\\end{alignat*}\n", + "$$\n", + "\n", + "Equations for the $\\boldsymbol{B}$ field:\n", + "$$\n", + "\\begin{alignat*}{3}\n", + "& \\frac{\\partial B_{xx}}{\\partial t} = i k_x (G_x + G_y + G_z) \\quad\n", + "&& \\frac{\\partial B_{xy}}{\\partial t} = -i k_y (E_{zx} + E_{zy} + E_{zz}) \\quad\n", + "&& \\frac{\\partial B_{xz}}{\\partial t} = i k_z (E_{yx} + E_{yy} + E_{yz}) \\\\[5pt]\n", + "& \\frac{\\partial B_{yx}}{\\partial t} = i k_x (E_{zx} + E_{zy} + E_{zz}) \\quad\n", + "&& \\frac{\\partial B_{yy}}{\\partial t} = i k_y (G_x + G_y + G_z) \\quad\n", + "&& \\frac{\\partial B_{yz}}{\\partial t} = -i k_z (E_{xx} + E_{xy} + E_{xz}) \\\\[5pt]\n", + "& \\frac{\\partial B_{zx}}{\\partial t} = -i k_x (E_{yx} + E_{yy} + E_{yz}) \\quad\n", + "&& \\frac{\\partial B_{zy}}{\\partial t} = i k_y (E_{xx} + E_{xy} + E_{xz}) \\quad\n", + "&& \\frac{\\partial B_{zz}}{\\partial t} = i k_z (G_x + G_y + G_z)\n", + "\\end{alignat*}\n", + "$$\n", + "\n", + "Equations for the $F$ field:\n", + "$$\n", + "\\frac{\\partial F_x}{\\partial t} = i k_x (E_{xx} + E_{xy} + E_{xz}) \\quad\n", + "\\frac{\\partial F_y}{\\partial t} = i k_y (E_{yx} + E_{yy} + E_{yz}) \\quad\n", + "\\frac{\\partial F_z}{\\partial t} = i k_z (E_{zx} + E_{zy} + E_{zz})\n", + "$$\n", + "\n", + "Equations for the $G$ field:\n", + "$$\n", + "\\frac{\\partial G_x}{\\partial t} = c^2 i k_x (B_{xx} + B_{xy} + B_{xz}) \\quad\n", + "\\frac{\\partial G_y}{\\partial t} = c^2 i k_y (B_{yx} + B_{yy} + B_{yz}) \\quad\n", + "\\frac{\\partial G_z}{\\partial t} = c^2 i k_z (B_{zx} + B_{zy} + B_{zz})\n", + "$$" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# indices 0 1 2 3 4 5 6 7 8\n", + "labels = ['xx', 'xy', 'xz', 'yx', 'yy', 'yz', 'zx', 'zy', 'zz']\n", + "\n", + "# E fields\n", + "Exx = sp.symbols(r'E_{xx}')\n", + "Exy = sp.symbols(r'E_{xy}')\n", + "Exz = sp.symbols(r'E_{xz}')\n", + "Eyx = sp.symbols(r'E_{yx}')\n", + "Eyy = sp.symbols(r'E_{yy}')\n", + "Eyz = sp.symbols(r'E_{yz}')\n", + "Ezx = sp.symbols(r'E_{zx}')\n", + "Ezy = sp.symbols(r'E_{zy}')\n", + "Ezz = sp.symbols(r'E_{zz}')\n", + "E = Matrix([[Exx],[Exy],[Exz],[Eyx],[Eyy],[Eyz],[Ezx],[Ezy],[Ezz]])\n", + "\n", + "# B fields\n", + "Bxx = sp.symbols(r'B_{xx}')\n", + "Bxy = sp.symbols(r'B_{xy}')\n", + "Bxz = sp.symbols(r'B_{xz}')\n", + "Byx = sp.symbols(r'B_{yx}')\n", + "Byy = sp.symbols(r'B_{yy}')\n", + "Byz = sp.symbols(r'B_{yz}')\n", + "Bzx = sp.symbols(r'B_{zx}')\n", + "Bzy = sp.symbols(r'B_{zy}')\n", + "Bzz = sp.symbols(r'B_{zz}')\n", + "B = Matrix([[Bxx],[Bxy],[Bxz],[Byx],[Byy],[Byz],[Bzx],[Bzy],[Bzz]])\n", + "\n", + "# F fields\n", + "Fx = sp.symbols(r'F_{x}')\n", + "Fy = sp.symbols(r'F_{y}')\n", + "Fz = sp.symbols(r'F_{z}')\n", + "F = Matrix([[Fx],[Fy],[Fz]])\n", + "\n", + "# G fields\n", + "Gx = sp.symbols(r'G_{x}')\n", + "Gy = sp.symbols(r'G_{y}')\n", + "Gz = sp.symbols(r'G_{z}')\n", + "G = Matrix([[Gx],[Gy],[Gz]])\n", + "\n", + "# dE/dt\n", + "dExx_dt = c**2 * I * kx * (Fx + Fy + Fz)\n", + "dExy_dt = c**2 * I * ky * (Bzx + Bzy + Bzz)\n", + "dExz_dt = - c**2 * I * kz * (Byx + Byy + Byz)\n", + "dEyx_dt = - c**2 * I * kx * (Bzx + Bzy + Bzz)\n", + "dEyy_dt = c**2 * I * ky * (Fx + Fy + Fz)\n", + "dEyz_dt = c**2 * I * kz * (Bxx + Bxy + Bxz)\n", + "dEzx_dt = c**2 * I * kx * (Byx + Byy + Byz)\n", + "dEzy_dt = - c**2 * I * ky * (Bxx + Bxy + Bxz)\n", + "dEzz_dt = c**2 * I * kz * (Fx + Fy + Fz)\n", + "dE_dt = Matrix([[dExx_dt],[dExy_dt],[dExz_dt],[dEyx_dt],[dEyy_dt],[dEyz_dt],[dEzx_dt],[dEzy_dt],[dEzz_dt]])\n", + "\n", + "# dB/dt\n", + "dBxx_dt = I * kx * (Gx + Gy + Gz)\n", + "dBxy_dt = - I * ky * (Ezx + Ezy + Ezz)\n", + "dBxz_dt = I * kz * (Eyx + Eyy + Eyz)\n", + "dByx_dt = I * kx * (Ezx + Ezy + Ezz)\n", + "dByy_dt = I * ky * (Gx + Gy + Gz)\n", + "dByz_dt = - I * kz * (Exx + Exy + Exz)\n", + "dBzx_dt = - I * kx * (Eyx + Eyy + Eyz)\n", + "dBzy_dt = I * ky * (Exx + Exy + Exz)\n", + "dBzz_dt = I * kz * (Gx + Gy + Gz)\n", + "dB_dt = Matrix([[dBxx_dt],[dBxy_dt],[dBxz_dt],[dByx_dt],[dByy_dt],[dByz_dt],[dBzx_dt],[dBzy_dt],[dBzz_dt]])\n", + "\n", + "# dF/dt\n", + "dFx_dt = I * kx * (Exx + Exy + Exz)\n", + "dFy_dt = I * ky * (Eyx + Eyy + Eyz)\n", + "dFz_dt = I * kz * (Ezx + Ezy + Ezz)\n", + "dF_dt = Matrix([[dFx_dt],[dFy_dt],[dFz_dt]])\n", + "\n", + "# dG/dt\n", + "dGx_dt = c**2 * I * kx * (Bxx + Bxy + Bxz)\n", + "dGy_dt = c**2 * I * ky * (Byx + Byy + Byz)\n", + "dGz_dt = c**2 * I * kz * (Bzx + Bzy + Bzz)\n", + "dG_dt = Matrix([[dGx_dt],[dGy_dt],[dGz_dt]])\n", + "\n", + "# d2E/dt2\n", + "d2Exx_dt2 = c**2 * I * kx * (dFx_dt + dFy_dt + dFz_dt)\n", + "d2Exy_dt2 = c**2 * I * ky * (dBzx_dt + dBzy_dt + dBzz_dt)\n", + "d2Exz_dt2 = - c**2 * I * kz * (dByx_dt + dByy_dt + dByz_dt)\n", + "d2Eyx_dt2 = - c**2 * I * kx * (dBzx_dt + dBzy_dt + dBzz_dt)\n", + "d2Eyy_dt2 = c**2 * I * ky * (dFx_dt + dFy_dt + dFz_dt)\n", + "d2Eyz_dt2 = c**2 * I * kz * (dBxx_dt + dBxy_dt + dBxz_dt)\n", + "d2Ezx_dt2 = c**2 * I * kx * (dByx_dt + dByy_dt + dByz_dt)\n", + "d2Ezy_dt2 = - c**2 * I * ky * (dBxx_dt + dBxy_dt + dBxz_dt)\n", + "d2Ezz_dt2 = c**2 * I * kz * (dFx_dt + dFy_dt + dFz_dt)\n", + "d2E_dt2 = Matrix([[d2Exx_dt2],[d2Exy_dt2],[d2Exz_dt2],[d2Eyx_dt2],[d2Eyy_dt2],[d2Eyz_dt2],[d2Ezx_dt2],[d2Ezy_dt2],[d2Ezz_dt2]])\n", + "\n", + "# d2B/dt2\n", + "d2Bxx_dt2 = I * kx * (dGx_dt + dGy_dt + dGz_dt)\n", + "d2Bxy_dt2 = - I * ky * (dEzx_dt + dEzy_dt + dEzz_dt)\n", + "d2Bxz_dt2 = I * kz * (dEyx_dt + dEyy_dt + dEyz_dt)\n", + "d2Byx_dt2 = I * kx * (dEzx_dt + dEzy_dt + dEzz_dt)\n", + "d2Byy_dt2 = I * ky * (dGx_dt + dGy_dt + dGz_dt)\n", + "d2Byz_dt2 = - I * kz * (dExx_dt + dExy_dt + dExz_dt)\n", + "d2Bzx_dt2 = - I * kx * (dEyx_dt + dEyy_dt + dEyz_dt)\n", + "d2Bzy_dt2 = I * ky * (dExx_dt + dExy_dt + dExz_dt)\n", + "d2Bzz_dt2 = I * kz * (dGx_dt + dGy_dt + dGz_dt)\n", + "d2B_dt2 = Matrix([[d2Bxx_dt2],[d2Bxy_dt2],[d2Bxz_dt2],[d2Byx_dt2],[d2Byy_dt2],[d2Byz_dt2],[d2Bzx_dt2],[d2Bzy_dt2],[d2Bzz_dt2]])\n", + "\n", + "# d2F/dt2\n", + "d2Fx_dt2 = I * kx * (dExx_dt + dExy_dt + dExz_dt)\n", + "d2Fy_dt2 = I * ky * (dEyx_dt + dEyy_dt + dEyz_dt)\n", + "d2Fz_dt2 = I * kz * (dEzx_dt + dEzy_dt + dEzz_dt)\n", + "d2F_dt2 = Matrix([[d2Fx_dt2],[d2Fy_dt2],[d2Fz_dt2]])\n", + "\n", + "# d2G/dt2\n", + "d2Gx_dt2 = c**2 * I * kx * (dBxx_dt + dBxy_dt + dBxz_dt)\n", + "d2Gy_dt2 = c**2 * I * ky * (dByx_dt + dByy_dt + dByz_dt)\n", + "d2Gz_dt2 = c**2 * I * kz * (dBzx_dt + dBzy_dt + dBzz_dt)\n", + "d2G_dt2 = Matrix([[d2Gx_dt2],[d2Gy_dt2],[d2Gz_dt2]])\n", + "\n", + "for i in range(dd):\n", + " d2E_dt2[i] = sp.expand(d2E_dt2[i])\n", + "\n", + "for i in range(dd):\n", + " d2B_dt2[i] = sp.expand(d2B_dt2[i])\n", + "\n", + "for i in range(3):\n", + " d2F_dt2[i] = sp.expand(d2F_dt2[i])\n", + " \n", + "for i in range(3):\n", + " d2G_dt2[i] = sp.expand(d2G_dt2[i])\n", + " \n", + "# Extended array for E and G\n", + "EG = zeros(DD,1)\n", + "for i in range(dd):\n", + " EG[i] = E[i]\n", + "for i in range(dd,DD):\n", + " EG[i] = G[i-dd]\n", + "\n", + "# dEG/dt\n", + "dEG_dt = zeros(DD,1)\n", + "for i in range(dd):\n", + " dEG_dt[i] = dE_dt[i]\n", + "for i in range(dd,DD):\n", + " dEG_dt[i] = dG_dt[i-dd]\n", + " \n", + "# d2EG/dt2\n", + "d2EG_dt2 = zeros(DD,1)\n", + "for i in range(dd):\n", + " d2EG_dt2[i] = d2E_dt2[i]\n", + "for i in range(dd,DD):\n", + " d2EG_dt2[i] = d2G_dt2[i-dd]\n", + " \n", + "# Extended array for B and F\n", + "BF = zeros(DD,1)\n", + "for i in range(dd):\n", + " BF[i] = B[i]\n", + "for i in range(dd,DD):\n", + " BF[i] = F[i-dd]\n", + "\n", + "# dBF/dt\n", + "dBF_dt = zeros(DD,1)\n", + "for i in range(dd):\n", + " dBF_dt[i] = dB_dt[i]\n", + "for i in range(dd,DD):\n", + " dBF_dt[i] = dF_dt[i-dd]\n", + "\n", + "# d2BF/dt2\n", + "d2BF_dt2 = zeros(DD,1)\n", + "for i in range(dd):\n", + " d2BF_dt2[i] = d2B_dt2[i]\n", + "for i in range(dd,DD):\n", + " d2BF_dt2[i] = d2F_dt2[i-dd]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Solve second-order ODEs for $\\boldsymbol{E}$, $\\boldsymbol{B}$, $F$ and $G$:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "print(r'Solve equations for E and G:')\n", + "EG_t, EG_new = evolve(EG, dEG_dt, d2EG_dt2)\n", + "\n", + "print(r'Solve equations for B and F:')\n", + "BF_t, BF_new = evolve(BF, dBF_dt, d2BF_dt2)\n", + "\n", + "# Check correctness by taking *first* derivative\n", + "# and comparing with initial right-hand side at time tn\n", + "# E,G\n", + "diff = EG_t.diff(t).subs(t, tn).subs(om, c*knorm).expand() - dEG_dt\n", + "diff.simplify()\n", + "if diff != zeros(DD,1):\n", + " print('Integration in time failed')\n", + " display(diff)\n", + "# B,F\n", + "diff = BF_t.diff(t).subs(t, tn).subs(om, c*knorm).expand() - dBF_dt\n", + "diff.simplify()\n", + "if diff != zeros(DD,1):\n", + " print('Integration in time failed')\n", + " display(diff)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Coefficients of PSATD equations in PML:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# Code generation\n", + "from sympy.codegen.ast import Assignment\n", + "\n", + "# 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11\n", + "# EG: Exx, Exy, Exz, Eyx, Eyy, Eyz, Ezx, Ezy, Ezz, Gx, Gy, Gz\n", + "# BF: Bxx, Bxy, Bxz, Byx, Byy, Byz, Bzx, Bzy, Bzz, Fx, Fy, Fz\n", + "\n", + "# Select update equation (left hand side)\n", + "X_new = BF_new[0]\n", + "\n", + "# Extract individual terms (right hand side)\n", + "for i in range(DD):\n", + " X = EG[i]\n", + " C = X_new.coeff(X, 1).simplify()\n", + " print(r'Coefficient multiplying ' + str(X))\n", + " display(C)\n", + " #print(ccode(Assignment(sp.symbols(r'LHS'), C)))\n", + "for i in range(DD):\n", + " X = BF[i]\n", + " C = X_new.coeff(X, 1).simplify()\n", + " print(r'Coefficient multiplying ' + str(X))\n", + " display(C)\n", + " #print(ccode(Assignment(sp.symbols(r'LHS'), C)))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/Tools/DevUtils/compute_domain.py b/Tools/DevUtils/compute_domain.py index d67cc86f53a..b54412639bd 100644 --- a/Tools/DevUtils/compute_domain.py +++ b/Tools/DevUtils/compute_domain.py @@ -11,7 +11,7 @@ The user specifies the minimal size of the physical domain and the resolution in each dimension, and the scripts computes: -- the number of cells and physical domain to satify the user-specified domain +- the number of cells and physical domain to satisfy the user-specified domain size and resolution AND make sure that the number of cells along each direction is a multiple of max_grid_size. - a starting point on how to parallelize on Cori KNL (number of nodes, etc.). diff --git a/Tools/DevUtils/update_benchmarks_from_azure_output.py b/Tools/DevUtils/update_benchmarks_from_azure_output.py index 28c3dc6b5f2..ec7b17d1050 100644 --- a/Tools/DevUtils/update_benchmarks_from_azure_output.py +++ b/Tools/DevUtils/update_benchmarks_from_azure_output.py @@ -51,7 +51,7 @@ # Raw Azure output comes with a prefix at the beginning of each line that we do # not need here. The first line that we will read is the prefix followed by the # "{" character, so we determine how long the prefix is by finding the last - # occurence of the "{" character in this line. + # occurrence of the "{" character in this line. azure_indent = line.rfind('{') first_line_read = True new_file_string += line[azure_indent:] diff --git a/Tools/LibEnsemble/run_libensemble_on_warpx.py b/Tools/LibEnsemble/run_libensemble_on_warpx.py index 1be152821f6..b86c0249d01 100644 --- a/Tools/LibEnsemble/run_libensemble_on_warpx.py +++ b/Tools/LibEnsemble/run_libensemble_on_warpx.py @@ -102,7 +102,7 @@ ('ramp_down_2', float, (1,)), # input parameter: position of the focusing lens. ('zlens_1', float, (1,)), - # Relative stength of the lens (1. is from + # Relative strength of the lens (1. is from # back-of-the-envelope calculation) ('adjust_factor', float, (1,)), ], diff --git a/Tools/PerformanceTests/functions_perftest.py b/Tools/PerformanceTests/functions_perftest.py index 938bcb974de..8d7f4e29246 100644 --- a/Tools/PerformanceTests/functions_perftest.py +++ b/Tools/PerformanceTests/functions_perftest.py @@ -41,13 +41,13 @@ def scale_n_cell(self, n_node=0): self.n_cell = n_cell_scaled def scale_n_cell(ncell, n_node): - ncell_scaled = ncell[:] - index_dim = 0 - while n_node > 1: - ncell_scaled[index_dim] *= 2 - n_node /= 2 - index_dim = (index_dim+1) % 3 - return ncell_scaled + ncell_scaled = ncell[:] + index_dim = 0 + while n_node > 1: + ncell_scaled[index_dim] *= 2 + n_node /= 2 + index_dim = (index_dim+1) % 3 + return ncell_scaled def store_git_hash(repo_path=None, filename=None, name=None): repo = git.Repo(path=repo_path) @@ -108,7 +108,7 @@ def run_batch(run_name, res_dir, bin_name, config_command, architecture='knl',\ def run_batch_nnode(test_list, res_dir, cwd, bin_name, config_command, batch_string, submit_job_command): # Clean res_dir if os.path.exists(res_dir): - shutil.rmtree(res_dir, ignore_errors=True) + shutil.rmtree(res_dir, ignore_errors=True) os.makedirs(res_dir) # Copy files to res_dir bin_dir = cwd + 'Bin/' @@ -154,7 +154,7 @@ def read_run_perf(filename, n_steps): '\nPPC::FieldGather.*',\ '\nPPC::ParticlePush.*',\ '\nPPC::Evolve::Copy.*',\ - '\nWarpX::EvolveEM().*',\ + '\nWarpX::Evolve().*',\ 'Checkpoint().*',\ 'WriteParticles().*',\ '\nVisMF::Write(FabArray).*',\ @@ -198,9 +198,9 @@ def extract_dataframe(filename, n_steps): # New, might break something line_match_WritePlotFile = re.search('\nDiagnostics::FilterComputePackFlush().*', search_area) if line_match_WritePlotFile is not None: - time_WritePlotFile = float(line_match_WritePlotFile.group(0).split()[3]) + time_WritePlotFile = float(line_match_WritePlotFile.group(0).split()[3]) else: - time_WritePlotFile = 0. + time_WritePlotFile = 0. # Get timers for all routines # Where to start and stop in the output_file partition_limit_start = 'NCalls Excl. Min Excl. Avg Excl. Max Max %' diff --git a/Tools/PerformanceTests/performance_log.txt b/Tools/PerformanceTests/performance_log.txt index 7f2d2453466..72fece34939 100644 --- a/Tools/PerformanceTests/performance_log.txt +++ b/Tools/PerformanceTests/performance_log.txt @@ -1,4 +1,4 @@ -## year month day run_name compiler architecture n_node n_mpi n_omp time_initialization time_one_iteration Redistribute FillBoundary ParallelCopy CurrentDeposition FieldGather ParthiclePush Copy EvolveEM Checkpoint WriteParticles Write_FabArray WriteMultiLevelPlotfile(unit: second) RedistributeMPI +## year month day run_name compiler architecture n_node n_mpi n_omp time_initialization time_one_iteration Redistribute FillBoundary ParallelCopy CurrentDeposition FieldGather ParthiclePush Copy Evolve Checkpoint WriteParticles Write_FabArray WriteMultiLevelPlotfile(unit: second) RedistributeMPI 2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.14 0.3986 0.1713 0.01719 0.01615 0.06987 0.03636 0.01901 0.01999 0.003602 0 0 0 0 0.007262 2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.39 0.4009 0.1712 0.01676 0.01583 0.07061 0.03684 0.01926 0.02011 0.003687 0 0 0 0 0.007841 2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.91 0.4024 0.1716 0.01826 0.01918 0.0703 0.0363 0.01912 0.01989 0.003017 0 0 0 0 0.007256 diff --git a/Tools/PerformanceTests/run_alltests.py b/Tools/PerformanceTests/run_alltests.py index af99df194e2..b1083fc6f45 100644 --- a/Tools/PerformanceTests/run_alltests.py +++ b/Tools/PerformanceTests/run_alltests.py @@ -259,7 +259,7 @@ def process_analysis(): log_line = '## year month day run_name compiler architecture n_node n_mpi ' +\ 'n_omp time_initialization time_one_iteration Redistribute '+\ 'FillBoundary ParallelCopy CurrentDeposition FieldGather '+\ - 'ParthiclePush Copy EvolveEM Checkpoint '+\ + 'ParticlePush Copy Evolve Checkpoint '+\ 'WriteParticles Write_FabArray '+\ 'WriteMultiLevelPlotfile '+\ 'RedistributeMPI(unit: second)\n' diff --git a/Tools/PerformanceTests/run_alltests_1node.py b/Tools/PerformanceTests/run_alltests_1node.py index 54900e9e06e..f112552b36e 100644 --- a/Tools/PerformanceTests/run_alltests_1node.py +++ b/Tools/PerformanceTests/run_alltests_1node.py @@ -26,7 +26,7 @@ # > --architecture=knl --mode=run --input_file=uniform_plasma # > --n_node=1 --log_file='my_performance_log.txt' -# ---- Running the pre-drefined automated tests ---- +# ---- Running the pre-defined automated tests ---- # Compile and run: # > python run_alltests_1node.py --automated --recompile # Just run: @@ -228,7 +228,7 @@ def process_analysis(): log_line = '## year month day input_file compiler architecture n_node n_mpi ' +\ 'n_omp time_initialization time_one_iteration Redistribute '+\ 'FillBoundary ParallelCopy CurrentDeposition FieldGather '+\ - 'ParthiclePush Copy EvolveEM Checkpoint '+\ + 'ParthiclePush Copy Evolve Checkpoint '+\ 'WriteParticles Write_FabArray '+\ 'WriteMultiLevelPlotfile(unit: second) '+\ 'RedistributeMPI\n' @@ -318,7 +318,7 @@ def process_analysis(): nrepeat = 4 legends = [ 'n_node', 'n_mpi', 'n_omp', 'time_initialization', 'time_one_iteration', \ 'Redistribute', 'FillBoundary', 'ParallelCopy', 'CurrentDeposition', \ - 'FieldGather', 'ParthiclePush', 'Copy', 'EvolveEM', 'Checkpoint', \ + 'FieldGather', 'ParthiclePush', 'Copy', 'Evolve', 'Checkpoint', \ 'WriteParticles', 'Write_FabArray', 'WriteMultiLevelPlotfile', \ 'RedistributeMPI'] date = np.loadtxt( filename, usecols = np.arange(0, 3 )) diff --git a/Tools/PerformanceTests/run_automated.py b/Tools/PerformanceTests/run_automated.py index 58fc02d6ac3..73ab79b00df 100644 --- a/Tools/PerformanceTests/run_automated.py +++ b/Tools/PerformanceTests/run_automated.py @@ -87,7 +87,7 @@ parser.add_argument('--n_node_list', dest='n_node_list', default=[], - help='list ofnumber of nodes for the runs', type=str) + help='list of number of nodes for the runs', type=str) parser.add_argument('--start_date', dest='start_date' ) parser.add_argument('--compiler', diff --git a/Tools/PostProcessing/plot_distribution_mapping.py b/Tools/PostProcessing/plot_distribution_mapping.py index 35fc2c197fa..19628551f26 100644 --- a/Tools/PostProcessing/plot_distribution_mapping.py +++ b/Tools/PostProcessing/plot_distribution_mapping.py @@ -98,7 +98,7 @@ def _get_costs_reduced_diagnostics(self, directory, prange): j_non_zero = j_blocks[j_blocks != 0] k_non_zero = k_blocks[k_blocks != 0] - # only one block in a dir - or smalles block size + # only one block in a dir - or smallest block size i_blocking_factor = 1 if len(i_non_zero) == 0 else i_non_zero.min() j_blocking_factor = 1 if len(j_non_zero) == 0 else j_non_zero.min() k_blocking_factor = 1 if len(k_non_zero) == 0 else k_non_zero.min() diff --git a/Tools/PostProcessing/plot_parallel.py b/Tools/PostProcessing/plot_parallel.py index e4b75c0e406..a4309b3896e 100644 --- a/Tools/PostProcessing/plot_parallel.py +++ b/Tools/PostProcessing/plot_parallel.py @@ -239,7 +239,7 @@ def reduce_evolved_quantity(z, q): file_list.sort() nfiles = len(file_list) -# Get list of particle speciess to plot +# Get list of particle species to plot pslist = get_species(file_list); rank = 0 diff --git a/Tools/Release/updatepyAMReX.py b/Tools/Release/updatepyAMReX.py new file mode 100755 index 00000000000..8ba10c895e0 --- /dev/null +++ b/Tools/Release/updatepyAMReX.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# +# Copyright 2021 Axel Huebl +# +# This file is part of WarpX. +# + +# This file is a maintainer tool to bump the pyAMReX version that we pull in +# when building WarpX. +# +import datetime +from pathlib import Path +import re +import sys + +import requests + +# Maintainer Inputs ########################################################### + +print("""Hi there, this is a WarpX maintainer tool to update the source +code of WarpX to a new commit/release of pyAMReX. +For it to work, you need write access on the source directory and +you should be working in a clean git branch without ongoing +rebase/merge/conflict resolves and without unstaged changes.""") + +# check source dir +REPO_DIR = Path(__file__).parent.parent.parent.absolute() +print(f"\nYour current source directory is: {REPO_DIR}") + +REPLY = input("Are you sure you want to continue? [y/N] ") +print() +if not REPLY in ["Y", "y"]: + print("You did not confirm with 'y', aborting.") + sys.exit(1) + + +# Current Versions ############################################################ + +# pyAMReX development HEAD +pyamrex_gh = requests.get('https://api.github.com/repos/AMReX-Codes/pyamrex/commits/development') +pyamrex_HEAD = pyamrex_gh.json()["sha"] + +# WarpX references to pyAMReX: cmake/dependencies/pyAMReX.cmake +pyamrex_cmake_path = str(REPO_DIR.joinpath("cmake/dependencies/pyAMReX.cmake")) +# branch/commit/tag (git fetcher) version +# set(WarpX_pyamrex_branch "development" ... +pyamrex_branch = f"unknown (format issue in {pyamrex_cmake_path})" +with open(pyamrex_cmake_path, encoding='utf-8') as f: + r_minimal = re.findall(r'.*set\(WarpX_pyamrex_branch\s+"(.+)"\s+.*', + f.read(), re.MULTILINE) + if len(r_minimal) >= 1: + pyamrex_branch = r_minimal[0] + +# minimal (external) version +# find_package(AMReX YY.MM CONFIG ... +pyamrex_minimal = f"unknown (format issue in {pyamrex_cmake_path})" +with open(pyamrex_cmake_path, encoding='utf-8') as f: + r_minimal = re.findall(r'.*find_package\(pyAMReX\s+(.+)\s+CONFIG\s+.*', + f.read(), re.MULTILINE) + if len(r_minimal) >= 1: + pyamrex_minimal = r_minimal[0] + + +# Ask for new ################################################################# + +print("""We will now run a few sed commands on your source directory. +Please answer the following questions about the version number +you want to require from pyAMReX:\n""") + +print(f"Currently, WarpX builds against this pyAMReX commit/branch/sha: {pyamrex_branch}") +print(f"pyAMReX HEAD commit (development branch): {pyamrex_HEAD}") +pyamrex_new_branch = input(f"Update pyAMReX commit/branch/sha: ").strip() +if not pyamrex_new_branch: + pyamrex_new_branch = pyamrex_branch + print(f"--> Nothing entered, will keep: {pyamrex_branch}") +print() + +print(f"Currently, a pre-installed pyAMReX is required at least at version: {pyamrex_minimal}") +today = datetime.date.today().strftime("%y.%m") +pyamrex_new_minimal = input(f"New minimal pyAMReX version (e.g. {today})? ").strip() +if not pyamrex_new_minimal: + pyamrex_new_minimal = pyamrex_minimal + print(f"--> Nothing entered, will keep: {pyamrex_minimal}") + +print() +print(f"New pyAMReX commit/branch/sha: {pyamrex_new_branch}") +print(f"New minimal pyAMReX version: {pyamrex_new_minimal}\n") + +REPLY = input("Is this information correct? Will now start updating! [y/N] ") +print() +if not REPLY in ["Y", "y"]: + print("You did not confirm with 'y', aborting.") + sys.exit(1) + + +# Updates ##################################################################### + +# WarpX references to pyAMReX: cmake/dependencies/pyAMReX.cmake +with open(pyamrex_cmake_path, encoding='utf-8') as f: + pyAMReX_cmake_content = f.read() + + # branch/commit/tag (git fetcher) version + # set(WarpX_pyamrex_branch "development" ... + pyAMReX_cmake_content = re.sub( + r'(.*set\(WarpX_pyamrex_branch\s+")(.+)("\s+.*)', + r'\g<1>{}\g<3>'.format(pyamrex_new_branch), + pyAMReX_cmake_content, flags = re.MULTILINE) + + # minimal (external) version + # find_package(AMReX YY.MM CONFIG ... + pyAMReX_cmake_content = re.sub( + r'(.*find_package\(pyAMReX\s+)(.+)(\s+CONFIG\s+.*)', + r'\g<1>{}\g<3>'.format(pyamrex_new_minimal), + pyAMReX_cmake_content, flags = re.MULTILINE) + +with open(pyamrex_cmake_path, "w", encoding='utf-8') as f: + f.write(pyAMReX_cmake_content) + + +# Epilogue #################################################################### + +print("""Done. Please check your source, e.g. via + git diff +now and commit the changes if no errors occurred.""") diff --git a/Tools/machines/adastra-cines/adastra_warpx.profile.example b/Tools/machines/adastra-cines/adastra_warpx.profile.example index 49f17d395e4..23441638893 100644 --- a/Tools/machines/adastra-cines/adastra_warpx.profile.example +++ b/Tools/machines/adastra-cines/adastra_warpx.profile.example @@ -21,6 +21,8 @@ module load cray-hdf5-parallel export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH +export PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3/bin:${PATH} + # optional: for Python bindings or libEnsemble module load cray-python/3.9.13.1 diff --git a/Tools/machines/adastra-cines/install_dependencies.sh b/Tools/machines/adastra-cines/install_dependencies.sh index e532ea33924..8a4cef4a2ec 100755 --- a/Tools/machines/adastra-cines/install_dependencies.sh +++ b/Tools/machines/adastra-cines/install_dependencies.sh @@ -101,7 +101,10 @@ rm -rf ${SW_DIR}/venvs/warpx-adastra python3 -m venv ${SW_DIR}/venvs/warpx-adastra source ${SW_DIR}/venvs/warpx-adastra/bin/activate python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas diff --git a/Tools/machines/crusher-olcf/install_dependencies.sh b/Tools/machines/crusher-olcf/install_dependencies.sh index e1f196a519b..ab5deb6b0a6 100755 --- a/Tools/machines/crusher-olcf/install_dependencies.sh +++ b/Tools/machines/crusher-olcf/install_dependencies.sh @@ -86,7 +86,10 @@ rm -rf ${SW_DIR}/venvs/warpx-crusher python3 -m venv ${SW_DIR}/venvs/warpx-crusher source ${SW_DIR}/venvs/warpx-crusher/bin/activate python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 9ef077c69ea..9460f9a5175 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -86,8 +86,14 @@ rm -rf ${SW_DIR}/venvs/warpx-frontier python3 -m venv ${SW_DIR}/venvs/warpx-frontier source ${SW_DIR}/venvs/warpx-frontier/bin/activate python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade cython +python3 -m pip install --upgrade setuptools +# cupy and h5py need an older Cython +# https://github.com/cupy/cupy/issues/4610 +# https://github.com/h5py/h5py/issues/2268 +python3 -m pip install --upgrade "cython<3.0" python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas python3 -m pip install --upgrade scipy @@ -97,6 +103,14 @@ python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +# cupy for ROCm +# https://docs.cupy.dev/en/stable/install.html#building-cupy-for-rocm-from-source +# https://github.com/cupy/cupy/issues/7830 +CC=cc CXX=CC \ +CUPY_INSTALL_USE_HIP=1 \ +ROCM_HOME=${ROCM_PATH} \ +HCC_AMDGPU_TARGET=${AMREX_AMD_ARCH} \ + python3 -m pip install -v cupy # optional: for libEnsemble python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) diff --git a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example index 62733f8e70f..9f41cc7e337 100644 --- a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example +++ b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example @@ -27,6 +27,7 @@ export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3/bin:${PATH} # optional: CCache #module load ccache # missing diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index 0f8b5c9b880..9334d0a2287 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -116,7 +116,10 @@ rm -rf ${SW_DIR}/venvs/warpx-gpu python3 -m venv ${SW_DIR}/venvs/warpx-gpu source ${SW_DIR}/venvs/warpx-gpu/bin/activate python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas diff --git a/Tools/machines/karolina-it4i/cleanup.sh b/Tools/machines/karolina-it4i/cleanup.sh new file mode 100755 index 00000000000..b4d46edd1e4 --- /dev/null +++ b/Tools/machines/karolina-it4i/cleanup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf ${HOME}/.spack ${WORK}/src/warpx ${WORK}/spack ${HOME}/.local/lib/python3.11 diff --git a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh b/Tools/machines/karolina-it4i/install_cpu_dependencies.sh deleted file mode 100755 index c6e858ba8a6..00000000000 --- a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was karolina_cpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_cpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="${HOME}/sw/karolina/cpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] -then - cd $HOME/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc -fi -rm -rf $HOME/src/c-blosc-cpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-cpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-cpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build - -# ADIOS2 -if [ -d $HOME/src/adios2 ] -then - cd $HOME/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 -fi -rm -rf $HOME/src/adios2-cpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-cpu-build --target install --parallel 16 -rm -rf $HOME/src/adios2-cpu-build - -# BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] -then - cd $HOME/src/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp -fi -rm -rf $HOME/src/blaspp-cpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-cpu-build -Duse_openmp=ON -Dcpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-cpu-build - -# LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] -then - cd $HOME/src/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp -fi -rm -rf $HOME/src/lapackpp-cpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-cpu-build - - -# Python ###################################################################### -# -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade virtualenv -python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-cpu -python3 -m venv ${SW_DIR}/venvs/warpx-cpu -source ${SW_DIR}/venvs/warpx-cpu/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/cpu -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh new file mode 100755 index 00000000000..0435b5e2926 --- /dev/null +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ################################# +# +set -eu -o pipefail + +# Check: ########################################################## +# +# Was karolina_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your karolina_warpx.profile file!" + echo "Please edit its line 2 to continue!" + return +fi + +# download and activate spack +# this might take about ~ 1 hour +if [ ! -d "$WORK/spack" ] +then + git clone -c feature.manyFiles=true -b v0.21.0 https://github.com/spack/spack.git $WORK/spack + source $WORK/spack/share/spack/setup-env.sh +else + # If the directory exists, checkout v0.21.0 branch + cd $WORK/spack + git checkout v0.21.0 + git pull origin v0.21.0 + source $WORK/spack/share/spack/setup-env.sh + + # Delete spack env if present + spack env deactivate || true + spack env rm -y warpx-karolina-cuda || true + + cd - +fi + +# create and activate the spack environment +spack env create warpx-karolina-cuda $WORK/src/warpx/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml +spack env activate warpx-karolina-cuda +spack install + +# Python ########################################################## +# +python -m pip install --user --upgrade pandas +python -m pip install --user --upgrade matplotlib +# optional +#python -m pip install --user --upgrade yt + +# install or update WarpX dependencies +python -m pip install --user --upgrade picmistandard==0.28.0 +python -m pip install --user --upgrade lasy + +# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +# python -m pip install --user --upgrade -r $WORK/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh b/Tools/machines/karolina-it4i/install_gpu_dependencies.sh deleted file mode 100755 index a12092c6e01..00000000000 --- a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was karolina_gpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_gpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="${HOME}/sw/karolina/gpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] -then - cd $HOME/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc -fi -rm -rf $HOME/src/c-blosc-gpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-gpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-gpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build - -# ADIOS2 -if [ -d $HOME/src/adios2 ] -then - cd $HOME/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 -fi -rm -rf $HOME/src/adios2-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-gpu-build --target install --parallel 12 -rm -rf $HOME/src/adios2-gpu-build - -# BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] -then - cd $HOME/src/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp -fi -rm -rf $HOME/src/blaspp-gpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/blaspp-gpu-build - -# LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] -then - cd $HOME/src/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp -fi -rm -rf $HOME/src/lapackpp-gpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/lapackpp-gpu-build - - -# Python ###################################################################### -# -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade virtualenv -python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-gpu -python3 -m venv ${SW_DIR}/venvs/warpx-gpu -source ${SW_DIR}/venvs/warpx-gpu/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch # CUDA 11.7 compatible wheel -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example deleted file mode 100644 index 66b2f67a8be..00000000000 --- a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example +++ /dev/null @@ -1,58 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module load GCCcore/11.3.0 -module load CMake/3.23.1-GCCcore-11.3.0 -module load OpenMPI/4.1.4-GCC-11.3.0 - -# optional: for QED support with detailed tables -module load Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -module load OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -# optional: CCache (not found) -#module load ccache - -# optional: for Python bindings or libEnsemble -module load Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/cpu/venvs/warpx-cpu" ] -then - source ${HOME}/sw/karolina/cpu/venvs/warpx-cpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -#alias getNode="salloc -N 1 --ntasks-per-node=4 -t 1:00:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" -# an alias to run a command on a batch node for up to 30min -# usage: runNode -#alias runNode="srun -N 1 --ntasks-per-node=4 -t 0:30:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) diff --git a/Tools/machines/karolina-it4i/karolina_gpu.qsub b/Tools/machines/karolina-it4i/karolina_gpu.qsub deleted file mode 100644 index 274184ed1ca..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu.qsub +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -l - -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Authors: Axel Huebl, Andrei Berceanu -# License: BSD-3-Clause-LBNL - -#PBS -q qgpu -#PBS -N WarpX -# Use two full nodes, 8 GPUs per node, 16 GPUs total -#PBS -l select=2:ncpus=128:ngpus=8:mpiprocs=8:ompthreads=16,walltime=00:10:00 -#PBS -A - -cd ${PBS_O_WORKDIR} - -# executable & inputs file or python interpreter & PICMI script here -EXE=./warpx.rz -INPUTS=inputs_rz - -# OpenMP threads per MPI rank -export OMP_NUM_THREADS=16 - -# run -mpirun -np ${PBS_NP} bash -c " - export CUDA_VISIBLE_DEVICES=\${OMPI_COMM_WORLD_LOCAL_RANK}; - ${EXE} ${INPUTS}" \ - > output.txt diff --git a/Tools/machines/karolina-it4i/karolina_gpu.sbatch b/Tools/machines/karolina-it4i/karolina_gpu.sbatch new file mode 100644 index 00000000000..6171ff03abc --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_gpu.sbatch @@ -0,0 +1,40 @@ +#!/bin/bash -l + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +#SBATCH --account= +#SBATCH --partition=qgpu +#SBATCH --time=00:10:00 +#SBATCH --job-name=WarpX +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=16 +#SBATCH --gpus-per-node=8 +#SBATCH --gpu-bind=single:1 + +#SBATCH --mail-type=ALL +# change me! +#SBATCH --mail-user=someone@example.com +#SBATCH --chdir=/scratch/project//it4i-/runs/warpx + +#SBATCH -o stdout_%j +#SBATCH -e stderr_%j + +# OpenMP threads per MPI rank +export OMP_NUM_THREADS=16 +export SRUN_CPUS_PER_TASK=16 + +# set user rights to u=rwx;g=r-x;o=--- +umask 0027 + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx.rz +INPUTS=./inputs_rz + +# run +srun -K1 ${EXE} ${INPUTS} diff --git a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example deleted file mode 100644 index f657916dfcd..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example +++ /dev/null @@ -1,62 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module purge -ml GCCcore/11.3.0 -ml CUDA/11.7.0 -ml OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 -ml CMake/3.23.1-GCCcore-11.3.0 - -# optional: for QED support with detailed tables -ml Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -ml OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -# optional: CCache (not found) -#ml ccache - -# optional: for Python bindings or libEnsemble -ml Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/gpu/venvs/warpx-gpu" ] -then - source ${HOME}/sw/karolina/gpu/venvs/warpx-gpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -alias getNode="qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00 -I" -# an alias to run a command on a batch node for up to 1hr -# usage: runNode -alias runNode='echo -e "#!/bin/bash\nmpirun -n 4 $1" | qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00' - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/karolina_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_warpx.profile.example new file mode 100644 index 00000000000..1a4eda19a23 --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_warpx.profile.example @@ -0,0 +1,65 @@ +# please set your project account, ie DD-N-N +export proj="" # change me! + +# Name and Path of this Script ################### (DO NOT change!) +export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) + +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file!" + echo "Please edit its line 2 to continue!" + return +fi + +# set env variable storing the path to the work directory +# please check if your project ID belongs to proj1, proj2, proj3 etc +export WORK="/mnt/proj/${proj,,}/${USER}" # change me! +mkdir -p WORK + +# clone warpx +# you can also clone your own fork here, eg git@github.com:/WarpX.git +if [ ! -d "$WORK/src/warpx" ] +then + git clone https://github.com/ECP-WarpX/WarpX.git $WORK/src/warpx +fi + +# load required modules +module purge +module load OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + +source $WORK/spack/share/spack/setup-env.sh && spack env activate warpx-karolina-cuda && { + echo "Spack environment 'warpx-karolina-cuda' activated successfully." +} || { + echo "Failed to activate Spack environment 'warpx-karolina-cuda'. Please run install_dependencies.sh." +} + +# Text Editor for Tools ########################## (edit this line) +# examples: "nano", "vim", "emacs -nw" or without terminal: "gedit" +#export EDITOR="nano" # change me! + +# allocate an interactive shell for one hour +# usage: getNode 2 # allocates two interactive nodes (default: 1) +function getNode() { + if [ -z "$1" ] ; then + numNodes=1 + else + numNodes=$1 + fi + export OMP_NUM_THREADS=16 + srun --time=1:00:00 --nodes=$numNodes --ntasks=$((8 * $numNodes)) --ntasks-per-node=8 --cpus-per-task=16 --exclusive --gpus-per-node=8 -p qgpu -A $proj --pty bash +} + +# Environment ##################################################### +# optimize CUDA compilation for A100 +export AMREX_CUDA_ARCH="8.0" +export SCRATCH="/scratch/project/${proj,,}/${USER}" + +# optimize CPU microarchitecture for AMD EPYC 7763 (zen3) +export CFLAGS="-march=znver3" +export CXXFLAGS="-march=znver3" + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml new file mode 100644 index 00000000000..1cb6a4ac209 --- /dev/null +++ b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml @@ -0,0 +1,80 @@ +spack: + specs: +# Karolina's openssl version is deprecated + - openssl certs=system + - pkgconfig + - ccache + - cmake@3.26.5 + - cuda@11.7.0 + - openmpi@4.1.4 +atomics + - fftw + - hdf5@1.14.0 + - adios2@2.9.2 ~mgard + - blaspp + - lapackpp + - boost@1.81.0 +program_options +atomic ~python + - python@3.11.6 + - py-pip + - py-setuptools + - py-wheel + - py-cython + - py-mpi4py + - py-numpy@1.24.2 + - openpmd-api@0.15.2 +python + - py-periodictable@1.5.0 + - py-h5py +# optional +# - py-libensemble +nlopt + + packages: + openssh: + externals: + - spec: openssh@7.4p1 + prefix: /usr + buildable: False + cuda: + externals: + - spec: cuda@11.7.0 + modules: + - CUDA/11.7.0 + buildable: False + mpi: + buildable: False + openmpi: + externals: + - spec: openmpi@4.1.4 +atomics +cuda %gcc@11.3.0 + modules: + - OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + libfabric: + externals: + - spec: libfabric@1.15.1 %gcc@11.3.0 + modules: + - libfabric/1.15.1-GCCcore-11.3.0 + buildable: False + all: + target: [zen3] + compiler: [gcc@11.3.0] + variants: +mpi ~fortran +cuda cuda_arch=80 + providers: + mpi: [openmpi@4.1.4] + cuda: [cuda@11.7.0] + + compilers: + - compiler: + modules: [GCCcore/11.3.0] + operating_system: centos7 + paths: + cc: /apps/all/GCCcore/11.3.0/bin/gcc + cxx: /apps/all/GCCcore/11.3.0/bin/g++ + f77: /apps/all/GCCcore/11.3.0/bin/gfortran + fc: /apps/all/GCCcore/11.3.0/bin/gfortran + spec: gcc@=11.3.0 + target: x86_64 + flags: {} + environment: {} + extra_rpaths: [] + + view: true + concretizer: + reuse: false + unify: true diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh index 21650f09ee0..fe8285b7501 100755 --- a/Tools/machines/lassen-llnl/install_v100_dependencies.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies.sh @@ -20,9 +20,11 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # Remove old dependencies ##################################################### # +SRC_DIR="/usr/workspace/${USER}/lassen/src" SW_DIR="/usr/workspace/${USER}/lassen/gpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} +mkdir -p ${SRC_DIR} # remove common user mistakes in python, located in .local instead of a venv python3 -m pip uninstall -qq -y pywarpx @@ -37,94 +39,103 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true build_dir=$(mktemp -d) # c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] +if [ -d ${SRC_DIR}/c-blosc ] then - cd $HOME/src/c-blosc + cd ${SRC_DIR}/c-blosc git fetch --prune git checkout v1.21.1 cd - else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc fi -cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake -S ${SRC_DIR}/c-blosc -B ${build_dir}/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 cmake --build ${build_dir}/c-blosc-lassen-build --target install --parallel 10 # HDF5 -if [ -d $HOME/src/hdf5 ] +if [ -d ${SRC_DIR}/hdf5 ] then - cd $HOME/src/hdf5 + cd ${SRC_DIR}/hdf5 git fetch --prune git checkout hdf5-1_14_1-2 cd - else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git $HOME/src/hdf5 + git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5 fi -cmake -S $HOME/src/hdf5 -B ${build_dir}/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 +cmake -S ${SRC_DIR}/hdf5 -B ${build_dir}/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 cmake --build ${build_dir}/hdf5-lassen-build --target install --parallel 10 # ADIOS2 -if [ -d $HOME/src/adios2 ] +if [ -d ${SRC_DIR}/adios2 ] then - cd $HOME/src/adios2 + cd ${SRC_DIR}/adios2 git fetch --prune git checkout v2.8.3 cd - else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 fi -cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake -S ${SRC_DIR}/adios2 -B ${build_dir}/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 cmake --build ${build_dir}/adios2-lassen-build --target install -j 10 # BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] +if [ -d ${SRC_DIR}/blaspp ] then - cd $HOME/src/blaspp + cd ${SRC_DIR}/blaspp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi -cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] +if [ -d ${SRC_DIR}/lapackpp ] then - cd $HOME/src/lapackpp + cd ${SRC_DIR}/lapackpp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 # Python ###################################################################### # +# sometimes, the Lassen PIP Index is down +export PIP_EXTRA_INDEX_URL="https://pypi.org/simple" + python3 -m pip install --upgrade --user virtualenv rm -rf ${SW_DIR}/venvs/warpx-lassen python3 -m venv ${SW_DIR}/venvs/warpx-lassen source ${SW_DIR}/venvs/warpx-lassen/bin/activate python3 -m pip install --upgrade pip python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade cython +python3 -m pip install --upgrade setuptools +# Older version for h4py +# https://github.com/h5py/h5py/issues/2268 +python3 -m pip install --upgrade "cython<3" python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas python3 -m pip install --upgrade -Ccompile-args="-j10" scipy python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py python3 -m pip install --upgrade openpmd-api +CC=mpicc H5PY_SETUP_REQUIRES=0 HDF5_DIR=${SW_DIR}/hdf5-1.14.1.2 HDF5_MPI=ON python3 -m pip install --upgrade h5py --no-cache-dir --no-build-isolation --no-binary h5py MPLLOCALFREETYPE=1 python3 -m pip install --upgrade matplotlib==3.2.2 # does not try to build freetype itself echo "matplotlib==3.2.2" > ${build_dir}/constraints.txt python3 -m pip install --upgrade -c ${build_dir}/constraints.txt yt # install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt # for ML dependencies, see install_v100_ml.sh diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh new file mode 100644 index 00000000000..916986ee119 --- /dev/null +++ b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was lassen_v100_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lassen_v100_warpx_toss3.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +SRC_DIR="/usr/workspace/${USER}/lassen-toss3/src" +SW_DIR="/usr/workspace/${USER}/lassen-toss3/gpu" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} +mkdir -p ${SRC_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + + +# General extra dependencies ################################################## +# + +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + +# c-blosc (I/O compression) +if [ -d ${SRC_DIR}/c-blosc ] +then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - +else + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi +cmake -S ${SRC_DIR}/c-blosc -B ${build_dir}/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-lassen-build --target install --parallel 10 + +# HDF5 +if [ -d ${SRC_DIR}/hdf5 ] +then + cd ${SRC_DIR}/hdf5 + git fetch --prune + git checkout hdf5-1_14_1-2 + cd - +else + git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5 +fi +cmake -S ${SRC_DIR}/hdf5 -B ${build_dir}/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 +cmake --build ${build_dir}/hdf5-lassen-build --target install --parallel 10 + +# ADIOS2 +if [ -d ${SRC_DIR}/adios2 ] +then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.8.3 + cd - +else + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +cmake -S ${SRC_DIR}/adios2 -B ${build_dir}/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-lassen-build --target install -j 10 + +# BLAS++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/blaspp ] +then + cd ${SRC_DIR}/blaspp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp +fi +cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/lapackpp ] +then + cd ${SRC_DIR}/lapackpp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp +fi +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so +cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 + + +# Python ###################################################################### +# +# sometimes, the Lassen PIP Index is down +export PIP_EXTRA_INDEX_URL="https://pypi.org/simple" + +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/warpx-lassen-toss3 +python3 -m venv ${SW_DIR}/venvs/warpx-lassen-toss3 +source ${SW_DIR}/venvs/warpx-lassen-toss3/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +# Older version for h4py +# https://github.com/h5py/h5py/issues/2268 +python3 -m pip install --upgrade "cython<3" +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +CMAKE_PREFIX_PATH=/usr/lib64:${CMAKE_PREFIX_PATH} python3 -m pip install --upgrade -Ccompile-args="-j10" -Csetup-args=-Dblas=BLAS -Csetup-args=-Dlapack=BLAS scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +CC=mpicc H5PY_SETUP_REQUIRES=0 HDF5_DIR=${SW_DIR}/hdf5-1.14.1.2 HDF5_MPI=ON python3 -m pip install --upgrade h5py --no-cache-dir --no-build-isolation --no-binary h5py +MPLLOCALFREETYPE=1 python3 -m pip install --upgrade matplotlib==3.2.2 # does not try to build freetype itself +echo "matplotlib==3.2.2" > ${build_dir}/constraints.txt +python3 -m pip install --upgrade -c ${build_dir}/constraints.txt yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt + +# for ML dependencies, see install_v100_ml.sh + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/lassen-llnl/install_v100_ml.sh b/Tools/machines/lassen-llnl/install_v100_ml.sh index 2ff90adb521..72a4d04f2f1 100755 --- a/Tools/machines/lassen-llnl/install_v100_ml.sh +++ b/Tools/machines/lassen-llnl/install_v100_ml.sh @@ -20,6 +20,9 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # Remove old dependencies ##################################################### # +SRC_DIR="/usr/workspace/${USER}/lassen/src" +mkdir -p ${SRC_DIR} + # remove common user mistakes in python, located in .local instead of a venv python3 -m pip uninstall -qqq -y torch 2>/dev/null || true @@ -29,21 +32,21 @@ python3 -m pip uninstall -qqq -y torch 2>/dev/null || true # for basic python dependencies, see install_v100_dependencies.sh # optional: for libEnsemble - WIP: issues with nlopt -# python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +# python3 -m pip install -r ${SRC_DIR}/warpx/Tools/LibEnsemble/requirements.txt # optional: for pytorch -if [ -d ${HOME}/src/pytorch ] +if [ -d ${SRC_DIR}/pytorch ] then - cd ${HOME}/src/pytorch + cd ${SRC_DIR}/pytorch git fetch git checkout . git checkout v2.0.1 git submodule update --init --recursive cd - else - git clone -b v2.0.1 --recurse-submodules https://github.com/pytorch/pytorch.git ${HOME}/src/pytorch + git clone -b v2.0.1 --recurse-submodules https://github.com/pytorch/pytorch.git ${SRC_DIR}/pytorch fi -cd ${HOME}/src/pytorch +cd ${SRC_DIR}/pytorch rm -rf build # see https://github.com/pytorch/pytorch/issues/97497#issuecomment-1499069641 @@ -61,6 +64,7 @@ rm -rf build cd - # optional: optimas dependencies (based on libEnsemble & ax->botorch->gpytorch->pytorch) +# TODO: scikit-learn needs a BLAS hint # commented because scikit-learn et al. compile > 2 hrs # please run manually on a login node if needed -#python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt +#python3 -m pip install -r ${SRC_DIR}/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example index dd938e85c11..4d4efbf917d 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example @@ -10,6 +10,7 @@ module load cuda/12.0.0 module load boost/1.70.0 # optional: for openPMD support +SRC_DIR="/usr/workspace/${USER}/lassen/src" SW_DIR="/usr/workspace/${USER}/lassen/gpu" export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH @@ -17,6 +18,8 @@ export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:$LD_LIBRARY_PATH +export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} +export PATH=${SW_DIR}/adios2-2.8.3/bin:$PATH # optional: for PSATD in RZ geometry support export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example new file mode 100644 index 00000000000..19c74348a99 --- /dev/null +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example @@ -0,0 +1,56 @@ +# please set your project account +#export proj="" # edit this and comment in + +# required dependencies +module load cmake/3.23.1 +module load gcc/11.2.1 +module load cuda/12.0.0 + +# optional: for QED lookup table generation support +module load boost/1.70.0 + +# optional: for openPMD support +SRC_DIR="/usr/workspace/${USER}/lassen-toss3/src" +SW_DIR="/usr/workspace/${USER}/lassen-toss3/gpu" +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:$LD_LIBRARY_PATH +export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} +export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} + +# optional: for PSATD in RZ geometry support +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH + +# optional: for Python bindings +module load python/3.8.2 + +if [ -d "${SW_DIR}/venvs/warpx-lassen-toss3" ] +then + source ${SW_DIR}/venvs/warpx-lassen-toss3/bin/activate +fi + +# optional: an alias to request an interactive node for two hours +alias getNode="bsub -G $proj -W 2:00 -nnodes 1 -Is /bin/bash" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="bsub -q debug -P $proj -W 2:00 -nnodes 1 -I" + +# fix system defaults: do not escape $ with a \ on tab completion +shopt -s direxpand + +# optimize CUDA compilation for V100 +export AMREX_CUDA_ARCH=7.0 +export CUDAARCHS=70 + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example index 9b9bf709ecc..472d0785bb2 100644 --- a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example +++ b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example @@ -21,6 +21,8 @@ export CMAKE_PREFIX_PATH=$HOME/sw/v100/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/v100/blaspp-master:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/v100/lapackpp-master:$CMAKE_PREFIX_PATH +export PATH=$HOME/sw/v100/adios2-2.8.3/bin:$PATH + # optional: CCache #module load ccache # missing diff --git a/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh b/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh new file mode 100644 index 00000000000..bbaf0ab8464 --- /dev/null +++ b/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, Marta Galbiati +# License: BSD-3-Clause-LBNL + +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was leonardo_gpu_warpx.profile sourced and configured correctly? +# + + +# Remove old dependencies ##################################################### +# +SW_DIR="$HOME/sw" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + + +# General extra dependencies ################################################## +# + +# ADIOS2 +if [ -d $HOME/src/adios2 ] +then + cd $HOME/src/adios2 + git fetch + git checkout master + git pull + cd - +else + git clone https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 +fi +rm -rf $HOME/src/adios2-gpu-build +cmake -S $HOME/src/adios2 -B $HOME/src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-master +cmake --build $HOME/src/adios2-gpu-build --target install -j 16 +rm -rf $HOME/src/adios2-gpu-build + + +# BLAS++ (for PSATD+RZ) +if [ -d $HOME/src/blaspp ] +then + cd $HOME/src/blaspp + git fetch + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp +fi +rm -rf $HOME/src/blaspp-gpu-build +CXX=$(which g++) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build $HOME/src/blaspp-gpu-build --target install --parallel 16 +rm -rf $HOME/src/blaspp-gpu-build + + +# LAPACK++ (for PSATD+RZ) +if [ -d $HOME/src/lapackpp ] +then + cd $HOME/src/lapackpp + git fetch + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp +fi +rm -rf $HOME/src/lapackpp-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build $HOME/src/lapackpp-gpu-build --target install --parallel 16 +rm -rf $HOME/src/lapackpp-gpu-build + + +# Python ###################################################################### +# +rm -rf ${SW_DIR}/venvs/warpx +python3 -m venv ${SW_DIR}/venvs/warpx +source ${SW_DIR}/venvs/warpx/bin/activate +python3 -m ensurepip --upgrade +python3 -m pip cache purge +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +MPICC="gcc -shared" python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +# optional: for libEnsemble +python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade torch # CUDA 11.8 compatible wheel +python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/leonardo-cineca/job.sh b/Tools/machines/leonardo-cineca/job.sh new file mode 100644 index 00000000000..a50c6e1995f --- /dev/null +++ b/Tools/machines/leonardo-cineca/job.sh @@ -0,0 +1,20 @@ +#!/usr/bin/bash +#SBATCH --time=02:00:00 +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=4 +#SBATCH --ntasks-per-socket=4 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus-per-node=4 +#SBATCH --gpus-per-task=1 +#SBATCH --mem=494000 +#SBATCH --partition=boost_usr_prod +#SBATCH --job-name= +#SBATCH --gres=gpu:4 +#SBATCH --err=job.err +#SBATCH --out=job.out +#SBATCH --account= +#SBATCH --mail-type=ALL +#SBATCH --mail-user= + +cd /leonardo_scratch/large/userexternal// +srun /leonardo/home/userexternal//src/warpx/build_gpu/bin/warpx.2d > output.txt diff --git a/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example b/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example new file mode 100644 index 00000000000..cffe565f9d7 --- /dev/null +++ b/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example @@ -0,0 +1,45 @@ +# required dependencies +module load profile/base +module load cmake/3.24.3 +module load gmp/6.2.1 +module load mpfr/4.1.0 +module load mpc/1.2.1 +module load gcc/11.3.0 +module load cuda/11.8 +module load zlib/1.2.13--gcc--11.3.0 +module load openmpi/4.1.4--gcc--11.3.0-cuda-11.8 + +# optional: for QED support with detailed tables +module load boost/1.80.0--openmpi--4.1.4--gcc--11.3.0 + +# optional: for openPMD and PSATD+RZ support +module load openblas/0.3.21--gcc--11.3.0 +export CMAKE_PREFIX_PATH=/leonardo/prod/spack/03/install/0.19/linux-rhel8-icelake/gcc-11.3.0/c-blosc-1.21.1-aifmix6v5lwxgt7rigwoebalrgbcnv26:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/adios2-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-master:$CMAKE_PREFIX_PATH + +export LD_LIBRARY_PATH=/leonardo/prod/spack/03/install/0.19/linux-rhel8-icelake/gcc-11.3.0/c-blosc-1.21.1-aifmix6v5lwxgt7rigwoebalrgbcnv26/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/adios2-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/lapackpp-master/lib64:$LD_LIBRARY_PATH + +export PATH=$HOME/sw/adios2-master/bin:$PATH + +# optional: for Python bindings or libEnsemble +module load python/3.10.8--gcc--11.3.0 + +if [ -d "$HOME/sw/venvs/warpx" ] +then + source $HOME/sw/venvs/warpx/bin/activate +fi + +# optimize CUDA compilation for A100 +export AMREX_CUDA_ARCH=8.0 + +# compiler environment hints +export CXX=$(which g++) +export CC=$(which gcc) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/lumi-csc/install_dependencies.sh b/Tools/machines/lumi-csc/install_dependencies.sh index 1cb1afb4fc4..0e466f1f57f 100755 --- a/Tools/machines/lumi-csc/install_dependencies.sh +++ b/Tools/machines/lumi-csc/install_dependencies.sh @@ -101,7 +101,10 @@ rm -rf ${SW_DIR}/venvs/warpx-lumi python3 -m venv ${SW_DIR}/venvs/warpx-lumi source ${SW_DIR}/venvs/warpx-lumi/bin/activate python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 31c133ed47a..74b8aa8df17 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -22,6 +22,7 @@ module load Boost/1.81.0-cpeCray-23.03 module load cray-hdf5/1.12.2.3 export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH +export PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble module load cray-python/3.9.13.1 diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index 6bfa47bc380..dbe59438a16 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -116,7 +116,10 @@ rm -rf ${SW_DIR}/venvs/warpx python3 -m venv ${SW_DIR}/venvs/warpx source ${SW_DIR}/venvs/warpx/bin/activate python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index a60c3ba7594..8dd64c56c58 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -116,7 +116,10 @@ rm -rf ${SW_DIR}/venvs/warpx python3 -m venv ${SW_DIR}/venvs/warpx source ${SW_DIR}/venvs/warpx/bin/activate python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas @@ -127,6 +130,7 @@ python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install cupy-cuda11x # CUDA 11.7 compatible wheel # optional: for libEnsemble python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index ab5a0c963b8..0150ded4839 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -11,10 +11,10 @@ module load cmake/3.22.0 module load cray-fftw/3.3.10.3 # optional: for QED support with detailed tables -export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-22.11/83104/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.80.0-ute7nbx4wmfrw53q7hyh6wyezub5ljry +export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 # optional: for openPMD and PSATD+RZ support -module load cray-hdf5-parallel/1.12.2.1 +module load cray-hdf5-parallel/1.12.2.7 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH @@ -25,8 +25,10 @@ export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} + # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-22.05/78535/spack/opt/spack/cray-sles15-zen3/gcc-11.2.0/ccache-4.5.1-ybl7xefvggn6hov4dsdxxnztji74tolj/bin:$PATH +export PATH=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8-eqk2d3bipbpkgwxq7ujlp6mckwal4dwz/bin:$PATH # optional: for Python bindings or libEnsemble module load cray-python/3.9.13.1 diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 09d6a61b458..13ab2ead605 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -9,10 +9,10 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo module load cmake/3.22.0 # optional: for QED support with detailed tables -export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-22.11/83104/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.80.0-ute7nbx4wmfrw53q7hyh6wyezub5ljry +export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 # optional: for openPMD and PSATD+RZ support -module load cray-hdf5-parallel/1.12.2.1 +module load cray-hdf5-parallel/1.12.2.7 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH @@ -23,8 +23,10 @@ export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/ export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} + # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-22.05/78535/spack/opt/spack/cray-sles15-zen3/gcc-11.2.0/ccache-4.5.1-ybl7xefvggn6hov4dsdxxnztji74tolj/bin:$PATH +export PATH=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8-eqk2d3bipbpkgwxq7ujlp6mckwal4dwz/bin:$PATH # optional: for Python bindings or libEnsemble module load cray-python/3.9.13.1 diff --git a/Tools/machines/quartz-llnl/install_dependencies.sh b/Tools/machines/quartz-llnl/install_dependencies.sh index c162ac5d390..2920b00663b 100755 --- a/Tools/machines/quartz-llnl/install_dependencies.sh +++ b/Tools/machines/quartz-llnl/install_dependencies.sh @@ -99,7 +99,10 @@ python3 -m venv ${SW_DIR}/venvs/warpx-quartz source ${SW_DIR}/venvs/warpx-quartz/bin/activate python3 -m pip install --upgrade pip python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas diff --git a/Tools/machines/quartz-llnl/quartz_warpx.profile.example b/Tools/machines/quartz-llnl/quartz_warpx.profile.example index 810005bafb7..a3646bfd557 100644 --- a/Tools/machines/quartz-llnl/quartz_warpx.profile.example +++ b/Tools/machines/quartz-llnl/quartz_warpx.profile.example @@ -18,6 +18,7 @@ module load hdf5-parallel/1.14.0 SW_DIR="/usr/workspace/${USER}/quartz" export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH +export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for PSATD in RZ geometry support export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH diff --git a/Tools/machines/summit-olcf/install_gpu_dependencies.sh b/Tools/machines/summit-olcf/install_gpu_dependencies.sh index 6bdcda92378..433b7ab0580 100755 --- a/Tools/machines/summit-olcf/install_gpu_dependencies.sh +++ b/Tools/machines/summit-olcf/install_gpu_dependencies.sh @@ -99,7 +99,10 @@ rm -rf ${SW_DIR}/venvs/warpx-summit python3 -m venv ${SW_DIR}/venvs/warpx-summit source ${SW_DIR}/venvs/warpx-summit/bin/activate python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas diff --git a/Tools/machines/summit-olcf/summit_warpx.profile.example b/Tools/machines/summit-olcf/summit_warpx.profile.example index 5d88bb9aeed..e41521b5815 100644 --- a/Tools/machines/summit-olcf/summit_warpx.profile.example +++ b/Tools/machines/summit-olcf/summit_warpx.profile.example @@ -11,7 +11,7 @@ module load nano # required dependencies module load cmake/3.20.2 module load gcc/9.3.0 -module load cuda/11.3.1 +module load cuda/11.7.1 # optional: faster re-builds module load ccache diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 8d873655fe0..886ee9c951f 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -1,4 +1,10 @@ macro(find_amrex) + # if pyAMReX is external, AMReX must be as well + if(DEFINED WarpX_pyamrex_internal AND NOT WarpX_pyamrex_internal) + set(WarpX_amrex_internal OFF CACHE BOOL + "Download & build AMReX" FORCE) + endif() + if(WarpX_amrex_src) message(STATUS "Compiling local AMReX ...") message(STATUS "AMReX source path: ${WarpX_amrex_src}") @@ -10,6 +16,7 @@ macro(find_amrex) message(STATUS "AMReX repository: ${WarpX_amrex_repo} (${WarpX_amrex_branch})") include(FetchContent) endif() + if(WarpX_amrex_internal OR WarpX_amrex_src) set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) @@ -243,7 +250,12 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 23.09 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} TINYP LSOLVERS) + find_package(AMReX 23.12 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + # note: TINYP skipped because user-configured and optional + + # AMReX CMake helper scripts + list(APPEND CMAKE_MODULE_PATH "${AMReX_DIR}/AMReXCMakeModules") + message(STATUS "AMReX: Found version '${AMReX_VERSION}'") endif() endmacro() @@ -257,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "b98bdae9fb67e5d9aafc488de92c53001bd323ec" +set(WarpX_amrex_branch "75571e2dcbf2417529c5ed8e24113580e8e1f3f1" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/FFT.cmake b/cmake/dependencies/FFT.cmake index 56dc396b31f..571006e8530 100644 --- a/cmake/dependencies/FFT.cmake +++ b/cmake/dependencies/FFT.cmake @@ -1,4 +1,4 @@ -if(WarpX_PSATD) +if(ABLASTR_FFT) # Helper Functions ############################################################ # option(WarpX_FFTW_IGNORE_OMP "Ignore FFTW3 OpenMP support, even if found" OFF) @@ -122,4 +122,4 @@ if(WarpX_PSATD) message(STATUS "FFTW: Did NOT search for OpenMP support (WarpX_COMPUTE!=OMP)") endif() endif() -endif(WarpX_PSATD) +endif(ABLASTR_FFT) diff --git a/cmake/dependencies/PICSAR.cmake b/cmake/dependencies/PICSAR.cmake index 8400dc06a4b..a1e93a9e33c 100644 --- a/cmake/dependencies/PICSAR.cmake +++ b/cmake/dependencies/PICSAR.cmake @@ -85,7 +85,7 @@ function(find_picsar) #message(STATUS "PICSAR: Using version '${PICSAR_VERSION}'") else() # not supported by PICSAR (yet) - #find_package(PICSAR 23.09 CONFIG REQUIRED QED) + #find_package(PICSAR 23.11 CONFIG REQUIRED QED) #message(STATUS "PICSAR: Found version '${PICSAR_VERSION}'") message(FATAL_ERROR "PICSAR: Cannot be used as externally installed " "library yet. " @@ -106,7 +106,7 @@ if(WarpX_QED) set(WarpX_picsar_repo "https://github.com/ECP-WarpX/picsar.git" CACHE STRING "Repository URI to pull and build PICSAR from if(WarpX_picsar_internal)") - set(WarpX_picsar_branch "23.09" + set(WarpX_picsar_branch "aa54e985398c1d575abc7e6737cdbc660a13765f" CACHE STRING "Repository branch for WarpX_picsar_repo if(WarpX_picsar_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index a36ef59d1f9..2ad63917965 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,8 +64,8 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 23.07 CONFIG REQUIRED) - message(STATUS "pyAMReX: Found version '${pyamrex_VERSION}'") + find_package(pyAMReX 23.12 CONFIG REQUIRED) + message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "development" +set(WarpX_pyamrex_branch "eb24d03fac522d36fb27d20c6e026d8b59dfa908" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index 8d097d869ff..e65cd206f34 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -37,7 +37,7 @@ function(find_pybind11) mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) endif() else() - find_package(pybind11 2.10.1 CONFIG REQUIRED) + find_package(pybind11 2.11.1 CONFIG REQUIRED) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") endif() endfunction() @@ -52,7 +52,7 @@ option(WarpX_pybind11_internal "Download & build pybind11" ON) set(WarpX_pybind11_repo "https://github.com/pybind/pybind11.git" CACHE STRING "Repository URI to pull and build pybind11 from if(WarpX_pybind11_internal)") -set(WarpX_pybind11_branch "v2.10.1" +set(WarpX_pybind11_branch "v2.11.1" CACHE STRING "Repository branch for WarpX_pybind11_repo if(WarpX_pybind11_internal)") diff --git a/pyproject.toml b/pyproject.toml index d7096f82d01..f53522b5622 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ requires = [ "setuptools>=42", "wheel", - "cmake>=3.20.0,<4.0.0" + "cmake>=3.20.0,<4.0.0", + "packaging>=23", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 7d2aa5376ad..60f14e1282d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ periodictable~=1.5 # PICMI # note: don't forget to update the version in Docs/requirements.txt, too -picmistandard==0.26.0 +picmistandard==0.28.0 # for development against an unreleased PICMI version, use: #picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python diff --git a/run_test.sh b/run_test.sh index 37b5d370667..deb4939da51 100755 --- a/run_test.sh +++ b/run_test.sh @@ -61,17 +61,14 @@ echo "cd $PWD" rm -rf py-venv python3 -m venv py-venv source py-venv/bin/activate -python3 -m pip install --upgrade pip setuptools wheel +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build packaging setuptools wheel python3 -m pip install --upgrade cmake -# setuptools/mp4py work-around, see -# https://github.com/mpi4py/mpi4py/pull/159 -# https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274 -export SETUPTOOLS_USE_DISTUTILS="stdlib" python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach b98bdae9fb67e5d9aafc488de92c53001bd323ec && cd - +cd amrex && git checkout --detach 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index 91c8f1ac828..5228b7b3521 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,3 @@ -from distutils.command.build import build -from distutils.command.clean import clean -from distutils.version import LooseVersion import os import platform import re @@ -9,6 +6,7 @@ import sys from setuptools import Extension, setup +from setuptools.command.build import build from setuptools.command.build_ext import build_ext @@ -26,10 +24,8 @@ def run(self): # by default, this stays around. we want to make sure generated # files like libwarpx.(1d|2d|rz|3d).(so|pyd) are always only the # ones we want to package and not ones from an earlier wheel's stage - c = clean(self.distribution) - c.all = True - c.finalize_options() - c.run() + if os.path.exists(self.build_base): + shutil.rmtree(self.build_base) # call superclass build.run(self) @@ -59,6 +55,8 @@ def __init__(self, name, sourcedir=''): class CMakeBuild(build_ext): def run(self): + from packaging.version import parse + try: out = subprocess.check_output(['cmake', '--version']) except OSError: @@ -67,11 +65,8 @@ def run(self): "extensions: " + ", ".join(e.name for e in self.extensions)) - cmake_version = LooseVersion(re.search( - r'version\s*([\d.]+)', - out.decode() - ).group(1)) - if cmake_version < '3.20.0': + cmake_version = parse(re.search(r"version\s*([\d.]+)", out.decode()).group(1)) + if cmake_version < parse("3.20.0"): raise RuntimeError("CMake >= 3.20.0 is required") for ext in self.extensions: @@ -283,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '23.09', + version = '23.12', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.',