From 1f0afdca5e1dee4a4fd7ddd0fd82f17c85acd92b Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 7 Nov 2023 16:08:52 -0500 Subject: [PATCH 01/33] Build libROM on Quartz (#208) * add libROM to CMakelists.txt * Add shell_script * Minor changes * Add module file --- CMakeLists.txt | 24 ++++++++++++++++ scripts/build_quartz_libROM.sh | 52 ++++++++++++++++++++++++++++++++++ scripts/modules.quartz | 14 +++++++++ 3 files changed, 90 insertions(+) create mode 100644 scripts/build_quartz_libROM.sh create mode 100644 scripts/modules.quartz diff --git a/CMakeLists.txt b/CMakeLists.txt index 177d29bf..f1deb89c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,6 +123,30 @@ if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) endif(${SCALAPACK_FOUND}) endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) +# libROM (optional) +set(USE_LIBROM False CACHE BOOL "Build with libROM") +set(LIBROM_PATH "" CACHE STRING "Path of libROM") +if(USE_LIBROM) + if(NOT LIBROM_PATH) + message(FATAL_ERROR "libROM PATH not specified.") + else(NOT LIBROM_PATH) + set(LIBROM_SCRIPTS_PATH ${LIBROM_PATH}/scripts) + message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") + message(STATUS "LIBROM_SCRIPTS_PATH: ${LIBROM_SCRIPTS_PATH}") + include(ExternalProject) + ExternalProject_Add( + libROM + SOURCE_DIR ${LIBROM_SCRIPTS_PATH} + CONFIGURE_COMMAND "" + BINARY_DIR ${LIBROM_PATH} + BUILD_COMMAND ${LIBROM_SCRIPTS_PATH}/compile.sh -t ${LIBROM_PATH}/cmake/toolchains/simple.cmake + INSTALL_COMMAND "" + ) + include_directories(${LIBROM_PATH}/lib) + link_directories(${LIBROM_PATH}/build/lib) + endif(NOT LIBROM_PATH) +endif(USE_LIBROM) + # ARPACK (optional) set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package") if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT) diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh new file mode 100644 index 00000000..dc2b689b --- /dev/null +++ b/scripts/build_quartz_libROM.sh @@ -0,0 +1,52 @@ +##! /bin/csh -f +## An example script to build on LLNL Peloton systems. +## For now, this script assumes intel/ mkl libraries are being used. + +# load some modules +source scripts/modules.quartz + +# set some environment variables. Set them explicitly or use loaded module path (preferred) +# Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid +# benign cmake warnings +#setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib +setenv SCALAPACK_ROOT ${MKLROOT} +setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 + +# We need to define the cmake blas vendor option here to find the right one. +set BLAS_VENDOR = Intel10_64lp + +# manually set the location of BLACS libraries for scalapack +set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 + +set MGMOL_ROOT = `pwd` + +set INSTALL_DIR = ${MGMOL_ROOT}/install_quartz +mkdir -p ${INSTALL_DIR} + +set BUILD_DIR = ${MGMOL_ROOT}/build_quartz +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# clone the libROM GitHub repo in BUILD_DIR +set USE_LIBROM="On" +set LIBROM_PATH = ${BUILD_DIR}/libROM +git clone https://github.com/LLNL/libROM + +# call cmake +cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_CXX_COMPILER=mpic++ \ + -DCMAKE_Fortran_COMPILER=mpif77 \ + -DMPIEXEC_NUMPROC_FLAG="-n" \ + -DBLA_VENDOR=${BLAS_VENDOR} \ + -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ + -DUSE_LIBROM=${USE_LIBROM} \ + -DLIBROM_PATH=${LIBROM_PATH} \ + -DCMAKE_BUILD_TYPE=DEBUG \ + .. + +# call make install +make -j +make install + +# -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ +# -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/scripts/modules.quartz b/scripts/modules.quartz new file mode 100644 index 00000000..220e0a31 --- /dev/null +++ b/scripts/modules.quartz @@ -0,0 +1,14 @@ +module load intel/2022.1.0 +module load hdf5-parallel/1.14.0 +#module load hdf5-parallel +#module load boost +module load mkl +module load cmake/3.14.5 +module load python +## manually add boost path +setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH + +#setenv MKLROOT $LIBRARY_PATH +#setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 +#setenv HDF5ROOT $LD_LIBRARY_PATH +#setenv HDF5ROOT From f7e1f22410e972823848e986a7b448d17717c1ae Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Tue, 7 Nov 2023 13:16:41 -0800 Subject: [PATCH 02/33] github ci workflow and docker image building (#215) * Update script to analyze forces on constraints * initial ci workflow files. * default branch name release. * create build directory for compilation. * minor fix --------- Co-authored-by: Jean-Luc Fattebert --- .github/workflows/ci.yml | 47 ++++++++++++++++++++++++ .github/workflows/docker_image.yml | 59 ++++++++++++++++++++++++++++++ docker/Dockerfile | 38 +++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docker_image.yml create mode 100644 docker/Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..a88d7b48 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI +on: + workflow_dispatch: {} + pull_request: + types: [opened, labeled, synchronize] + branches: + - release + # push: + # branches: + # - release + +jobs: + docker-image: + uses: ./.github/workflows/docker_image.yml + build: + runs-on: ubuntu-latest + needs: [docker-image] + container: + image: ghcr.io/llnl/mgmol/mgmol_env:latest + options: --user 1001 --privileged + volumes: + - /mnt:/mnt + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ github.token }} + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 10 + - name: Check out pylibROM + uses: actions/checkout@v1 + with: + submodules: 'true' + - name: Git Submodules status + run: | + git submodule status + - name: cmake + run: | + mkdir build + cd build + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 + - name: make + run: | + cd build && make -j 4 + diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml new file mode 100644 index 00000000..90a2447b --- /dev/null +++ b/.github/workflows/docker_image.yml @@ -0,0 +1,59 @@ +name: docker-image +on: + workflow_call: + +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: llnl/mgmol/mgmol_env + DOCKERPATH: docker + +jobs: + docker-ci: + runs-on: ubuntu-latest + name: "docker env" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: Ana06/get-changed-files@v2.2.0 + id: files + - name: DockerPATH configuration + run: echo "DOCKERPATH=$DOCKERPATH" + - name: DockerPATH - check if files in docker path changed + if: contains(steps.files.outputs.all,env.DOCKERPATH) || contains(steps.files.outputs.all,'docker_image.yml') + run: | + echo "CI container needs rebuilding..." + echo "CI_NEEDS_REBUILD=true" >> $GITHUB_ENV + - name: Log into registry ${{ env.REGISTRY }} + if: env.CI_NEEDS_REBUILD + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + if: env.CI_NEEDS_REBUILD + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: type=sha + flavor: latest=true + - name: Build Container motd + if: env.CI_NEEDS_REBUILD + run: | + echo "#!/bin/bash" > ${{env.DOCKERPATH}}/motd.sh + echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo mgmol_env/CI Development Container" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo \"Revision: `echo ${GITHUB_SHA} | cut -c1-8`\"" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh + chmod 755 ${{env.DOCKERPATH}}/motd.sh + cat ${{env.DOCKERPATH}}/motd.sh + - name: Docker Image - Build and push + if: env.CI_NEEDS_REBUILD + uses: docker/build-push-action@v3 + with: + push: true + context: ${{ env.DOCKERPATH }} + tags: ${{ steps.meta.outputs.tags }} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..4ccd50ac --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:22.04 + +ENV ENVDIR=env + +# install sudo +RUN apt-get -yq update && apt-get -yq install sudo + +WORKDIR /$ENVDIR + +# install packages +RUN sudo apt-get install -yq git +RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpich-dev +RUN sudo apt-get install -yq libboost-all-dev +RUN sudo apt-get install -yq vim +RUN sudo apt-get install -yq git-lfs +RUN sudo apt-get install -yq valgrind +RUN sudo apt-get install -yq wget +RUN sudo apt-get install -yq astyle + +# install lldb and gdb for debugging +RUN sudo apt-get install -yq lldb gdb + +RUN sudo apt-get clean -q + +# NOTE: currently docker does not have a way to set environment variable with a command output. +# The following environment variable should use $(uname -m) for different architecture. +# ENV SCALAPACK_ROOT=/usr/lib/aarch64-linux-gnu/ +ENV SCALAPACK_ROOT=/usr/lib/x86_64-linux-gnu/ + +# create and switch to a user +ENV USERNAME=test +RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME +RUN adduser $USERNAME sudo +USER $USERNAME +WORKDIR /home/$USERNAME + From 5ec2dbedd62a3f674c3ce1fcaaf90950dcf01410 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Tue, 9 Apr 2024 05:40:11 -0700 Subject: [PATCH 03/33] Base for librom-mgmol interface development (#219) * changed the Signal.h header file name. * additional routine to find scalapack library, if nothing is found. * ci workflow with test and format * changed branch name * ctest verbose * added --oversubscribe to cmake MPIEXEC_PREFLAGS variable. * parallel hdf5 package fix. also librom is included. * ci workflow only checks the first 20 tests. * minor fix * rom-fpmd driver and librom dependency in cmake. librom must be compiled before mgmol cmake. * cmake fix and toolchain file for LC quartz. * bug fix on FindSCALAPACK.cmake * librom-mgmol installation script for LC quartz. --- .github/workflows/ci.yml | 45 ++++++++--- CMakeLists.txt | 20 +---- cmake_modules/FindSCALAPACK.cmake | 21 ++++- cmake_modules/FindlibROM.cmake | 11 +++ cmake_toolchains/quartz.default.cmake | 6 ++ docker/Dockerfile | 24 ++++-- scripts/build_quartz_libROM.sh | 69 +++++++++-------- scripts/build_ubuntu22_openmpi.sh | 2 +- scripts/modules.quartz | 21 +++-- src/CMakeLists.txt | 21 +++++ src/MGmol.cc | 2 +- src/md.cc | 2 +- src/rom_main.cc | 102 +++++++++++++++++++++++++ src/tools/{Signal.h => Signal_mgmol.h} | 4 +- src/tools/Timeout.h | 2 +- 15 files changed, 274 insertions(+), 78 deletions(-) create mode 100644 cmake_modules/FindlibROM.cmake create mode 100644 cmake_toolchains/quartz.default.cmake create mode 100644 src/rom_main.cc rename src/tools/{Signal.h => Signal_mgmol.h} (94%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a88d7b48..05942636 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: pull_request: types: [opened, labeled, synchronize] branches: - - release + - 'ROMFPMD' # push: # branches: # - release @@ -25,23 +25,48 @@ jobs: uses: styfle/cancel-workflow-action@0.11.0 with: access_token: ${{ github.token }} - - name: Set Swap Space - uses: pierotofy/set-swap-space@master - with: - swap-size-gb: 10 - - name: Check out pylibROM + # - name: Set Swap Space + # uses: pierotofy/set-swap-space@master + # with: + # swap-size-gb: 10 + - name: Check out mgmol uses: actions/checkout@v1 with: submodules: 'true' - - name: Git Submodules status - run: | - git submodule status - name: cmake run: | mkdir build cd build - cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" - name: make run: | cd build && make -j 4 + - name: test + run: | + cd build && ctest --no-compress-output -V -T Test -I 1,20,1 + # code-style: + # runs-on: ubuntu-latest + # needs: [docker-image] + # container: + # image: ghcr.io/llnl/mgmol/mgmol_env:latest + # options: --user 1001 --privileged + # volumes: + # - /mnt:/mnt + # steps: + # - name: Cancel previous runs + # uses: styfle/cancel-workflow-action@0.11.0 + # with: + # access_token: ${{ github.token }} + # - name: Check out mgmol + # uses: actions/checkout@v1 + # with: + # submodules: 'true' + # - name: cmake + # run: | + # mkdir build + # cd build + # cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON + # - name: make + # run: | + # cd build && make format diff --git a/CMakeLists.txt b/CMakeLists.txt index f1deb89c..6898f97b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,24 +127,12 @@ endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) set(USE_LIBROM False CACHE BOOL "Build with libROM") set(LIBROM_PATH "" CACHE STRING "Path of libROM") if(USE_LIBROM) + message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") if(NOT LIBROM_PATH) - message(FATAL_ERROR "libROM PATH not specified.") - else(NOT LIBROM_PATH) - set(LIBROM_SCRIPTS_PATH ${LIBROM_PATH}/scripts) - message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") - message(STATUS "LIBROM_SCRIPTS_PATH: ${LIBROM_SCRIPTS_PATH}") - include(ExternalProject) - ExternalProject_Add( - libROM - SOURCE_DIR ${LIBROM_SCRIPTS_PATH} - CONFIGURE_COMMAND "" - BINARY_DIR ${LIBROM_PATH} - BUILD_COMMAND ${LIBROM_SCRIPTS_PATH}/compile.sh -t ${LIBROM_PATH}/cmake/toolchains/simple.cmake - INSTALL_COMMAND "" - ) - include_directories(${LIBROM_PATH}/lib) - link_directories(${LIBROM_PATH}/build/lib) + message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.") endif(NOT LIBROM_PATH) + + find_package(libROM REQUIRED) endif(USE_LIBROM) # ARPACK (optional) diff --git a/cmake_modules/FindSCALAPACK.cmake b/cmake_modules/FindSCALAPACK.cmake index dde12c26..74e58e7d 100644 --- a/cmake_modules/FindSCALAPACK.cmake +++ b/cmake_modules/FindSCALAPACK.cmake @@ -7,7 +7,7 @@ if(DEFINED ENV{SCALAPACK_ROOT}) endif(DEFINED ENV{SCALAPACK_ROOT}) if(SCALAPACK_ROOT) - set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT}) + set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT} ${SCALAPACK_ROOT}/lib/intel64) list(APPEND _SCALAPACK_SEARCHES ${_SCALAPACK_SEARCH_DIR}) endif(SCALAPACK_ROOT) @@ -29,13 +29,28 @@ if(NOT SCALAPACK_LIBRARY) endforeach() endif() -unset(SCALAPACK_NAMES) - mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR) include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) +# Search for some default library paths +if (NOT SCALAPACK_FOUND) + find_library(SCALAPACK_LIBRARY + NAMES ${SCALAPACK_NAMES} + PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib + /opt/local/lib /opt/sw/lib /sw/lib + ENV LD_LIBRARY_PATH + ENV DYLD_FALLBACK_LIBRARY_PATH + ENV DYLD_LIBRARY_PATH + ENV SCALAPACKDIR + ENV BLACSDIR) + + FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) +endif() + +unset(SCALAPACK_NAMES) + if(SCALAPACK_FOUND) # Only Intel's scalapack requires an include directory if(SCALAPACK_INCLUDE_DIR) diff --git a/cmake_modules/FindlibROM.cmake b/cmake_modules/FindlibROM.cmake new file mode 100644 index 00000000..9fed8fb6 --- /dev/null +++ b/cmake_modules/FindlibROM.cmake @@ -0,0 +1,11 @@ +if(NOT LIBROM_PATH) + message(FATAL_ERROR "LIBROM_PATH not specified.") +endif(NOT LIBROM_PATH) + +find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib") +find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib") + +mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES) \ No newline at end of file diff --git a/cmake_toolchains/quartz.default.cmake b/cmake_toolchains/quartz.default.cmake new file mode 100644 index 00000000..9901dcd6 --- /dev/null +++ b/cmake_toolchains/quartz.default.cmake @@ -0,0 +1,6 @@ +set(CMAKE_C_COMPILER mpicc) +set(CMAKE_CXX_COMPILER mpicxx) +set(CMAKE_Fortran_COMPILER mpif90) + +set(SCALAPACK_ROOT $ENV{MKLROOT}) +set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4ccd50ac..a097df9b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,23 +10,35 @@ WORKDIR /$ENVDIR # install packages RUN sudo apt-get install -yq git RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake -RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpich-dev +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev RUN sudo apt-get install -yq libboost-all-dev RUN sudo apt-get install -yq vim RUN sudo apt-get install -yq git-lfs RUN sudo apt-get install -yq valgrind RUN sudo apt-get install -yq wget -RUN sudo apt-get install -yq astyle +### clang-format seems to be updated to 14.0. Not using it for now. +# RUN sudo apt-get install -yq clang-format # install lldb and gdb for debugging RUN sudo apt-get install -yq lldb gdb RUN sudo apt-get clean -q -# NOTE: currently docker does not have a way to set environment variable with a command output. -# The following environment variable should use $(uname -m) for different architecture. -# ENV SCALAPACK_ROOT=/usr/lib/aarch64-linux-gnu/ -ENV SCALAPACK_ROOT=/usr/lib/x86_64-linux-gnu/ +ENV LIB_DIR=/$ENVDIR/dependencies +WORKDIR $LIB_DIR + +# cmake toolchain file for librom +RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\ +set(CMAKE_CXX_COMPILER mpicxx)\n\ +set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake +ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake + +# install libROM for scaleupROM +RUN sudo git clone https://github.com/LLNL/libROM.git +WORKDIR ./libROM/build +# libROM without MFEM. +RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF +RUN sudo make -j 16 # create and switch to a user ENV USERNAME=test diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh index dc2b689b..458dd662 100644 --- a/scripts/build_quartz_libROM.sh +++ b/scripts/build_quartz_libROM.sh @@ -1,52 +1,61 @@ -##! /bin/csh -f +#!/bin/bash ## An example script to build on LLNL Peloton systems. ## For now, this script assumes intel/ mkl libraries are being used. -# load some modules +## load some modules source scripts/modules.quartz -# set some environment variables. Set them explicitly or use loaded module path (preferred) -# Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid -# benign cmake warnings -#setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib -setenv SCALAPACK_ROOT ${MKLROOT} -setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 - -# We need to define the cmake blas vendor option here to find the right one. -set BLAS_VENDOR = Intel10_64lp - -# manually set the location of BLACS libraries for scalapack -set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 - -set MGMOL_ROOT = `pwd` - -set INSTALL_DIR = ${MGMOL_ROOT}/install_quartz +## set some environment variables. Set them explicitly or use loaded module path (preferred) +## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid +## benign cmake warnings +##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib +#setenv SCALAPACK_ROOT ${MKLROOT} +#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 +# +## We need to define the cmake blas vendor option here to find the right one. +#set BLAS_VENDOR = Intel10_64lp +# +## manually set the location of BLACS libraries for scalapack +#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 + +MGMOL_ROOT="$(pwd)" + +INSTALL_DIR=${MGMOL_ROOT}/install_quartz mkdir -p ${INSTALL_DIR} -set BUILD_DIR = ${MGMOL_ROOT}/build_quartz +BUILD_DIR=${MGMOL_ROOT}/build_quartz mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} # clone the libROM GitHub repo in BUILD_DIR -set USE_LIBROM="On" -set LIBROM_PATH = ${BUILD_DIR}/libROM +USE_LIBROM="On" +LIBROM_PATH=${BUILD_DIR}/libROM git clone https://github.com/LLNL/libROM +cd libROM +#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake +./scripts/compile.sh +cd ${BUILD_DIR} # call cmake -cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ - -DCMAKE_CXX_COMPILER=mpic++ \ - -DCMAKE_Fortran_COMPILER=mpif77 \ - -DMPIEXEC_NUMPROC_FLAG="-n" \ - -DBLA_VENDOR=${BLAS_VENDOR} \ - -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DUSE_LIBROM=${USE_LIBROM} \ -DLIBROM_PATH=${LIBROM_PATH} \ - -DCMAKE_BUILD_TYPE=DEBUG \ .. +# -DCMAKE_CXX_COMPILER=mpic++ \ +# -DCMAKE_Fortran_COMPILER=mpif77 \ +# -DMPIEXEC_NUMPROC_FLAG="-n" \ +# -DBLA_VENDOR=${BLAS_VENDOR} \ +# -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +# -DCMAKE_BUILD_TYPE=DEBUG \ + # call make install -make -j -make install +make -j 16 +### Currently libROM does not have the installation procedure, +### so copying binary file to installation directory will disrupt the relative path to libROM.so, +### causing a run-time error. +#make install # -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ # -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/scripts/build_ubuntu22_openmpi.sh b/scripts/build_ubuntu22_openmpi.sh index 23f510b6..a0ca470a 100755 --- a/scripts/build_ubuntu22_openmpi.sh +++ b/scripts/build_ubuntu22_openmpi.sh @@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ + -DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ .. # call make install diff --git a/scripts/modules.quartz b/scripts/modules.quartz index 220e0a31..82433acb 100644 --- a/scripts/modules.quartz +++ b/scripts/modules.quartz @@ -1,12 +1,19 @@ -module load intel/2022.1.0 +### choose either gcc or intel +#module load intel/2022.1.0 +module load gcc/11.2.1 + +module load cmake module load hdf5-parallel/1.14.0 -#module load hdf5-parallel -#module load boost -module load mkl -module load cmake/3.14.5 +module load boost + +### choose either one +module load mkl-interfaces +#module load mkl + module load python -## manually add boost path -setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH + +### manually add boost path +#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH #setenv MKLROOT $LIBRARY_PATH #setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b7aec664..1575135f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -193,3 +193,24 @@ endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-opt DESTINATION bin) +# build ROM executable +if(USE_LIBROM) + add_executable(mgmol-rom rom_main.cc) + target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS} ${LIBROM_INCLUDES}) + + target_link_libraries(mgmol-rom mgmol_src ${link_libs}) + target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_HL_LIBRARIES}) + target_link_libraries(mgmol-rom ${BLAS_LIBRARIES}) + target_link_libraries(mgmol-rom ${LAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${Boost_LIBRARIES}) + target_link_libraries(mgmol-rom ${LIBROM_LIB}) + if (${OPENMP_CXX_FOUND}) + target_link_libraries(mgmol-rom OpenMP::OpenMP_CXX) + endif() + if(${MGMOL_WITH_LIBXC}) + target_link_libraries(mgmol-rom ${LIBXC_DIR}/lib/libxc.a) + endif (${MGMOL_WITH_LIBXC}) + install(TARGETS mgmol-rom DESTINATION bin) +endif(USE_LIBROM) diff --git a/src/MGmol.cc b/src/MGmol.cc index b43c70e3..0d1703d9 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -106,7 +106,7 @@ extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; extern Timer updateCenters_tm; -#include "Signal.h" +#include "Signal_mgmol.h" std::set Signal::recv_; template diff --git a/src/md.cc b/src/md.cc index 1adab46b..6b43048e 100644 --- a/src/md.cc +++ b/src/md.cc @@ -31,7 +31,7 @@ #include "ProjectedMatricesMehrstellen.h" #include "ProjectedMatricesSparse.h" #include "Rho.h" -#include "Signal.h" +#include "Signal_mgmol.h" #include "SpreadsAndCenters.h" #include "tools.h" diff --git a/src/rom_main.cc b/src/rom_main.cc new file mode 100644 index 00000000..c7ec278e --- /dev/null +++ b/src/rom_main.cc @@ -0,0 +1,102 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// +// main.cc +// +// Description: +// Real grid, finite difference, molecular dynamics program +// for with nonorthogonal localized orbitals. +// +// Uses Mehrstellen operators, multigrid accelerations, and +// non-local pseudopotentials. +// +// Includes LDA and PBE exchange and correlation functionals. +// +// Units: +// Potentials, eigenvalues and operators in Rydberg +// Energies in Hartree +// +#include +#include +#include +#include +using namespace std; + +#ifdef _OPENMP +#include +#endif + +#ifdef USE_CNR +#include +#endif + +#include + +#include "Control.h" +#include "DistMatrix.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "MatricesBlacsContext.h" +#include "Mesh.h" +#include "PackedCommunicationBuffer.h" +#include "ReplicatedWorkSpace.h" +#include "SparseDistMatrix.h" +#include "magma_singleton.h" +#include "tools.h" + +#include +#include +#include + +#include +namespace po = boost::program_options; + +#include "librom.h" + +//#include "MemTrack.h" + +int main(int argc, char** argv) +{ + // change handling of memory allocation errors + set_new_handler(noMoreMemory); + + cout.sync_with_stdio(); + + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Initialization failed!!!" << endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + MPI_Comm_rank(MPI_COMM_WORLD, &mype); + assert(mype > -1); + onpe0 = (mype == 0); + + CAROM::Vector librom_vector(10, false); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Finalize failed!!!" << endl; + } + + time_t tt; + time(&tt); + if (onpe0) cout << " Run ended at " << ctime(&tt) << endl; + + // MemTrack::TrackDumpBlocks(); + + // MemTrack::TrackListMemoryUsage(); + + return 0; +} diff --git a/src/tools/Signal.h b/src/tools/Signal_mgmol.h similarity index 94% rename from src/tools/Signal.h rename to src/tools/Signal_mgmol.h index 8e8d0a21..245a1ca2 100644 --- a/src/tools/Signal.h +++ b/src/tools/Signal_mgmol.h @@ -7,7 +7,7 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// Adapted from Jeep: Signal.h,v 1.5 2002/06/28 20:50:33 +// Adapted from Jeep: Signal_mgmol.h,v 1.5 2002/06/28 20:50:33 // The Signal class is a utility to catch UNIX signals. // A set of flags is maintained to remeber which signals were caught. @@ -15,7 +15,7 @@ // access the flag set, reset flags, or interrogate flags. // The Signal class can be used in an application by declaring // -// #include "Signal.h" +// #include "Signal_mgmol.h" // set Signal::recv_; // // A signal can be registered using, e.g. diff --git a/src/tools/Timeout.h b/src/tools/Timeout.h index 18940a78..db60d719 100644 --- a/src/tools/Timeout.h +++ b/src/tools/Timeout.h @@ -17,7 +17,7 @@ #include #include "MPIdata.h" -#include "Signal.h" +#include "Signal_mgmol.h" #if PCS #include From b436fe5fa586110581cffecc8d8ab481ee5ebab9 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Thu, 2 May 2024 13:02:57 -0700 Subject: [PATCH 04/33] Update ROMFPMD branch (#233) * Fix build with HDF5P on (#231) * LoadOrbitalsFromRestartFile and miscellaneous changes (#226) * Signal.h -> mgmol_Signal.h * Carbyne example files. * fixed FindSCALAPACK.cmake to search for default paths * MGmol::loadOrbitalFromRestartFile * nullptr initialization --------- Co-authored-by: Jean-Luc Fattebert --- CMakeLists.txt | 5 +- examples/Carbyne/carbyne.cfg | 26 ++++++ examples/Carbyne/carbyne.in | 14 ++++ scripts/build_condo-mod.sh | 1 + src/Ions.cc | 26 +++--- src/MGmol.cc | 2 +- src/MGmol.h | 4 +- src/OrbitalsExtrapolation.h | 2 +- src/md.cc | 86 +++++++++++++++++++- src/tools/Timeout.h | 2 +- src/tools/{Signal_mgmol.h => mgmol_Signal.h} | 4 +- 11 files changed, 151 insertions(+), 21 deletions(-) create mode 100644 examples/Carbyne/carbyne.cfg create mode 100644 examples/Carbyne/carbyne.in rename src/tools/{Signal_mgmol.h => mgmol_Signal.h} (94%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6898f97b..859d3377 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,6 @@ else() #search for HDF5 if(MGMOL_USE_HDF5P) message(STATUS "Use HDF5 parallel capability") set(HDF5_PREFER_PARALLEL True) - add_definitions(-DMGMOL_USE_HDF5P) endif() message(STATUS "HDF5_ROOT: ${HDF5_ROOT}") find_package(HDF5 REQUIRED COMPONENTS C HL) @@ -73,6 +72,10 @@ else() #search for HDF5 message(FATAL_ERROR "Required HDF5 package not found.") endif (${HDF5_FOUND}) endif() +if(MGMOL_USE_HDF5P) + add_definitions(-DMGMOL_USE_HDF5P) +endif() + set(MGMOL_WITH_LIBXC FALSE CACHE BOOL "Compile with LIBXC") if(${MGMOL_WITH_LIBXC}) diff --git a/examples/Carbyne/carbyne.cfg b/examples/Carbyne/carbyne.cfg new file mode 100644 index 00000000..d153e6f6 --- /dev/null +++ b/examples/Carbyne/carbyne.cfg @@ -0,0 +1,26 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=200 +atol=1.e-8 +[Orbitals] +initial_type=Fourier +[Restart] +output_level=3 diff --git a/examples/Carbyne/carbyne.in b/examples/Carbyne/carbyne.in new file mode 100644 index 00000000..9f4e975a --- /dev/null +++ b/examples/Carbyne/carbyne.in @@ -0,0 +1,14 @@ +H00 1 -0.0000 -0.0000 15.2674 +C01 2 -0.0000 0.0000 13.2519 +C02 2 -0.0000 0.0000 10.9495 +C03 2 -0.0000 -0.0000 8.4221 +C04 2 0.0000 0.0000 6.0897 +C05 2 -0.0000 0.0000 3.5892 +C06 2 -0.0000 -0.0000 1.2470 +C07 2 0.0000 -0.0000 -1.2469 +C08 2 0.0000 -0.0000 -3.5891 +C09 2 -0.0000 -0.0000 -6.0897 +C10 2 -0.0000 0.0000 -8.4221 +C11 2 0.0000 -0.0000 -10.9495 +C12 2 0.0000 0.0000 -13.2520 +H13 1 0.0000 0.0000 -15.2675 diff --git a/scripts/build_condo-mod.sh b/scripts/build_condo-mod.sh index f7be0c3b..d9abe034 100755 --- a/scripts/build_condo-mod.sh +++ b/scripts/build_condo-mod.sh @@ -21,6 +21,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DCMAKE_CXX_COMPILER=mpiCC \ -DCMAKE_Fortran_COMPILER=mpif77 \ -DBLA_VENDOR=${BLAS_VENDOR} \ + -DMGMOL_USE_HDF5P=OFF \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ -DSCALAPACK_LIBRARY="${SCALAPACK_DIR}/lib/libscalapack.a;/lib64/libgfortran.so.3" \ diff --git a/src/Ions.cc b/src/Ions.cc index ca5120ab..e2cc9916 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -47,9 +47,10 @@ double Ions::max_Vl_radius_ = -1.; double Ions::max_Vnl_radius_ = -1.; template -void writeData2d(hid_t file_id, std::string datasetname, std::vector& data, - const int n, T element) +void writeData2d(HDFrestart& h5f_file, std::string datasetname, + std::vector& data, const size_t n, T element) { + hid_t file_id = h5f_file.file_id(); #ifdef MGMOL_USE_HDF5P if (h5f_file.useHdf5p()) { @@ -67,7 +68,7 @@ void writeData2d(hid_t file_id, std::string datasetname, std::vector& data, else #endif { - size_t dims[2] = { data.size()/n, n }; + size_t dims[2] = { data.size() / n, n }; mgmol_tools::write2d(file_id, datasetname, data, dims); } } @@ -753,7 +754,7 @@ void Ions::writeAtomicNumbers(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Atomic_numbers"); - writeData2d(file_id, datasetname, data, 1, -1); + writeData2d(h5f_file, datasetname, data, 1, -1); } } @@ -788,12 +789,11 @@ void Ions::writeAtomNames(HDFrestart& h5f_file) // write data hid_t file_id = h5f_file.file_id(); - if (file_id >= 0) { std::string datasetname("/Atomic_names"); std::string empty; - writeData2d(file_id, datasetname, data, 1, empty); + writeData2d(h5f_file, datasetname, data, 1, empty); } } @@ -864,7 +864,7 @@ void Ions::writeLockedAtomNames(HDFrestart& h5f_file) { std::string datasetname("/LockedAtomsNames"); std::string empty; - writeData2d(file_id, datasetname, data, 1, empty); + writeData2d(h5f_file, datasetname, data, 1, empty); } } @@ -900,7 +900,7 @@ void Ions::writeAtomicIDs(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Atomic_IDs"); - writeData2d(file_id, datasetname, data, 1, -1); + writeData2d(h5f_file, datasetname, data, 1, -1); } } @@ -937,7 +937,7 @@ void Ions::writeAtomicNLprojIDs(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/AtomicNLproj_IDs"); - writeData2d(file_id, datasetname, data, 1, -1); + writeData2d(h5f_file, datasetname, data, 1, -1); } } @@ -975,7 +975,7 @@ void Ions::writePositions(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Ionic_positions"); - writeData2d(file_id, datasetname, data, 3, 1.e32); + writeData2d(h5f_file, datasetname, data, 3, 1.e32); } } @@ -1138,7 +1138,7 @@ void Ions::writeVelocities(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Ionic_velocities"); - writeData2d(file_id, datasetname, data, 3, 1.e32); + writeData2d(h5f_file, datasetname, data, 3, 1.e32); } } @@ -1184,7 +1184,7 @@ void Ions::writeRandomStates(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Ionic_RandomStates"); - writeData2d(file_id, datasetname, data, 3, (unsigned short)0); + writeData2d(h5f_file, datasetname, data, 3, (unsigned short)0); } } @@ -1343,7 +1343,7 @@ void Ions::writeForces(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Ionic_forces"); - writeData2d(file_id, datasetname, data, 3, 1.e32); + writeData2d(h5f_file, datasetname, data, 3, 1.e32); } } diff --git a/src/MGmol.cc b/src/MGmol.cc index 0d1703d9..0af62ebc 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -106,7 +106,7 @@ extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; extern Timer updateCenters_tm; -#include "Signal_mgmol.h" +#include "mgmol_Signal.h" std::set Signal::recv_; template diff --git a/src/MGmol.h b/src/MGmol.h index 23b507c4..e63e7069 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -102,7 +102,7 @@ class MGmol : public MGmolInterface double total_energy_; ConstraintSet* constraints_; - OrbitalsExtrapolation* orbitals_extrapol_; + OrbitalsExtrapolation* orbitals_extrapol_ = nullptr; float md_time_; int md_iteration_; @@ -301,6 +301,8 @@ class MGmol : public MGmolInterface { forces_->force(orbitals, ions); } + + OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); }; // Instantiate static variables here to avoid clang warnings template diff --git a/src/OrbitalsExtrapolation.h b/src/OrbitalsExtrapolation.h index 9f2348f7..47c1a21b 100644 --- a/src/OrbitalsExtrapolation.h +++ b/src/OrbitalsExtrapolation.h @@ -44,7 +44,7 @@ class OrbitalsExtrapolation virtual short getNumOrbitalExtrapolations() { return 0; } protected: - OrbitalsType* orbitals_minus1_; + OrbitalsType* orbitals_minus1_ = nullptr; }; #endif diff --git a/src/md.cc b/src/md.cc index 6b43048e..ec288839 100644 --- a/src/md.cc +++ b/src/md.cc @@ -31,7 +31,7 @@ #include "ProjectedMatricesMehrstellen.h" #include "ProjectedMatricesSparse.h" #include "Rho.h" -#include "Signal_mgmol.h" +#include "mgmol_Signal.h" #include "SpreadsAndCenters.h" #include "tools.h" @@ -674,5 +674,89 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) delete orbitals_extrapol_; } +template +OrbitalsType* MGmol::loadOrbitalFromRestartFile(const std::string filename) +{ + MGmol_MPI& mmpi(*(MGmol_MPI::instance())); + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + /* For now, we only consider double-precision hdf5 I/O. */ + assert(ct.restart_info > 3); + assert((ct.AtomsDynamic() == AtomsDynamicType::MD) || (ct.AtomsDynamic() == AtomsDynamicType::Quench)); + + HDFrestart h5file(filename, myPEenv, ct.out_restart_file_type); + int ierr; + + OrbitalsType *restart_orbitals = nullptr; + + /* This corresponds to MGmol::initial */ + { + // Copy from current orbital, instead of constructing brand-new one + restart_orbitals = new OrbitalsType("ForLoading", *current_orbitals_, false); + + /* This corresponds to MGmol::read_restart_data */ + { + ierr = restart_orbitals->read_func_hdf5(h5file); + } // read_restart_data + } // initial() + + /* This corresponds to MGmol::md */ + { + int flag_extrapolated_data = 0; + if (onpe0) + { + flag_extrapolated_data + = h5file.dset_exists("ExtrapolatedFunction0000"); + if (flag_extrapolated_data == 0) + flag_extrapolated_data + = h5file.dset_exists("ExtrapolatedFunction0"); + } + MPI_Bcast(&flag_extrapolated_data, 1, MPI_INT, 0, comm_); + + /* + If extrapolated function exists, + then function is set as previous orbitals, + while the extrapolated function is set as the current orbitals. + This is how the restart file is saved via dumprestartFile. + */ + if (flag_extrapolated_data) + { + orbitals_extrapol_ = OrbitalsExtrapolationFactory::create( + ct.WFExtrapolation()); + + if (onpe0) os_ << "Create new orbitals_minus1..." << std::endl; + + orbitals_extrapol_->setupPreviousOrbitals(&restart_orbitals, + proj_matrices_, lrs_, local_cluster_, currentMasks_, + corrMasks_, h5file); + } + + /* main workflow delete h5f_file_ here, meaning the loading is over. */ + } // md() + + ierr = h5file.close(); + mmpi.allreduce(&ierr, 1, MPI_MIN); + if (ierr < 0) + { + if (onpe0) + (*MPIdata::serr) + << "loadRestartFile: cannot close file..." << std::endl; + return nullptr; + } + + /* + In returning restart_orbitals, + we hope that the wavefunctions in restart_orbitals are all set. + At least the following functions should return proper data loaded from the file: + + restart_orbitals->getLocNumpt() + restart_orbitals->chromatic_number() + restart_orbitals->getPsi(idx) (for int idx) + */ + return restart_orbitals; +} + template class MGmol; template class MGmol; diff --git a/src/tools/Timeout.h b/src/tools/Timeout.h index db60d719..25751b2a 100644 --- a/src/tools/Timeout.h +++ b/src/tools/Timeout.h @@ -17,7 +17,7 @@ #include #include "MPIdata.h" -#include "Signal_mgmol.h" +#include "mgmol_Signal.h" #if PCS #include diff --git a/src/tools/Signal_mgmol.h b/src/tools/mgmol_Signal.h similarity index 94% rename from src/tools/Signal_mgmol.h rename to src/tools/mgmol_Signal.h index 245a1ca2..33c47b3e 100644 --- a/src/tools/Signal_mgmol.h +++ b/src/tools/mgmol_Signal.h @@ -7,7 +7,7 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// Adapted from Jeep: Signal_mgmol.h,v 1.5 2002/06/28 20:50:33 +// Adapted from Jeep: Signal.h,v 1.5 2002/06/28 20:50:33 // The Signal class is a utility to catch UNIX signals. // A set of flags is maintained to remeber which signals were caught. @@ -15,7 +15,7 @@ // access the flag set, reset flags, or interrogate flags. // The Signal class can be used in an application by declaring // -// #include "Signal_mgmol.h" +// #include "mgmol_Signal.h" // set Signal::recv_; // // A signal can be registered using, e.g. From 4ae35994862453bb0e7badf6230b11bcc88b17bd Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Thu, 2 May 2024 13:33:25 -0700 Subject: [PATCH 05/33] Parsing ROM related options (#228) * librom compilation setup. * rom config options. * parsing rom config option to Control. --- .gitignore | 1 + CMakeLists.txt | 7 +++++ src/CMakeLists.txt | 14 +++++++-- src/Control.cc | 67 ++++++++++++++++++++++++++++++++++++++++++ src/Control.h | 12 ++++++++ src/MGmol_prototypes.h | 6 ++++ src/mgmol_config.h.in | 19 ++++++++++++ src/read_config.cc | 24 +++++++++++++++ src/rom_Control.h | 42 ++++++++++++++++++++++++++ 9 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 .gitignore create mode 100644 src/mgmol_config.h.in create mode 100644 src/rom_Control.h diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..81c6b154 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +src/mgmol_config.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 859d3377..75752a3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,10 @@ if(USE_LIBROM) endif(NOT LIBROM_PATH) find_package(libROM REQUIRED) + + if(libROM_FOUND) + set(MGMOL_HAS_LIBROM 1) + endif(libROM_FOUND) endif(USE_LIBROM) # ARPACK (optional) @@ -261,6 +265,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra") include_directories("${PROJECT_SOURCE_DIR}/src/tools") include_directories("${PROJECT_SOURCE_DIR}/src") +include_directories("${LIBROM_PATH}/lib") +link_libraries(${LIBROM_LIB}) + # add subdirectories for source files, tests add_subdirectory(src) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1575135f..962c163c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,7 @@ +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h.in + ${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h @ONLY) + add_subdirectory(DistMatrix) add_subdirectory(linear_algebra) add_subdirectory(local_matrices) @@ -8,7 +11,8 @@ add_subdirectory(radial) add_subdirectory(sparse_linear_algebra) add_subdirectory(tools) -set(link_libs mgmol_distmatrix +set(link_libs + mgmol_distmatrix mgmol_linear_algebra mgmol_local_matrices mgmol_numerical_kernels @@ -161,7 +165,11 @@ add_library(mgmol_src ${SOURCES}) target_include_directories(mgmol_src PRIVATE ${HDF5_INCLUDE_DIRS}) target_include_directories(mgmol_src PRIVATE ${Boost_INCLUDE_DIRS}) -target_include_directories (mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +if(USE_LIBROM) + target_include_directories (mgmol_src PRIVATE ${LIBROM_INCLUDES}) + target_link_libraries(mgmol_src ${LIBROM_LIB}) +endif(USE_LIBROM) target_link_libraries(mgmol_src ${link_libs}) if(${MGMOL_WITH_MAGMA}) @@ -196,7 +204,7 @@ install(TARGETS mgmol-opt DESTINATION bin) # build ROM executable if(USE_LIBROM) add_executable(mgmol-rom rom_main.cc) - target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS} ${LIBROM_INCLUDES}) + target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(mgmol-rom mgmol_src ${link_libs}) target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES}) diff --git a/src/Control.cc b/src/Control.cc index 7d8dc33f..ce3023e3 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1917,6 +1917,10 @@ void Control::setOptions(const boost::program_options::variables_map& vm) // synchronize all processors sync(); + +#ifdef MGMOL_HAS_LIBROM + setROMOptions(vm); +#endif } int Control::checkOptions() @@ -2051,3 +2055,66 @@ void Control::printPoissonOptions(std::ostream& os) } os << std::endl; } + +void Control::setROMOptions(const boost::program_options::variables_map& vm) +{ + printWithTimeStamp("Control::setROMOptions()...", std::cout); + + if (onpe0) + { + std::string str = vm["ROM.stage"].as(); + if (str.compare("offline") == 0) + rom_pri_option.rom_stage = ROMStage::OFFLINE; + else if (str.compare("online") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE; + else if (str.compare("build") == 0) + rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("none") == 0) + rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; + + rom_pri_option.restart_file_fmt = vm["ROM.offline.restart_filefmt"].as(); + rom_pri_option.restart_file_minidx = vm["ROM.offline.restart_min_idx"].as(); + rom_pri_option.restart_file_maxidx = vm["ROM.offline.restart_max_idx"].as(); + rom_pri_option.basis_file = vm["ROM.offline.basis_file"].as(); + + rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); + } // onpe0 + + // synchronize all processors + syncROMOptions(); +} + +void Control::syncROMOptions() +{ + if (onpe0 && verbose > 0) + (*MPIdata::sout) << "Control::syncROMOptions()" << std::endl; + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + + mmpi.bcast(rom_pri_option.restart_file_fmt, comm_global_); + mmpi.bcast(rom_pri_option.basis_file, comm_global_); + + auto bcast_check = [](int mpirc) { + if (mpirc != MPI_SUCCESS) + { + (*MPIdata::sout) << "MPI Bcast of Control failed!!!" << std::endl; + MPI_Abort(comm_global_, 2); + } + }; + + short rom_stage = (short)static_cast(rom_pri_option.rom_stage); + int mpirc; + mpirc = MPI_Bcast(&rom_stage, 1, MPI_SHORT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.restart_file_minidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.restart_file_maxidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_); + bcast_check(mpirc); + + rom_pri_option.rom_stage = static_cast(rom_stage); +} \ No newline at end of file diff --git a/src/Control.h b/src/Control.h index f1fd8779..1ef7a047 100644 --- a/src/Control.h +++ b/src/Control.h @@ -13,6 +13,10 @@ #include "Species.h" #include "Timeout.h" +/* enumeration and option variables for libROM */ +#include "mgmol_config.h" +#include "rom_Control.h" + #include #include #include @@ -220,6 +224,9 @@ class Control void printRestartLink(); + /* libROM related options */ + ROMPrivateOptions rom_pri_option; + public: static Control* instance() { @@ -707,6 +714,11 @@ class Control } bool AtomsMove() { return (atoms_dyn_ != 0); } + + /* ROM-related options */ + void setROMOptions(const boost::program_options::variables_map& vm); + void syncROMOptions(); + const ROMPrivateOptions getROMOptions() { return rom_pri_option; } }; #endif diff --git a/src/MGmol_prototypes.h b/src/MGmol_prototypes.h index 8c47c388..207201d7 100644 --- a/src/MGmol_prototypes.h +++ b/src/MGmol_prototypes.h @@ -10,9 +10,11 @@ #ifndef MGMOL_PROTOTYPES_H #define MGMOL_PROTOTYPES_H +#include "mgmol_config.h" #include "global.h" #include +namespace po = boost::program_options; class Ions; class KBPsiMatrixSparse; @@ -24,4 +26,8 @@ int read_config(int argc, char** argv, boost::program_options::variables_map& vm, std::string& input_file, std::string& lrs_filename, std::string& constraints_filename, float& total_spin, bool& with_spin, bool& tcheck); +#ifdef MGMOL_HAS_LIBROM +void setupROMConfigOption(po::options_description &rom_cfg); +#endif + #endif diff --git a/src/mgmol_config.h.in b/src/mgmol_config.h.in new file mode 100644 index 00000000..0b099f6c --- /dev/null +++ b/src/mgmol_config.h.in @@ -0,0 +1,19 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// Description: A configuration header for mgmol. +// + +#ifndef INCLUDED_MGMOL_CONFIG_H +#define INCLUDED_MGMOL_CONFIG_H + +/* Have google test library. */ +#cmakedefine MGMOL_HAS_LIBROM + +#endif diff --git a/src/read_config.cc b/src/read_config.cc index 534981d8..5a1437df 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -14,6 +14,7 @@ #include #include #include +#include "MGmol_prototypes.h" #include namespace po = boost::program_options; @@ -201,6 +202,10 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(""), "Output file for dumping cluster information in vtk format"); +#ifdef MGMOL_HAS_LIBROM + setupROMConfigOption(config); +#endif + // Hidden options, will be allowed in config file, but will not be // shown to the user. po::options_description hidden("Hidden options"); @@ -409,3 +414,22 @@ int read_config(int argc, char** argv, po::variables_map& vm, return 0; } + +#ifdef MGMOL_HAS_LIBROM +void setupROMConfigOption(po::options_description &rom_cfg) +{ + rom_cfg.add_options() + ("ROM.stage", po::value()->default_value("none"), + "ROM workflow stage: offline; build; online; none.") + ("ROM.offline.restart_filefmt", po::value(), + "File name format to read for snapshots.") + ("ROM.offline.restart_min_idx", po::value(), + "Minimum index for snapshot file format.") + ("ROM.offline.restart_max_idx", po::value(), + "Maximum index for snapshot file format.") + ("ROM.offline.basis_file", po::value(), + "File name for libROM snapshot/POD matrices.") + ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), + "Save libROM snapshot file at FOM simulation."); +} +#endif \ No newline at end of file diff --git a/src/rom_Control.h b/src/rom_Control.h new file mode 100644 index 00000000..d1fd6854 --- /dev/null +++ b/src/rom_Control.h @@ -0,0 +1,42 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_CONTROL_H +#define ROM_CONTROL_H + +#include +#include +#include +#include +#include + +enum class ROMStage +{ + OFFLINE, + ONLINE, + RESTORE, // TODO(kevin): what stage is this? + BUILD, + UNSUPPORTED +}; + +/* Stored as a private member variable of Control class */ +struct ROMPrivateOptions +{ + ROMStage rom_stage = ROMStage::UNSUPPORTED; + + std::string restart_file_fmt = ""; + int restart_file_minidx = -1; + int restart_file_maxidx = -1; + std::string basis_file = ""; + + /* save librom snapshot matrix at FOM simulation. */ + bool save_librom_snapshot = false; +}; + +#endif // ROM_CONTROL_H From 14941c53be182fc0d4830a0212a354c8328f7b00 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 1 May 2024 16:27:16 -0700 Subject: [PATCH 06/33] FOM orbital snapshot saving workflow --- src/CMakeLists.txt | 1 + src/MGmol.cc | 14 ++++++++++- src/MGmol.h | 5 ++++ src/md.cc | 12 +++++++++ src/rom.cc | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 src/rom.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 962c163c..1bad52e7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -159,6 +159,7 @@ set(SOURCES magma_singleton.cc ChebyshevApproximation.cc ChebyshevApproximationInterface.cc + rom.cc ) add_library(mgmol_src ${SOURCES}) diff --git a/src/MGmol.cc b/src/MGmol.cc index 0af62ebc..501578b8 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1148,7 +1148,19 @@ void MGmol::cleanup() if (ierr < 0) os_ << "WARNING: writing restart data failed!!!" << std::endl; - } + +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0 && ct.AtomsDynamic() == AtomsDynamicType::Quench) + { + ierr = save_orbital_snapshot( + filename, *current_orbitals_); + + if (ierr < 0) + os_ << "WARNING: writing ROM snapshot data failed!!!" << std::endl; + } +#endif + } // if (ct.out_restart_info > 0 && !ct.AtomsMove()) MPI_Barrier(comm_); closing_tm_.stop(); diff --git a/src/MGmol.h b/src/MGmol.h index e63e7069..2a0d0783 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -10,6 +10,8 @@ #ifndef MGMOL_H #define MGMOL_H +#include "mgmol_config.h" + #include "Energy.h" #include "GridFuncVector.h" #include "Hamiltonian.h" @@ -303,6 +305,9 @@ class MGmol : public MGmolInterface } OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); +#ifdef MGMOL_HAS_LIBROM + int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); +#endif }; // Instantiate static variables here to avoid clang warnings template diff --git a/src/md.cc b/src/md.cc index ec288839..4cb1ae10 100644 --- a/src/md.cc +++ b/src/md.cc @@ -641,6 +641,18 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) count++; } +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0) + { + int ierr = save_orbital_snapshot( + ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); + + if (ierr < 0) + os_ << "WARNING md(): writing ROM snapshot data failed!!!" << std::endl; + } +#endif + printWithTimeStamp("dumped restart file...", std::cout); } diff --git a/src/rom.cc b/src/rom.cc new file mode 100644 index 00000000..f269f4b8 --- /dev/null +++ b/src/rom.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "mgmol_config.h" +#ifdef MGMOL_HAS_LIBROM + +#include "LocGridOrbitals.h" +#include "MGmol.h" + +#include "librom.h" + +#include +#include +#include +#include + +// Save the wavefunction snapshots +template +int MGmol::save_orbital_snapshot(std::string file_path, OrbitalsType& orbitals) +{ + std::string snapshot_filename = file_path; + struct stat s; + if (stat(file_path.c_str(), &s) == 0) + { + if (s.st_mode & S_IFDIR) + snapshot_filename = file_path + "/orbital"; + else if (s.st_mode & S_IFREG) + { + snapshot_filename = file_path + "_orbital"; + } + else + { + std::cout << file_path << " exists but is not a directory or a file." << std::endl; + return 1; + } + } + + const int dim = orbitals.getLocNumpt(); + const int totalSamples = orbitals.chromatic_number(); + + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, snapshot_filename); + + for (int i = 0; i < totalSamples; ++i) + basis_generator.takeSample(orbitals.getPsi(i)); + + basis_generator.writeSnapshot(); + + return 0; +} + +template class MGmol; +template class MGmol; + +#endif // MGMOL_HAS_LIBROM From a36d2c7016a5eaa49ee7cf4c7cf6ae89dcd97da0 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 2 May 2024 15:03:07 -0700 Subject: [PATCH 07/33] added braces --- src/rom.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rom.cc b/src/rom.cc index f269f4b8..a554d2cf 100644 --- a/src/rom.cc +++ b/src/rom.cc @@ -29,7 +29,9 @@ int MGmol::save_orbital_snapshot(std::string file_path, OrbitalsTy if (stat(file_path.c_str(), &s) == 0) { if (s.st_mode & S_IFDIR) + { snapshot_filename = file_path + "/orbital"; + } else if (s.st_mode & S_IFREG) { snapshot_filename = file_path + "_orbital"; From 4d9f5f3d2fd70161a1f71a3a93f8fa5faa1322cc Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Fri, 3 May 2024 10:28:51 -0700 Subject: [PATCH 08/33] ROM workflow for restart file collection and POD basis training (#230) * rom load restart file and POD basis training. * example rom config file for carbyne. * update docker container with hdf5-tools --- docker/Dockerfile | 2 +- examples/Carbyne/carbyne.rom.cfg | 34 ++++++++ src/CMakeLists.txt | 5 ++ src/rom_main.cc | 145 ++++++++++++++++++++----------- src/rom_workflows.cc | 71 +++++++++++++++ src/rom_workflows.h | 41 +++++++++ 6 files changed, 244 insertions(+), 54 deletions(-) create mode 100644 examples/Carbyne/carbyne.rom.cfg create mode 100644 src/rom_workflows.cc create mode 100644 src/rom_workflows.h diff --git a/docker/Dockerfile b/docker/Dockerfile index a097df9b..8b1ffa94 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,7 +14,7 @@ RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack- RUN sudo apt-get install -yq libboost-all-dev RUN sudo apt-get install -yq vim RUN sudo apt-get install -yq git-lfs -RUN sudo apt-get install -yq valgrind +RUN sudo apt-get install -yq valgrind hdf5-tools RUN sudo apt-get install -yq wget ### clang-format seems to be updated to 14.0. Not using it for now. # RUN sudo apt-get install -yq clang-format diff --git a/examples/Carbyne/carbyne.rom.cfg b/examples/Carbyne/carbyne.rom.cfg new file mode 100644 index 00000000..cb0cd295 --- /dev/null +++ b/examples/Carbyne/carbyne.rom.cfg @@ -0,0 +1,34 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=5 +atol=1.e-8 +[Orbitals] +initial_type=Fourier +[Restart] +output_level=4 +input_level=4 +input_filename=snapshot0_000 + +[ROM.offline] +restart_filefmt=snapshot0_%03d +restart_min_idx=0 +restart_max_idx=1 +basis_file=carom diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1bad52e7..e33d03d2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -162,6 +162,11 @@ set(SOURCES rom.cc ) +if(USE_LIBROM) + list(APPEND SOURCES + rom_workflows.cc) +endif(USE_LIBROM) + add_library(mgmol_src ${SOURCES}) target_include_directories(mgmol_src PRIVATE ${HDF5_INCLUDE_DIRS}) diff --git a/src/rom_main.cc b/src/rom_main.cc index c7ec278e..dc877a58 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -23,76 +23,115 @@ // Potentials, eigenvalues and operators in Rydberg // Energies in Hartree // -#include -#include -#include -#include -using namespace std; - -#ifdef _OPENMP -#include -#endif - -#ifdef USE_CNR -#include -#endif - -#include - -#include "Control.h" -#include "DistMatrix.h" -#include "ExtendedGridOrbitals.h" -#include "LocGridOrbitals.h" -#include "MGmol.h" -#include "MGmol_MPI.h" -#include "MPIdata.h" -#include "MatricesBlacsContext.h" -#include "Mesh.h" -#include "PackedCommunicationBuffer.h" -#include "ReplicatedWorkSpace.h" -#include "SparseDistMatrix.h" -#include "magma_singleton.h" -#include "tools.h" - -#include -#include -#include - -#include -namespace po = boost::program_options; - -#include "librom.h" - -//#include "MemTrack.h" +#include "rom_workflows.h" -int main(int argc, char** argv) +// A helper function +template +std::ostream& operator<<(std::ostream& os, const std::vector& v) { - // change handling of memory allocation errors - set_new_handler(noMoreMemory); - - cout.sync_with_stdio(); + copy(v.begin(), v.end(), std::ostream_iterator(std::cout, " ")); + return os; +} +int main(int argc, char** argv) +{ int mpirc = MPI_Init(&argc, &argv); if (mpirc != MPI_SUCCESS) { - cerr << "MPI Initialization failed!!!" << endl; + std::cerr << "MPI Initialization failed!!!" << std::endl; MPI_Abort(MPI_COMM_WORLD, 0); } - MPI_Comm_rank(MPI_COMM_WORLD, &mype); - assert(mype > -1); - onpe0 = (mype == 0); - CAROM::Vector librom_vector(10, false); + MPI_Comm comm = MPI_COMM_WORLD; + + mgmol_init(comm); + + // read runtime parameters + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + bool tcheck = false; + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // use configure file if it can be found + // std::string config_filename("mgmol.cfg"); + + // read options from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin, tcheck); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout); + else + mgmol + = new MGmol(global_comm, *MPIdata::sout); + + unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; + double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; + const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; + Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); + + mgmol->setupFromInput(input_filename); + + if (ct.isLocMode() || ct.init_loc == 1) mgmol->setupLRs(lrs_filename); + + mgmol->setupConstraintsFromInput(constraints_filename); + + mgmol_setup(); + + if (!tcheck) + { + mgmol->setup(); + + if (ct.isLocMode()) + readRestartFiles(mgmol); + else + readRestartFiles(mgmol); + } + else + { + *MPIdata::sout << " Input parameters OK\n"; + } + delete mgmol; + + } // close main scope + + mgmol_finalize(); mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { - cerr << "MPI Finalize failed!!!" << endl; + std::cerr << "MPI Finalize failed!!!" << std::endl; } time_t tt; time(&tt); - if (onpe0) cout << " Run ended at " << ctime(&tt) << endl; + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; // MemTrack::TrackDumpBlocks(); diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc new file mode 100644 index 00000000..4594dd3a --- /dev/null +++ b/src/rom_workflows.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "rom_workflows.h" +#include +#include +#include + +template +std::string string_format( const std::string& format, Args ... args ) +{ + int size_s = std::snprintf( nullptr, 0, format.c_str(), args ... ) + 1; // Extra space for '\0' + if( size_s <= 0 ){ throw std::runtime_error( "Error during formatting." ); } + auto size = static_cast( size_s ); + std::unique_ptr buf( new char[ size ] ); + std::snprintf( buf.get(), size, format.c_str(), args ... ); + return std::string( buf.get(), buf.get() + size - 1 ); // We don't want the '\0' inside +} + +template +void readRestartFiles(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + ROMPrivateOptions rom_options = ct.getROMOptions(); + assert(rom_options.restart_file_minidx >= 0); + assert(rom_options.restart_file_maxidx >= 0); + const int minidx = rom_options.restart_file_minidx; + const int maxidx = rom_options.restart_file_maxidx; + const int num_restart = maxidx - minidx + 1; + + MGmol *mgmol = static_cast *>(mgmol_); + OrbitalsType *orbitals = nullptr; + std::string filename; + + /* Read the first snapshot to determin dimension and number of snapshots */ + filename = string_format(rom_options.restart_file_fmt, minidx); + orbitals = mgmol->loadOrbitalFromRestartFile(filename); + const int dim = orbitals->getLocNumpt(); + const int chrom_num = orbitals->chromatic_number(); + const int totalSamples = orbitals->chromatic_number() * num_restart; + delete orbitals; + + /* Initialize libROM classes */ + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, rom_options.basis_file); + + /* Collect the restart files */ + for (int k = minidx; k <= maxidx; k++) + { + filename = string_format(rom_options.restart_file_fmt, k); + orbitals = mgmol->loadOrbitalFromRestartFile(filename); + assert(dim == orbitals->getLocNumpt()); + assert(chrom_num == orbitals->chromatic_number()); + + for (int i = 0; i < chrom_num; ++i) + basis_generator.takeSample(orbitals->getPsi(i)); + + delete orbitals; + } + basis_generator.writeSnapshot(); + basis_generator.endSamples(); +} + +template void readRestartFiles(MGmolInterface *mgmol_); +template void readRestartFiles(MGmolInterface *mgmol_); \ No newline at end of file diff --git a/src/rom_workflows.h b/src/rom_workflows.h new file mode 100644 index 00000000..dfb22b79 --- /dev/null +++ b/src/rom_workflows.h @@ -0,0 +1,41 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_WORKFLOWS_H +#define ROM_WORKFLOWS_H + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "Mesh.h" +#include "mgmol_run.h" +#include "tools.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +namespace po = boost::program_options; + +#include "librom.h" + +template +void readRestartFiles(MGmolInterface *mgmol_); + +#endif // ROM_WORKFLOWS_H From ec0cb30d91ad4b51c80aeed88d4228c25fde6b04 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Mon, 20 May 2024 11:57:57 -0700 Subject: [PATCH 09/33] set default values for ROM config options. (#235) --- src/read_config.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/read_config.cc b/src/read_config.cc index 5a1437df..6ce97648 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -421,13 +421,13 @@ void setupROMConfigOption(po::options_description &rom_cfg) rom_cfg.add_options() ("ROM.stage", po::value()->default_value("none"), "ROM workflow stage: offline; build; online; none.") - ("ROM.offline.restart_filefmt", po::value(), + ("ROM.offline.restart_filefmt", po::value()->default_value(""), "File name format to read for snapshots.") - ("ROM.offline.restart_min_idx", po::value(), + ("ROM.offline.restart_min_idx", po::value()->default_value(-1), "Minimum index for snapshot file format.") - ("ROM.offline.restart_max_idx", po::value(), + ("ROM.offline.restart_max_idx", po::value()->default_value(-1), "Maximum index for snapshot file format.") - ("ROM.offline.basis_file", po::value(), + ("ROM.offline.basis_file", po::value()->default_value(""), "File name for libROM snapshot/POD matrices.") ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), "Save libROM snapshot file at FOM simulation."); From 29c250254c80eb6b3188437b5a6b7568ef9dc6d5 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Wed, 22 May 2024 11:39:21 -0700 Subject: [PATCH 10/33] Merging PR #234 and #236 to ROMFPMD (#237) * Clean up class Rho (#234) * Clean up class MVPSolver (#236) --------- Co-authored-by: Jean-Luc Fattebert --- src/DavidsonSolver.cc | 2 - src/MGmol.h | 1 + src/MVPSolver.cc | 87 ++++------ src/MVPSolver.h | 15 +- src/Rho.cc | 371 +++--------------------------------------- src/Rho.h | 29 +--- 6 files changed, 67 insertions(+), 438 deletions(-) diff --git a/src/DavidsonSolver.cc b/src/DavidsonSolver.cc index f21cf7fa..c53e11c0 100644 --- a/src/DavidsonSolver.cc +++ b/src/DavidsonSolver.cc @@ -598,7 +598,6 @@ int DavidsonSolver::solve( rho_->computeRho( orbitals, work_orbitals, dm11, dm12, dm21, dm22); - rho_->rescaleTotalCharge(); mgmol_strategy_->update_pot(vh_init, ions_); @@ -663,7 +662,6 @@ int DavidsonSolver::solve( rho_->computeRho( orbitals, work_orbitals, dm11, dm12, dm21, dm22); - rho_->rescaleTotalCharge(); } } // inner iterations diff --git a/src/MGmol.h b/src/MGmol.h index 2a0d0783..b9d7c854 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -305,6 +305,7 @@ class MGmol : public MGmolInterface } OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); + #ifdef MGMOL_HAS_LIBROM int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); #endif diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 2f8060b7..8ac6a0be 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -52,18 +52,22 @@ MVPSolver::MVPSolver(MPI_Comm comm, std::ostream& os, os_(os), n_inner_steps_(n_inner_steps), use_old_dm_(use_old_dm), - ions_(ions) + ions_(ions), + numst_(numst) { - history_length_ = 2; - eks_history_.resize(history_length_, 100000.); + Control& ct = *(Control::instance()); + if (onpe0 && ct.verbose > 0) + { + os_ << "MVPSolver..." << std::endl; + if (use_old_dm_) os_ << "MVPSolver uses old DM..." << std::endl; + } rho_ = rho; energy_ = energy; electrostat_ = electrostat; mgmol_strategy_ = mgmol_strategy; - numst_ = numst; - work_ = new MatrixType("workMVP", numst_, numst_); + work_ = new MatrixType("workMVP", numst_, numst_); proj_mat_work_ = new ProjectedMatrices(numst_, false, kbT); proj_mat_work_->setup(global_indexes); @@ -123,9 +127,9 @@ void MVPSolver::buildTarget_MVP( { target_tm_.start(); - if (onpe0) std::cout << "buildTarget_MVP()..." << std::endl; - Control& ct = *(Control::instance()); + if (onpe0 && ct.verbose > 1) + std::cout << "buildTarget_MVP()..." << std::endl; proj_mat_work_->assignH(h11); @@ -135,10 +139,8 @@ void MVPSolver::buildTarget_MVP( // // compute target density matrix, with occupations>0 in numst only // - // if( onpe0 )os_<<"Compute XN..."<setHB2H(); - // if( onpe0 )os_<<"Compute DM..."<updateDM(orbitals_index); target = proj_mat_work_->dm(); @@ -168,18 +170,12 @@ int MVPSolver::solve(OrbitalsType& orbitals) os_ << "---------------------------------------------------------------" "-" << std::endl; - os_ << "Update DM functions using MVP Solver..." << std::endl; + os_ << "Update DM using MVP Solver..." << std::endl; os_ << "---------------------------------------------------------------" "-" << std::endl; } - // step for numerical derivative - - // double nel=orbitals.projMatrices()->getNel(); - // if( onpe0 ) - // os_<<"NEW algorithm: Number of electrons at start = "<::solve(OrbitalsType& orbitals) energy_->saveVofRho(); } - ProjectedMatricesInterface* current_proj_mat - = (inner_it == 0) ? orbitals.getProjMatrices() : proj_mat_work_; + ProjectedMatrices* current_proj_mat + = (inner_it == 0) + ? dynamic_cast*>( + orbitals.getProjMatrices()) + : proj_mat_work_; - double ts0; - double e0 = 0.; const int printE = (ct.verbose > 1) ? 1 : 0; // compute h11 for the current potential by adding local part to @@ -239,49 +236,32 @@ int MVPSolver::solve(OrbitalsType& orbitals) MatrixType h11(h11_nl); mgmol_strategy_->addHlocal2matrix(orbitals, orbitals, h11); + current_proj_mat->assignH(h11); + current_proj_mat->setHB2H(); + if (inner_it == 0) { - // orbitals are new, so a few things need to recomputed - ProjectedMatrices* projmatrices - = dynamic_cast*>( - orbitals.getProjMatrices()); - - projmatrices->assignH(h11); - projmatrices->setHB2H(); - if (use_old_dm_) { - dmInit = projmatrices->dm(); + dmInit = current_proj_mat->dm(); } - - ts0 = evalEntropyMVP(projmatrices, true, os_); - e0 = energy_->evaluateTotal( - ts0, projmatrices, orbitals, printE, os_); } else { - dmInit = proj_mat_work_->dm(); - - proj_mat_work_->assignH(h11); - proj_mat_work_->setHB2H(); - - ts0 = evalEntropyMVP(proj_mat_work_, true, os_); - e0 = energy_->evaluateTotal( - ts0, proj_mat_work_, orbitals, printE, os_); } - // N x N target... - // proj_mat_work_->setHiterativeIndex(orbitals.getIterativeIndex(), - // pot.getIterativeIndex()); + const double ts0 = evalEntropyMVP(current_proj_mat, true, os_); + const double e0 = energy_->evaluateTotal( + ts0, current_proj_mat, orbitals, printE, os_); MatrixType target("target", numst_, numst_); - MatrixType delta_dm("delta_dm", numst_, numst_); buildTarget_MVP(h11, s11, target); if (use_old_dm_ || inner_it > 0) { + MatrixType delta_dm("delta_dm", numst_, numst_); delta_dm = target; delta_dm -= dmInit; @@ -291,18 +271,16 @@ int MVPSolver::solve(OrbitalsType& orbitals) // evaluate free energy at beta=1 // if (onpe0 && ct.verbose > 2) - std::cout << "Target energy..." << std::endl; + std::cout << "MVP --- Target energy..." << std::endl; proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); proj_mat_work_->computeOccupationsFromDM(); if (ct.verbose > 2) current_proj_mat->printOccupations(os_); - double nel = proj_mat_work_->getNel(); + const double nel = proj_mat_work_->getNel(); if (onpe0 && ct.verbose > 1) - os_ << "Number of electrons at beta=1 : " << nel + os_ << "MVP --- Number of electrons at beta=1 : " << nel << std::endl; - // if( onpe0 )os_<<"Rho..."<computeRho(orbitals, target); - rho_->rescaleTotalCharge(); mgmol_strategy_->update_pot(vh_init, ions_); @@ -323,13 +301,13 @@ int MVPSolver::solve(OrbitalsType& orbitals) ts1, proj_mat_work_, orbitals, ct.verbose - 1, os_); // line minimization - double beta + const double beta = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); if (onpe0 && ct.verbose > 0) { os_ << std::setprecision(12); - os_ << std::fixed << "Inner iteration " << inner_it + os_ << std::fixed << "MVP inner iteration " << inner_it << ", E0=" << e0 << ", E1=" << e1; os_ << std::scientific << ", E0'=" << de0 << " -> beta=" << beta; @@ -355,14 +333,13 @@ int MVPSolver::solve(OrbitalsType& orbitals) os_ << "Number of electrons for interpolated DM = " << pnel << std::endl; } - // if( onpe0 )os_<<"Rho..."<computeRho(orbitals, *work_); - rho_->rescaleTotalCharge(); } } // inner iterations + // set DM to latest iteration value ProjectedMatrices* projmatrices = dynamic_cast*>( orbitals.getProjMatrices()); @@ -375,11 +352,9 @@ int MVPSolver::solve(OrbitalsType& orbitals) if (onpe0 && ct.verbose > 1) { os_ << "---------------------------------------------------------------" - "-" << std::endl; os_ << "End MVP Solver..." << std::endl; os_ << "---------------------------------------------------------------" - "-" << std::endl; } solve_tm_.stop(); diff --git a/src/MVPSolver.h b/src/MVPSolver.h index a3c29d1d..8e7a7c70 100644 --- a/src/MVPSolver.h +++ b/src/MVPSolver.h @@ -24,27 +24,22 @@ template class MVPSolver { private: - MPI_Comm comm_; + const MPI_Comm comm_; std::ostream& os_; - short n_inner_steps_; + const short n_inner_steps_; - bool use_old_dm_; + const bool use_old_dm_; Ions& ions_; + int numst_; + Rho* rho_; Energy* energy_; Electrostatic* electrostat_; - int history_length_; - std::vector eks_history_; - MGmol* mgmol_strategy_; - double de_old_; - double de_; - - int numst_; MatrixType* work_; ProjectedMatrices* proj_mat_work_; diff --git a/src/Rho.cc b/src/Rho.cc index 823e4478..b456f6c1 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -28,8 +28,6 @@ template Timer Rho::compute_tm_("Rho::compute"); template Timer Rho::compute_blas_tm_("Rho::compute_usingBlas"); -// template -// Timer Rho::compute_offdiag_tm_("Rho::compute_offdiag"); #ifdef HAVE_MAGMA template @@ -46,10 +44,6 @@ Rho::Rho() myspin_ = mmpi.myspin(); nspin_ = mmpi.nspin(); - // default values for block sizes - // block_functions_ = 8; - // block_space_ = 256; - Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); np_ = mygrid.size(); @@ -77,31 +71,6 @@ void Rho::setup(const OrthoType orbitals_type, orbitals_indexes_ = orbitals_indexes; } -template -void Rho::extrapolate() -{ - double minus = -1; - double two = 2.; - if (rho_minus1_.empty()) - { - rho_minus1_.resize(nspin_); - rho_minus1_[myspin_].resize(np_); - memcpy(&rho_minus1_[myspin_][0], &rho_[myspin_][0], - np_ * sizeof(RHODTYPE)); - return; - } - RHODTYPE* tmp = new RHODTYPE[np_]; - memcpy(tmp, &rho_[myspin_][0], np_ * sizeof(RHODTYPE)); - - LinearAlgebraUtils::MPscal(np_, two, &rho_[myspin_][0]); - LinearAlgebraUtils::MPaxpy( - np_, minus, &rho_minus1_[myspin_][0], &rho_[myspin_][0]); - - memcpy(&rho_minus1_[myspin_][0], tmp, np_ * sizeof(RHODTYPE)); - - delete[] tmp; -} - template void Rho::axpyRhoc(const double alpha, RHODTYPE* rhoc) { @@ -144,8 +113,6 @@ void Rho::update(OrbitalsType& current_orbitals) computeRho(current_orbitals); - rescaleTotalCharge(); - assert(iterative_index_ >= 0); update_tm_.stop(); @@ -234,68 +201,6 @@ void Rho::rescaleTotalCharge() #endif } -// template -// int Rho::setupSubdomainData(const int iloc, -// const vector& vorbitals, -// const ProjectedMatricesInterface* const projmatrices, -// vector& melements, vector>& vmpsi) -//{ -// // printWithTimeStamp("Rho::setupSubdomainData()...",cout); -// -// const short norb = (short)vorbitals.size(); -// vmpsi.resize(norb); -// -// const vector& loc_indexes(orbitals_indexes_[iloc]); -// const int n_colors = (int)loc_indexes.size(); -// -// if (n_colors == 0) return 0; -// -// vector mycolors; -// mycolors.reserve(n_colors); -// for (int icolor = 0; icolor < n_colors; icolor++) -// if (loc_indexes[icolor] != -1) mycolors.push_back(icolor); -// const int nmycolors = (int)mycolors.size(); -// -// for (short j = 0; j < norb; j++) -// vmpsi[j].resize(nmycolors); -// -// short j = 0; -// for (typename vector::const_iterator it = vorbitals.begin(); -// it != vorbitals.end(); ++it) -// { -// for (int color = 0; color < nmycolors; color++) -// { -// vmpsi[j][color] = (*it)->getPsi(mycolors[color], iloc); -// assert(vmpsi[j][color] != NULL); -// } -// j++; -// } -// -// SquareLocalMatrices& -// localX(projmatrices->getLocalX()); const MATDTYPE* const localX_iloc = -// localX.getSubMatrix(iloc); melements.clear(); melements.resize(nmycolors * -// nmycolors); -// -// for (int i = 0; i < nmycolors; i++) -// { -// const int icolor = mycolors[i]; -// if (norb == 1) -// { -// melements[i * nmycolors + i] -// = localX_iloc[icolor + n_colors * icolor]; -// } -// const int jmax = (norb == 1) ? i : nmycolors; -// for (int j = 0; j < jmax; j++) -// { -// int jcolor = mycolors[j]; -// melements[j * nmycolors + i] -// = localX_iloc[icolor + n_colors * jcolor]; -// } -// } -// -// return nmycolors; -//} - template void Rho::accumulateCharge(const double alpha, const short ix_max, const ORBDTYPE* const psii, const ORBDTYPE* const psij, @@ -305,235 +210,6 @@ void Rho::accumulateCharge(const double alpha, const short ix_max, plrho[ix] += (RHODTYPE)(alpha * (double)psii[ix] * (double)psij[ix]); } -// template -// void Rho::computeRhoSubdomain( -// const int iloc_init, const int iloc_end, const OrbitalsType& orbitals) -//{ -// assert(orbitals_type_ == OrthoType::Eigenfunctions -// || orbitals_type_ == OrthoType::Nonorthogonal); -// -// compute_tm_.start(); -// -// Mesh* mymesh = Mesh::instance(); -// -// const int loc_numpt = mymesh->locNumpt(); -// -// RHODTYPE* const prho = &rho_[myspin_][0]; -// -// vector vorbitals; -// vorbitals.push_back(&orbitals); -// -// const double nondiagfactor = 2.; -// -// for (int iloc = iloc_init; iloc < iloc_end; iloc++) -// { -// const int istart = iloc * loc_numpt; -// -// RHODTYPE* const lrho = &prho[istart]; -// -// vector melements; -// vector> vmpsi; -// -// const int nmycolors = setupSubdomainData( -// iloc, vorbitals, orbitals.projMatrices(), melements, vmpsi); -// assert(vmpsi.size() == 1); -// vector mpsi = vmpsi[0]; -// -// const int nblocks_color = nmycolors / block_functions_; -// const int max_icolor = (nmycolors % block_functions_ == 0) -// ? nmycolors -// : nblocks_color * block_functions_; -// const int missed_rows = nmycolors - max_icolor; -// //(*MPIdata::sout)<<"max_icolor="<::computeRhoSubdomain: -// //missed_rows="<::computeRhoSubdomainOffDiagBlock()...",cout); -// -// Mesh* mymesh = Mesh::instance(); -// -// const int loc_numpt = mymesh->locNumpt(); -// -// RHODTYPE* const prho = &rho_[myspin_][0]; -// -// for (int iloc = iloc_init; iloc < iloc_end; iloc++) -// { -// const int istart = iloc * loc_numpt; -// -// RHODTYPE* const lrho = &prho[istart]; -// -// vector melements; -// vector> vmpsi; -// -// const int nmycolors = setupSubdomainData( -// iloc, vorbitals, projmatrices, melements, vmpsi); -// const int nblocks_color = nmycolors / block_functions_; -// const int max_icolor = (nmycolors % block_functions_ == 0) -// ? nmycolors -// : nblocks_color * block_functions_; -// const int missed_rows = nmycolors - max_icolor; -// //(*MPIdata::sout)<<"max_icolor="<::computeRhoSubdomainOffDiagBlock: -// //missed_rows="<& occ); - // void computeRhoSubdomainOffDiagBlock(const int iloc_init, - // const int iloc_end, - // const std::vector& vorbitals, - // const ProjectedMatricesInterface* const); void computeRhoSubdomainUsingBlas3(const int iloc_init, const int iloc_end, const OrbitalsType& orbitals1, const OrbitalsType& orbitals2); void computeRhoSubdomainUsingBlas3( @@ -64,11 +53,6 @@ class Rho void accumulateCharge(const double alpha, const short ix_max, const ORBDTYPE* const psii, const ORBDTYPE* const psij, RHODTYPE* const plrho); - // int setupSubdomainData(const int iloc, - // const std::vector& vorbitals, - // const ProjectedMatricesInterface* const projmatrices, - // std::vector& melements, - // std::vector>& mpsi); void computeRho(OrbitalsType& orbitals); void computeRho( @@ -76,15 +60,17 @@ class Rho void gatherSpin(); + void resetValues(); + + void rescaleTotalCharge(); + public: // electronic density on grid std::vector> rho_; - std::vector> rho_minus1_; Rho(); ~Rho(){}; - void rescaleTotalCharge(); void setup( const OrthoType orbitals_type, const std::vector>&); void setVerbosityLevel(const int vlevel) { verbosity_level_ = vlevel; } @@ -105,18 +91,11 @@ class Rho void initUniform(); int getIterativeIndex() const { return iterative_index_; } int readRestart(HDFrestart& file); - void extrapolate(); void axpyRhoc(const double alpha, RHODTYPE* rhoc); template double dotWithRho(const T2* const func) const; - // void setupBlockSizes(const int block_functions, const int block_space) - //{ - // block_functions_ = block_functions; - // block_space_ = block_space; - //} - static void printTimers(std::ostream& os); }; From 24b26b6dcbea1bf337ab22299f465747ea4914fb Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Thu, 13 Jun 2024 19:01:54 -0700 Subject: [PATCH 11/33] Update ROMFPMD branch from release (#255) * Reenable testShortSighted test (#248) * Make new driver to check input (#247) * clean up/reorganize main.cc * use shared_ptr in class MGmol * Add possible periodic dimensions to xyz2in.py (#249) * Add possible periodic dimensions to xyz2in.py * Remove unused/untested option extrapolateH (#250) * Exit with failure if density off by more than 2% (#251) * Exit with failure if density off by more than 2% * adapt SiH4 test to catch that * fix bug in DFTsolver that was leading to wrong density * Example driver (#252) * add example driver, showing use of MGmol as a force/energy computational engine * clean up related functions in class Ions * loadOrbitalsFromRestartFile -> loadRestartFile * update the rom main driver. * update rom main driver again * temporary fix before the next PR --------- Co-authored-by: Jean-Luc Fattebert --- CMakeLists.txt | 4 +- drivers/CMakeLists.txt | 11 ++ drivers/check_input.cc | 93 ++++++++++++++ drivers/example1.cc | 167 +++++++++++++++++++++++++ src/CMakeLists.txt | 16 +-- src/DFTsolver.cc | 4 +- src/Electrostatic.cc | 4 +- src/Ions.cc | 191 ++++++++++++++-------------- src/Ions.h | 28 +++-- src/MGmol.cc | 241 ++++++++++++++++++------------------ src/MGmol.h | 69 ++++++----- src/MGmolInterface.h | 6 + src/MGmol_prototypes.h | 2 +- src/OrbitalsExtrapolation.h | 2 - src/Rho.cc | 6 + src/computeHij.cc | 11 +- src/lbfgsrlx.cc | 7 +- src/main.cc | 62 +++------- src/md.cc | 93 +++----------- src/mlwf.cc | 6 +- src/quench.cc | 47 +++---- src/readInput.cc | 4 +- src/read_config.cc | 7 +- src/rom_main.cc | 44 +++---- src/rom_workflows.cc | 4 +- src/runfire.cc | 11 +- src/setup.cc | 14 ++- tests/CMakeLists.txt | 2 +- tests/SiH4/testSiH4.py | 13 +- util/xyz2in.py | 46 +++++-- 30 files changed, 729 insertions(+), 486 deletions(-) create mode 100644 drivers/CMakeLists.txt create mode 100644 drivers/check_input.cc create mode 100644 drivers/example1.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 75752a3a..6c48a2e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,7 +178,7 @@ if(${MGMOL_WITH_CLANG_FORMAT}) find_package(CLANG_FORMAT) if(${CLANG_FORMAT_FOUND}) message(STATUS "Indent with clang-format") - file(GLOB_RECURSE FORMAT_SOURCES src/*.cc src/*.h tests/*.cc tests/*.h) + file(GLOB_RECURSE FORMAT_SOURCES src/*.cc src/*.h tests/*.cc tests/*.h drivers/*.cc) add_custom_target(format COMMAND ${CLANG_FORMAT_EXECUTABLE} -i -style=file ${FORMAT_SOURCES} DEPENDS ${FORMAT_SOURCES}) @@ -271,5 +271,7 @@ link_libraries(${LIBROM_LIB}) # add subdirectories for source files, tests add_subdirectory(src) +add_subdirectory(drivers) + add_subdirectory(tests) diff --git a/drivers/CMakeLists.txt b/drivers/CMakeLists.txt new file mode 100644 index 00000000..4eb1c741 --- /dev/null +++ b/drivers/CMakeLists.txt @@ -0,0 +1,11 @@ +include_directories( ${CMAKE_SOURCE_DIR}/src ) + +add_executable(check_input check_input.cc) + +add_executable(example1 example1.cc) + +target_include_directories(check_input PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(example1 PRIVATE ${Boost_INCLUDE_DIRS}) + +target_link_libraries(check_input mgmol_src) +target_link_libraries(example1 mgmol_src) diff --git a/drivers/check_input.cc b/drivers/check_input.cc new file mode 100644 index 00000000..14dbaa09 --- /dev/null +++ b/drivers/check_input.cc @@ -0,0 +1,93 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + mgmol_init(comm); + + // read runtime parameters + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read options from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + *MPIdata::sout << " Input parameters OK\n"; + + delete mgmol; + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + return 0; +} diff --git a/drivers/example1.cc b/drivers/example1.cc new file mode 100644 index 00000000..561f5e08 --- /dev/null +++ b/drivers/example1.cc @@ -0,0 +1,167 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // compute energy and forces using all MPI tasks + // expect positions to be replicated on all MPI tasks + std::vector forces; + mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e33d03d2..e602101f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -191,13 +191,15 @@ install(TARGETS mgmol_src DESTINATION lib) add_executable(mgmol-opt main.cc) target_include_directories (mgmol-opt PRIVATE ${Boost_INCLUDE_DIRS}) -target_link_libraries(mgmol-opt mgmol_src ${link_libs}) -target_link_libraries(mgmol-opt ${SCALAPACK_LIBRARIES}) -target_link_libraries(mgmol-opt ${HDF5_LIBRARIES}) -target_link_libraries(mgmol-opt ${HDF5_HL_LIBRARIES}) -target_link_libraries(mgmol-opt ${BLAS_LIBRARIES}) -target_link_libraries(mgmol-opt ${LAPACK_LIBRARIES}) -target_link_libraries(mgmol-opt ${Boost_LIBRARIES}) +target_link_libraries(mgmol_src ${link_libs}) +target_link_libraries(mgmol_src ${SCALAPACK_LIBRARIES}) +target_link_libraries(mgmol_src ${LAPACK_LIBRARIES}) +target_link_libraries(mgmol_src ${BLAS_LIBRARIES}) +target_link_libraries(mgmol_src ${HDF5_LIBRARIES}) +target_link_libraries(mgmol_src ${HDF5_HL_LIBRARIES}) +target_link_libraries(mgmol_src ${Boost_LIBRARIES}) + +target_link_libraries(mgmol-opt mgmol_src) if (${OPENMP_CXX_FOUND}) target_link_libraries(mgmol-opt OpenMP::OpenMP_CXX) endif() diff --git a/src/DFTsolver.cc b/src/DFTsolver.cc index c1484160..91c4a0f6 100644 --- a/src/DFTsolver.cc +++ b/src/DFTsolver.cc @@ -339,13 +339,15 @@ int DFTsolver::solve(OrbitalsType& orbitals, else { bool updateDM = false; + bool updatedS = false; if (!ct.fullyOccupied()) { orbitals.computeGramAndInvS(); dm_strategy_->dressDM(); updateDM = true; + updatedS = true; } - orbitals.orthonormalizeLoewdin(true, nullptr, updateDM); + orbitals.orthonormalizeLoewdin(updatedS, nullptr, updateDM); orbitals_stepper_->restartMixing(); } diff --git a/src/Electrostatic.cc b/src/Electrostatic.cc index b0afeb76..80cb754b 100644 --- a/src/Electrostatic.cc +++ b/src/Electrostatic.cc @@ -36,7 +36,7 @@ Timer Electrostatic::solve_tm_("Electrostatic::solve"); Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], const double screening_const) - : laptype_(lap_type) + : laptype_(lap_type), poisson_solver_(nullptr) { assert(bcPoisson[0] >= 0); assert(bcPoisson[1] >= 0); @@ -166,6 +166,8 @@ Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], Electrostatic::~Electrostatic() { + assert(poisson_solver_ != nullptr); + delete poisson_solver_; if (grhod_ != nullptr) delete grhod_; if (grhoc_ != nullptr) delete grhoc_; diff --git a/src/Ions.cc b/src/Ions.cc index 53ee0cf4..940e12e8 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -10,6 +10,7 @@ #include "Ions.h" #include "Control.h" #include "HDFrestart.h" +#include "MGmol_MPI.h" #include "MGmol_blas1.h" #include "MPIdata.h" #include "Mesh.h" @@ -169,12 +170,9 @@ void Ions::computeMaxNumProjs() << " initialized" << std::endl; #endif - std::vector::iterator ion = list_ions_.begin(); - while (ion != list_ions_.end()) + for (auto& ion : list_ions_) { - max_num_proj_ = std::max(max_num_proj_, (*ion)->nProjectors()); - - ion++; + max_num_proj_ = std::max(max_num_proj_, ion->nProjectors()); } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -210,11 +208,9 @@ void Ions::setup() if (ct.verbose > 0) printWithTimeStamp("Ions::setup()... individual ions...", std::cout); - std::vector::iterator ion = list_ions_.begin(); - while (ion != list_ions_.end()) + for (auto& ion : list_ions_) { - (*ion)->setup(); - ion++; + ion->setup(); } setMapVL(); @@ -243,7 +239,7 @@ Ions::~Ions() void Ions::setupListOverlappingIons() { Control& ct = *(Control::instance()); - if (ct.verbose > 0) + if (ct.verbose > 1) printWithTimeStamp("Ions::setupListOverlappingIons()...", std::cout); overlappingNL_ions_.clear(); @@ -269,7 +265,7 @@ void Ions::setupListOverlappingIons() void Ions::setupInteractingIons() { Control& ct = *(Control::instance()); - if (ct.verbose > 0) + if (ct.verbose > 1) printWithTimeStamp("Ions::setupInteractingIons()...", std::cout); ions_setupInteractingIons_tm.start(); @@ -1569,7 +1565,7 @@ Ion* Ions::findLocalIon(const int index) const return nullptr; } -void Ions::setPositions(const std::vector& tau) +void Ions::setLocalPositions(const std::vector& tau) { assert(tau.size() == 3 * local_ions_.size()); @@ -1586,69 +1582,6 @@ void Ions::setPositions(const std::vector& tau) setup_ = false; } -void Ions::get_positions(std::vector>& rr) const -{ - assert(rr.size() == local_ions_.size()); - if (local_ions_.empty()) return; - std::vector tau(3); - int i = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) - { - tau[0] = (*ion)->position(0); - tau[1] = (*ion)->position(1); - tau[2] = (*ion)->position(2); - rr[i] = tau; - ion++; - i++; - } -} -void Ions::set_positions(const std::vector>& rr) -{ - assert(rr.size() == local_ions_.size()); - - if (local_ions_.empty()) return; - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) - { - assert(rr[i].size() == 3); - (*ion)->setPosition(rr[i][0], rr[i][1], rr[i][2]); - ion++; - i++; - } -} -void Ions::get_forces(std::vector>& ff) const -{ - assert(ff.size() == local_ions_.size()); - - if (local_ions_.empty()) return; - int i = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) - { - ff[i][0] = (*ion)->force(0); - ff[i][1] = (*ion)->force(1); - ff[i][2] = (*ion)->force(2); - ion++; - i++; - } -} -void Ions::set_forces(const std::vector>& ff) -{ - assert(ff.size() == local_ions_.size()); - - if (local_ions_.empty()) return; - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) - { - (*ion)->setForce(ff[i][0], ff[i][1], ff[i][2]); - ion++; - i++; - } -} - int Ions::readAtoms(const std::string& filename, const bool cell_relative) { @@ -1785,6 +1718,7 @@ int Ions::readAtomsFromXYZ( } ++count; } + delete tfile; } mmpi.bcastGlobal(&count, 1); @@ -1793,6 +1727,17 @@ int Ions::readAtomsFromXYZ( mmpi.bcastGlobal(&crds[0], 3 * natoms); mmpi.bcastGlobal(&spec[0], natoms); + return setAtoms(crds, spec); +} + +int Ions::setAtoms( + const std::vector& crds, const std::vector& spec) +{ + MGmol_MPI& mmpi(*(MGmol_MPI::instance())); + Control& ct(*(Control::instance())); + + const int natoms = crds.size() / 3; + double velocity[3] = { 0., 0., 0. }; bool locked = false; for (int ia = 0; ia < natoms; ++ia) @@ -1820,7 +1765,7 @@ int Ions::readAtomsFromXYZ( } if (spname.compare("") == 0) { - (*MPIdata::serr) << "Ions::readAtomsFromXYZ() --- ERROR: unknown " + (*MPIdata::serr) << "Ions::setAtoms() --- ERROR: unknown " "species for atomic number " << spec[ia] << std::endl; return -1; @@ -1841,7 +1786,7 @@ int Ions::readAtomsFromXYZ( // Populate list_ions_ list // std::cout<<"crds: "<set_here(true); + new_ion->set_here(true); local_ions_.push_back(new_ion); } else - (new_ion)->set_here(false); + new_ion->set_here(false); } else { @@ -2193,7 +2134,7 @@ void Ions::setVelocities(const std::vector& tau0, } } -void Ions::getPositions(std::vector& tau) const +void Ions::getLocalPositions(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); @@ -2207,6 +2148,42 @@ void Ions::getPositions(std::vector& tau) const } } +void Ions::getPositions(std::vector& tau) +{ + std::vector tau_local(3 * local_ions_.size()); + + getLocalPositions(tau_local); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.allGatherV(tau_local, tau); +} + +void Ions::getAtomicNumbers(std::vector& atnumbers) +{ + std::vector local_atnumbers; + + for (auto& ion : local_ions_) + { + local_atnumbers.push_back(ion->atomic_number()); + } + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.allGatherV(local_atnumbers, atnumbers); +} + +void Ions::getForces(std::vector& forces) +{ + std::vector forces_local(3 * local_ions_.size()); + + getLocalForces(forces_local); + + int n = getNumIons(); + forces.resize(3 * n); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.allGatherV(forces_local, forces); +} + void Ions::setTau0() { assert(tau0_.size() == 3 * local_ions_.size()); @@ -2236,6 +2213,20 @@ void Ions::setPositionsToTau0() } } +void Ions::setPositions( + const std::vector& tau, const std::vector& anumbers) +{ + assert(tau.size() == anumbers.size() * 3); + + // clear previous data + clearLists(); + + num_ions_ = setAtoms(tau, anumbers); + + // setup required after updating local ions positions + setup(); +} + void Ions::setVelocitiesToVel() { assert(velocity_.size() == 3 * local_ions_.size()); @@ -2249,7 +2240,7 @@ void Ions::setVelocitiesToVel() } } -void Ions::getForces(std::vector& tau) const +void Ions::getLocalForces(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); @@ -2380,12 +2371,10 @@ double Ions::computeMaxNLprojRadius() const { double radius = 0.; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + for (auto& iion : local_ions_) { - double r = (*iion)->radiusNLproj(); + double r = iion->radiusNLproj(); radius = r > radius ? r : radius; - iion++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -3061,7 +3050,7 @@ void Ions::updateForcesInteractingIons() MGmol_MPI& mmpi(*(MGmol_MPI::instance())); // get computed forces into fion_ - getForces(fion_); + getLocalForces(fion_); // initialize with local names and forces DistributedIonicData forces_data(local_names_, fion_); @@ -3136,6 +3125,18 @@ void Ions::updateTaupInteractingIons() } } +void Ions::clearLists() +{ + local_ions_.clear(); + std::vector::iterator ion = list_ions_.begin(); + while (ion != list_ions_.end()) + { + delete *ion; + ion++; + } + list_ions_.clear(); +} + // update list of local ions void Ions::updateListIons() { @@ -3160,15 +3161,7 @@ void Ions::updateListIons() // Note: this is based on data from MD std::vectors // First cleanup list_ions_ - local_ions_.clear(); - // delete current ions from list - std::vector::iterator ion = list_ions_.begin(); - while (ion != list_ions_.end()) - { - delete *ion; - ion++; - } - list_ions_.clear(); + clearLists(); // Update list starting with local ion data. // This enables overlapping data accumulation with communication. diff --git a/src/Ions.h b/src/Ions.h index 2351d678..772c51d1 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -16,7 +16,6 @@ #include #include -#include "DataDistribution.h" #include "DistributedIonicData.h" #include "Ion.h" #include "hdf5.h" @@ -37,7 +36,12 @@ class Ions const std::vector& species_; std::vector list_ions_; - std::vector local_ions_; // centered in local sub-domain + + /* + * ions located in local sub-domain + */ + std::vector local_ions_; + std::vector interacting_ions_; // for ion-ion interactions std::vector overlappingNL_ions_; // with projectors overlapping local sub-domain @@ -62,7 +66,6 @@ class Ions void readRestartPositions(HDFrestart& h5_file); int read1atom(std::ifstream* tfile, const bool cell_relative); - // void associate2PE(); void setupInteractingIons(); void setupListOverlappingIons(); void setMapVL(); @@ -158,6 +161,7 @@ class Ions void gatherForces( std::vector& forces, const int root, const MPI_Comm comm) const; bool hasLockedAtoms() const; + void clearLists(); public: Ions(const double lat[3], const std::vector& sp); @@ -247,11 +251,7 @@ class Ions // check if ion is in list of local ions bool isLocal(const std::string& ion_name) const; - void setPositions(const std::vector& tau); - void get_positions(std::vector>& r) const; - void set_positions(const std::vector>& r); - void get_forces(std::vector>& f) const; - void set_forces(const std::vector>& f); + void setLocalPositions(const std::vector& tau); void lockAtom(const std::string& name); @@ -260,6 +260,8 @@ class Ions int readAtoms(const std::string& filename, const bool cell_relative); int readAtoms(std::ifstream* tfile, const bool cell_relative); void initFromRestartFile(HDFrestart& h5_file); + int setAtoms( + const std::vector& crds, const std::vector& spec); int getNValenceElectrons() const; void syncForces(); @@ -268,9 +270,15 @@ class Ions void setTau0(); void setPositionsToTau0(); void setVelocitiesToVel(); + void setPositions( + const std::vector& tau, const std::vector& anumbers); + + void getLocalPositions(std::vector& tau) const; + void getPositions(std::vector& tau); + void getAtomicNumbers(std::vector& atnumbers); - void getPositions(std::vector& tau) const; - void getForces(std::vector& tau) const; + void getForces(std::vector& forces); + void getLocalForces(std::vector& tau) const; void syncData(const std::vector& sp); // void syncNames(const int nions, std::vector& local_names, // std::vector& names); diff --git a/src/MGmol.cc b/src/MGmol.cc index 39f30378..c91120e9 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -113,73 +113,29 @@ template MGmol::MGmol(MPI_Comm comm, std::ostream& os, std::string input_filename, std::string lrs_filename, std::string constraints_filename) - : os_(os) + : os_(os), comm_(comm) { - comm_ = comm; - - constraints_ = new ConstraintSet(); + constraints_.reset(new ConstraintSet()); mgmol::out = &os; - geom_optimizer_ = nullptr; - local_cluster_ = nullptr; - proj_matrices_ = nullptr; - dm_strategy_ = nullptr; - - h5f_file_ = nullptr; - - aomm_ = nullptr; - - spreadf_ = nullptr; - - spread_penalty_ = nullptr; - - orbitals_precond_ = nullptr; - - forces_ = nullptr; - - energy_ = nullptr; + current_orbitals_ = nullptr; setupFromInput(input_filename); + /* + * Extra setup if using localization regions + */ setupLRs(lrs_filename); - setupConstraintsFromInput(constraints_filename); - - setup(); + if (!constraints_filename.empty()) + setupConstraintsFromInput(constraints_filename); } template MGmol::~MGmol() { - delete electrostat_; - delete rho_; - delete constraints_; - delete xcongrid_; - delete energy_; - if (hamiltonian_ != nullptr) delete hamiltonian_; - if (geom_optimizer_ != nullptr) delete geom_optimizer_; - - if (currentMasks_ != nullptr) delete currentMasks_; - if (corrMasks_ != nullptr) delete corrMasks_; - - if (aomm_ != nullptr) delete aomm_; - - delete current_orbitals_; - delete ions_; - delete g_kbpsi_; - - delete proj_matrices_; - if (local_cluster_ != nullptr) delete local_cluster_; - - if (h5f_file_ != nullptr) delete h5f_file_; - - if (spreadf_ != nullptr) delete spreadf_; - - if (spread_penalty_ != nullptr) delete spread_penalty_; - - delete forces_; - if (dm_strategy_ != nullptr) delete dm_strategy_; + if (current_orbitals_) delete current_orbitals_; } template <> @@ -192,18 +148,17 @@ void MGmol::initialMasks() if (ct.verbose > 0) printWithTimeStamp("MGmol::initialMasks()...", os_); - currentMasks_ = new MasksSet(false, ct.getMGlevels()); + currentMasks_ + = std::shared_ptr(new MasksSet(false, ct.getMGlevels())); currentMasks_->setup(lrs_); - corrMasks_ = new MasksSet(true, 0); + corrMasks_ = std::shared_ptr(new MasksSet(true, 0)); corrMasks_->setup(lrs_); } template <> void MGmol::initialMasks() { - currentMasks_ = nullptr; - corrMasks_ = nullptr; } template @@ -227,7 +182,7 @@ int MGmol::initial() pb::Lap* lapop = ct.Mehrstellen() ? hamiltonian_->lapOper() : nullptr; - g_kbpsi_ = new KBPsiMatrixSparse(lapop); + g_kbpsi_.reset(new KBPsiMatrixSparse(lapop)); check_anisotropy(); @@ -269,7 +224,7 @@ int MGmol::initial() // set number of iterations to 10. if (ct.load_balancing_alpha > 0.0) { - local_cluster_ = new ClusterOrbitals(lrs_); + local_cluster_.reset(new ClusterOrbitals(lrs_)); local_cluster_->setup(); local_cluster_->computeClusters(ct.load_balancing_max_iterations); } @@ -289,29 +244,31 @@ int MGmol::initial() { #ifdef HAVE_MAGMA if (use_replicated_matrix) - proj_matrices_ = new ProjectedMatricesMehrstellen( - ct.numst, with_spin, ct.occ_width); + proj_matrices_.reset( + new ProjectedMatricesMehrstellen( + ct.numst, with_spin, ct.occ_width)); else #endif - proj_matrices_ = new ProjectedMatricesMehrstellen< + proj_matrices_.reset(new ProjectedMatricesMehrstellen< dist_matrix::DistMatrix>( - ct.numst, with_spin, ct.occ_width); + ct.numst, with_spin, ct.occ_width)); } else if (ct.short_sighted) - proj_matrices_ = new ProjectedMatricesSparse( - ct.numst, ct.occ_width, lrs_, local_cluster_); + proj_matrices_.reset(new ProjectedMatricesSparse( + ct.numst, ct.occ_width, lrs_, local_cluster_.get())); else #ifdef HAVE_MAGMA if (use_replicated_matrix) - proj_matrices_ = new ProjectedMatrices( - ct.numst, with_spin, ct.occ_width); + proj_matrices_.reset(new ProjectedMatrices( + ct.numst, with_spin, ct.occ_width)); else #endif - proj_matrices_ - = new ProjectedMatrices>( - ct.numst, with_spin, ct.occ_width); + proj_matrices_.reset( + new ProjectedMatrices>( + ct.numst, with_spin, ct.occ_width)); - forces_ = new Forces(hamiltonian_, rho_, proj_matrices_); + forces_.reset(new Forces( + hamiltonian_.get(), rho_.get(), proj_matrices_.get())); if (ct.verbose > 0) printWithTimeStamp( @@ -326,8 +283,8 @@ int MGmol::initial() printWithTimeStamp("MGmol::initial(), create T...", os_); current_orbitals_ = new OrbitalsType("Primary", mygrid, mymesh->subdivx(), - ct.numst, ct.bcWF, proj_matrices_, lrs_, currentMasks_, corrMasks_, - local_cluster_, true); + ct.numst, ct.bcWF, proj_matrices_.get(), lrs_, currentMasks_.get(), + corrMasks_.get(), local_cluster_.get(), true); increaseMemorySlotsForOrbitals(); @@ -372,7 +329,7 @@ int MGmol::initial() "MGmol::initial(), init wf and masks...", os_); // Make temp mask for initial random wave functions - if (ct.init_loc == 1 && currentMasks_ != nullptr) + if (ct.init_loc == 1 && currentMasks_) { float cut_init = ct.initRadius(); assert(cut_init > 0.); @@ -383,7 +340,7 @@ int MGmol::initial() current_orbitals_->initWF(lrs_); // initialize masks again - if (ct.init_loc == 1 && currentMasks_ != nullptr) + if (ct.init_loc == 1 && currentMasks_) { currentMasks_->initialize(lrs_, 0); } @@ -427,9 +384,10 @@ int MGmol::initial() } if (ct.verbose > 0) printWithTimeStamp("Initialize XC functional...", os_); - xcongrid_ = XCfunctionalFactory::create( - ct.xctype, mmpi.nspin(), *rho_, pot); - assert(xcongrid_ != nullptr); + xcongrid_ + = std::shared_ptr(XCfunctionalFactory::create( + ct.xctype, mmpi.nspin(), *rho_, pot)); + assert(xcongrid_); // initialize nl potentials with restart values if possible // if( ct.restart_info>1 ) @@ -473,7 +431,7 @@ int MGmol::initial() { Vector3D origin(mygrid.origin(0), mygrid.origin(1), mygrid.origin(2)); Vector3D ll(mygrid.ll(0), mygrid.ll(1), mygrid.ll(2)); - spreadf_ = new SpreadsAndCenters(origin, ll); + spreadf_.reset(new SpreadsAndCenters(origin, ll)); } bool energy_with_spread_penalty = false; @@ -481,26 +439,30 @@ int MGmol::initial() { if (ct.isSpreadFunctionalVolume()) { - spread_penalty_ = new SpreadPenaltyVolume(spreadf_, - ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor(), - ct.spreadPenaltyDampingFactor()); + spread_penalty_.reset( + new SpreadPenaltyVolume(spreadf_.get(), + ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor(), + ct.spreadPenaltyDampingFactor())); } else if (ct.isSpreadFunctionalEnergy()) { energy_with_spread_penalty = true; - spread_penalty_ = new EnergySpreadPenalty(spreadf_, - ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor()); + spread_penalty_.reset( + new EnergySpreadPenalty(spreadf_.get(), + ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor())); } else - spread_penalty_ = new SpreadPenalty(spreadf_, - ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor(), - ct.spreadPenaltyDampingFactor()); + spread_penalty_.reset( + new SpreadPenalty(spreadf_.get(), + ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor(), + ct.spreadPenaltyDampingFactor())); } - SpreadPenaltyInterface* spread_penalty + std::shared_ptr> spread_penalty = energy_with_spread_penalty ? spread_penalty_ : nullptr; - energy_ = new Energy( - mygrid, *ions_, pot, *electrostat_, *rho_, *xcongrid_, spread_penalty); + energy_ = std::shared_ptr>( + new Energy(mygrid, *ions_, pot, *electrostat_, *rho_, + *xcongrid_, spread_penalty.get())); if (ct.verbose > 0) printWithTimeStamp("Setup matrices...", os_); @@ -509,15 +471,16 @@ int MGmol::initial() // HMVP algorithm requires that H is initialized #ifdef HAVE_MAGMA if (use_replicated_matrix) - dm_strategy_ - = DMStrategyFactory::create(comm_, - os_, *ions_, rho_, energy_, electrostat_, this, proj_matrices_, - current_orbitals_); + dm_strategy_.reset( + DMStrategyFactory::create(comm_, + os_, *ions_, rho_.get(), energy_.get(), electrostat_.get(), + this, proj_matrices_.get(), current_orbitals_)); else #endif - dm_strategy_ = DMStrategyFactory>::create(comm_, os_, *ions_, rho_, - energy_, electrostat_, this, proj_matrices_, current_orbitals_); + dm_strategy_.reset(DMStrategyFactory>::create(comm_, os_, *ions_, + rho_.get(), energy_.get(), electrostat_.get(), this, + proj_matrices_.get(), current_orbitals_)); // theta = invB * Hij proj_matrices_->updateThetaAndHB(); @@ -609,7 +572,7 @@ void MGmol::finalEnergy() // Get the total energy const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] total_energy_ = energy_->evaluateTotal( - ts, proj_matrices_, *current_orbitals_, 2, os_); + ts, proj_matrices_.get(), *current_orbitals_, 2, os_); } template @@ -624,7 +587,7 @@ void MGmol::printMM() ProjectedMatrices>* projmatrices = dynamic_cast< ProjectedMatrices>*>( - proj_matrices_); + proj_matrices_.get()); assert(projmatrices != nullptr); projmatrices->printHamiltonianMM(tfileh); } @@ -697,7 +660,7 @@ void MGmol::write_header() ct.printPoissonOptions(os_); } // onpe0 - if (current_orbitals_ != nullptr && ct.verbose > 0) + if (current_orbitals_ && ct.verbose > 0) { current_orbitals_->printNumst(os_); current_orbitals_->printChromaticNumber(os_); @@ -802,9 +765,9 @@ void MGmol::printEigAndOcc() #ifdef HAVE_MAGMA // try with ReplicatedMatrix first { - ProjectedMatrices* projmatrices - = dynamic_cast*>( - proj_matrices_); + std::shared_ptr> projmatrices + = std::dynamic_pointer_cast< + ProjectedMatrices>(proj_matrices_); if (projmatrices) { projmatrices->printEigenvalues(os_); @@ -815,10 +778,10 @@ void MGmol::printEigAndOcc() #endif if (!printflag) { - ProjectedMatrices>* - projmatrices - = dynamic_cast< - ProjectedMatrices>*>( + std::shared_ptr< + ProjectedMatrices>> + projmatrices = std::dynamic_pointer_cast< + ProjectedMatrices>>( proj_matrices_); assert(projmatrices); @@ -1062,21 +1025,23 @@ double MGmol::get_evnl(const Ions& ions) double val; if (ct.short_sighted) { - ProjectedMatricesSparse* projmatrices - = dynamic_cast(proj_matrices_); + std::shared_ptr projmatrices + = std::dynamic_pointer_cast( + proj_matrices_); assert(projmatrices); - val = g_kbpsi_->getEvnl(ions, projmatrices); + val = g_kbpsi_->getEvnl(ions, projmatrices.get()); } else { - ProjectedMatrices>* projmatrices - = dynamic_cast< - ProjectedMatrices>*>( + std::shared_ptr< + ProjectedMatrices>> + projmatrices = std::dynamic_pointer_cast< + ProjectedMatrices>>( proj_matrices_); assert(projmatrices); - val = g_kbpsi_->getEvnl(ions, projmatrices); + val = g_kbpsi_->getEvnl(ions, projmatrices.get()); } evnl_tm_.stop(); @@ -1102,11 +1067,11 @@ void MGmol::setup() printWithTimeStamp("MGmol::setup()...", os_); if (ct.verbose > 0) printWithTimeStamp("Setup VH...", os_); - electrostat_ = new Electrostatic( - ct.getPoissonFDtype(), ct.bcPoisson, ct.screening_const); + electrostat_ = std::shared_ptr(new Electrostatic( + ct.getPoissonFDtype(), ct.bcPoisson, ct.screening_const)); electrostat_->setup(ct.vh_init); - rho_ = new Rho(); + rho_ = std::shared_ptr>(new Rho()); rho_->setVerbosityLevel(ct.verbose); #ifdef HAVE_MAGMA @@ -1205,17 +1170,18 @@ template void MGmol::setGamma( const pb::Lap& lapOper, const Potentials& pot) { - assert(orbitals_precond_ != nullptr); + assert(orbitals_precond_); Control& ct = *(Control::instance()); - orbitals_precond_->setGamma(lapOper, pot, ct.getMGlevels(), proj_matrices_); + orbitals_precond_->setGamma( + lapOper, pot, ct.getMGlevels(), proj_matrices_.get()); } template void MGmol::precond_mg(OrbitalsType& phi) { - assert(orbitals_precond_ != nullptr); + assert(orbitals_precond_); orbitals_precond_->precond_mg(phi); } @@ -1446,11 +1412,44 @@ template void MGmol::addResidualSpreadPenalty( OrbitalsType& phi, OrbitalsType& res) { - assert(spread_penalty_ != nullptr); + assert(spread_penalty_); spread_penalty_->addResidual(phi, res); } +template +void MGmol::getAtomicPositions(std::vector& tau) +{ + ions_->getPositions(tau); +} + +template +void MGmol::getAtomicNumbers(std::vector& an) +{ + ions_->getAtomicNumbers(an); +} + +template +double MGmol::evaluateEnergyAndForces( + const std::vector& tau, std::vector& atnumbers, + std::vector& forces) +{ + assert(tau.size() == 3 * atnumbers.size()); + + Control& ct = *(Control::instance()); + + ions_->setPositions(tau, atnumbers); + + double eks = 0.; + quench(current_orbitals_, *ions_, ct.max_electronic_steps, 20, eks); + + force(*current_orbitals_, *ions_); + + ions_->getForces(forces); + + return eks; +} + template class MGmol; template class MGmol; template int MGmol::initial(); diff --git a/src/MGmol.h b/src/MGmol.h index 1ed04041..ef7a020b 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -44,6 +44,7 @@ class IonicAlgorithm; #include "AOMMprojector.h" #include "ClusterOrbitals.h" #include "DMStrategy.h" +#include "Energy.h" #include "ExtendedGridOrbitals.h" #include "Forces.h" #include "Ions.h" @@ -55,6 +56,8 @@ class IonicAlgorithm; #include "SpreadPenaltyInterface.h" #include "SpreadsAndCenters.h" +#include + template class MGmol : public MGmolInterface { @@ -63,48 +66,49 @@ class MGmol : public MGmolInterface MPI_Comm comm_; - XConGrid* xcongrid_; + std::shared_ptr xcongrid_; OrbitalsType* current_orbitals_; - AOMMprojector* aomm_; + std::shared_ptr aomm_; - Ions* ions_; + std::shared_ptr ions_; - Rho* rho_; + std::shared_ptr> rho_; - Energy* energy_; + std::shared_ptr> energy_; - Hamiltonian* hamiltonian_; + std::shared_ptr> hamiltonian_; - Forces* forces_; + std::shared_ptr> forces_; - MasksSet* currentMasks_; - MasksSet* corrMasks_; + std::shared_ptr currentMasks_; + std::shared_ptr corrMasks_; - // ProjectedMatrices* proj_matrices_; - ProjectedMatricesInterface* proj_matrices_; + std::shared_ptr proj_matrices_; - IonicAlgorithm* geom_optimizer_; + std::shared_ptr> geom_optimizer_; std::shared_ptr lrs_; - ClusterOrbitals* local_cluster_; + std::shared_ptr local_cluster_; + + std::shared_ptr g_kbpsi_; - KBPsiMatrixSparse* g_kbpsi_; + std::shared_ptr> spreadf_; - SpreadsAndCenters* spreadf_; + std::shared_ptr> spread_penalty_; - SpreadPenaltyInterface* spread_penalty_; + std::shared_ptr> dm_strategy_; - DMStrategy* dm_strategy_; + std::shared_ptr h5f_file_; - HDFrestart* h5f_file_; + std::shared_ptr> orbitals_precond_; double total_energy_; - ConstraintSet* constraints_; + std::shared_ptr constraints_; - OrbitalsExtrapolation* orbitals_extrapol_ = nullptr; + std::shared_ptr> orbitals_extrapol_; float md_time_; int md_iteration_; @@ -168,17 +172,30 @@ class MGmol : public MGmolInterface static Timer comp_res_tm_; static Timer init_nuc_tm_; - OrbitalsPreconditioning* orbitals_precond_; - public: - Electrostatic* electrostat_; + std::shared_ptr electrostat_; MGmol(MPI_Comm comm, std::ostream& os, std::string input_filename, std::string lrs_filename, std::string constraints_filename); ~MGmol() override; + /* access functions */ + OrbitalsType* getOrbitals() { return current_orbitals_; } + std::shared_ptr> getHamiltonian() { return hamiltonian_; } + void run() override; + + double evaluateEnergyAndForces(const std::vector& tau, + std::vector& atnumbers, std::vector& forces); + + /* + * get internal atomic positions + */ + void getAtomicPositions(std::vector& tau); + + void getAtomicNumbers(std::vector& an); + void initNuc(Ions& ions); void initKBR(); @@ -276,10 +293,6 @@ class MGmol : public MGmolInterface double get_evnl(const Ions& ions); void sebprintPositions(); void sebprintForces(); - void get_positions(std::vector>& r); - void set_positions(std::vector>& r); - void get_forces(std::vector>& f); - void set_forces(std::vector>& f); int nions() { return ions_->getNumIons(); } double getTotalEnergy(); void cleanup(); @@ -306,7 +319,7 @@ class MGmol : public MGmolInterface forces_->force(orbitals, ions); } - OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); + void loadRestartFile(const std::string filename); #ifdef MGMOL_HAS_LIBROM int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index 9932dd9f..046459d6 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -11,6 +11,7 @@ #define MGMOLINTERFACE_H #include +#include class MGmolInterface { @@ -24,6 +25,11 @@ class MGmolInterface virtual int setupConstraintsFromInput(const std::string input_file) = 0; virtual void setup() = 0; virtual void run() = 0; + virtual double evaluateEnergyAndForces(const std::vector& tau, + std::vector& atnumbers, std::vector& forces) + = 0; + virtual void getAtomicPositions(std::vector& tau) = 0; + virtual void getAtomicNumbers(std::vector& an) = 0; }; #endif diff --git a/src/MGmol_prototypes.h b/src/MGmol_prototypes.h index 207201d7..02e37890 100644 --- a/src/MGmol_prototypes.h +++ b/src/MGmol_prototypes.h @@ -25,7 +25,7 @@ double getLAeigen(const double tol, const int maxit, Ions& ions); int read_config(int argc, char** argv, boost::program_options::variables_map& vm, std::string& input_file, std::string& lrs_filename, std::string& constraints_filename, - float& total_spin, bool& with_spin, bool& tcheck); + float& total_spin, bool& with_spin); #ifdef MGMOL_HAS_LIBROM void setupROMConfigOption(po::options_description &rom_cfg); #endif diff --git a/src/OrbitalsExtrapolation.h b/src/OrbitalsExtrapolation.h index 47c1a21b..e2d0ff0c 100644 --- a/src/OrbitalsExtrapolation.h +++ b/src/OrbitalsExtrapolation.h @@ -26,8 +26,6 @@ class OrbitalsExtrapolation virtual ~OrbitalsExtrapolation(); - virtual bool extrapolatedH() const { return false; } - virtual void clearOldOrbitals(); bool getRestartData(OrbitalsType& orbitals); virtual void setupPreviousOrbitals(OrbitalsType** orbitals, diff --git a/src/Rho.cc b/src/Rho.cc index b456f6c1..ca0f20ea 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -175,6 +175,12 @@ void Rho::rescaleTotalCharge() (*MPIdata::sout) << " Rescaling factor: " << t1 << std::endl; (*MPIdata::sout) << " Num. electrons: " << nel << std::endl; } + if (fabs(t1 - 1.) > 0.02) + { + std::cerr << "Error on density too large to continue. Abort." + << std::endl; + mmpi.abort(); + } for (int ispin = 0; ispin < nspin; ispin++) LinearAlgebraUtils::MPscal( diff --git a/src/computeHij.cc b/src/computeHij.cc index 4942959a..75b8ef86 100644 --- a/src/computeHij.cc +++ b/src/computeHij.cc @@ -280,13 +280,14 @@ template void MGmol::getKBPsiAndHij(OrbitalsType& orbitals, Ions& ions, KBPsiMatrixSparse* kbpsi, dist_matrix::DistMatrix& hij) { - getKBPsiAndHij(orbitals, orbitals, ions, kbpsi, proj_matrices_, hij); + getKBPsiAndHij(orbitals, orbitals, ions, kbpsi, proj_matrices_.get(), hij); } template void MGmol::getKBPsiAndHij(OrbitalsType& orbitals, Ions& ions) { - getKBPsiAndHij(orbitals, orbitals, ions, g_kbpsi_, proj_matrices_); + getKBPsiAndHij( + orbitals, orbitals, ions, g_kbpsi_.get(), proj_matrices_.get()); } template @@ -396,7 +397,7 @@ template void MGmol::getHpsiAndTheta( Ions& ions, OrbitalsType& phi, OrbitalsType& hphi) { - getHpsiAndTheta(ions, phi, hphi, g_kbpsi_); + getHpsiAndTheta(ions, phi, hphi, g_kbpsi_.get()); } template @@ -434,10 +435,10 @@ void MGmol::getHpsiAndTheta(Ions& ions, OrbitalsType& phi, proj_matrices_->clearSparseH(); // Compute the contribution of the non-local potential into sh - kbpsi->computeHvnlMatrix(ions, proj_matrices_); + kbpsi->computeHvnlMatrix(ions, proj_matrices_.get()); // add local part of H to sh - hamiltonian_->addHlocalij(phi, proj_matrices_); + hamiltonian_->addHlocalij(phi, proj_matrices_.get()); energy_->saveVofRho(); diff --git a/src/lbfgsrlx.cc b/src/lbfgsrlx.cc index 3881abab..b2aff795 100644 --- a/src/lbfgsrlx.cc +++ b/src/lbfgsrlx.cc @@ -36,15 +36,14 @@ void MGmol::lbfgsrlx(OrbitalsType** orbitals, Ions& ions) Control& ct = *(Control::instance()); LBFGS lbfgs(orbitals, ions, *rho_, *constraints_, lrs_, - local_cluster_, *currentMasks_, *corrMasks_, *electrostat_, ct.dt, + local_cluster_.get(), *currentMasks_, *corrMasks_, *electrostat_, ct.dt, *this); DFTsolver::resetItCount(); - lbfgs.init(h5f_file_); + lbfgs.init(h5f_file_.get()); - delete h5f_file_; - h5f_file_ = nullptr; + h5f_file_.reset(); // additional quench to compensate random start if (ct.restart_info < 3) diff --git a/src/main.cc b/src/main.cc index f6f396d2..3fcbf791 100644 --- a/src/main.cc +++ b/src/main.cc @@ -29,39 +29,16 @@ #include "MGmol.h" #include "MGmol_MPI.h" #include "MPIdata.h" -#include "Mesh.h" #include "mgmol_run.h" -#include "tools.h" #include -#include #include -#include -#include #include #include -#include - #include namespace po = boost::program_options; -//#include "MemTrack.h" - -/* -void trapfpe () { - feenableexcept(FE_INVALID|FE_DIVBYZERO|FE_OVERFLOW); -} -*/ - -// A helper function -template -std::ostream& operator<<(std::ostream& os, const std::vector& v) -{ - copy(v.begin(), v.end(), std::ostream_iterator(std::cout, " ")); - return os; -} - int main(int argc, char** argv) { int mpirc = MPI_Init(&argc, &argv); @@ -73,33 +50,37 @@ int main(int argc, char** argv) MPI_Comm comm = MPI_COMM_WORLD; + /* + * Initialize general things, like magma, openmp, IO, ... + */ mgmol_init(comm); - // read runtime parameters + /* + * read runtime parameters + */ std::string input_filename(""); std::string lrs_filename; std::string constraints_filename(""); - bool tcheck = false; float total_spin = 0.; bool with_spin = false; po::variables_map vm; - // use configure file if it can be found - // std::string config_filename("mgmol.cfg"); - - // read options from PE0 only + // read from PE0 only if (MPIdata::onpe0) { read_config(argc, argv, vm, input_filename, lrs_filename, - constraints_filename, total_spin, with_spin, tcheck); + constraints_filename, total_spin, with_spin); } MGmol_MPI::setup(comm, std::cout, with_spin); MGmol_MPI& mmpi = *(MGmol_MPI::instance()); MPI_Comm global_comm = mmpi.commGlobal(); + /* + * Setup control struct with run time parameters + */ Control::setup(global_comm, with_spin, total_spin); Control& ct = *(Control::instance()); @@ -108,11 +89,6 @@ int main(int argc, char** argv) int ret = ct.checkOptions(); if (ret < 0) return ret; - unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; - double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; - const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; - Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); - mmpi.bcastGlobal(input_filename); mmpi.bcastGlobal(lrs_filename); @@ -126,14 +102,10 @@ int main(int argc, char** argv) mgmol = new MGmol(global_comm, *MPIdata::sout, input_filename, lrs_filename, constraints_filename); - if (!tcheck) - { - mgmol->run(); - } - else - { - *MPIdata::sout << " Input parameters OK\n"; - } + mgmol->setup(); + + mgmol->run(); + delete mgmol; } // close main scope @@ -150,9 +122,5 @@ int main(int argc, char** argv) time(&tt); if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; - // MemTrack::TrackDumpBlocks(); - - // MemTrack::TrackListMemoryUsage(); - return 0; } diff --git a/src/md.cc b/src/md.cc index 089deb82..939c6e62 100644 --- a/src/md.cc +++ b/src/md.cc @@ -91,11 +91,6 @@ void MGmol::postWFextrapolation(OrbitalsType* orbitals) else proj_matrices_->setDMto2InvS(); } - else - { - if (orbitals_extrapol_->extrapolatedH()) - dm_strategy_->update(*orbitals); - } } template @@ -137,8 +132,8 @@ OrbitalsType* MGmol::new_orbitals_with_current_LRs(bool setup) // need to build new orbitals as masks have changed OrbitalsType* new_orbitals = new OrbitalsType("NewMasks", mygrid, - mymesh->subdivx(), ct.numst, ct.bcWF, proj_matrices_, lrs_, - currentMasks_, corrMasks_, local_cluster_, setup); + mymesh->subdivx(), ct.numst, ct.bcWF, proj_matrices_.get(), lrs_, + currentMasks_.get(), corrMasks_.get(), local_cluster_.get(), setup); return new_orbitals; } @@ -317,8 +312,8 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) int size_tau = (int)tau0.size(); DFTsolver::resetItCount(); - orbitals_extrapol_ = OrbitalsExtrapolationFactory::create( - ct.WFExtrapolation()); + orbitals_extrapol_.reset(OrbitalsExtrapolationFactory::create( + ct.WFExtrapolation())); MD_IonicStepper* stepper = new MD_IonicStepper( ct.dt, atmove, tau0, taup, taum, fion, pmass, rand_states); @@ -349,8 +344,9 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) constraints_->enforceConstraints(20); stepper->updateTau(); - ions.setPositions(tau0); + ions.setLocalPositions(tau0); + // setup required after updating local ions positions ions.setup(); } @@ -376,8 +372,8 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (onpe0) os_ << "Create new orbitals_minus1..." << std::endl; orbitals_extrapol_->setupPreviousOrbitals(¤t_orbitals_, - proj_matrices_, lrs_, local_cluster_, currentMasks_, - corrMasks_, *h5f_file_); + proj_matrices_.get(), lrs_, local_cluster_.get(), + currentMasks_.get(), corrMasks_.get(), *h5f_file_); // need to reset a few things as we just read new orbitals (*orbitals)->computeGramAndInvS(); @@ -395,8 +391,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) moveVnuc(ions); } - delete h5f_file_; - h5f_file_ = nullptr; + h5f_file_.reset(); } // additional SC steps to compensate random start @@ -437,7 +432,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ct.adaptiveLRs()) { assert(lrs_); - adaptLR(spreadf_, nullptr); + adaptLR(spreadf_.get(), nullptr); last_move_is_small = lrs_->moveIsSmall(); @@ -495,7 +490,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) force(**orbitals, ions); // set fion - ions.getForces(fion); + ions.getLocalForces(fion); // constraints need to be added again and setup as atoms // may have moved and local atoms are not the same anymore @@ -682,11 +677,11 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) } delete stepper; - delete orbitals_extrapol_; + orbitals_extrapol_.reset(); } template -OrbitalsType* MGmol::loadOrbitalFromRestartFile( +void MGmol::loadRestartFile( const std::string filename) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -700,53 +695,15 @@ OrbitalsType* MGmol::loadOrbitalFromRestartFile( || (ct.AtomsDynamic() == AtomsDynamicType::Quench)); HDFrestart h5file(filename, myPEenv, ct.out_restart_file_type); - int ierr; - - OrbitalsType* restart_orbitals = nullptr; - - /* This corresponds to MGmol::initial */ - { - // Copy from current orbital, instead of constructing brand-new one - restart_orbitals - = new OrbitalsType("ForLoading", *current_orbitals_, false); - /* This corresponds to MGmol::read_restart_data */ - { - ierr = restart_orbitals->read_func_hdf5(h5file); - } // read_restart_data - } // initial() - - /* This corresponds to MGmol::md */ + int ierr = read_restart_data(h5file, *rho_, *current_orbitals_); + if (ierr < 0) { - int flag_extrapolated_data = 0; if (onpe0) - { - flag_extrapolated_data - = h5file.dset_exists("ExtrapolatedFunction0000"); - if (flag_extrapolated_data == 0) - flag_extrapolated_data - = h5file.dset_exists("ExtrapolatedFunction0"); - } - MPI_Bcast(&flag_extrapolated_data, 1, MPI_INT, 0, comm_); - - /* - If extrapolated function exists, - then function is set as previous orbitals, - while the extrapolated function is set as the current orbitals. - This is how the restart file is saved via dumprestartFile. + (*MPIdata::serr) << "loadRestartFile: failed to read the restart file." << std::endl; - For now, we just enforce to not use the restart files with extrapolation. - */ - if (flag_extrapolated_data) - { - if (onpe0) - (*MPIdata::serr) << "loadRestartFile: does not support restart files with extrapolation." << std::endl; - - global_exit(0); - } - - /* main workflow delete h5f_file_ here, meaning the loading is over. */ - } // md() + global_exit(0); + } ierr = h5file.close(); mmpi.allreduce(&ierr, 1, MPI_MIN); @@ -755,20 +712,10 @@ OrbitalsType* MGmol::loadOrbitalFromRestartFile( if (onpe0) (*MPIdata::serr) << "loadRestartFile: cannot close file..." << std::endl; - return nullptr; + return; } - /* - In returning restart_orbitals, - we hope that the wavefunctions in restart_orbitals are all set. - At least the following functions should return proper data loaded from - the file: - - restart_orbitals->getLocNumpt() - restart_orbitals->chromatic_number() - restart_orbitals->getPsi(idx) (for int idx) - */ - return restart_orbitals; + return; } template class MGmol; diff --git a/src/mlwf.cc b/src/mlwf.cc index cc4fdeb8..3efdfeb2 100644 --- a/src/mlwf.cc +++ b/src/mlwf.cc @@ -279,9 +279,9 @@ int MGmol::get_NOLMO(NOLMOTransform& noot, OrbitalsType& orbitals, } sincos.clear(); - ProjectedMatrices>* projmatrices - = dynamic_cast< - ProjectedMatrices>*>( + std::shared_ptr>> + projmatrices = std::dynamic_pointer_cast< + ProjectedMatrices>>( proj_matrices_); assert(projmatrices); diff --git a/src/quench.cc b/src/quench.cc index de01dad3..614aa636 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -326,8 +326,9 @@ bool MGmol::rotateStatesPairsOverlap( spreadf2st.print(os_); orbitals.computeGramAndInvS(); - ProjectedMatricesSparse* projmatrices - = dynamic_cast(proj_matrices_); + std::shared_ptr projmatrices + = std::dynamic_pointer_cast( + proj_matrices_); if (onpe0) std::cout << "Gram Matrix for pair of states after transformation:" << std::endl; @@ -398,7 +399,7 @@ void MGmol::disentangleOrbitals(OrbitalsType& orbitals, template <> void MGmol::applyAOMMprojection(LocGridOrbitals& orbitals) { - aomm_ = new AOMMprojector(orbitals, lrs_); + aomm_.reset(new AOMMprojector(orbitals, lrs_)); aomm_->projectOut(orbitals); } @@ -422,8 +423,9 @@ int MGmol::outerSolve(LocGridOrbitals& orbitals, case OuterSolverType::ABPG: case OuterSolverType::NLCG: { - DFTsolver solver(hamiltonian_, proj_matrices_, - energy_, electrostat_, this, ions, rho_, dm_strategy_, os_); + DFTsolver solver(hamiltonian_.get(), + proj_matrices_.get(), energy_.get(), electrostat_.get(), this, + ions, rho_.get(), dm_strategy_.get(), os_); retval = solver.solve( orbitals, work_orbitals, ions, max_steps, iprint, last_eks); @@ -433,9 +435,9 @@ int MGmol::outerSolve(LocGridOrbitals& orbitals, case OuterSolverType::PolakRibiere: { - PolakRibiereSolver solver(hamiltonian_, - proj_matrices_, energy_, electrostat_, this, ions, rho_, - dm_strategy_, os_); + PolakRibiereSolver solver(hamiltonian_.get(), + proj_matrices_.get(), energy_.get(), electrostat_.get(), this, + ions, rho_.get(), dm_strategy_.get(), os_); retval = solver.solve( orbitals, work_orbitals, ions, max_steps, iprint, last_eks); @@ -466,8 +468,9 @@ int MGmol::outerSolve(OrbitalsType& orbitals, case OuterSolverType::ABPG: case OuterSolverType::NLCG: { - DFTsolver solver(hamiltonian_, proj_matrices_, - energy_, electrostat_, this, ions, rho_, dm_strategy_, os_); + DFTsolver solver(hamiltonian_.get(), + proj_matrices_.get(), energy_.get(), electrostat_.get(), this, + ions, rho_.get(), dm_strategy_.get(), os_); retval = solver.solve( orbitals, work_orbitals, ions, max_steps, iprint, last_eks); @@ -477,9 +480,9 @@ int MGmol::outerSolve(OrbitalsType& orbitals, case OuterSolverType::PolakRibiere: { - PolakRibiereSolver solver(hamiltonian_, - proj_matrices_, energy_, electrostat_, this, ions, rho_, - dm_strategy_, os_); + PolakRibiereSolver solver(hamiltonian_.get(), + proj_matrices_.get(), energy_.get(), electrostat_.get(), this, + ions, rho_.get(), dm_strategy_.get(), os_); retval = solver.solve( orbitals, work_orbitals, ions, max_steps, iprint, last_eks); @@ -499,8 +502,9 @@ int MGmol::outerSolve(OrbitalsType& orbitals, #else DavidsonSolver> #endif - solver(os_, *ions_, hamiltonian_, rho_, energy_, electrostat_, - this, gids, ct.dm_mix, with_spin); + solver(os_, *ions_, hamiltonian_.get(), rho_.get(), + energy_.get(), electrostat_.get(), this, gids, ct.dm_mix, + with_spin); retval = solver.solve(orbitals, work_orbitals); break; @@ -558,9 +562,9 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, applyAOMMprojection(*orbitals); } - orbitals_precond_ = new OrbitalsPreconditioning(); + orbitals_precond_.reset(new OrbitalsPreconditioning()); orbitals_precond_->setup( - *orbitals, ct.getMGlevels(), ct.lap_type, currentMasks_, lrs_); + *orbitals, ct.getMGlevels(), ct.lap_type, currentMasks_.get(), lrs_); // solve electronic structure problem // (inner iterations) @@ -570,11 +574,9 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, if (ct.use_kernel_functions) { - delete aomm_; - aomm_ = nullptr; + aomm_.reset(); } - delete orbitals_precond_; - orbitals_precond_ = nullptr; + orbitals_precond_.reset(); // Get the n.l. energy // TODO: Fix bug where energy vs. time output is incorrect if get_evnl is @@ -599,7 +601,8 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, << " TS [Ha] = " << ts << std::endl; } } - last_eks = energy_->evaluateTotal(ts, proj_matrices_, *orbitals, 2, os_); + last_eks + = energy_->evaluateTotal(ts, proj_matrices_.get(), *orbitals, 2, os_); if (ct.computeCondGramMD()) { diff --git a/src/readInput.cc b/src/readInput.cc index b0a47411..79fbecf1 100644 --- a/src/readInput.cc +++ b/src/readInput.cc @@ -175,7 +175,7 @@ int MGmol::readCoordinates( // setup ions const std::vector& sp(ct.getSpecies()); - ions_ = new Ions(lattice, sp); + ions_.reset(new Ions(lattice, sp)); if (ct.restart_info > 0 && ct.override_restart == 0) // read restart ionic positions @@ -217,7 +217,7 @@ int MGmol::readCoordinates( // setup ions const std::vector& sp(ct.getSpecies()); - ions_ = new Ions(lattice, sp); + ions_.reset(new Ions(lattice, sp)); if (ct.restart_info > 0 && ct.override_restart == 0) // read restart ionic positions diff --git a/src/read_config.cc b/src/read_config.cc index 6ce97648..74533dfb 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -21,8 +21,7 @@ namespace po = boost::program_options; int read_config(int argc, char** argv, po::variables_map& vm, std::string& input_file, std::string& lrs_filename, - std::string& constraints_filename, float& total_spin, bool& with_spin, - bool& tcheck) + std::string& constraints_filename, float& total_spin, bool& with_spin) { // use configure file if it can be found // std::string config_filename("mgmol.cfg"); @@ -378,10 +377,6 @@ int read_config(int argc, char** argv, po::variables_map& vm, return 0; } - if (vm.count("check")) - { - tcheck = true; - } if (vm.count("spin")) { total_spin = vm["spin"].as(); diff --git a/src/rom_main.cc b/src/rom_main.cc index dc877a58..3718078f 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -50,7 +50,6 @@ int main(int argc, char** argv) std::string input_filename(""); std::string lrs_filename; std::string constraints_filename(""); - bool tcheck = false; float total_spin = 0.; bool with_spin = false; @@ -64,7 +63,7 @@ int main(int argc, char** argv) if (MPIdata::onpe0) { read_config(argc, argv, vm, input_filename, lrs_filename, - constraints_filename, total_spin, with_spin, tcheck); + constraints_filename, total_spin, with_spin); } MGmol_MPI::setup(comm, std::cout, with_spin); @@ -79,6 +78,11 @@ int main(int argc, char** argv) int ret = ct.checkOptions(); if (ret < 0) return ret; + unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; + double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; + const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; + Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); + mmpi.bcastGlobal(input_filename); mmpi.bcastGlobal(lrs_filename); @@ -86,37 +90,19 @@ int main(int argc, char** argv) { MGmolInterface* mgmol; if (ct.isLocMode()) - mgmol = new MGmol(global_comm, *MPIdata::sout); + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); else - mgmol - = new MGmol(global_comm, *MPIdata::sout); - - unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; - double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; - const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; - Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); - - mgmol->setupFromInput(input_filename); - - if (ct.isLocMode() || ct.init_loc == 1) mgmol->setupLRs(lrs_filename); + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); - mgmol->setupConstraintsFromInput(constraints_filename); + mgmol->setup(); - mgmol_setup(); - - if (!tcheck) - { - mgmol->setup(); - - if (ct.isLocMode()) - readRestartFiles(mgmol); - else - readRestartFiles(mgmol); - } + if (ct.isLocMode()) + readRestartFiles(mgmol); else - { - *MPIdata::sout << " Input parameters OK\n"; - } + readRestartFiles(mgmol); + delete mgmol; } // close main scope diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 4594dd3a..1788281d 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -40,7 +40,7 @@ void readRestartFiles(MGmolInterface *mgmol_) /* Read the first snapshot to determin dimension and number of snapshots */ filename = string_format(rom_options.restart_file_fmt, minidx); - orbitals = mgmol->loadOrbitalFromRestartFile(filename); + mgmol->loadRestartFile(filename); const int dim = orbitals->getLocNumpt(); const int chrom_num = orbitals->chromatic_number(); const int totalSamples = orbitals->chromatic_number() * num_restart; @@ -54,7 +54,7 @@ void readRestartFiles(MGmolInterface *mgmol_) for (int k = minidx; k <= maxidx; k++) { filename = string_format(rom_options.restart_file_fmt, k); - orbitals = mgmol->loadOrbitalFromRestartFile(filename); + mgmol->loadRestartFile(filename); assert(dim == orbitals->getLocNumpt()); assert(chrom_num == orbitals->chromatic_number()); diff --git a/src/runfire.cc b/src/runfire.cc index d71c52a8..3239e1b1 100644 --- a/src/runfire.cc +++ b/src/runfire.cc @@ -33,13 +33,12 @@ void MGmol::runfire(OrbitalsType** orbitals, Ions& ions) DFTsolver::resetItCount(); - orbitals_extrapol_ = OrbitalsExtrapolationFactory::create( - ct.WFExtrapolation()); + orbitals_extrapol_.reset(OrbitalsExtrapolationFactory::create( + ct.WFExtrapolation())); - fire.init(h5f_file_); + fire.init(h5f_file_.get()); - delete h5f_file_; - h5f_file_ = nullptr; + h5f_file_.reset(); // additional quench to compensate random start if (ct.restart_info < 3) @@ -129,7 +128,7 @@ void MGmol::runfire(OrbitalsType** orbitals, Ions& ions) } // end for steps - delete orbitals_extrapol_; + orbitals_extrapol_.reset(); // final dump if (ct.out_restart_info > 0) diff --git a/src/setup.cc b/src/setup.cc index 62d9f835..f022678c 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -28,7 +28,15 @@ int MGmol::setupFromInput(const std::string filename) MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - hamiltonian_ = new Hamiltonian(); + /* + * Setup global mesh for calculations + */ + unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; + double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; + const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; + Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); + + hamiltonian_.reset(new Hamiltonian()); Potentials& pot = hamiltonian_->potential(); ct.registerPotentials(pot); @@ -43,8 +51,8 @@ int MGmol::setupFromInput(const std::string filename) const pb::PEenv& myPEenv = mymesh->peenv(); if (ct.restart_info > 0) - h5f_file_ - = new HDFrestart(ct.restart_file, myPEenv, ct.restart_file_type); + h5f_file_.reset( + new HDFrestart(ct.restart_file, myPEenv, ct.restart_file_type)); int status = readCoordinates(filename, false); if (status == -1) return -1; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4c68adb7..fe7b571b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -482,7 +482,7 @@ add_test(NAME ChebyshevMVP ${CMAKE_CURRENT_SOURCE_DIR}/Chebyshev/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -if(NOT ${MAGMA_FOUND}) +if(NOT ${MGMOL_WITH_MAGMA}) add_test(NAME testShortSighted COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/test.py ${MPIEXEC} --oversubscribe ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} diff --git a/tests/SiH4/testSiH4.py b/tests/SiH4/testSiH4.py index 1a0965ac..c7f81bdc 100755 --- a/tests/SiH4/testSiH4.py +++ b/tests/SiH4/testSiH4.py @@ -44,16 +44,21 @@ lines=output.split(b'\n') tol = 5.e-3 +found_forces = False for line in lines: - num_matches = line.count(b'%%') - if num_matches: + if line.count(b'%%'): print(line) - num_matches = line.count(b'##') - if num_matches: + if line.count(b'##'): words=line.split() if len(words)==8: print(line) + found_forces = True for i in range(5,8): if abs(eval(words[i]))>tol: sys.exit(1) +if (not found_forces): + print("no forces found") + sys.exit(1) + +sys.exit(0) diff --git a/util/xyz2in.py b/util/xyz2in.py index 3555e138..bb2e022d 100644 --- a/util/xyz2in.py +++ b/util/xyz2in.py @@ -5,9 +5,11 @@ # This file is part of MGmol. For details, see https://github.com/llnl/mgmol. # Please also read this link https://github.com/llnl/mgmol/LICENSE # -# Python program to convert .xyz file into mgmol input +# Python program to convert .xyz file into mgmol input coordinates file +# Optional arguments: [lx ly lz] to define box size and map all atoms into +# box (0,0,0)-(lx,ly,lz) using periodic boundary conditions # -# use: python coords.xyz > coords.in +# use: python coords.xyz [lx ly lz] > coords.in #------------------------------------------------------------------------------- import sys, string @@ -15,6 +17,16 @@ #read file ifile=open(sys.argv[1],'r') + +lx = 0. +ly = 0. +lz = 0. +if( len(sys.argv) > 2 ): + lx = eval(sys.argv[2]) + ly = eval(sys.argv[3]) + lz = eval(sys.argv[4]) + + lines=ifile.readlines() count=0 @@ -22,13 +34,31 @@ dummy=0 #unused flag set to 0 for line in lines: ## loop over lines of file - words=line.split() - if len(words)>1: - if words[0][0:1]!='#': - name=words[0]+str(count) + if count>1: + words=line.split() + if len(words)>1: + name=words[0]+str(count-2) x=eval(words[1])*ang2bohr y=eval(words[2])*ang2bohr z=eval(words[3])*ang2bohr - + + if lx > 0.: + if x<0: + x = x +lx + if x>lx: + x = x -lx + + if ly > 0.: + if y<0: + y = y +ly + if y>ly: + y = y -ly + + if lz > 0.: + if z<0: + z = z +lz + if z>lz: + z = z -lz + print(name,'\t',dummy,'\t',x,'\t',y,'\t',z,'\t',movable) - count=count+1 + count=count+1 From 63587f6cd5080a99286b757471ae4b1fa95bc54f Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Mon, 17 Jun 2024 09:44:09 -0700 Subject: [PATCH 12/33] Snapshot collection/POD training for potential (#254) * offline workflow with potential POD training * update rom main driver again * ci workflow include librom in compilation * add config option --- .github/workflows/ci.yml | 2 +- src/Control.cc | 13 ++++++++++ src/read_config.cc | 6 +++-- src/rom_Control.h | 8 ++++++ src/rom_workflows.cc | 55 ++++++++++++++++++++++++++++++++-------- src/rom_workflows.h | 1 + 6 files changed, 71 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05942636..beec14ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: run: | mkdir build cd build - cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" -DUSE_LIBROM=On -DLIBROM_PATH=/env/dependencies/libROM - name: make run: | cd build && make -j 4 diff --git a/src/Control.cc b/src/Control.cc index ce3023e3..ae8caf4e 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2077,6 +2077,14 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.restart_file_maxidx = vm["ROM.offline.restart_max_idx"].as(); rom_pri_option.basis_file = vm["ROM.offline.basis_file"].as(); + str = vm["ROM.offline.variable"].as(); + if (str.compare("orbitals") == 0) + rom_pri_option.variable = ROMVariable::ORBITALS; + else if (str.compare("potential") == 0) + rom_pri_option.variable = ROMVariable::POTENTIAL; + else + rom_pri_option.variable = ROMVariable::NONE; + rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); } // onpe0 @@ -2116,5 +2124,10 @@ void Control::syncROMOptions() mpirc = MPI_Bcast(&rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_); bcast_check(mpirc); + short rom_var = (short)static_cast(rom_pri_option.variable); + mpirc = MPI_Bcast(&rom_var, 1, MPI_SHORT, 0, comm_global_); + bcast_check(mpirc); + rom_pri_option.rom_stage = static_cast(rom_stage); + rom_pri_option.variable = static_cast(rom_var); } \ No newline at end of file diff --git a/src/read_config.cc b/src/read_config.cc index 74533dfb..159e8668 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -425,6 +425,8 @@ void setupROMConfigOption(po::options_description &rom_cfg) ("ROM.offline.basis_file", po::value()->default_value(""), "File name for libROM snapshot/POD matrices.") ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), - "Save libROM snapshot file at FOM simulation."); + "Save libROM snapshot file at FOM simulation.") + ("ROM.offline.variable", po::value()->default_value(""), + "FOM variable to perform POD: either orbitals or potential."); } -#endif \ No newline at end of file +#endif diff --git a/src/rom_Control.h b/src/rom_Control.h index d1fd6854..753a2e68 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -25,6 +25,13 @@ enum class ROMStage UNSUPPORTED }; +enum class ROMVariable +{ + ORBITALS, + POTENTIAL, + NONE +}; + /* Stored as a private member variable of Control class */ struct ROMPrivateOptions { @@ -34,6 +41,7 @@ struct ROMPrivateOptions int restart_file_minidx = -1; int restart_file_maxidx = -1; std::string basis_file = ""; + ROMVariable variable=ROMVariable::NONE; /* save librom snapshot matrix at FOM simulation. */ bool save_librom_snapshot = false; diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 1788281d..b73f1898 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -27,7 +27,14 @@ template void readRestartFiles(MGmolInterface *mgmol_) { Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + ROMPrivateOptions rom_options = ct.getROMOptions(); + /* type of variable we intend to run POD */ + ROMVariable rom_var = rom_options.variable; + + /* number of restart files, start/end indices */ assert(rom_options.restart_file_minidx >= 0); assert(rom_options.restart_file_maxidx >= 0); const int minidx = rom_options.restart_file_minidx; @@ -35,20 +42,37 @@ void readRestartFiles(MGmolInterface *mgmol_) const int num_restart = maxidx - minidx + 1; MGmol *mgmol = static_cast *>(mgmol_); - OrbitalsType *orbitals = nullptr; + OrbitalsType *orbitals = mgmol->getOrbitals(); + Potentials& pot = mgmol->getHamiltonian()->potential(); std::string filename; - /* Read the first snapshot to determin dimension and number of snapshots */ - filename = string_format(rom_options.restart_file_fmt, minidx); - mgmol->loadRestartFile(filename); - const int dim = orbitals->getLocNumpt(); + /* Determine basis prefix, dimension, and sample size */ + std::string basis_prefix = rom_options.basis_file; + int dim; + int totalSamples = num_restart; const int chrom_num = orbitals->chromatic_number(); - const int totalSamples = orbitals->chromatic_number() * num_restart; - delete orbitals; + switch (rom_var) + { + case ROMVariable::ORBITALS: + basis_prefix += "_orbitals"; + dim = orbitals->getLocNumpt(); + /* if orbitals, each sample have chromatic number of wave functions */ + totalSamples *= orbitals->chromatic_number(); + break; + + case ROMVariable::POTENTIAL: + basis_prefix += "_potential"; + dim = pot.size(); + break; + + default: + ct.global_exit(-1); + break; + } /* Initialize libROM classes */ CAROM::Options svd_options(dim, totalSamples, 1); - CAROM::BasisGenerator basis_generator(svd_options, false, rom_options.basis_file); + CAROM::BasisGenerator basis_generator(svd_options, false, basis_prefix); /* Collect the restart files */ for (int k = minidx; k <= maxidx; k++) @@ -58,10 +82,19 @@ void readRestartFiles(MGmolInterface *mgmol_) assert(dim == orbitals->getLocNumpt()); assert(chrom_num == orbitals->chromatic_number()); - for (int i = 0; i < chrom_num; ++i) - basis_generator.takeSample(orbitals->getPsi(i)); + switch (rom_var) + { + case ROMVariable::ORBITALS: + for (int i = 0; i < chrom_num; ++i) + basis_generator.takeSample(orbitals->getPsi(i)); + break; - delete orbitals; + case ROMVariable::POTENTIAL: + basis_prefix += "_potential"; + /* we save total potential for now */ + basis_generator.takeSample(pot.vtot()); + break; + } } basis_generator.writeSnapshot(); basis_generator.endSamples(); diff --git a/src/rom_workflows.h b/src/rom_workflows.h index dfb22b79..d45abb92 100644 --- a/src/rom_workflows.h +++ b/src/rom_workflows.h @@ -13,6 +13,7 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" #include "LocGridOrbitals.h" +#include "Potentials.h" #include "MGmol.h" #include "MGmol_MPI.h" #include "MPIdata.h" From 58b9abe293a082893d7180c4f7826dcf1dc7ab51 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Fri, 19 Jul 2024 11:02:53 -0700 Subject: [PATCH 13/33] Poisson rom (#257) * rom workflow for buildROMPoissonOperator * change from total to hartree potential * test routine for Poisson ROM operator. * POD reconstruction * test Poisson ROM operator verified. * buildROMPoissonOperator updated. * ci test for Poisson ROM operator --- .github/workflows/ci.yml | 14 +- src/Control.cc | 7 + src/Electrostatic.h | 2 + src/Hartree_CG.h | 7 + src/PCGSolver.h | 2 + src/Poisson.h | 6 + src/read_config.cc | 4 +- src/rom_Control.h | 4 + src/rom_main.cc | 38 ++- src/rom_workflows.cc | 341 ++++++++++++++++++++++++- src/rom_workflows.h | 7 + tests/ROM/test_rom_poisson/carbyne.cfg | 63 +++++ tests/ROM/test_rom_poisson/carbyne.in | 14 + 13 files changed, 497 insertions(+), 12 deletions(-) create mode 100644 tests/ROM/test_rom_poisson/carbyne.cfg create mode 100644 tests/ROM/test_rom_poisson/carbyne.in diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beec14ef..5e94113f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,15 +35,21 @@ jobs: submodules: 'true' - name: cmake run: | - mkdir build - cd build + mkdir ${GITHUB_WORKSPACE}/build + cd ${GITHUB_WORKSPACE}/build cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" -DUSE_LIBROM=On -DLIBROM_PATH=/env/dependencies/libROM - name: make run: | - cd build && make -j 4 + cd ${GITHUB_WORKSPACE}/build && make -j 4 - name: test run: | - cd build && ctest --no-compress-output -V -T Test -I 1,20,1 + cd ${GITHUB_WORKSPACE}/build && ctest --no-compress-output -V -T Test -I 1,20,1 + - name: test ROM Poisson operator + run: | + cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson + ln -s ${GITHUB_WORKSPACE}/build/src/mgmol-rom . + ln -s ${GITHUB_WORKSPACE}/potentials/* . + mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.cfg -i carbyne.in # code-style: # runs-on: ubuntu-latest # needs: [docker-image] diff --git a/src/Control.cc b/src/Control.cc index ae8caf4e..d1c4c672 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2069,6 +2069,8 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.rom_stage = ROMStage::ONLINE; else if (str.compare("build") == 0) rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("test_poisson") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_POISSON; else if (str.compare("none") == 0) rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; @@ -2086,6 +2088,8 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.variable = ROMVariable::NONE; rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); + + rom_pri_option.num_potbasis = vm["ROM.basis.number_of_potential_basis"].as(); } // onpe0 // synchronize all processors @@ -2130,4 +2134,7 @@ void Control::syncROMOptions() rom_pri_option.rom_stage = static_cast(rom_stage); rom_pri_option.variable = static_cast(rom_var); + + mpirc = MPI_Bcast(&rom_pri_option.num_potbasis, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); } \ No newline at end of file diff --git a/src/Electrostatic.h b/src/Electrostatic.h index a3b86879..785bc909 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -46,6 +46,8 @@ class Electrostatic ~Electrostatic(); static Timer solve_tm() { return solve_tm_; } + Poisson* getPoissonSolver() { return poisson_solver_; } + void setup(const short max_sweeps); void setupPB(const double e0, const double rho0, const double drho0, Potentials& pot); diff --git a/src/Hartree_CG.h b/src/Hartree_CG.h index 78e9da65..efa50919 100644 --- a/src/Hartree_CG.h +++ b/src/Hartree_CG.h @@ -40,6 +40,13 @@ class Hartree_CG : public Poisson void solve(const pb::GridFunc& rho, const pb::GridFunc& rhoc) override; + + void applyOperator(pb::GridFunc &vh, + pb::GridFunc &lhs) override + { + T *oper = poisson_solver_->getOperator(); + oper->apply(vh, lhs); + } }; #endif diff --git a/src/PCGSolver.h b/src/PCGSolver.h index 8ba76158..d38787f1 100644 --- a/src/PCGSolver.h +++ b/src/PCGSolver.h @@ -97,6 +97,8 @@ class PCGSolver double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } + T* getOperator() { return &oper_; } + // Destructor ~PCGSolver() { clear(); } }; diff --git a/src/Poisson.h b/src/Poisson.h index 87107542..14716cda 100644 --- a/src/Poisson.h +++ b/src/Poisson.h @@ -88,6 +88,12 @@ class Poisson : public PoissonInterface Int_vhrho_ = vel * vh_->gdot(rho); Int_vhrhoc_ = vel * vh_->gdot(rhoc); } + + virtual void applyOperator(pb::GridFunc &vh, pb::GridFunc &lhs) + { + std::cerr << "ERROR: Abstract method Poisson::applyOperator()" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } }; #endif diff --git a/src/read_config.cc b/src/read_config.cc index 159e8668..14057553 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -427,6 +427,8 @@ void setupROMConfigOption(po::options_description &rom_cfg) ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), "Save libROM snapshot file at FOM simulation.") ("ROM.offline.variable", po::value()->default_value(""), - "FOM variable to perform POD: either orbitals or potential."); + "FOM variable to perform POD: either orbitals or potential.") + ("ROM.basis.number_of_potential_basis", po::value()->default_value(-1), + "Number of potential POD basis to build Hartree potential ROM operator."); } #endif diff --git a/src/rom_Control.h b/src/rom_Control.h index 753a2e68..2e4a1c71 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -22,6 +22,7 @@ enum class ROMStage ONLINE, RESTORE, // TODO(kevin): what stage is this? BUILD, + TEST_POISSON, UNSUPPORTED }; @@ -45,6 +46,9 @@ struct ROMPrivateOptions /* save librom snapshot matrix at FOM simulation. */ bool save_librom_snapshot = false; + + /* options for ROM building */ + int num_potbasis = -1; }; #endif // ROM_CONTROL_H diff --git a/src/rom_main.cc b/src/rom_main.cc index 3718078f..c54bcd37 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -86,6 +86,10 @@ int main(int argc, char** argv) mmpi.bcastGlobal(input_filename); mmpi.bcastGlobal(lrs_filename); + /* Get ROM driver mode */ + ROMPrivateOptions rom_options = ct.getROMOptions(); + ROMStage rom_stage = rom_options.rom_stage; + // Enter main scope { MGmolInterface* mgmol; @@ -98,10 +102,36 @@ int main(int argc, char** argv) mgmol->setup(); - if (ct.isLocMode()) - readRestartFiles(mgmol); - else - readRestartFiles(mgmol); + switch (rom_stage) + { + case (ROMStage::OFFLINE): + if (ct.isLocMode()) + readRestartFiles(mgmol); + else + readRestartFiles(mgmol); + break; + + case (ROMStage::BUILD): + if (ct.isLocMode()) + buildROMPoissonOperator(mgmol); + else + buildROMPoissonOperator(mgmol); + break; + + case (ROMStage::TEST_POISSON): + if (ct.isLocMode()) + testROMPoissonOperator(mgmol); + else + testROMPoissonOperator(mgmol); + break; + + default: + std::cerr << "rom_main error: Unknown ROM stage" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + break; + } + + delete mgmol; diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index b73f1898..d9629c07 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -8,6 +8,7 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "rom_workflows.h" +#include "Electrostatic.h" #include #include #include @@ -91,8 +92,8 @@ void readRestartFiles(MGmolInterface *mgmol_) case ROMVariable::POTENTIAL: basis_prefix += "_potential"; - /* we save total potential for now */ - basis_generator.takeSample(pot.vtot()); + /* we save hartree potential */ + basis_generator.takeSample(pot.vh_rho()); break; } } @@ -100,5 +101,339 @@ void readRestartFiles(MGmolInterface *mgmol_) basis_generator.endSamples(); } +template +void buildROMPoissonOperator(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + /* type of variable we intend to run POD */ + ROMVariable rom_var = rom_options.variable; + if (rom_var != ROMVariable::POTENTIAL) + { + std::cerr << "buildROMPoissonOperator error: ROM variable must be POTENTIAL to run this stage!\n" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + /* Load Hartree potential basis matrix */ + std::string basis_file = rom_options.basis_file; + const int num_pot_basis = rom_options.num_potbasis; + CAROM::BasisReader basis_reader(basis_file); + CAROM::Matrix *pot_basis = basis_reader.getSpatialBasis(num_pot_basis); + + /* Load PoissonSolver pointer */ + MGmol *mgmol = static_cast *>(mgmol_); + Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); + + /* GridFunc initialization inputs */ + const pb::Grid &grid(poisson->vh().grid()); + short bc[3]; + for (int d = 0; d < 3; d++) + bc[d] = poisson->vh().bc(d); + + /* Initialize ROM matrix (undistributed) */ + CAROM::Matrix pot_rom(num_pot_basis, num_pot_basis, false); + + pb::GridFunc col_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc opcol_gf(grid, bc[0], bc[1], bc[2]); + CAROM::Vector op_col(pot_basis->numRows(), true); + CAROM::Vector rom_col(num_pot_basis, false); + for (int c = 0; c < num_pot_basis; c++) + { + /* copy c-th column librom vector to GridFunc gf_col */ + CAROM::Vector *col = pot_basis->getColumn(c); + col_gf.assign(col->getData(), 'd'); + + /* apply Laplace operator */ + poisson->applyOperator(col_gf, opcol_gf); + + /* get librom view-vector of gf_opcol */ + opcol_gf.init_vect(op_col.getData(), 'd'); + + /* Compute basis projection of the column */ + /* Resulting vector is undistributed */ + pot_basis->transposeMult(op_col, rom_col); + + /* libROM matrix is row-major, so data copy is necessary */ + for (int r = 0; r < num_pot_basis; r++) + pot_rom(r, c) = rom_col(r); + + delete col; + } // for (int c = 0; c < num_pot_basis; c++) + + /* Save ROM operator */ + // write the file from PE0 only + if (MPIdata::onpe0) + { + std::string rom_oper = "pot_rom_oper.h5"; + CAROM::HDFDatabase h5_helper; + h5_helper.create(rom_oper); + h5_helper.putInteger("number_of_potential_basis", num_pot_basis); + h5_helper.putDoubleArray("potential_rom_operator", pot_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* save the inverse as well */ + pot_rom.inverse(); + h5_helper.putDoubleArray("potential_rom_inverse", pot_rom.getData(), + num_pot_basis * num_pot_basis, false); + + h5_helper.close(); + } +} + +template +void testROMPoissonOperator(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + + /* Load MGmol pointer and Potentials */ + MGmol *mgmol = static_cast *>(mgmol_); + Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + const int dim = pot.size(); + printf("pot size: %d\n", dim); + + /* GridFunc initialization inputs */ + const pb::Grid &grid(poisson->vh().grid()); + short bc[3]; + for (int d = 0; d < 3; d++) + bc[d] = poisson->vh().bc(d); + + /* fictitious snapshot numbers */ + const int nsnapshot = 3; + + /* Set compensating charges to zero now */ + pb::GridFunc rhoc(grid, bc[0], bc[1], bc[2]); + rhoc = 0.0; + + /* Generate fictitious right-hand sides and snapshots */ + std::vector> rhs(nsnapshot), fom_sol(nsnapshot); + for (int s = 0; s < nsnapshot; s++) + { + rhs[s].resize(dim); + for (int d = 0; d < dim; d++) + rhs[s][d] = ran0(); + + /* average out for periodic bc */ + pb::GridFunc rhs_gf(grid, bc[0], bc[1], bc[2]); + rhs_gf.assign(rhs[s].data(), 'd'); + double avg = rhs_gf.get_average(); + rhs_gf -= avg; + + /* copy back to rhs */ + rhs_gf.init_vect(rhs[s].data(), 'd'); + + poisson->solve(rhs_gf, rhoc); + + fom_sol[s].resize(dim); + poisson->vh().init_vect(fom_sol[s].data(), 'd'); + + /* check if the solution is correct */ + pb::GridFunc res(grid, bc[0], bc[1], bc[2]); + pb::GridFunc sol_gf(grid, bc[0], bc[1], bc[2]); + sol_gf.assign(fom_sol[s].data()); + /* apply Laplace operator */ + poisson->applyOperator(sol_gf, res); + /* FD operator scales rhs by 4pi */ + res.axpy(- 4. * M_PI, rhs_gf); + printf("FOM res norm: %.3e\n", res.norm2()); + } + + /* Initialize libROM classes */ + std::string basis_prefix = "test_poisson"; + CAROM::Options svd_options(dim, nsnapshot, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, basis_prefix); + + /* Collect snapshots and train POD basis */ + for (int s = 0; s < nsnapshot; s++) + basis_generator.takeSample(fom_sol[s].data()); + basis_generator.endSamples(); + + /* Load POD basis. We use the maximum number of basis vectors. */ + const CAROM::Matrix *pot_basis = basis_generator.getSpatialBasis(); + + /* Check if full projection preserves FOM solution */ + for (int c = 0; c < nsnapshot; c++) + { + CAROM::Vector *fom_sol_vec = nullptr; + /* get librom view-vector of fom_sol */ + fom_sol_vec = new CAROM::Vector(fom_sol[c].data(), pot_basis->numRows(), true, false); + + CAROM::Vector *rom_proj = pot_basis->transposeMult(*fom_sol_vec); + CAROM::Vector *reconstruct = pot_basis->mult(*rom_proj); + + /* error on libROM side */ + CAROM::Vector *librom_error = reconstruct->minus(fom_sol_vec); + printf("librom reconstruction error: %.3e\n", librom_error->norm()); + + /* error on mgmol side */ + pb::GridFunc recon_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc fom_gf(grid, bc[0], bc[1], bc[2]); + recon_gf.assign(reconstruct->getData(), 'd'); + fom_gf.assign(fom_sol[c].data(), 'd'); + recon_gf -= fom_gf; + printf("mgmol reconstruction error: %.3e\n", recon_gf.norm2()); + + delete fom_sol_vec; + delete rom_proj; + delete reconstruct; + delete librom_error; + } + + /* Check FOM axpy is equivalent to ROM axpy */ + for (int s = 0; s < nsnapshot; s++) + { + CAROM::Vector fom_res(pot_basis->numRows(), true); + CAROM::Vector rom_res(nsnapshot, false); + CAROM::Vector fom_rhs(pot_basis->numRows(), true); + CAROM::Vector rom_rhs(nsnapshot, false); + + pb::GridFunc res(grid, bc[0], bc[1], bc[2]); + pb::GridFunc fomsol_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc mgmol_rhs(grid, bc[0], bc[1], bc[2]); + fomsol_gf.assign(fom_sol[s].data(), 'd'); + mgmol_rhs.assign(rhs[s].data(), 'd'); + + /* apply Laplace operator */ + poisson->applyOperator(fomsol_gf, res); + + /* get librom view-vector of fom_res */ + res.init_vect(fom_res.getData(), 'd'); + pot_basis->transposeMult(fom_res, rom_res); + + /* get librom view-vector of fom_rhs */ + mgmol_rhs.init_vect(fom_rhs.getData(), 'd'); + pot_basis->transposeMult(fom_rhs, rom_rhs); + + /* ROM residual: FD operator scales rhs by 4pi */ + rom_rhs *= 4. * M_PI; + rom_res -= rom_rhs; + printf("ROM res norm: %.3e\n", rom_res.norm()); + + /* FOM residual: FD operator scales rhs by 4pi */ + res.axpy(- 4. * M_PI, mgmol_rhs); + printf("FOM res norm: %.3e\n", res.norm2()); + + /* projection of the residual */ + res.init_vect(fom_res.getData(), 'd'); + CAROM::Vector *res_proj = pot_basis->transposeMult(fom_res); + printf("FOM res projection norm: %.3e\n", res_proj->norm()); + + delete res_proj; + } + + /* Initialize Projection ROM matrix (undistributed) */ + CAROM::Matrix pot_rom(nsnapshot, nsnapshot, false); + + /* Build Projection of Poisson operator */ + for (int c = 0; c < nsnapshot; c++) + { + pb::GridFunc col_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc opcol_gf(grid, bc[0], bc[1], bc[2]); + CAROM::Vector op_col(pot_basis->numRows(), true); + + /* copy c-th column librom vector to GridFunc gf_col */ + CAROM::Vector *col = pot_basis->getColumn(c); + col_gf.assign(col->getData(), 'd'); + + /* apply Laplace operator */ + poisson->applyOperator(col_gf, opcol_gf); + + /* get librom view-vector of gf_opcol */ + opcol_gf.init_vect(op_col.getData(), 'd'); + + /* Compute basis projection of the column */ + /* Resulting vector is undistributed */ + CAROM::Vector *rom_col = pot_basis->transposeMult(op_col); + + /* libROM matrix is row-major, so data copy is necessary */ + for (int r = 0; r < nsnapshot; r++) + pot_rom(r, c) = (*rom_col)(r); + + delete col; + delete rom_col; + } // for (int c = 0; c < num_pot_basis; c++) + + /* Inverse of the projection ROM matrix */ + CAROM::Matrix pot_rom_inv(pot_rom); + pot_rom_inv.inverse(); + + /* Check the inverse */ + CAROM::Matrix *identity = pot_rom_inv.mult(pot_rom); + printf("pot_rom_inv * pot_rom = identity\n"); + for (int i = 0; i < nsnapshot; i++) + { + for (int j = 0; j < nsnapshot; j++) + printf("%.3e\t", identity->item(i, j)); + printf("\n"); + } + delete identity; + + /* Test with sample RHS. ROM must be able to 100% reproduce the FOM solution. */ + std::vector rom_sol(0), rom_rhs(0); + std::vector> test_sol(nsnapshot); + for (int s = 0; s < nsnapshot; s++) + { + /* get librom view-vector of rhs[s] */ + CAROM::Vector fom_rhs(rhs[s].data(), dim, true, false); + + /* project onto POD basis */ + rom_rhs.push_back(pot_basis->transposeMult(fom_rhs)); + + /* FOM FD operator scales rhs by 4pi */ + *rom_rhs.back() *= 4. * M_PI; + + /* solve ROM */ + rom_sol.push_back(pot_rom_inv.mult(*rom_rhs.back())); + + /* check ROM solution */ + CAROM::Vector &res(*pot_rom.mult(*rom_sol.back())); + res -= *rom_rhs.back(); + printf("rom res norm: %.3e\n", res.norm()); + + /* initialize lift-up FOM solution */ + test_sol[s].resize(dim); + /* get librom view-vector of test_sol[s] */ + CAROM::Vector test_sol_vec(test_sol[s].data(), dim, true, false); + pot_basis->mult(*rom_sol.back(), test_sol_vec); + } + + /* Compute relative errors */ + for (int s = 0; s < nsnapshot; s++) + { + pb::GridFunc testsol_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc fomsol_gf(grid, bc[0], bc[1], bc[2]); + + testsol_gf.assign(test_sol[s].data(), 'd'); + fomsol_gf.assign(fom_sol[s].data(), 'd'); + + testsol_gf -= fomsol_gf; + double rel_error = testsol_gf.norm2() / fomsol_gf.norm2(); + printf("%d-th sample relative error: %.3e\n", s, rel_error); + + if (rel_error > 1.0e-9) + abort(); + } + + /* clean up pointers */ + for (int s = 0; s < nsnapshot; s++) + { + delete rom_sol[s]; + delete rom_rhs[s]; + } +} + template void readRestartFiles(MGmolInterface *mgmol_); -template void readRestartFiles(MGmolInterface *mgmol_); \ No newline at end of file +template void readRestartFiles(MGmolInterface *mgmol_); + +template void buildROMPoissonOperator(MGmolInterface *mgmol_); +template void buildROMPoissonOperator(MGmolInterface *mgmol_); + +template void testROMPoissonOperator(MGmolInterface *mgmol_); +template void testROMPoissonOperator(MGmolInterface *mgmol_); \ No newline at end of file diff --git a/src/rom_workflows.h b/src/rom_workflows.h index d45abb92..2399d065 100644 --- a/src/rom_workflows.h +++ b/src/rom_workflows.h @@ -35,8 +35,15 @@ namespace po = boost::program_options; #include "librom.h" +#include "utils/HDFDatabase.h" template void readRestartFiles(MGmolInterface *mgmol_); +template +void buildROMPoissonOperator(MGmolInterface *mgmol_); + +template +void testROMPoissonOperator(MGmolInterface *mgmol_); + #endif // ROM_WORKFLOWS_H diff --git a/tests/ROM/test_rom_poisson/carbyne.cfg b/tests/ROM/test_rom_poisson/carbyne.cfg new file mode 100644 index 00000000..6aac926c --- /dev/null +++ b/tests/ROM/test_rom_poisson/carbyne.cfg @@ -0,0 +1,63 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Poisson] +FDtype=4th +#max_steps_initial=99 +#max_steps=99 +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +#type=QUENCH +type=MD +[Quench] +#solver=PR +max_steps=300 +atol=1.e-8 +[MD] +num_steps=3000 +dt=40. +print_interval=1 +[Orbitals] +initial_type=Fourier +nempty=10 +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 + +[Thermostat] +type=Langevin +temperature=300. +relax_time=1000. + +[Restart] +#input_filename=init_cond_144proc +#input_level=4 +output_level=4 +interval=1 + +[ROM] +stage=test_poisson + +[ROM.offline] +restart_filefmt=case-300K/snapshot%05d +restart_min_idx=800 +restart_max_idx=1999 +basis_file=basis_300K_2/test_300K +variable=potential + diff --git a/tests/ROM/test_rom_poisson/carbyne.in b/tests/ROM/test_rom_poisson/carbyne.in new file mode 100644 index 00000000..9f4e975a --- /dev/null +++ b/tests/ROM/test_rom_poisson/carbyne.in @@ -0,0 +1,14 @@ +H00 1 -0.0000 -0.0000 15.2674 +C01 2 -0.0000 0.0000 13.2519 +C02 2 -0.0000 0.0000 10.9495 +C03 2 -0.0000 -0.0000 8.4221 +C04 2 0.0000 0.0000 6.0897 +C05 2 -0.0000 0.0000 3.5892 +C06 2 -0.0000 -0.0000 1.2470 +C07 2 0.0000 -0.0000 -1.2469 +C08 2 0.0000 -0.0000 -3.5891 +C09 2 -0.0000 -0.0000 -6.0897 +C10 2 -0.0000 0.0000 -8.4221 +C11 2 0.0000 -0.0000 -10.9495 +C12 2 0.0000 0.0000 -13.2520 +H13 1 0.0000 0.0000 -15.2675 From 75b41753a092281eecec38468f93064e47c01d4e Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Wed, 14 Aug 2024 04:43:14 -0700 Subject: [PATCH 14/33] ROM Pinned H2O (#271) * Reenable testShortSighted test (#248) * Make new driver to check input (#247) * clean up/reorganize main.cc * use shared_ptr in class MGmol * Add possible periodic dimensions to xyz2in.py (#249) * Add possible periodic dimensions to xyz2in.py * Remove unused/untested option extrapolateH (#250) * Exit with failure if density off by more than 2% (#251) * Exit with failure if density off by more than 2% * adapt SiH4 test to catch that * fix bug in DFTsolver that was leading to wrong density * Example driver (#252) * add example driver, showing use of MGmol as a force/energy computational engine * clean up related functions in class Ions * loadOrbitalsFromRestartFile -> loadRestartFile (#253) * Add SG15 PBE potential for N (#258) * Update 2-pyridone example (#259) * Adjust verbosity in some functions (#260) * Add new example: pinned H2O (#261) * Print out eigenvalues out of MVP solver (#262) Previously, the wrong eigenvalues (0) were printed out because eigenvalues outside solver were not up-to-date. * Add Li2 example with local GTH potential (#263) * Fix LBFGS termination when converged (#264) * Remove unused code to extrapolate rho (#265) * Fix and test EnergyAndForces interface (#266) * Atomic potentials were not updated when atomic positions were changed * Added test to make sure energies and forces are the same after positions move by one mesh spacing * Add new functionality to compute energy and forces (#267) * use specified initial conditions for wavefunctions --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Kevin" Seung Whan Chung --- examples/2-pyridone/2-pyridone.cfg | 26 +- examples/2-pyridone/README | 10 +- examples/2-pyridone/coords.in | 25 +- examples/Li2GTH/README | 11 + examples/Li2GTH/davidson.cfg | 46 + examples/Li2GTH/li2.xyz | 4 + examples/PinnedH2O/README | 3 + examples/PinnedH2O/coords.in | 3 + examples/PinnedH2O/mgmol.cfg | 35 + potentials/pseudo.Li_GTH_PBE | 332 +++ potentials/pseudo.N_ONCV_PBE_SG15 | 1863 +++++++++++++++++ src/Control.cc | 36 - src/Control.h | 1 - src/DavidsonSolver.cc | 12 +- src/DensityMatrix.cc | 4 +- src/Hartree_CG.cc | 4 +- src/IonicAlgorithm.cc | 2 +- src/LBFGS.cc | 2 +- src/MGmol.cc | 53 +- src/MGmol.h | 22 +- src/MGmolInterface.h | 7 + src/MVPSolver.cc | 9 +- src/Orbitals.h | 2 + src/ProjectedMatrices.h | 6 + src/ProjectedMatricesInterface.h | 2 +- src/lbfgsrlx.cc | 12 +- src/md.cc | 37 +- src/quench.cc | 24 +- tests/CMakeLists.txt | 21 + tests/EnergyAndForces/coords.in | 2 + tests/EnergyAndForces/lrs.in | 6 + tests/EnergyAndForces/mgmol.cfg | 33 + tests/EnergyAndForces/test.py | 87 + tests/EnergyAndForces/testEnergyAndForces.cc | 198 ++ tests/MVP/test.py | 24 + tests/WFEnergyAndForces/mgmol.cfg | 28 + tests/WFEnergyAndForces/sih4.xyz | 8 + tests/WFEnergyAndForces/test.py | 91 + .../testWFEnergyAndForces.cc | 210 ++ util/generateLiLocalGTHpseudo.py | 65 + 40 files changed, 3221 insertions(+), 145 deletions(-) create mode 100644 examples/Li2GTH/README create mode 100644 examples/Li2GTH/davidson.cfg create mode 100644 examples/Li2GTH/li2.xyz create mode 100644 examples/PinnedH2O/README create mode 100644 examples/PinnedH2O/coords.in create mode 100644 examples/PinnedH2O/mgmol.cfg create mode 100644 potentials/pseudo.Li_GTH_PBE create mode 100644 potentials/pseudo.N_ONCV_PBE_SG15 create mode 100644 tests/EnergyAndForces/coords.in create mode 100644 tests/EnergyAndForces/lrs.in create mode 100644 tests/EnergyAndForces/mgmol.cfg create mode 100755 tests/EnergyAndForces/test.py create mode 100644 tests/EnergyAndForces/testEnergyAndForces.cc create mode 100644 tests/WFEnergyAndForces/mgmol.cfg create mode 100644 tests/WFEnergyAndForces/sih4.xyz create mode 100755 tests/WFEnergyAndForces/test.py create mode 100644 tests/WFEnergyAndForces/testWFEnergyAndForces.cc create mode 100644 util/generateLiLocalGTHpseudo.py diff --git a/examples/2-pyridone/2-pyridone.cfg b/examples/2-pyridone/2-pyridone.cfg index 82457d9b..23f53496 100644 --- a/examples/2-pyridone/2-pyridone.cfg +++ b/examples/2-pyridone/2-pyridone.cfg @@ -1,5 +1,6 @@ verbosity=1 xcFunctional=PBE +FDtype=4th [Mesh] nx=128 ny=128 @@ -12,30 +13,17 @@ lx=24. ly=24. lz=12. [Potentials] -pseudopotential=pseudo.N_soft_pbe -pseudopotential=pseudo.O_pbe -pseudopotential=pseudo.C_pbe -pseudopotential=pseudo.H_pbe -[Poisson] -solver=MG -max_steps_initial=10 -max_steps=10 -bcx=periodic -bcy=periodic -bcz=periodic +pseudopotential=pseudo.N_ONCV_PBE_SG15 +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=QUENCH [Quench] -step_length=1.5 max_steps=200 atol=1.e-7 -num_lin_iterations=2 -ortho_freq=20 -interval_print_residual=10 [Orbitals] initial_type=Gaussian initial_width=1.5 -[ProjectedMatrices] -solver=exact -[Restart] -output_type=single_file +[Poisson] +FDtype=4th diff --git a/examples/2-pyridone/README b/examples/2-pyridone/README index 351b7012..663f5855 100644 --- a/examples/2-pyridone/README +++ b/examples/2-pyridone/README @@ -1,6 +1,6 @@ -ln -s ../../potentials/pseudo.N_soft_pbe -ln -s ../../potentials/pseudo.O_pbe -ln -s ../../potentials/pseudo.C_pbe -ln -s ../../potentials/pseudo.H_pbe +ln -s ../../potentials/pseudo.N_ONCV_PBE_SG15 +ln -s ../../potentials/pseudo.O_ONCV_PBE_SG15 +ln -s ../../potentials/pseudo.C_ONCV_PBE_SG15 +ln -s ../../potentials/pseudo.H_ONCV_PBE_SG15 -srun -ppdebug -n32 /nfs/tmp2/jeanluc/SVN/MGmol/mgmol/branches/bin/mgmol-pel -c 2-pyridone.cfg -l lrs.in -i coords.in +srun -ppdebug -n32 mgmol-opt -c 2-pyridone.cfg -l lrs.in -i coords.in diff --git a/examples/2-pyridone/coords.in b/examples/2-pyridone/coords.in index 911b5cdf..390ff902 100644 --- a/examples/2-pyridone/coords.in +++ b/examples/2-pyridone/coords.in @@ -1,13 +1,12 @@ -N1 1 10.0818 7.9149 6. 1 -O1 2 14.2787 7.9126 6. 1 -C1 3 7.9625 9.1867 6. 0 -C2 3 7.8388 11.8070 6. 0 -C3 3 10.1050 13.1610 6. 0 -C4 3 12.3830 11.8680 6. 0 -C5 3 12.2810 9.2255 6. 0 -H1 4 6.2219 8.0878 6. 1 -H2 4 6.0028 12.7142 6. 1 -H3 4 10.0754 15.2192 6. 1 -H4 4 14.2295 12.7482 6. 1 -H5 4 10.2332 6.0038 6. 1 - +N1 1 10.0818 7.9149 6. +O1 2 14.2787 7.9126 6. +C1 3 7.9625 9.1867 6. +C2 3 7.8388 11.8070 6. +C3 3 10.1050 13.1610 6. +C4 3 12.3830 11.8680 6. +C5 3 12.2810 9.2255 6. +H1 4 6.2219 8.0878 6. +H2 4 6.0028 12.7142 6. +H3 4 10.0754 15.2192 6. +H4 4 14.2295 12.7482 6. +H5 4 10.2332 6.0038 6. diff --git a/examples/Li2GTH/README b/examples/Li2GTH/README new file mode 100644 index 00000000..f411a01e --- /dev/null +++ b/examples/Li2GTH/README @@ -0,0 +1,11 @@ +#Run MGmol +mpirun -np 4 mgmol-opt -c davidson.cfg -i li2.xyz > davidson.out + +#extract visit files from HDF5 file +python read_hdf5.py -bov li2.h5 Vtotal + +python read_hdf5.py -bov li2.h50 Function0002 + +Note: the eigenfunctions corresponding to the 3 occupied states are stored +in functions 2, 3 and 4 because they are the result of diagonalizing the DM +in Davidson algorithm, not H. diff --git a/examples/Li2GTH/davidson.cfg b/examples/Li2GTH/davidson.cfg new file mode 100644 index 00000000..d211b812 --- /dev/null +++ b/examples/Li2GTH/davidson.cfg @@ -0,0 +1,46 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=96 +ny=96 +nz=96 +[Domain] +ox=-9. +oy=-9. +oz=-9. +lx=18. +ly=18. +lz=18. +[Potentials] +pseudopotential=pseudo.Li_GTH_PBE +[Poisson] +solver=CG +bcx=0 +bcy=0 +bcz=0 +[Run] +type=QUENCH +[Quench] +solver=Davidson +max_steps=200 +atol=1.e-8 +preconditioner_num_levels=3 +[Orbitals] +nempty=2 +initial_type=random +temperature=300. +bcx=0 +bcy=0 +bcz=0 +[ProjectedMatrices] +solver=exact +[DensityMatrix] +nb_inner_it=1 +mixing=1. +[Restart] +output_level=3 +output_filename=li2.h5 +output_type=single_file +[Coloring] +scope=global diff --git a/examples/Li2GTH/li2.xyz b/examples/Li2GTH/li2.xyz new file mode 100644 index 00000000..ccf3a3f1 --- /dev/null +++ b/examples/Li2GTH/li2.xyz @@ -0,0 +1,4 @@ +2 +Experimental geometry (Menconi and Tozer, Chem. Phys. Lett. 2002) +Li 1.3365 0.0 0.0 +Li -1.3365 0.0 0.0 diff --git a/examples/PinnedH2O/README b/examples/PinnedH2O/README new file mode 100644 index 00000000..12ca9138 --- /dev/null +++ b/examples/PinnedH2O/README @@ -0,0 +1,3 @@ +Single water molecule MD, with Oxigen atom pinned to (0,0,0) coordinate: + +mpirun -np 4 mgmol-opt -c mgmol.cfg -i coords.in diff --git a/examples/PinnedH2O/coords.in b/examples/PinnedH2O/coords.in new file mode 100644 index 00000000..307e170c --- /dev/null +++ b/examples/PinnedH2O/coords.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 -0.45 1.42 -1.07 1 +H2 2 -0.45 -1.48 -0.97 1 diff --git a/examples/PinnedH2O/mgmol.cfg b/examples/PinnedH2O/mgmol.cfg new file mode 100644 index 00000000..32325e47 --- /dev/null +++ b/examples/PinnedH2O/mgmol.cfg @@ -0,0 +1,35 @@ +verbosity=1 +xcFunctional=PBE +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=50 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=1000. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=0 diff --git a/potentials/pseudo.Li_GTH_PBE b/potentials/pseudo.Li_GTH_PBE new file mode 100644 index 00000000..7da944e0 --- /dev/null +++ b/potentials/pseudo.Li_GTH_PBE @@ -0,0 +1,332 @@ +# Goedecker, Teter, Hutter, Phys. rev. B 54 (3), 1996 +# Parameters from M. Krack, Theor. Chem. Acc. 114, 2005 +# Short description of the species type. One line only! +LithiumGTH_PBE +# +White +#radii of balls and covalent bonds +0.4 1.0 +# Nlcc flag +0 +# Atomic number +3 +# Atomic mass +3.0 +# Number of valence electrons +3.0 +# Gaussian core charge parameter rc +1. +# Number of potentials +1 +# l-value for state which is local +0 0 +# Local potential radius +3. +# Non-local potential radius +3. +# number of points in radial grid +301 +# log mesh parameter +0. +# radial grid, reference state, and potential for l=0 +0.0 -20.06528920602149 +0.01 -20.054252432503986 +0.02 -20.021181949571293 +0.03 -19.966197051499293 +0.04 -19.889495835016746 +0.05 -19.791354114111485 +0.06 -19.672123911517346 +0.07 -19.53223153752974 +0.08 -19.372175269684018 +0.09 -19.192522649601603 +0.1 -18.993907415941983 +0.11 -18.777026094871008 +0.12 -18.54263427174855 +0.13 -18.291542569831762 +0.14 -18.02461236366824 +0.15 -17.742751256501393 +0.16 -17.446908352415903 +0.17 -17.138069355105003 +0.18 -16.81725152603506 +0.19 -16.485498535412276 +0.2 -16.14387523971814 +0.21 -15.7934624196748 +0.22 -15.435351512330925 +0.23 -15.07063937052802 +0.24 -14.700423082322942 +0.25 -14.325794882014955 +0.26 -13.947837183265412 +0.27 -13.567617763419314 +0.28 -13.186185126555326 +0.29 -12.80456407102134 +0.3 -12.423751485274781 +0.31 -12.0447123937598 +0.32 -11.668376272337989 +0.33 -11.295633650466765 +0.34 -10.927333014911554 +0.35 -10.564278027307214 +0.36 -10.207225065372501 +0.37 -9.856881095051628 +0.38 -9.513901878330472 +0.39 -9.178890518973706 +0.4 -8.852396345973204 +0.41 -8.534914132107685 +0.42 -8.226883642706916 +0.43 -7.928689507508834 +0.44 -7.64066140641021 +0.45 -7.363074557955612 +0.46 -7.096150497598659 +0.47 -6.840058131114424 +0.48 -6.594915047052536 +0.49 -6.3607890708040955 +0.5 -6.1377000417180545 +0.51 -5.925621793748402 +0.52 -5.724484319343899 +0.53 -5.534176095708496 +0.54 -5.354546552160849 +0.55 -5.1854086571031885 +0.56 -5.026541603068161 +0.57 -4.877693568441651 +0.58 -4.73858453475222 +0.59 -4.60890913886544 +0.6 -4.488339540014026 +0.61 -4.376528282322026 +0.62 -4.273111134331415 +0.63 -4.17770988800058 +0.64 -4.089935100703225 +0.65 -4.009388764900398 +0.66 -3.935666891373983 +0.67 -3.868361993183916 +0.68 -3.807065458829906 +0.69 -3.751369804448444 +0.7 -3.7008707962442955 +0.71 -3.655169435730016 +0.72 -3.6138738017150143 +0.73 -3.5766007443361536 +0.74 -3.5429774277437494 +0.75 -3.512642719340314 +0.76 -3.4852484247051203 +0.77 -3.4604603685176145 +0.78 -3.4379593229091037 +0.79 -3.4174417857190456 +0.8 -3.3986206121036857 +0.81 -3.38122550383682 +0.82 -3.3650033614510035 +0.83 -3.3497185050904883 +0.84 -3.335152770582715 +0.85 -3.3211054877824484 +0.86 -3.307393348702202 +0.87 -3.293850173314738 +0.88 -3.2803265812005225 +0.89 -3.2666895774169027 +0.9 -3.2528220610898493 +0.91 -3.2386222652767795 +0.92 -3.224003136624447 +0.93 -3.208891663253622 +0.94 -3.1932281591472704 +0.95 -3.1769655131062664 +0.96 -3.160068410071886 +0.97 -3.142512532302873 +0.98 -3.1242837475424827 +0.99 -3.10537729092335 +1.0 -3.085796946940759 +1.01 -3.065554237383791 +1.02 -3.0446676206540504 +1.03 -3.0231617074287267 +1.04 -3.001066497143526 +1.05 -2.978416639286402 +1.06 -2.9552507230094722 +1.07 -2.931610598088238 +1.08 -2.9075407297881393 +1.09 -2.8830875897419252 +1.1 -2.8582990845006977 +1.11 -2.833224022999286 +1.12 -2.8079116237754826 +1.13 -2.7824110624044724 +1.14 -2.7567710592563195 +1.15 -2.7310395073568614 +1.16 -2.705263139831839 +1.17 -2.6794872361411723 +1.18 -2.653755366065358 +1.19 -2.628109170188997 +1.2 -2.60258817543732 +1.21 -2.57722964405972 +1.22 -2.552068454319101 +1.23 -2.527137011036365 +1.24 -2.50246518405455 +1.25 -2.4780802726257742 +1.26 -2.4540069936848927 +1.27 -2.430267491955173 +1.28 -2.4068813698318525 +1.29 -2.383865735007489 +1.3 -2.361235263837057 +1.31 -2.339002278488971 +1.32 -2.31717683598915 +1.33 -2.2957668273371157 +1.34 -2.274778084954434 +1.35 -2.2542144968149556 +1.36 -2.2340781257018256 +1.37 -2.2143693321366866 +1.38 -2.1950868996304913 +1.39 -2.176228161011706 +1.4 -2.1577891246951135 +1.41 -2.1397645998619574 +1.42 -2.1221483196287054 +1.43 -2.1049330613864585 +1.44 -2.0881107635950986 +1.45 -2.07167263841506 +1.46 -2.055609279654423 +1.47 -2.03991076559945 +1.48 -2.0245667563822187 +1.49 -2.0095665856193814 +1.5 -1.9948993461310054 +1.51 -1.9805539696177794 +1.52 -1.96651930023847 +1.53 -1.95278416208739 +1.54 -1.9393374206237712 +1.55 -1.9261680381514323 +1.56 -1.9132651234880866 +1.57 -1.900617975999247 +1.58 -1.8882161242021371 +1.59 -1.876049359170528 +1.6 -1.8641077629922906 +1.61 -1.8523817325479044 +1.62 -1.8408619988905475 +1.63 -1.8295396425169552 +1.64 -1.8184061048233324 +1.65 -1.807453196042526 +1.66 -1.796673099957719 +1.67 -1.786058375684427 +1.68 -1.7756019568068189 +1.69 -1.7652971481466952 +1.7 -1.7551376204340732 +1.71 -1.7451174031375227 +1.72 -1.7352308757004553 +1.73 -1.7254727574166897 +1.74 -1.7158380961650346 +1.75 -1.7063222562085625 +1.76 -1.696920905249867 +1.77 -1.6876300009190652 +1.78 -1.6784457768568064 +1.79 -1.6693647285401818 +1.8 -1.6603835989853395 +1.81 -1.6514993644468863 +1.82 -1.6427092202208944 +1.83 -1.6340105666456095 +1.84 -1.625400995381811 +1.85 -1.6168782760432898 +1.86 -1.608440343237089 +1.87 -1.6000852840630355 +1.88 -1.5918113261126972 +1.89 -1.5836168259992183 +1.9 -1.5755002584415392 +1.91 -1.567460205919256 +1.92 -1.5594953489078422 +1.93 -1.5516044566980827 +1.94 -1.54378637879836 +1.95 -1.5360400369138487 +1.96 -1.5283644174926794 +1.97 -1.5207585648257078 +1.98 -1.513221574683613 +1.99 -1.5057525884726553 +2.0 -1.4983507878884341 +2.01 -1.4910153900454863 +2.02 -1.4837456430593685 +2.03 -1.476540822057088 +2.04 -1.4694002255912106 +2.05 -1.4623231724327952 +2.06 -1.4553089987182684 +2.07 -1.4483570554256544 +2.08 -1.441466706155941 +2.09 -1.4346373251959794 +2.1 -1.4278682958400157 +2.11 -1.4211590089477848 +2.12 -1.4145088617179962 +2.13 -1.407917256657042 +2.14 -1.4013836007237732 +2.15 -1.3949073046322673 +2.16 -1.38848778229559 +2.17 -1.3821244503946528 +2.18 -1.3758167280573612 +2.19 -1.3695640366343098 +2.2 -1.3633657995583632 +2.21 -1.3572214422764575 +2.22 -1.3511303922429727 +2.23 -1.3450920789649687 +2.24 -1.3391059340904876 +2.25 -1.3331713915319947 +2.26 -1.3272878876178418 +2.27 -1.321454861265409 +2.28 -1.3156717541702987 +2.29 -1.3099380110066239 +2.3 -1.304253079634056 +2.31 -1.2986164113078702 +2.32 -1.2930274608887506 +2.33 -1.2874856870495992 +2.34 -1.2819905524770356 +2.35 -1.2765415240656612 +2.36 -1.271138073103527 +2.37 -1.2657796754475625 +2.38 -1.260465811688003 +2.39 -1.2551959673011142 +2.4 -1.2499696327897274 +2.41 -1.244786303811294 +2.42 -1.2396454812933442 +2.43 -1.2345466715363633 +2.44 -1.229489386304243 +2.45 -1.224473142902546 +2.46 -1.219497464244928 +2.47 -1.214561878908113 +2.48 -1.2096659211758896 +2.49 -1.204809131072622 +2.5 -1.1999910543868184 +2.51 -1.1952112426853008 +2.52 -1.1904692533185557 +2.53 -1.1857646494178307 +2.54 -1.1810969998845513 +2.55 -1.1764658793726301 +2.56 -1.1718708682642154 +2.57 -1.1673115526394273 +2.58 -1.1627875242406034 +2.59 -1.158298380431558 +2.6 -1.1538437241523383 +2.61 -1.149423163869939 +2.62 -1.1450363135254051 +2.63 -1.1406827924777474 +2.64 -1.1363622254450418 +2.65 -1.1320742424430907 +2.66 -1.1278184787219725 +2.67 -1.1235945747008054 +2.68 -1.1194021759010047 +2.69 -1.1152409328783188 +2.7 -1.1111105011538749 +2.71 -1.107010541144477 +2.72 -1.1029407180923525 +2.73 -1.0989007019945478 +2.74 -1.094890167532131 +2.75 -1.0909087939993736 +2.76 -1.0869562652330373 +2.77 -1.0830322695418986 +2.78 -1.079136499636622 +2.79 -1.0752686525600812 +2.8 -1.0714284296182166 +2.81 -1.067615536311507 +2.82 -1.0638296822671227 +2.83 -1.0600705811718185 +2.84 -1.0563379507056208 +2.85 -1.0526315124763437 +2.86 -1.0489509919549826 +2.87 -1.0452961184120013 +2.88 -1.0416666248545523 +2.89 -1.0380622479646329 +2.9 -1.0344827280382114 +2.91 -1.0309278089253149 +2.92 -1.0273972379711012 +2.93 -1.0238907659579113 +2.94 -1.0204081470483046 +2.95 -1.0169491387290766 +2.96 -1.013513501756257 +2.97 -1.0101010001010782 +2.98 -1.0067114008969125 +2.99 -1.0033444743871645 +3.0 -0.9999999938741118 diff --git a/potentials/pseudo.N_ONCV_PBE_SG15 b/potentials/pseudo.N_ONCV_PBE_SG15 new file mode 100644 index 00000000..958c0c8b --- /dev/null +++ b/potentials/pseudo.N_ONCV_PBE_SG15 @@ -0,0 +1,1863 @@ +# This pseudopotential file has been produced using the code +# ONCVPSP (Optimized Norm-Conservinng Vanderbilt PSeudopotential) +# scalar-relativistic version 2.1.1, 03/26/2014 by D. R. Hamann +# as distributed by Mat-Sim Research at www.mat-simresearch.com. +# Reference: +# D. R. Hamann, Phys. Rev. B 88, 085117 (2013) +# DOI:https://doi.org/10.1103/PhysRevB.88.085117 +# +# Pseudopotential generation was done by M. Schlipf and F. Gygi using +# the input parameters obtained from their optimization algorithm. +# Reference: +# M. Schlipf and F. Gygi, Computer Physics Communications (2015) +# DOI: 10.1016/j.cpc.2015.05.011 +# http://dx.doi.org/10.1016/j.cpc.2015.05.011 +# http://www.quantum-simulation.org +# License: +# This work is licensed under the Creative Commons Attribution-ShareAlike +# 4.0 International License. To view a copy of this license, visit +# http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to +# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. +# +# Format: +# The format of this file has been adapted to match MGmol pseudopotential +# parser using the script convertONCVPSP_SG15.py +# +N_ONCV_PBE-1 +# +color +#radii of balls and covalent bonds +-1. -1. +# Nlcc flag +0 +# Atomic number +7 +# Atomic mass +14.00699997 +# Number of valence electrons +5 +#Gaussian core charge parameter rc +1.0 +# Number of potentials +3 +# l-value for state which is local, then type of potential format +2 3 +# Local potential radius +3.2 +# Non-local potential radius +3.2 +# number of points in radial grid +602 +# VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# l, nproj +0 2 0.7070940539E+01 0.9781080649E+00 +1 2 -0.4781783452E+01 -0.1328201236E+01 +# l= 0 +0.0 -8.3833937254 3.4232195615 +0.01 -8.3799978758 3.4202071705 +0.02 -8.3697860653 3.4111530231 +0.03 -8.3527825341 3.3960738696 +0.04 -8.3290115228 3.3749864603 +0.05 -8.2985069887 3.3479144681 +0.06 -8.261312626 3.3148887067 +0.07 -8.2174818893 3.2759474053 +0.08 -8.1670780196 3.231136536 +0.09 -8.1101740718 3.1805101875 +0.1 -8.0468529415 3.1241309807 +0.11 -7.9772073885 3.0620705184 +0.12 -7.9013400567 2.9944098623 +0.13 -7.8193634857 2.9212400287 +0.14 -7.7314001139 2.8426624957 +0.15 -7.6375822685 2.7587897114 +0.16 -7.5380521411 2.6697455952 +0.17 -7.4329617463 2.5756660223 +0.18 -7.3224728588 2.4766992817 +0.19 -7.2067569298 2.3730065 +0.2 -7.0859949741 2.264762015 +0.21 -6.9603774349 2.1521537041 +0.22 -6.8301040119 2.0353832393 +0.23 -6.6953834613 1.9146662776 +0.24 -6.5564333561 1.7902325648 +0.25 -6.4134798193 1.6623259662 +0.26 -6.2667572033 1.5312043795 +0.27 -6.1165077488 1.3971395792 +0.28 -5.962981185 1.2604169264 +0.29 -5.8064342997 1.1213349889 +0.3 -5.647130465 0.98020504493 +0.31 -5.4853391206 0.83735047259 +0.32 -5.321335212 0.69310601673 +0.33 -5.1553985987 0.54781696226 +0.34 -4.9878134137 0.4018381796 +0.35 -4.8188673807 0.2555330423 +0.36 -4.6488511141 0.10927228913 +0.37 -4.4780573629 -0.036567271748 +0.38 -4.3067802363 -0.18160413005 +0.39 -4.1353144013 -0.32545351173 +0.4 -3.9639542476 -0.46772893557 +0.41 -3.7929930387 -0.60804381665 +0.42 -3.6227220421 -0.74601313801 +0.43 -3.4534296479 -0.88125517478 +0.44 -3.2854004768 -1.0133932584 +0.45 -3.1189144859 -1.1420575692 +0.46 -2.9542460734 -1.2668869442 +0.47 -2.7916631913 -1.387530686 +0.48 -2.6314265018 -1.5036503015 +0.49 -2.47378844 -1.6149214048 +0.5 -2.3189924173 -1.7210353819 +0.51 -2.1672720468 -1.8217009988 +0.52 -2.0188503481 -1.9166460389 +0.53 -1.8739388537 -2.0056191124 +0.54 -1.732737225 -2.0883905138 +0.55 -1.5954322069 -2.1647542719 +0.56 -1.4621973571 -2.2345287551 +0.57 -1.333192259 -2.2975581981 +0.58 -1.2085621978 -2.353713346 +0.59 -1.0884375079 -2.4028926755 +0.6 -0.97293360827 -2.4450223075 +0.61 -0.86215002573 -2.4800577568 +0.62 -0.75617104038 -2.5079826487 +0.63 -0.65506469693 -2.5288104095 +0.64 -0.55888310603 -2.5425835395 +0.65 -0.46766273408 -2.5493728888 +0.66 -0.38142374876 -2.5492785866 +0.67 -0.30017057762 -2.5424287393 +0.68 -0.22389217436 -2.5289786283 +0.69 -0.15256244154 -2.5091096125 +0.7 -0.086139788103 -2.4830294406 +0.71 -0.024568342949 -2.4509697004 +0.72 0.032221809773 -2.413184916 +0.73 0.084314093363 -2.3699510915 +0.74 0.1318047363 -2.321563976 +0.75 0.17480239707 -2.2683377698 +0.76 0.2134272741 -2.2106030175 +0.77 0.24781028338 -2.1487046133 +0.78 0.27809232097 -2.0829999018 +0.79 0.30442342743 -2.0138566245 +0.8 0.32696191973 -1.9416508145 +0.81 0.34587349606 -1.8667646538 +0.82 0.36133031921 -1.7895843109 +0.83 0.37351008427 -1.7104977709 +0.84 0.38259507609 -1.6298926758 +0.85 0.38877122239 -1.5481541871 +0.86 0.39222714765 -1.4656628877 +0.87 0.39315323345 -1.3827927341 +0.88 0.39174069021 -1.2999090738 +0.89 0.3881806455 -1.21736674 +0.9 0.38266325363 -1.1355082343 +0.91 0.3753768312 -1.05466201 +0.92 0.36650702291 -0.97514086392 +0.93 0.35623600185 -0.8972404476 +0.94 0.34474170806 -0.82123790516 +0.95 0.33219712893 -0.74739064427 +0.96 0.31876980308 -0.67593583254 +0.97 0.30462077846 -0.60708800794 +0.98 0.28990423397 -0.54103907779 +0.99 0.27476701087 -0.47795782396 +1.0 0.25934807756 -0.41798919045 +1.01 0.24377810285 -0.3612539062 +1.02 0.22817911217 -0.30784832508 +1.03 0.21266470226 -0.2578456722 +1.04 0.19733797262 -0.21129138178 +1.05 0.18229298665 -0.1682077754 +1.06 0.16761409975 -0.12859294839 +1.07 0.15337584366 -0.092421151151 +1.08 0.1396438166 -0.059645387463 +1.09 0.12647159361 -0.030191037026 +1.1 0.11390353081 -0.0039630063463 +1.11 0.10197406871 0.019155281934 +1.12 0.090708799358 0.039298886262 +1.13 0.080123593815 0.056619471672 +1.14 0.070227673946 0.071275245192 +1.15 0.061024372085 0.083429790064 +1.16 0.052511962988 0.093250798533 +1.17 0.044682028129 0.10090978584 +1.18 0.037521888086 0.10657912289 +1.19 0.031015059574 0.11043138827 +1.2 0.025141213318 0.11263866157 +1.21 0.019876005667 0.11336980971 +1.22 0.015192955473 0.11279033889 +1.23 0.011063595893 0.11106306695 +1.24 0.007456655257 0.10834247812 +1.25 0.0043400745021 0.10477759314 +1.26 0.0016808247737 0.10051281117 +1.27 -0.00055510433714 0.095680812463 +1.28 -0.0024016580764 0.090406927903 +1.29 -0.0038929622779 0.084811273797 +1.3 -0.0050625477429 0.078998968474 +1.31 -0.0059432416054 0.073068149935 +1.32 -0.0065674921377 0.067110734764 +1.33 -0.0069657771649 0.061201995595 +1.34 -0.007167828516 0.055412646763 +1.35 -0.0072020326121 0.049803640184 +1.36 -0.0070943360586 0.044422493293 +1.37 -0.0068701508663 0.039313829408 +1.38 -0.0065520891543 0.034508390981 +1.39 -0.0061611134842 0.030030366818 +1.4 -0.005717227564 0.025899372872 +1.41 -0.005236852799 0.022121815245 +1.42 -0.0047362333134 0.018703666441 +1.43 -0.0042285949508 0.01564086524 +1.44 -0.0037256393417 0.012925597893 +1.45 -0.003238024767 0.010546723262 +1.46 -0.0027732187555 0.0084862043738 +1.47 -0.0023389767364 0.0067264437607 +1.48 -0.0019396582406 0.005243996278 +1.49 -0.0015795366936 0.0040158727896 +1.5 -0.0012608949043 0.0030165453042 +1.51 -0.00098475359848 0.0022205393899 +1.52 -0.00075153947202 0.0016016330403 +1.53 -0.00055956428127 0.0011349570223 +1.54 -0.000407575033 0.00079484813589 +1.55 -0.00029180979095 0.00055922039698 +1.56 -0.00020917873683 0.00040490419124 +1.57 -0.00015459824577 0.00031332706041 +1.58 -0.00012325344328 0.00026554043958 +1.59 -0.00011211107894 0.00025459512511 +1.6 -0.00011091815786 0.00025685343044 +1.61 -8.0065645976e-05 0.00018636643746 +1.62 -2.207959279e-05 5.1598480417e-05 +1.63 6.3416160474e-06 -1.4718026015e-05 +1.64 3.4369124819e-06 -7.9766051653e-06 +1.65 0.0 0.0 +1.66 0.0 0.0 +1.67 0.0 0.0 +1.68 0.0 0.0 +1.69 0.0 0.0 +1.7 0.0 0.0 +1.71 0.0 0.0 +1.72 0.0 0.0 +1.73 0.0 0.0 +1.74 0.0 0.0 +1.75 0.0 0.0 +1.76 0.0 0.0 +1.77 0.0 0.0 +1.78 0.0 0.0 +1.79 0.0 0.0 +1.8 0.0 0.0 +1.81 0.0 0.0 +1.82 0.0 0.0 +1.83 0.0 0.0 +1.84 0.0 0.0 +1.85 0.0 0.0 +1.86 0.0 0.0 +1.87 0.0 0.0 +1.88 0.0 0.0 +1.89 0.0 0.0 +1.9 0.0 0.0 +1.91 0.0 0.0 +1.92 0.0 0.0 +1.93 0.0 0.0 +1.94 0.0 0.0 +1.95 0.0 0.0 +1.96 0.0 0.0 +1.97 0.0 0.0 +1.98 0.0 0.0 +1.99 0.0 0.0 +2.0 0.0 0.0 +2.01 0.0 0.0 +2.02 0.0 0.0 +2.03 0.0 0.0 +2.04 0.0 0.0 +2.05 0.0 0.0 +2.06 0.0 0.0 +2.07 0.0 0.0 +2.08 0.0 0.0 +2.09 0.0 0.0 +2.1 0.0 0.0 +2.11 0.0 0.0 +2.12 0.0 0.0 +2.13 0.0 0.0 +2.14 0.0 0.0 +2.15 0.0 0.0 +2.16 0.0 0.0 +2.17 0.0 0.0 +2.18 0.0 0.0 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 1 +0.0 0.0 0.0 +0.01 0.31691115296 0.12833055002 +0.02 0.6327506292 0.25591217486 +0.03 0.94645136007 0.38199950664 +0.04 1.2569554737 0.50585428246 +0.05 1.5632188455 0.62674887265 +0.06 1.8642155902 0.74396977966 +0.07 2.1589424785 0.85682109725 +0.08 2.4464232565 0.96462791932 +0.09 2.7257128515 1.0667396871 +0.1 2.9959014448 1.1625334629 +0.11 3.2561183922 1.2514171183 +0.12 3.5055359765 1.3328324232 +0.13 3.7433729717 1.4062580228 +0.14 3.9688980035 1.4712122871 +0.15 4.1814326898 1.5272560198 +0.16 4.3803545423 1.5739950083 +0.17 4.5650996186 1.6110824023 +0.18 4.7351649045 1.6382209014 +0.19 4.8901104195 1.6551647388 +0.2 5.0295610238 1.6617214382 +0.21 5.1532079278 1.6577533385 +0.22 5.2608098792 1.643178856 +0.23 5.3521940304 1.6179734804 +0.24 5.4272564708 1.5821704817 +0.25 5.4859624266 1.5358613221 +0.26 5.5283461091 1.4791957464 +0.27 5.5545102276 1.4123815589 +0.28 5.5646251474 1.3356840555 +0.29 5.5589277078 1.2494251212 +0.3 5.5377196963 1.1539819761 +0.31 5.5013659832 1.0497855704 +0.32 5.4502923332 0.93731863025 +0.33 5.3849828862 0.81711334783 +0.34 5.3059773248 0.68974871947 +0.35 5.213867786 0.55584758189 +0.36 5.1092954099 0.41607324677 +0.37 4.9929467086 0.27112589469 +0.38 4.8655496976 0.12173868067 +0.39 4.7278696661 -0.031326572994 +0.4 4.5807049337 -0.18728376199 +0.41 4.4248823343 -0.34532720879 +0.42 4.261252569 -0.50463655721 +0.43 4.0906854444 -0.66438183174 +0.44 3.9140650229 -0.82372863041 +0.45 3.732284716 -0.98184341479 +0.46 3.5462423515 -1.137898858 +0.47 3.3568352506 -1.2910792079 +0.48 3.1649554901 -1.4405854704 +0.49 2.9714847265 -1.5856410667 +0.5 2.777289713 -1.7254967608 +0.51 2.5832178663 -1.8594355206 +0.52 2.3900927615 -1.9867774392 +0.53 2.1987094003 -2.1068848753 +0.54 2.009831375 -2.2191655283 +0.55 1.8241857462 -2.323077914 +0.56 1.6424607668 -2.4181337401 +0.57 1.4653019142 -2.5039020266 +0.58 1.293309626 -2.5800113322 +0.59 1.1270361291 -2.6461528615 +0.6 0.96698445288 -2.702081178 +0.61 0.81360483138 -2.7476175365 +0.62 0.66729559136 -2.782648464 +0.63 0.52840007985 -2.8071282798 +0.64 0.39720703887 -2.8210779659 +0.65 0.27395097135 -2.8245839931 +0.66 0.15881083858 -2.8177985492 +0.67 0.051911531399 -2.8009369687 +0.68 -0.04667512666 -2.774275508 +0.69 -0.13692897946 -2.7381486826 +0.7 -0.21888098639 -2.6929468757 +0.71 -0.29261008059 -2.6391119188 +0.72 -0.35824128404 -2.5771334414 +0.73 -0.4159431628 -2.5075445643 +0.74 -0.46592500191 -2.4309173651 +0.75 -0.50843386132 -2.3478577659 +0.76 -0.54375116939 -2.2590003427 +0.77 -0.57218933031 -2.165003103 +0.78 -0.59408818976 -2.0665419618 +0.79 -0.60981137125 -1.9643051699 +0.8 -0.61974254036 -1.8589877089 +0.81 -0.62428163146 -1.7512857073 +0.82 -0.62384107136 -1.6418909303 +0.83 -0.61884203343 -1.5314853966 +0.84 -0.60971075517 -1.4207361709 +0.85 -0.59687495066 -1.3102903826 +0.86 -0.58076034815 -1.2007705148 +0.87 -0.56178738113 -1.0927700081 +0.88 -0.54036805961 -0.98684921825 +0.89 -0.51690304588 -0.88353176365 +0.9 -0.49177895717 -0.78330129473 +0.91 -0.46536591463 -0.68659871302 +0.92 -0.43801535576 -0.59381986204 +0.93 -0.41005812411 -0.50531370759 +0.94 -0.38180284687 -0.42138101873 +0.95 -0.3535346078 -0.3422735548 +0.96 -0.32551478108 -0.26819491238 +0.97 -0.29797801258 -0.1992975509 +0.98 -0.27113326694 -0.13568562865 +0.99 -0.24516387242 -0.077416374488 +1.0 -0.22022736766 -0.024501296954 +1.01 -0.19645591928 0.023091770031 +1.02 -0.17395706242 0.065436168456 +1.03 -0.15281569619 0.10264273095 +1.04 -0.13309095544 0.13486073225 +1.05 -0.11482209959 0.16226970049 +1.06 -0.098028572297 0.18507736826 +1.07 -0.082711657304 0.20351587633 +1.08 -0.068857068133 0.21783821631 +1.09 -0.056433565659 0.22831418337 +1.1 -0.04539878345 0.23522480425 +1.11 -0.035699969459 0.23885904541 +1.12 -0.027275757447 0.23951189031 +1.13 -0.020055775351 0.23748125528 +1.14 -0.013962010699 0.23306834239 +1.15 -0.0089106101841 0.2265749932 +1.16 -0.0048126887966 0.21830334176 +1.17 -0.0015766618338 0.20854486815 +1.18 0.00088988846604 0.19758215521 +1.19 0.0026797111161 0.18568844316 +1.2 0.0038846691896 0.17312405795 +1.21 0.0045919134973 0.16012585041 +1.22 0.0048855679028 0.14691636088 +1.23 0.004847386319 0.13370571947 +1.24 0.0045491243956 0.12066933844 +1.25 0.0040581363722 0.10796761423 +1.26 0.0034376915115 0.095746225461 +1.27 0.0027393344079 0.084115610895 +1.28 0.0020090206457 0.073169616761 +1.29 0.001288444668 0.062988588425 +1.3 0.00060672379237 0.053618373655 +1.31 -1.1366147491e-05 0.045092609726 +1.32 -0.00054582135205 0.037431155887 +1.33 -0.00098759768257 0.030625723663 +1.34 -0.0013297945509 0.024661714734 +1.35 -0.001571493513 0.019508874959 +1.36 -0.0017177187464 0.015122411378 +1.37 -0.0017749428805 0.011452922158 +1.38 -0.0017547314206 0.0084389831708 +1.39 -0.0016697251667 0.0060163835385 +1.4 -0.0015344078581 0.0041157761059 +1.41 -0.0013638321981 0.0026676915805 +1.42 -0.0011732258973 0.0016011633474 +1.43 -0.00097689571102 0.00084843776735 +1.44 -0.00078726131467 0.00034630915422 +1.45 -0.00061636763112 3.2136810107e-05 +1.46 -0.00047079465995 -0.00014219650548 +1.47 -0.00035797587788 -0.00022811628966 +1.48 -0.00027793709055 -0.00025754617817 +1.49 -0.00023007362243 -0.00026174974903 +1.5 -0.00021697247884 -0.00026976292795 +1.51 -0.00020908394519 -0.00026344496824 +1.52 -0.00011615291035 -0.0001468686914 +1.53 -2.9555906996e-06 -3.7564455382e-06 +1.54 1.2960730815e-05 1.6280198529e-05 +1.55 0.0 0.0 +1.56 0.0 0.0 +1.57 0.0 0.0 +1.58 0.0 0.0 +1.59 0.0 0.0 +1.6 0.0 0.0 +1.61 0.0 0.0 +1.62 0.0 0.0 +1.63 0.0 0.0 +1.64 0.0 0.0 +1.65 0.0 0.0 +1.66 0.0 0.0 +1.67 0.0 0.0 +1.68 0.0 0.0 +1.69 0.0 0.0 +1.7 0.0 0.0 +1.71 0.0 0.0 +1.72 0.0 0.0 +1.73 0.0 0.0 +1.74 0.0 0.0 +1.75 0.0 0.0 +1.76 0.0 0.0 +1.77 0.0 0.0 +1.78 0.0 0.0 +1.79 0.0 0.0 +1.8 0.0 0.0 +1.81 0.0 0.0 +1.82 0.0 0.0 +1.83 0.0 0.0 +1.84 0.0 0.0 +1.85 0.0 0.0 +1.86 0.0 0.0 +1.87 0.0 0.0 +1.88 0.0 0.0 +1.89 0.0 0.0 +1.9 0.0 0.0 +1.91 0.0 0.0 +1.92 0.0 0.0 +1.93 0.0 0.0 +1.94 0.0 0.0 +1.95 0.0 0.0 +1.96 0.0 0.0 +1.97 0.0 0.0 +1.98 0.0 0.0 +1.99 0.0 0.0 +2.0 0.0 0.0 +2.01 0.0 0.0 +2.02 0.0 0.0 +2.03 0.0 0.0 +2.04 0.0 0.0 +2.05 0.0 0.0 +2.06 0.0 0.0 +2.07 0.0 0.0 +2.08 0.0 0.0 +2.09 0.0 0.0 +2.1 0.0 0.0 +2.11 0.0 0.0 +2.12 0.0 0.0 +2.13 0.0 0.0 +2.14 0.0 0.0 +2.15 0.0 0.0 +2.16 0.0 0.0 +2.17 0.0 0.0 +2.18 0.0 0.0 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# local +0.0 -9.7953079069E+00 +0.01 -1.0400311409E+01 +0.02 -1.0888406632E+01 +0.03 -1.1200435759E+01 +0.04 -1.1277240974E+01 +0.05 -1.1209255048E+01 +0.06 -1.1082805427E+01 +0.07 -1.0947620831E+01 +0.08 -1.0824723658E+01 +0.09 -1.0719296734E+01 +0.1 -1.0629697231E+01 +0.11 -1.0552200672E+01 +0.12 -1.0483027171E+01 +0.13 -1.0419016416E+01 +0.14 -1.0357741770E+01 +0.15 -1.0297434680E+01 +0.16 -1.0236868006E+01 +0.17 -1.0175247464E+01 +0.18 -1.0112121292E+01 +0.19 -1.0047305980E+01 +0.2 -9.9808234246E+00 +0.21 -9.9128463822E+00 +0.22 -9.8436503701E+00 +0.23 -9.7735719697E+00 +0.24 -9.7029738376E+00 +0.25 -9.6322169180E+00 +0.26 -9.5616399874E+00 +0.27 -9.4915460360E+00 +0.28 -9.4221946690E+00 +0.29 -9.3537991265E+00 +0.3 -9.2865265731E+00 +0.31 -9.2205003077E+00 +0.32 -9.1558027948E+00 +0.33 -9.0924785976E+00 +0.34 -9.0305368263E+00 +0.35 -8.9699529387E+00 +0.36 -8.9106696470E+00 +0.37 -8.8525976536E+00 +0.38 -8.7956163315E+00 +0.39 -8.7395746729E+00 +0.4 -8.6842935256E+00 +0.41 -8.6295690309E+00 +0.42 -8.5751777523E+00 +0.43 -8.5208835233E+00 +0.44 -8.4664458092E+00 +0.45 -8.4116291209E+00 +0.46 -8.3562127953E+00 +0.47 -8.3000003168E+00 +0.48 -8.2428271911E+00 +0.49 -8.1845673552E+00 +0.5 -8.1251364881E+00 +0.51 -8.0644926540E+00 +0.52 -8.0026346953E+00 +0.53 -7.9395985751E+00 +0.54 -7.8754509157E+00 +0.55 -7.8102832830E+00 +0.56 -7.7442045501E+00 +0.57 -7.6773346132E+00 +0.58 -7.6097981896E+00 +0.59 -7.5417198577E+00 +0.6 -7.4732201798E+00 +0.61 -7.4044126046E+00 +0.62 -7.3354020632E+00 +0.63 -7.2662835375E+00 +0.64 -7.1971420148E+00 +0.65 -7.1280529085E+00 +0.66 -7.0590822441E+00 +0.67 -6.9902877566E+00 +0.68 -6.9217198555E+00 +0.69 -6.8534226258E+00 +0.7 -6.7854345116E+00 +0.71 -6.7177895103E+00 +0.72 -6.6505178739E+00 +0.73 -6.5836468129E+00 +0.74 -6.5172011142E+00 +0.75 -6.4512036077E+00 +0.76 -6.3856756197E+00 +0.77 -6.3206373121E+00 +0.78 -6.2561079364E+00 +0.79 -6.1921060357E+00 +0.8 -6.1286495929E+00 +0.81 -6.0657561346E+00 +0.82 -6.0034427989E+00 +0.83 -5.9417263711E+00 +0.84 -5.8806232961E+00 +0.85 -5.8201496694E+00 +0.86 -5.7603212132E+00 +0.87 -5.7011532385E+00 +0.88 -5.6426605986E+00 +0.89 -5.5848576333E+00 +0.9 -5.5277581083E+00 +0.91 -5.4713751498E+00 +0.92 -5.4157211748E+00 +0.93 -5.3608078208E+00 +0.94 -5.3066458721E+00 +0.95 -5.2532451864E+00 +0.96 -5.2006146500E+00 +0.97 -5.1487620332E+00 +0.98 -5.0976939521E+00 +0.99 -5.0474158034E+00 +1.0 -4.9979316802E+00 +1.01 -4.9492442981E+00 +1.02 -4.9013549271E+00 +1.03 -4.8542634146E+00 +1.04 -4.8079678026E+00 +1.05 -4.7624645403E+00 +1.06 -4.7177483280E+00 +1.07 -4.6738120477E+00 +1.08 -4.6306468941E+00 +1.09 -4.5882416969E+00 +1.1 -4.5465833716E+00 +1.11 -4.5056567463E+00 +1.12 -4.4654448539E+00 +1.13 -4.4259298481E+00 +1.14 -4.3870946833E+00 +1.15 -4.3489228924E+00 +1.16 -4.3113985506E+00 +1.17 -4.2745064577E+00 +1.18 -4.2382318993E+00 +1.19 -4.2025606063E+00 +1.2 -4.1674787716E+00 +1.21 -4.1329731249E+00 +1.22 -4.0990307195E+00 +1.23 -4.0656388980E+00 +1.24 -4.0327855198E+00 +1.25 -4.0004586688E+00 +1.26 -3.9686466511E+00 +1.27 -3.9373381834E+00 +1.28 -3.9065221765E+00 +1.29 -3.8761876992E+00 +1.3 -3.8463241824E+00 +1.31 -3.8169211999E+00 +1.32 -3.7879684651E+00 +1.33 -3.7594560262E+00 +1.34 -3.7313740213E+00 +1.35 -3.7037127811E+00 +1.36 -3.6764629142E+00 +1.37 -3.6496150905E+00 +1.38 -3.6231602853E+00 +1.39 -3.5970896520E+00 +1.4 -3.5713944446E+00 +1.41 -3.5460663178E+00 +1.42 -3.5210969494E+00 +1.43 -3.4964783706E+00 +1.44 -3.4722028189E+00 +1.45 -3.4482626548E+00 +1.46 -3.4246506993E+00 +1.47 -3.4013597263E+00 +1.48 -3.3783830415E+00 +1.49 -3.3557139795E+00 +1.5 -3.3333461989E+00 +1.51 -3.3112736518E+00 +1.52 -3.2894903707E+00 +1.53 -3.2679910124E+00 +1.54 -3.2467701693E+00 +1.55 -3.2258229108E+00 +1.56 -3.2051442058E+00 +1.57 -3.1847290067E+00 +1.58 -3.1645723052E+00 +1.59 -3.1446693226E+00 +1.6 -3.1250152668E+00 +1.61 -3.1056053810E+00 +1.62 -3.0864350882E+00 +1.63 -3.0674999620E+00 +1.64 -3.0487957324E+00 +1.65 -3.0303182431E+00 +1.66 -3.0120634156E+00 +1.67 -2.9940271521E+00 +1.68 -2.9762055892E+00 +1.69 -2.9585948974E+00 +1.7 -2.9411913974E+00 +1.71 -2.9239914167E+00 +1.72 -2.9069914394E+00 +1.73 -2.8901879713E+00 +1.74 -2.8735776456E+00 +1.75 -2.8571571408E+00 +1.76 -2.8409232236E+00 +1.77 -2.8248727423E+00 +1.78 -2.8090025834E+00 +1.79 -2.7933097620E+00 +1.8 -2.7777912717E+00 +1.81 -2.7624442908E+00 +1.82 -2.7472659091E+00 +1.83 -2.7322534622E+00 +1.84 -2.7174041320E+00 +1.85 -2.7027153831E+00 +1.86 -2.6881845381E+00 +1.87 -2.6738091286E+00 +1.88 -2.6595866351E+00 +1.89 -2.6455146440E+00 +1.9 -2.6315907931E+00 +1.91 -2.6178127142E+00 +1.92 -2.6041781993E+00 +1.93 -2.5906849203E+00 +1.94 -2.5773307925E+00 +1.95 -2.5641135986E+00 +1.96 -2.5510312912E+00 +1.97 -2.5380818020E+00 +1.98 -2.5252630984E+00 +1.99 -2.5125732639E+00 +2.0 -2.5000102769E+00 +2.01 -2.4875723360E+00 +2.02 -2.4752575202E+00 +2.03 -2.4630640399E+00 +2.04 -2.4509901208E+00 +2.05 -2.4390339638E+00 +2.06 -2.4271939251E+00 +2.07 -2.4154682479E+00 +2.08 -2.4038553365E+00 +2.09 -2.3923535617E+00 +2.1 -2.3809613030E+00 +2.11 -2.3696770644E+00 +2.12 -2.3584992450E+00 +2.13 -2.3474264006E+00 +2.14 -2.3364570512E+00 +2.15 -2.3255897148E+00 +2.16 -2.3148230361E+00 +2.17 -2.3041555626E+00 +2.18 -2.2935859647E+00 +2.19 -2.2831129150E+00 +2.2 -2.2727350358E+00 +2.21 -2.2624511056E+00 +2.22 -2.2522598146E+00 +2.23 -2.2421599154E+00 +2.24 -2.2321502244E+00 +2.25 -2.2222294734E+00 +2.26 -2.2123965330E+00 +2.27 -2.2026502440E+00 +2.28 -2.1929894132E+00 +2.29 -2.1834129847E+00 +2.3 -2.1739198255E+00 +2.31 -2.1645088428E+00 +2.32 -2.1551790170E+00 +2.33 -2.1459292535E+00 +2.34 -2.1367585500E+00 +2.35 -2.1276659213E+00 +2.36 -2.1186503127E+00 +2.37 -2.1097108018E+00 +2.38 -2.1008464331E+00 +2.39 -2.0920562046E+00 +2.4 -2.0833392519E+00 +2.41 -2.0746946489E+00 +2.42 -2.0661214596E+00 +2.43 -2.0576188552E+00 +2.44 -2.0491859464E+00 +2.45 -2.0408218556E+00 +2.46 -2.0325257888E+00 +2.47 -2.0242968934E+00 +2.48 -2.0161343429E+00 +2.49 -2.0080373776E+00 +2.5 -2.0000051821E+00 +2.51 -1.9920369736E+00 +2.52 -1.9841320267E+00 +2.53 -1.9762895631E+00 +2.54 -1.9685088371E+00 +2.55 -1.9607891574E+00 +2.56 -1.9531297830E+00 +2.57 -1.9455299990E+00 +2.58 -1.9379891482E+00 +2.59 -1.9305065270E+00 +2.6 -1.9230814452E+00 +2.61 -1.9157132799E+00 +2.62 -1.9084013649E+00 +2.63 -1.9011450292E+00 +2.64 -1.8939436842E+00 +2.65 -1.8867967011E+00 +2.66 -1.8797034234E+00 +2.67 -1.8726632960E+00 +2.68 -1.8656757242E+00 +2.69 -1.8587400729E+00 +2.7 -1.8518558088E+00 +2.71 -1.8450223684E+00 +2.72 -1.8382391535E+00 +2.73 -1.8315056327E+00 +2.74 -1.8248212788E+00 +2.75 -1.8181855313E+00 +2.76 -1.8115978559E+00 +2.77 -1.8050577615E+00 +2.78 -1.7985647255E+00 +2.79 -1.7921182080E+00 +2.8 -1.7857177520E+00 +2.81 -1.7793628661E+00 +2.82 -1.7730530256E+00 +2.83 -1.7667877827E+00 +2.84 -1.7605666767E+00 +2.85 -1.7543892206E+00 +2.86 -1.7482549508E+00 +2.87 -1.7421634433E+00 +2.88 -1.7361142500E+00 +2.89 -1.7301068892E+00 +2.9 -1.7241409715E+00 +2.91 -1.7182160709E+00 +2.92 -1.7123317389E+00 +2.93 -1.7064875664E+00 +2.94 -1.7006831640E+00 +2.95 -1.6949181227E+00 +2.96 -1.6891920072E+00 +2.97 -1.6835044635E+00 +2.98 -1.6778551046E+00 +2.99 -1.6722435230E+00 +3.0 -1.6666693437E+00 +3.01 -1.6611322150E+00 +3.02 -1.6556317677E+00 +3.03 -1.6501675996E+00 +3.04 -1.6447393891E+00 +3.05 -1.6393467861E+00 +3.06 -1.6339894265E+00 +3.07 -1.6286669522E+00 +3.08 -1.6233790509E+00 +3.09 -1.6181253872E+00 +3.1 -1.6129056026E+00 +3.11 -1.6077193828E+00 +3.12 -1.6025664196E+00 +3.13 -1.5974463932E+00 +3.14 -1.5923589527E+00 +3.15 -1.5873038213E+00 +3.16 -1.5822806958E+00 +3.17 -1.5772892645E+00 +3.18 -1.5723292057E+00 +3.19 -1.5674002544E+00 +3.2 -1.5625021199E+00 +3.21 -1.5576344984E+00 +3.22 -1.5527970977E+00 +3.23 -1.5479896601E+00 +3.24 -1.5432119082E+00 +3.25 -1.5384635475E+00 +3.26 -1.5337443106E+00 +3.27 -1.5290539473E+00 +3.28 -1.5243921936E+00 +3.29 -1.5197587649E+00 +3.3 -1.5151534139E+00 +3.31 -1.5105758996E+00 +3.32 -1.5060259698E+00 +3.33 -1.5015033514E+00 +3.34 -1.4970078128E+00 +3.35 -1.4925391227E+00 +3.36 -1.4880970407E+00 +3.37 -1.4836813055E+00 +3.38 -1.4792916976E+00 +3.39 -1.4749279962E+00 +3.4 -1.4705899722E+00 +3.41 -1.4662773767E+00 +3.42 -1.4619899986E+00 +3.43 -1.4577276285E+00 +3.44 -1.4534900480E+00 +3.45 -1.4492770216E+00 +3.46 -1.4450883429E+00 +3.47 -1.4409238146E+00 +3.48 -1.4367832287E+00 +3.49 -1.4326663637E+00 +3.5 -1.4285730146E+00 +3.51 -1.4245029971E+00 +3.52 -1.4204561129E+00 +3.53 -1.4164321554E+00 +3.54 -1.4124309178E+00 +3.55 -1.4084522296E+00 +3.56 -1.4044959016E+00 +3.57 -1.4005617428E+00 +3.58 -1.3966495421E+00 +3.59 -1.3927591429E+00 +3.6 -1.3888903647E+00 +3.61 -1.3850430270E+00 +3.62 -1.3812169299E+00 +3.63 -1.3774119129E+00 +3.64 -1.3736278098E+00 +3.65 -1.3698644486E+00 +3.66 -1.3661216462E+00 +3.67 -1.3623992310E+00 +3.68 -1.3586970529E+00 +3.69 -1.3550149479E+00 +3.7 -1.3513527506E+00 +3.71 -1.3477102763E+00 +3.72 -1.3440873909E+00 +3.73 -1.3404839380E+00 +3.74 -1.3368997612E+00 +3.75 -1.3333346896E+00 +3.76 -1.3297885764E+00 +3.77 -1.3262612817E+00 +3.78 -1.3227526565E+00 +3.79 -1.3192625488E+00 +3.8 -1.3157907932E+00 +3.81 -1.3123372674E+00 +3.82 -1.3089018292E+00 +3.83 -1.3054843366E+00 +3.84 -1.3020846342E+00 +3.85 -1.2987025869E+00 +3.86 -1.2953380691E+00 +3.87 -1.2919909450E+00 +3.88 -1.2886610792E+00 +3.89 -1.2853483151E+00 +3.9 -1.2820525436E+00 +3.91 -1.2787736362E+00 +3.92 -1.2755114636E+00 +3.93 -1.2722658895E+00 +3.94 -1.2690367790E+00 +3.95 -1.2658240235E+00 +3.96 -1.2626274997E+00 +3.97 -1.2594470844E+00 +3.98 -1.2562826417E+00 +3.99 -1.2531340562E+00 +4.0 -1.2500012187E+00 +4.01 -1.2468840119E+00 +4.02 -1.2437823182E+00 +4.03 -1.2406960031E+00 +4.04 -1.2376249674E+00 +4.05 -1.2345691024E+00 +4.06 -1.2315282961E+00 +4.07 -1.2285024364E+00 +4.08 -1.2254913921E+00 +4.09 -1.2224950757E+00 +4.1 -1.2195133805E+00 +4.11 -1.2165461998E+00 +4.12 -1.2135934242E+00 +4.13 -1.2106549331E+00 +4.14 -1.2077306417E+00 +4.15 -1.2048204481E+00 +4.16 -1.2019242507E+00 +4.17 -1.1990419438E+00 +4.18 -1.1961734148E+00 +4.19 -1.1933185819E+00 +4.2 -1.1904773482E+00 +4.21 -1.1876496167E+00 +4.22 -1.1848352867E+00 +4.23 -1.1820342502E+00 +4.24 -1.1792464301E+00 +4.25 -1.1764717336E+00 +4.26 -1.1737100682E+00 +4.27 -1.1709613393E+00 +4.28 -1.1682254410E+00 +4.29 -1.1655023010E+00 +4.3 -1.1627918312E+00 +4.31 -1.1600939433E+00 +4.32 -1.1574085490E+00 +4.33 -1.1547355427E+00 +4.34 -1.1520748576E+00 +4.35 -1.1494264097E+00 +4.36 -1.1467901148E+00 +4.37 -1.1441658889E+00 +4.38 -1.1415536337E+00 +4.39 -1.1389532790E+00 +4.4 -1.1363647480E+00 +4.41 -1.1337879605E+00 +4.42 -1.1312228362E+00 +4.43 -1.1286692854E+00 +4.44 -1.1261272320E+00 +4.45 -1.1235966071E+00 +4.46 -1.1210773343E+00 +4.47 -1.1185693371E+00 +4.48 -1.1160725347E+00 +4.49 -1.1135868436E+00 +4.5 -1.1111122031E+00 +4.51 -1.1086485404E+00 +4.52 -1.1061957826E+00 +4.53 -1.1037538568E+00 +4.54 -1.1013226758E+00 +4.55 -1.0989021820E+00 +4.56 -1.0964923080E+00 +4.57 -1.0940929841E+00 +4.58 -1.0917041410E+00 +4.59 -1.0893257018E+00 +4.6 -1.0869575968E+00 +4.61 -1.0845997688E+00 +4.62 -1.0822521513E+00 +4.63 -1.0799146781E+00 +4.64 -1.0775872830E+00 +4.65 -1.0752698851E+00 +4.66 -1.0729624349E+00 +4.67 -1.0706648699E+00 +4.68 -1.0683771270E+00 +4.69 -1.0660991430E+00 +4.7 -1.0638308489E+00 +4.71 -1.0615721790E+00 +4.72 -1.0593230825E+00 +4.73 -1.0570834992E+00 +4.74 -1.0548533689E+00 +4.75 -1.0526326312E+00 +4.76 -1.0504212151E+00 +4.77 -1.0482190695E+00 +4.78 -1.0460261407E+00 +4.79 -1.0438423714E+00 +4.8 -1.0416677041E+00 +4.81 -1.0395020809E+00 +4.82 -1.0373454313E+00 +4.83 -1.0351977142E+00 +4.84 -1.0330588748E+00 +4.85 -1.0309288585E+00 +4.86 -1.0288076105E+00 +4.87 -1.0266950722E+00 +4.88 -1.0245911830E+00 +4.89 -1.0224959010E+00 +4.9 -1.0204091741E+00 +4.91 -1.0183309501E+00 +4.92 -1.0162611767E+00 +4.93 -1.0141997958E+00 +4.94 -1.0121467543E+00 +4.95 -1.0101020102E+00 +4.96 -1.0080655139E+00 +4.97 -1.0060372156E+00 +4.98 -1.0040170655E+00 +4.99 -1.0020050068E+00 +5.0 -1.0000009915E+00 +5.01 -9.9800497853E-01 +5.02 -9.9601692053E-01 +5.03 -9.9403677005E-01 +5.04 -9.9206447965E-01 +5.05 -9.9009999481E-01 +5.06 -9.8814327013E-01 +5.07 -9.8619426653E-01 +5.08 -9.8425293881E-01 +5.09 -9.8231924171E-01 +5.1 -9.8039313002E-01 +5.11 -9.7847455227E-01 +5.12 -9.7656346372E-01 +5.13 -9.7465982794E-01 +5.14 -9.7276360183E-01 +5.15 -9.7087474224E-01 +5.16 -9.6899320607E-01 +5.17 -9.6711894568E-01 +5.18 -9.6525191502E-01 +5.19 -9.6339208106E-01 +5.2 -9.6153940271E-01 +5.21 -9.5969383884E-01 +5.22 -9.5785534834E-01 +5.23 -9.5602388818E-01 +5.24 -9.5419940913E-01 +5.25 -9.5238188227E-01 +5.26 -9.5057126840E-01 +5.27 -9.4876752834E-01 +5.28 -9.4697062287E-01 +5.29 -9.4518051281E-01 +5.3 -9.4339714846E-01 +5.31 -9.4162050123E-01 +5.32 -9.3985053520E-01 +5.33 -9.3808721299E-01 +5.34 -9.3633049724E-01 +5.35 -9.3458035056E-01 +5.36 -9.3283672935E-01 +5.37 -9.3109959783E-01 +5.38 -9.2936892590E-01 +5.39 -9.2764467793E-01 +5.4 -9.2592681829E-01 +5.41 -9.2421531134E-01 +5.42 -9.2251012024E-01 +5.43 -9.2081120058E-01 +5.44 -9.1911852851E-01 +5.45 -9.1743207008E-01 +5.46 -9.1575179130E-01 +5.47 -9.1407765821E-01 +5.48 -9.1240963683E-01 +5.49 -9.1074768644E-01 +5.5 -9.0909177628E-01 +5.51 -9.0744187842E-01 +5.52 -9.0579796048E-01 +5.53 -9.0415999006E-01 +5.54 -9.0252793478E-01 +5.55 -9.0090176200E-01 +5.56 -8.9928142906E-01 +5.57 -8.9766691560E-01 +5.58 -8.9605819074E-01 +5.59 -8.9445522360E-01 +5.6 -8.9285798330E-01 +5.61 -8.9126643897E-01 +5.62 -8.8968055584E-01 +5.63 -8.8810030060E-01 +5.64 -8.8652565061E-01 +5.65 -8.8495657644E-01 +5.66 -8.8339304863E-01 +5.67 -8.8183503776E-01 +5.68 -8.8028251439E-01 +5.69 -8.7873544257E-01 +5.7 -8.7719379623E-01 +5.71 -8.7565755125E-01 +5.72 -8.7412667957E-01 +5.73 -8.7260115311E-01 +5.74 -8.7108094382E-01 +5.75 -8.6956602361E-01 +5.76 -8.6805635630E-01 +5.77 -8.6655192092E-01 +5.78 -8.6505269276E-01 +5.79 -8.6355864504E-01 +5.8 -8.6206975101E-01 +5.81 -8.6058598390E-01 +5.82 -8.5910731697E-01 +5.83 -8.5763371456E-01 +5.84 -8.5616515888E-01 +5.85 -8.5470162543E-01 +5.86 -8.5324308868E-01 +5.87 -8.5178952313E-01 +5.88 -8.5034090325E-01 +5.89 -8.4889720355E-01 +5.9 -8.4745838969E-01 +5.91 -8.4602444519E-01 +5.92 -8.4459534656E-01 +5.93 -8.4317106947E-01 +5.94 -8.4175158961E-01 +5.95 -8.4033688264E-01 +5.96 -8.3892692424E-01 +5.97 -8.3752168212E-01 +5.98 -8.3612113938E-01 +5.99 -8.3472527429E-01 +6.0 -8.3333406366E-01 +6.01 -8.3194748432E-01 diff --git a/src/Control.cc b/src/Control.cc index d1c4c672..a798b5db 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -997,42 +997,6 @@ void Control::readRestartInfo(std::ifstream* tfile) } } -void Control::readRestartOutputInfo(std::ifstream* tfile) -{ - const std::string zero = "0"; - const std::string one = "1"; - if (tfile != nullptr) - { - // Read in the output restart filename - std::string filename; - (*tfile) >> filename; - // int dpcs_chkpoint=0; - if (zero.compare(filename) == 0) // no restart dump - out_restart_info = 0; - else - { - if (one.compare(filename) == 0) - { // automatic naming of dump - filename = "snapshot"; - out_restart_file_naming_strategy = 1; - } - (*tfile) >> out_restart_info; - (*tfile) >> out_restart_file_type; - //(*tfile)>>dpcs_chkpoint; - // timeout_.set(dpcs_chkpoint); - } - - out_restart_file.assign(run_directory_); - out_restart_file.append("/"); - out_restart_file.append(filename); - (*MPIdata::sout) << "Output restart file: " << out_restart_file - << " with info level " << out_restart_info - << std::endl; - //(*MPIdata::sout)<<"Time for DPCS checkpoint: - //"<::solve( // orthonormalization over outer iterations (MD or geometry optimization) // jlf, 01/02/2021 if (mmpi.PE0() && ct.verbose > 1) - os_ << "Orthonormalize wavefunctions ate start of DavidsonSolver" + os_ << "Orthonormalize wavefunctions at start of DavidsonSolver" << std::endl; orbitals.orthonormalizeLoewdin(false, nullptr, false); @@ -418,7 +418,7 @@ int DavidsonSolver::solve( ProjectedMatrices* proj_matN = dynamic_cast*>( orbitals.getProjMatrices()); - if (mmpi.PE0() && ct.verbose > 1) + if (mmpi.PE0() && ct.verbose > 0) { os_ << "###########################" << std::endl; os_ << "DavidsonSolver -> Iteration " << outer_it << std::endl; @@ -498,7 +498,7 @@ int DavidsonSolver::solve( // if( onpe0 )os_<<"Matrices..."<printMatrices(os_); - ts0 = evalEntropy(projmatrices, true, os_); + ts0 = evalEntropy(projmatrices, (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( ts0, projmatrices, orbitals, printE, os_); @@ -547,7 +547,7 @@ int DavidsonSolver::solve( proj_mat2N_->assignBlocksH(h11, h12, h21, h22); proj_mat2N_->setHB2H(); - ts0 = evalEntropy(proj_mat2N_.get(), true, os_); + ts0 = evalEntropy(proj_mat2N_.get(), (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( ts0, proj_mat2N_.get(), orbitals, printE, os_); } @@ -627,7 +627,7 @@ int DavidsonSolver::solve( // line minimization beta = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); - if (mmpi.PE0() && ct.verbose > 1) + if (mmpi.PE0() && ct.verbose > 0) { os_ << std::setprecision(12); os_ << "ts1=" << ts1 << std::endl; @@ -690,7 +690,7 @@ int DavidsonSolver::solve( os_ << "Total occupations for top half states=" << std::setprecision(15) << tot << std::endl; } - if (mmpi.PE0() && ct.verbose > 0) + if (mmpi.PE0() && ct.verbose > 1) { os_ << std::setprecision(15) << "Last level occupancy = " << eval[numst_] << std::endl; diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 32877d01..87b30224 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -118,13 +118,13 @@ void DensityMatrix::build( template void DensityMatrix::build(const int new_orbitals_index) { - //#ifdef PRINT_OPERATIONS MGmol_MPI& mmpi = *(MGmol_MPI::instance()); +#ifdef PRINT_OPERATIONS if (mmpi.PE0()) std::cout << "DensityMatrix::build() for diagonal occupation..." << std::endl; - //#endif +#endif if (!occ_uptodate_ && mmpi.instancePE0()) std::cout << "Warning: occupations not up to date to build DM!!!" << std::endl; diff --git a/src/Hartree_CG.cc b/src/Hartree_CG.cc index db156a9f..f9bf3860 100644 --- a/src/Hartree_CG.cc +++ b/src/Hartree_CG.cc @@ -90,8 +90,10 @@ void Hartree_CG::solve( double residual_reduction = poisson_solver_->getResidualReduction(); double final_residual = poisson_solver_->getFinalResidual(); + const bool large_residual + = (residual_reduction > 1.e-3 || final_residual > 1.e-3); - if (onpe0) + if (onpe0 && (large_residual || ct.verbose > 1)) (*MPIdata::sout) << setprecision(2) << scientific << "Hartree_CG: residual reduction = " << residual_reduction diff --git a/src/IonicAlgorithm.cc b/src/IonicAlgorithm.cc index d2cea35a..37a077e4 100644 --- a/src/IonicAlgorithm.cc +++ b/src/IonicAlgorithm.cc @@ -98,7 +98,7 @@ void IonicAlgorithm::init(HDFrestart* h5f_file) template int IonicAlgorithm::quenchElectrons(const int itmax, double& etot) { - int ret = mgmol_strategy_.quench(*orbitals_, ions_, itmax, 0, etot); + int ret = mgmol_strategy_.quench(**orbitals_, ions_, itmax, 0, etot); return ret; } diff --git a/src/LBFGS.cc b/src/LBFGS.cc index 0837a0e1..9154b7b3 100644 --- a/src/LBFGS.cc +++ b/src/LBFGS.cc @@ -97,7 +97,7 @@ int LBFGS::quenchElectrons(const int itmax, double& etot) { etot_i_[0] = etot_i_[1]; etot_i_[1] = etot_i_[2]; - int ret = mgmol_strategy_.quench(*orbitals_, ions_, itmax, 0, etot_i_[2]); + int ret = mgmol_strategy_.quench(**orbitals_, ions_, itmax, 0, etot_i_[2]); etot = etot_i_[2]; return ret; diff --git a/src/MGmol.cc b/src/MGmol.cc index c91120e9..5eb37c2c 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -521,7 +521,8 @@ void MGmol::run() switch (ct.AtomsDynamic()) { case AtomsDynamicType::Quench: - quench(current_orbitals_, *ions_, ct.max_electronic_steps, 20, eks); + quench( + *current_orbitals_, *ions_, ct.max_electronic_steps, 20, eks); // Forces for the last states force(*current_orbitals_, *ions_); @@ -1097,25 +1098,21 @@ void MGmol::setup() } template -void MGmol::cleanup() +void MGmol::dumpRestart() { - closing_tm_.start(); - - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); - Control& ct = *(Control::instance()); - - printTimers(); + Control& ct = *(Control::instance()); // Save data to restart file - if (ct.out_restart_info > 0 && !ct.AtomsMove()) + if (ct.out_restart_info > 0) { + Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); unsigned gdim[3] = { mygrid.gdim(0), mygrid.gdim(1), mygrid.gdim(2) }; + const pb::PEenv& myPEenv = mymesh->peenv(); // create restart file std::string filename(std::string(ct.out_restart_file)); - filename += "0"; + if (ct.out_restart_file_naming_strategy) filename += "0"; HDFrestart h5restartfile( filename, myPEenv, gdim, ct.out_restart_file_type); @@ -1136,7 +1133,22 @@ void MGmol::cleanup() os_ << "WARNING: writing ROM snapshot data failed!!!" << std::endl; } #endif - } // if (ct.out_restart_info > 0 && !ct.AtomsMove()) + } +} +template +void MGmol::cleanup() +{ + closing_tm_.start(); + + Control& ct = *(Control::instance()); + + printTimers(); + + // Save data to restart file + if (!ct.AtomsMove()) + { + dumpRestart(); + } MPI_Barrier(comm_); closing_tm_.stop(); @@ -1433,6 +1445,14 @@ template double MGmol::evaluateEnergyAndForces( const std::vector& tau, std::vector& atnumbers, std::vector& forces) +{ + return evaluateEnergyAndForces(current_orbitals_, tau, atnumbers, forces); +} + +template +double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, + const std::vector& tau, std::vector& atnumbers, + std::vector& forces) { assert(tau.size() == 3 * atnumbers.size()); @@ -1440,10 +1460,13 @@ double MGmol::evaluateEnergyAndForces( ions_->setPositions(tau, atnumbers); - double eks = 0.; - quench(current_orbitals_, *ions_, ct.max_electronic_steps, 20, eks); + moveVnuc(*ions_); + + double eks = 0.; + OrbitalsType* dorbitals = dynamic_cast(orbitals); + quench(*dorbitals, *ions_, ct.max_electronic_steps, 20, eks); - force(*current_orbitals_, *ions_); + force(*dorbitals, *ions_); ions_->getForces(forces); diff --git a/src/MGmol.h b/src/MGmol.h index ef7a020b..0e70d6b7 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -133,7 +133,7 @@ class MGmol : public MGmolInterface KBPsiMatrixSparse* kbpsi, dist_matrix::DistMatrix& hij); void computeHnlPhiAndAdd2HPhi(Ions& ions, OrbitalsType& phi, OrbitalsType& hphi, const KBPsiMatrixSparse* const kbpsi); - int dumprestartFile(OrbitalsType** orbitals, Ions& ions, + int dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count); @@ -182,13 +182,19 @@ class MGmol : public MGmolInterface /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } - std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getHamiltonian() + { + return hamiltonian_; + } void run() override; double evaluateEnergyAndForces(const std::vector& tau, std::vector& atnumbers, std::vector& forces); + double evaluateEnergyAndForces(Orbitals*, const std::vector& tau, + std::vector& atnumbers, std::vector& forces); + /* * get internal atomic positions */ @@ -246,7 +252,7 @@ class MGmol : public MGmolInterface void update_pot(const pb::GridFunc& vh_init, const Ions& ions); void update_pot(const Ions& ions); - int quench(OrbitalsType* orbitals, Ions& ions, const int max_steps, + int quench(OrbitalsType& orbitals, Ions& ions, const int max_steps, const int iprint, double& last_eks); int outerSolve(OrbitalsType& orbitals, OrbitalsType& work_orbitals, Ions& ions, const int max_steps, const int iprint, double& last_eks); @@ -319,8 +325,18 @@ class MGmol : public MGmolInterface forces_->force(orbitals, ions); } + /* + * simply dump current state + */ + void dumpRestart(); + void loadRestartFile(const std::string filename); + std::shared_ptr getProjectedMatrices() + { + return proj_matrices_; + } + #ifdef MGMOL_HAS_LIBROM int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); #endif diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index 046459d6..af7be67b 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -28,8 +28,15 @@ class MGmolInterface virtual double evaluateEnergyAndForces(const std::vector& tau, std::vector& atnumbers, std::vector& forces) = 0; + virtual double evaluateEnergyAndForces(Orbitals*, + const std::vector& tau, std::vector& atnumbers, + std::vector& forces) + = 0; virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; + virtual std::shared_ptr getProjectedMatrices() + = 0; + virtual void dumpRestart() = 0; }; #endif diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 63ddf690..7feb564d 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -59,7 +59,8 @@ MVPSolver::MVPSolver(MPI_Comm comm, std::ostream& os, if (onpe0 && ct.verbose > 0) { os_ << "MVPSolver..." << std::endl; - if (use_old_dm_) os_ << "MVPSolver uses old DM..." << std::endl; + if (use_old_dm_ && ct.verbose > 1) + os_ << "MVPSolver uses old DM..." << std::endl; } rho_ = rho; @@ -252,8 +253,9 @@ int MVPSolver::solve(OrbitalsType& orbitals) dmInit = proj_mat_work_->dm(); } - const double ts0 = evalEntropyMVP(current_proj_mat, true, os_); - const double e0 = energy_->evaluateTotal( + const double ts0 + = evalEntropyMVP(current_proj_mat, (ct.verbose > 1), os_); + const double e0 = energy_->evaluateTotal( ts0, current_proj_mat, orbitals, printE, os_); MatrixType target("target", numst_, numst_); @@ -345,6 +347,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) = dynamic_cast*>( orbitals.getProjMatrices()); projmatrices->setDM(*work_, orbitals.getIterativeIndex()); + projmatrices->setEigenvalues(proj_mat_work_->getEigenvalues()); } // Generate new density diff --git a/src/Orbitals.h b/src/Orbitals.h index d58f46bb..bc524aff 100644 --- a/src/Orbitals.h +++ b/src/Orbitals.h @@ -26,6 +26,8 @@ class Orbitals Orbitals() { iterative_index_ = -10; } + virtual ~Orbitals(){}; + Orbitals(const Orbitals& A, const bool copy_data) { if (copy_data) diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index 702637f4..6725b827 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -409,6 +409,12 @@ class ProjectedMatrices : public ProjectedMatricesInterface { dm_->setMatrix(mat, orbitals_index); } + void setEigenvalues(const std::vector& eigenvalues) + { + memcpy(eigenvalues_.data(), eigenvalues.data(), + eigenvalues.size() * sizeof(double)); + } + const std::vector& getEigenvalues() const { return eigenvalues_; } void computeGenEigenInterval(std::vector& interval, const int maxits, const double padding = 0.01); DensityMatrix& getDM() { return *dm_; } diff --git a/src/ProjectedMatricesInterface.h b/src/ProjectedMatricesInterface.h index 4041ba76..653b386d 100644 --- a/src/ProjectedMatricesInterface.h +++ b/src/ProjectedMatricesInterface.h @@ -71,7 +71,7 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction Control& ct = *(Control::instance()); nel_ = ct.getNelSpin(); MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - if (mmpi.PE0()) + if (mmpi.PE0() && ct.verbose > 1) std::cout << "ProjectedMatricesInterface: nel_=" << nel_ << std::endl; }; diff --git a/src/lbfgsrlx.cc b/src/lbfgsrlx.cc index b2aff795..6698eef0 100644 --- a/src/lbfgsrlx.cc +++ b/src/lbfgsrlx.cc @@ -77,6 +77,7 @@ void MGmol::lbfgsrlx(OrbitalsType** orbitals, Ions& ions) lbfgs.computeForces(); + // check for convergence int flag_convF = lbfgs.checkTolForces(ct.tol_forces); int conv = 0; @@ -87,10 +88,8 @@ void MGmol::lbfgsrlx(OrbitalsType** orbitals, Ions& ions) os_ << endl << endl << "LBFGS: convergence in forces has been achieved. " - "stopping ..." << endl; } - conv = 1; } else { @@ -116,10 +115,17 @@ void MGmol::lbfgsrlx(OrbitalsType** orbitals, Ions& ions) // Write down positions and displacements ions.printPositions(os_); + // early termination if convergence achieved + if (flag_convF) + { + if (onpe0) os_ << "LBFGS: stopping ..." << endl; + break; + } + + // reset iterations if last step failed if (conv != 0) { if (onpe0) os_ << "LBFGS Geometry optimization stopped" << endl; - // break; if (onpe0) os_ << "LBFGS Geometry optimization reset" << endl; lbfgs.reset(ct.dt); } diff --git a/src/md.cc b/src/md.cc index 939c6e62..354f3f47 100644 --- a/src/md.cc +++ b/src/md.cc @@ -226,7 +226,7 @@ void checkMaxForces(const std::vector& fion, } template -int MGmol::dumprestartFile(OrbitalsType** orbitals, Ions& ions, +int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -255,7 +255,7 @@ int MGmol::dumprestartFile(OrbitalsType** orbitals, Ions& ions, { if (onpe0) (*MPIdata::serr) - << "dumprestartFile: cannot write ...previous_orbitals..." + << "dumpMDrestartFile: cannot write ...previous_orbitals..." << std::endl; return ierr; } @@ -269,7 +269,7 @@ int MGmol::dumprestartFile(OrbitalsType** orbitals, Ions& ions, if (ierr < 0) { if (onpe0) - (*MPIdata::serr) << "dumprestartFile: cannot write " + (*MPIdata::serr) << "dumpMDrestartFile: cannot write " "...ExtrapolatedFunction..." << std::endl; return ierr; @@ -282,7 +282,7 @@ int MGmol::dumprestartFile(OrbitalsType** orbitals, Ions& ions, { if (onpe0) (*MPIdata::serr) - << "dumprestartFile: cannot close file..." << std::endl; + << "dumpMDrestartFile: cannot close file..." << std::endl; return ierr; } @@ -398,7 +398,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ct.restart_info < 3) { double eks = 0.; - quench(*orbitals, ions, ct.max_electronic_steps, 20, eks); + quench(**orbitals, ions, ct.max_electronic_steps, 20, eks); } ct.max_changes_pot = 0; @@ -426,7 +426,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) bool last_move_is_small = true; do { - retval = quench(*orbitals, ions, ct.max_electronic_steps, 0, eks); + retval = quench(**orbitals, ions, ct.max_electronic_steps, 0, eks); // update localization regions if (ct.adaptiveLRs()) @@ -576,18 +576,8 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) ct.steps = md_iteration_; -#if EXTRAPOLATE_RHO - if (onpe0) os_ << "Extrapolate rho..." << std::endl; - rho_->axpyRhoc(-1., rhoc_); - rho_->extrapolate(); -#endif - moveVnuc(ions); -#if EXTRAPOLATE_RHO - rho_->axpyRhoc(1., rhoc_); -#endif - if (!small_move) { orbitals_extrapol_->clearOldOrbitals(); @@ -624,12 +614,12 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) while (ierr < 0 && count < DUMP_MAX_NUM_TRY) { dump_tm_.start(); - ierr = dumprestartFile( + ierr = dumpMDrestartFile( orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) std::cout - << "dumprestartFile() failed... try again..." + << "dumpMDrestartFile() failed... try again..." << std::endl; if (ierr < 0) sleep(1.); count++; @@ -662,12 +652,12 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) while (ierr < 0 && count < DUMP_MAX_NUM_TRY) { dump_tm_.start(); - ierr = dumprestartFile( + ierr = dumpMDrestartFile( orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) - std::cout << "dumprestartFile() failed... try again..." + std::cout << "dumpMDrestartFile() failed... try again..." << std::endl; if (ierr < 0) sleep(1.); count++; @@ -681,8 +671,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) } template -void MGmol::loadRestartFile( - const std::string filename) +void MGmol::loadRestartFile(const std::string filename) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); Control& ct = *(Control::instance()); @@ -700,7 +689,9 @@ void MGmol::loadRestartFile( if (ierr < 0) { if (onpe0) - (*MPIdata::serr) << "loadRestartFile: failed to read the restart file." << std::endl; + (*MPIdata::serr) + << "loadRestartFile: failed to read the restart file." + << std::endl; global_exit(0); } diff --git a/src/quench.cc b/src/quench.cc index 614aa636..59a16d50 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -519,7 +519,7 @@ int MGmol::outerSolve(OrbitalsType& orbitals, } template -int MGmol::quench(OrbitalsType* orbitals, Ions& ions, +int MGmol::quench(OrbitalsType& orbitals, Ions& ions, const int max_inner_steps, const int iprint, double& last_eks) { assert(max_inner_steps > -1); @@ -543,33 +543,33 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, } // get actual indexes of stored functions - const std::vector>& gids(orbitals->getOverlappingGids()); + const std::vector>& gids(orbitals.getOverlappingGids()); g_kbpsi_->setup(*ions_); electrostat_->setup(ct.vh_its); rho_->setup(ct.getOrthoType(), gids); - OrbitalsType work_orbitals("Work", *orbitals); + OrbitalsType work_orbitals("Work", orbitals); - orbitals->setDataWithGhosts(); - orbitals->trade_boundaries(); + orbitals.setDataWithGhosts(); + orbitals.trade_boundaries(); - disentangleOrbitals(*orbitals, work_orbitals, ions, max_steps); + disentangleOrbitals(orbitals, work_orbitals, ions, max_steps); // setup "kernel" functions for AOMM algorithm if (ct.use_kernel_functions) { - applyAOMMprojection(*orbitals); + applyAOMMprojection(orbitals); } orbitals_precond_.reset(new OrbitalsPreconditioning()); orbitals_precond_->setup( - *orbitals, ct.getMGlevels(), ct.lap_type, currentMasks_.get(), lrs_); + orbitals, ct.getMGlevels(), ct.lap_type, currentMasks_.get(), lrs_); // solve electronic structure problem // (inner iterations) retval = outerSolve( - *orbitals, work_orbitals, ions, max_steps, iprint, last_eks); + orbitals, work_orbitals, ions, max_steps, iprint, last_eks); if (retval == -1) return -1; if (ct.use_kernel_functions) @@ -602,7 +602,7 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, } } last_eks - = energy_->evaluateTotal(ts, proj_matrices_.get(), *orbitals, 2, os_); + = energy_->evaluateTotal(ts, proj_matrices_.get(), orbitals, 2, os_); if (ct.computeCondGramMD()) { @@ -615,7 +615,7 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, if (ct.isLocMode() || ct.isSpreadFunctionalActive()) { // build matrices necessary to compute spreads and centers - spreadf_->computePositionMatrix(*orbitals, work_orbitals); + spreadf_->computePositionMatrix(orbitals, work_orbitals); if (ct.verbose > 0 || !ct.AtomsMove()) { @@ -628,7 +628,7 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, { if (ct.wannier_transform_type >= 1) { - wftransform(orbitals, &work_orbitals, ions); + wftransform(&orbitals, &work_orbitals, ions); } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fe7b571b..11125081 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -246,6 +246,10 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) +add_executable(testEnergyAndForces + ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) +add_executable(testWFEnergyAndForces + ${CMAKE_SOURCE_DIR}/tests/WFEnergyAndForces/testWFEnergyAndForces.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload @@ -334,6 +338,21 @@ add_test(NAME testGramMatrix add_test(NAME testDensityMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testDensityMatrix) +add_test(NAME testEnergyAndForces + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testEnergyAndForces + ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/coords.in + ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/lrs.in + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testWFEnergyAndForces + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testWFEnergyAndForces + ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/sih4.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -518,6 +537,8 @@ target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX) target_link_libraries(testSuperSampling PRIVATE MPI::MPI_CXX) target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) +target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/EnergyAndForces/coords.in b/tests/EnergyAndForces/coords.in new file mode 100644 index 00000000..cda898b9 --- /dev/null +++ b/tests/EnergyAndForces/coords.in @@ -0,0 +1,2 @@ +N1 1 0. 0. -1.0345 +N2 1 0. 0. 1.0345 diff --git a/tests/EnergyAndForces/lrs.in b/tests/EnergyAndForces/lrs.in new file mode 100644 index 00000000..9136a500 --- /dev/null +++ b/tests/EnergyAndForces/lrs.in @@ -0,0 +1,6 @@ +0.00 0.7 1.4 +0.00 -0.7 -1.4 +0.00 0.7 -1.4 +0.00 -0.7 1.4 +0.46 0.0 0.0 +-0.46 0.0 0.0 diff --git a/tests/EnergyAndForces/mgmol.cfg b/tests/EnergyAndForces/mgmol.cfg new file mode 100644 index 00000000..cf23889e --- /dev/null +++ b/tests/EnergyAndForces/mgmol.cfg @@ -0,0 +1,33 @@ +verbosity=1 +xcFunctional=LDA +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.N_ONCVPSP_LDA +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=100 +atol=1.e-9 +step_length=2. +ortho_freq=10 +[Orbitals] +initial_type=Gaussian +initial_width=1.5 +temperature=10. +nempty=1 +[Restart] +output_level=0 +[DensityMatrix] +mixing=0.5 diff --git a/tests/EnergyAndForces/test.py b/tests/EnergyAndForces/test.py new file mode 100755 index 00000000..e066bfc6 --- /dev/null +++ b/tests/EnergyAndForces/test.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test EnergyAndForces...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-6): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-5] +inp = sys.argv[nargs-4] +coords = sys.argv[nargs-3] +print("coordinates file: %s"%coords) +lrs = sys.argv[-2] + +#create links to potentials files +dst = 'pseudo.N_ONCVPSP_LDA' +src = sys.argv[-1] + '/' + dst + +if not os.path.exists(dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run +command = "{} {} -c {} -i {} -l {}".format(mpicmd,exe,inp,coords,lrs) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#analyse output +energies=[] +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'achieved'): + energies.append(energy) + +flag=0 +forces=[] +for line in lines: + if flag>0: + print(line) + words=line.split(b' ') + forces.append(words[1].decode()) + forces.append(words[2].decode()) + forces.append(words[3].decode()) + flag=flag-1 + if line.count(b'Forces:'): + flag=2 + + +print("Check energies...") +print( energies ) +if len(energies)<2: + print("Expected two converged energies") + sys.exit(1) + +tol = 1.e-6 +diff=eval(energies[1])-eval(energies[0]) +print(diff) +if abs(diff)>tol: + print("Energies differ: {} vs {} !!!".format(energies[0],energies[1])) + sys.exit(1) + +print("Check forces...") +print(forces) +flag=0 +for i in range(6): + diff=eval(forces[i+6])-eval(forces[i]) + print(diff) + if abs(diff)>1.e-3: + print("Forces difference larger than tol") + flag=1 +if flag>0: + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/EnergyAndForces/testEnergyAndForces.cc b/tests/EnergyAndForces/testEnergyAndForces.cc new file mode 100644 index 00000000..c8f7ab95 --- /dev/null +++ b/tests/EnergyAndForces/testEnergyAndForces.cc @@ -0,0 +1,198 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // compute energy and forces using all MPI tasks + // expect positions to be replicated on all MPI tasks + std::vector forces; + double eks + = mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + // move atoms one mesh spacing in all directions + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double hspacing[3] + = { mygrid.hgrid(0), mygrid.hgrid(1), mygrid.hgrid(2) }; + int i = 0; + for (auto& pos : positions) + { + pos += hspacing[i % 3]; + i++; + } + + // evaluate energy and forces again + eks = mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} diff --git a/tests/MVP/test.py b/tests/MVP/test.py index 708fd45c..bd5d708e 100644 --- a/tests/MVP/test.py +++ b/tests/MVP/test.py @@ -70,6 +70,30 @@ print("Found HDF5 error") sys.exit(1) +flag = 0 +eigenvalues=[] +for line in lines: + if line.count(b'FERMI'): + flag = 0 + if flag==1: + words=line.split() + for w in words: + eigenvalues.append(eval(w)) + if line.count(b'Eigenvalues'): + flag = 1 + eigenvalues=[] + +print(eigenvalues) +tol = 1.e-4 +eigenvalue0 = -0.208 +if abs(eigenvalues[0]-eigenvalue0)>tol: + print("Expected eigenvalue 0 to be {}".format(eigenvalue0)) + sys.exit(1) +eigenvalue50 = 0.208 +if abs(eigenvalues[50]-eigenvalue50)>tol: + print("Expected eigenvalue 50 to be {}".format(eigenvalue50)) + sys.exit(1) + niterations = len(energies) print("MVP solver ran for {} iterations".format(niterations)) if niterations>180: diff --git a/tests/WFEnergyAndForces/mgmol.cfg b/tests/WFEnergyAndForces/mgmol.cfg new file mode 100644 index 00000000..d543b626 --- /dev/null +++ b/tests/WFEnergyAndForces/mgmol.cfg @@ -0,0 +1,28 @@ +verbosity=2 +xcFunctional=LDA +[Mesh] +nx=40 +ny=40 +nz=40 +[Domain] +ox=-6.75 +oy=-6.75 +oz=-6.75 +lx=13.5 +ly=13.5 +lz=13.5 +[Potentials] +pseudopotential=pseudo.Si +pseudopotential=pseudo.H +[Run] +type=QUENCH +[Quench] +max_steps=50 +atol=1.e-9 +num_lin_iterations=2 +[Orbitals] +initial_type=Gaussian +initial_width=2. +[Restart] +output_level=3 +output_filename=WF diff --git a/tests/WFEnergyAndForces/sih4.xyz b/tests/WFEnergyAndForces/sih4.xyz new file mode 100644 index 00000000..b3f921e3 --- /dev/null +++ b/tests/WFEnergyAndForces/sih4.xyz @@ -0,0 +1,8 @@ +5 +SiH4 molecule (coordinates in Angstrom) +Si 0.0 0.0 0.0 +H 0.885 0.885 0.885 +H -0.885 -0.885 0.885 +H -0.885 0.885 -0.885 +H 0.885 -0.885 -0.885 + diff --git a/tests/WFEnergyAndForces/test.py b/tests/WFEnergyAndForces/test.py new file mode 100755 index 00000000..45420ddf --- /dev/null +++ b/tests/WFEnergyAndForces/test.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test WFEnergyAndForces...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-4): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-4] +inp = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.Si' +dst2 = 'pseudo.H' +src1 = sys.argv[nargs-1] + '/' + dst1 +src2 = sys.argv[nargs-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#analyse output +energies=[] +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'achieved'): + energies.append(energy) + +flag=0 +forces=[] +for line in lines: + if flag>0: + print(line) + words=line.split(b' ') + forces.append(words[1].decode()) + forces.append(words[2].decode()) + forces.append(words[3].decode()) + flag=flag-1 + if line.count(b'Forces:'): + flag=2 + + +print("Check energies...") +print( energies ) +if len(energies)<2: + print("Expected two converged energies") + sys.exit(1) + +tol = 1.e-6 +diff=eval(energies[1])-eval(energies[0]) +print(diff) +if abs(diff)>tol: + print("Energies differ: {} vs {} !!!".format(energies[0],energies[1])) + sys.exit(1) + +print("Check forces...") +print(forces) +flag=0 +for i in range(6): + diff=eval(forces[i+6])-eval(forces[i]) + print(diff) + if abs(diff)>1.e-3: + print("Forces difference larger than tol") + flag=1 +if flag>0: + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc new file mode 100644 index 00000000..c9e28d48 --- /dev/null +++ b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc @@ -0,0 +1,210 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // compute energy and forces using all MPI tasks + // expect positions to be replicated on all MPI tasks + std::vector forces; + double eks + = mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + mgmol->dumpRestart(); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + // compute energy and forces again using wavefunctions + // from previous call + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + std::shared_ptr projmatrices + = mgmol->getProjectedMatrices(); + + ExtendedGridOrbitals orbitals("new_orbitals", mygrid, mymesh->subdivx(), + ct.numst, ct.bcWF, projmatrices.get(), nullptr, nullptr, nullptr, + nullptr); + + const pb::PEenv& myPEenv = mymesh->peenv(); + HDFrestart h5file("WF", myPEenv, ct.out_restart_file_type); + orbitals.read_hdf5(h5file); + + // + // evaluate energy and forces again + // + + // convergence should be really quick since we start with an initial + // guess which is the solution + ct.max_electronic_steps = 5; + eks = mgmol->evaluateEnergyAndForces( + &orbitals, positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} diff --git a/util/generateLiLocalGTHpseudo.py b/util/generateLiLocalGTHpseudo.py new file mode 100644 index 00000000..1b553370 --- /dev/null +++ b/util/generateLiLocalGTHpseudo.py @@ -0,0 +1,65 @@ +# Generate local potential according to +# Goedecker, Teter, Hutter, Phys. rev. B 54 (3), 1996 +# Parameters from M. Krack, Theor. Chem. Acc. 114, 2005 +from math import exp, erf, sqrt, pi + +#coefficients for H +rloc = 0.4 +c1 = -14.081155 +c2 = 9.626220 +c3 = -1.783616 +c4 = 0.085152 +zion = 3. +anumber = 3 +name = "LithiumGTH_PBE" +mass = 3. + +def radialfunction(r): + alpha = (r/rloc)**2 + val = exp(-0.5*alpha)*(c1+c2*alpha+c3*alpha*alpha+c4*alpha*alpha*alpha) + if r>1.e-8: + val = val - zion*erf(r/(sqrt(2.)*rloc))/r + else: + #print("special case for r = {}".format(r)) + val = val -zion*sqrt(2.)/(sqrt(pi)*rloc) + + return val + +npts = 301 + +#header +print("# Short description of the species type. One line only!") +print(name) +print("#") +print("White") +print("#radii of balls and covalent bonds") +print("0.4 1.0") +print("# Nlcc flag") +print("0") +print("# Atomic number") +print(anumber) +print("# Atomic mass") +print(mass) +print("# Number of valence electrons") +print(zion) +print("# Gaussian core charge parameter rc") +print("1.") +print("# Number of potentials") +print("1") +print("# l-value for state which is local") +print("0 0") +print("# Local potential radius") +print("3.") +print("# Non-local potential radius") +print("3.") +print("# number of points in radial grid") +print(npts) +print("# log mesh parameter") +print("0.") +print("# radial grid, reference state, and potential for l=0") + +#potential +for i in range(npts): + r = round(0.01*i,2) + f = radialfunction(r) + print("{} {}".format(r,f)) From 1140370f41408ef9dd07393cdaba25bdafaf3ce7 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 20 Aug 2024 05:25:32 -0700 Subject: [PATCH 15/33] ROM offline calculations for MD snapshots (#272) * Add libROM snapshot frequency option * make install enabled * Add scripts * Add postprocessing script --- examples/PinnedH2O/job.basis | 32 +++++++++++++++++ examples/PinnedH2O/job.rom | 35 +++++++++++++++++++ examples/PinnedH2O/mgmol_rom.cfg | 38 +++++++++++++++++++++ examples/PinnedH2O/postprocess_PinnedH2O.py | 27 +++++++++++++++ scripts/build_quartz_libROM.sh | 2 +- src/Control.cc | 6 +++- src/md.cc | 26 ++++++++------ src/read_config.cc | 2 ++ src/rom_Control.h | 1 + 9 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 examples/PinnedH2O/job.basis create mode 100644 examples/PinnedH2O/job.rom create mode 100644 examples/PinnedH2O/mgmol_rom.cfg create mode 100644 examples/PinnedH2O/postprocess_PinnedH2O.py diff --git a/examples/PinnedH2O/job.basis b/examples/PinnedH2O/job.basis new file mode 100644 index 00000000..8f8c79f6 --- /dev/null +++ b/examples/PinnedH2O/job.basis @@ -0,0 +1,32 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20240815 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples + +set snapshot_files = "" +set increment_md_steps = 1 +set num_md_steps = 500 + +foreach k (`seq $increment_md_steps $increment_md_steps $num_md_steps`) + set snapshot_files = "$snapshot_files MD_mdstep${k}_snapshot" +end +echo "Snapshot files: $snapshot_files" + +set basis_file = "PinnedH2O_orbitals_basis" + +srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_Pinned_H2O.out + +date diff --git a/examples/PinnedH2O/job.rom b/examples/PinnedH2O/job.rom new file mode 100644 index 00000000..854c56d8 --- /dev/null +++ b/examples/PinnedH2O/job.rom @@ -0,0 +1,35 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 1:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20240815 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set datadir = $maindir/examples/PinnedH2O + +set cfg_rom = mgmol_rom.cfg +#cp $datadir/$cfg_rom . + +cp $datadir/coords.in . + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg_rom -i coords.in > rom_PinnedH2O.out + +date diff --git a/examples/PinnedH2O/mgmol_rom.cfg b/examples/PinnedH2O/mgmol_rom.cfg new file mode 100644 index 00000000..332dd77d --- /dev/null +++ b/examples/PinnedH2O/mgmol_rom.cfg @@ -0,0 +1,38 @@ +verbosity=1 +xcFunctional=PBE +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=1000. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 +[ROM.offline] +save_librom_snapshot=true +librom_snapshot_freq=1 diff --git a/examples/PinnedH2O/postprocess_PinnedH2O.py b/examples/PinnedH2O/postprocess_PinnedH2O.py new file mode 100644 index 00000000..142a34ac --- /dev/null +++ b/examples/PinnedH2O/postprocess_PinnedH2O.py @@ -0,0 +1,27 @@ +import subprocess +import re + +print("\\begin{tabular}{|c||c|c|c|c|c|c|c|}") +print("\\hline") +print("$k$ & $\\varepsilon = 10^{-1}$ & $\\varepsilon = 10^{-2}$ & $\\varepsilon = 10^{-3}$ & $\\varepsilon = 10^{-4}$ & $\\varepsilon = 10^{-5}$ & Snapshots \\\\") +print("\\hline") + +pattern = r"For energy fraction: \d+\.\d+, take first (\d+) of \d+ basis vectors" +for t in range(10): + k = 50*(t+1) + snapshots = 4*k + grep_command = f"grep 'take first' basis_1_{k}_Pinned_H2O.out" + result = subprocess.run(grep_command, shell=True, capture_output=True, text=True) + matches = re.findall(pattern, result.stdout) + energy_fractions = { + "0.9": matches[0], + "0.99": matches[1], + "0.999": matches[2], + "0.9999": matches[3], + "0.99999": matches[4], + } + line = f"{k} & {energy_fractions['0.9']} & {energy_fractions['0.99']} & {energy_fractions['0.999']} & {energy_fractions['0.9999']} & {energy_fractions['0.99999']} & {snapshots} \\\\" + print(line) + +print("\\hline") +print("\\end{tabular}") diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh index 458dd662..7092f3d5 100644 --- a/scripts/build_quartz_libROM.sh +++ b/scripts/build_quartz_libROM.sh @@ -55,7 +55,7 @@ make -j 16 ### Currently libROM does not have the installation procedure, ### so copying binary file to installation directory will disrupt the relative path to libROM.so, ### causing a run-time error. -#make install +make install # -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ # -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/src/Control.cc b/src/Control.cc index a798b5db..56272996 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2052,6 +2052,7 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.variable = ROMVariable::NONE; rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); + rom_pri_option.librom_snapshot_freq = vm["ROM.offline.librom_snapshot_freq"].as(); rom_pri_option.num_potbasis = vm["ROM.basis.number_of_potential_basis"].as(); } // onpe0 @@ -2092,6 +2093,9 @@ void Control::syncROMOptions() mpirc = MPI_Bcast(&rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_); bcast_check(mpirc); + mpirc = MPI_Bcast(&rom_pri_option.librom_snapshot_freq, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + short rom_var = (short)static_cast(rom_pri_option.variable); mpirc = MPI_Bcast(&rom_var, 1, MPI_SHORT, 0, comm_global_); bcast_check(mpirc); @@ -2101,4 +2105,4 @@ void Control::syncROMOptions() mpirc = MPI_Bcast(&rom_pri_option.num_potbasis, 1, MPI_INT, 0, comm_global_); bcast_check(mpirc); -} \ No newline at end of file +} diff --git a/src/md.cc b/src/md.cc index 354f3f47..ed1e5969 100644 --- a/src/md.cc +++ b/src/md.cc @@ -406,6 +406,9 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) bool extrapolated_flag = true; if (ct.dt <= 0.) extrapolated_flag = false; + int librom_snapshot_freq = ct.getROMOptions().librom_snapshot_freq; + if (librom_snapshot_freq == -1) librom_snapshot_freq = ct.md_print_freq; + MDfiles md_files; // main MD iteration loop @@ -625,21 +628,22 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) count++; } + printWithTimeStamp("dumped restart file...", std::cout); + } + #ifdef MGMOL_HAS_LIBROM - // Save orbital snapshots - if (ct.getROMOptions().save_librom_snapshot > 0) - { - int ierr = save_orbital_snapshot( - ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); + // Save orbital snapshots + if (md_iteration_ % librom_snapshot_freq == 0 + && ct.getROMOptions().save_librom_snapshot > 0) + { + int ierr = save_orbital_snapshot( + ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); - if (ierr < 0) - os_ << "WARNING md(): writing ROM snapshot data failed!!!" << std::endl; - } + if (ierr < 0) + os_ << "WARNING md(): writing ROM snapshot data failed!!!" << std::endl; + } #endif - printWithTimeStamp("dumped restart file...", std::cout); - } - md_iterations_tm.stop(); } // md loop diff --git a/src/read_config.cc b/src/read_config.cc index 14057553..40194678 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -426,6 +426,8 @@ void setupROMConfigOption(po::options_description &rom_cfg) "File name for libROM snapshot/POD matrices.") ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), "Save libROM snapshot file at FOM simulation.") + ("ROM.offline.librom_snapshot_freq", po::value()->default_value(-1), + "Frequency of saving libROM snapshot file at FOM simulation.") ("ROM.offline.variable", po::value()->default_value(""), "FOM variable to perform POD: either orbitals or potential.") ("ROM.basis.number_of_potential_basis", po::value()->default_value(-1), diff --git a/src/rom_Control.h b/src/rom_Control.h index 2e4a1c71..67cb85a6 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -46,6 +46,7 @@ struct ROMPrivateOptions /* save librom snapshot matrix at FOM simulation. */ bool save_librom_snapshot = false; + int librom_snapshot_freq = -1; /* options for ROM building */ int num_potbasis = -1; From c40089b1a76b244bb4d989dced9b9637ad70fb94 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Mon, 16 Sep 2024 13:49:03 -0700 Subject: [PATCH 16/33] ROM hyper reduction for density evaluation (#268) * base for rho hyperreduction test * compute rho on sample points, on carom space. * computeRhoOnSamplePts checked. does not seem to have the factor 2.. * exact equivalence cannot be accomplished due to rescaleTotalCharge. adjusting tolerance. * save hyperreduction operator in build rom phase. --- src/Control.cc | 2 + src/MGmol.h | 6 +- src/Rho.h | 2 + src/rom_Control.h | 1 + src/rom_main.cc | 7 + src/rom_workflows.cc | 297 ++++++++++++++++++++++++++++++++++++++++++- src/rom_workflows.h | 9 ++ 7 files changed, 319 insertions(+), 5 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index 56272996..2231813e 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2035,6 +2035,8 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.rom_stage = ROMStage::BUILD; else if (str.compare("test_poisson") == 0) rom_pri_option.rom_stage = ROMStage::TEST_POISSON; + else if (str.compare("test_rho") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_RHO; else if (str.compare("none") == 0) rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; diff --git a/src/MGmol.h b/src/MGmol.h index 0e70d6b7..ab07703d 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -182,10 +182,8 @@ class MGmol : public MGmolInterface /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } - std::shared_ptr> getHamiltonian() - { - return hamiltonian_; - } + std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getRho() { return rho_; } void run() override; diff --git a/src/Rho.h b/src/Rho.h index ea5069da..11063914 100644 --- a/src/Rho.h +++ b/src/Rho.h @@ -71,6 +71,8 @@ class Rho Rho(); ~Rho(){}; + const OrthoType getOrthoType() { return orbitals_type_; } + void setup( const OrthoType orbitals_type, const std::vector>&); void setVerbosityLevel(const int vlevel) { verbosity_level_ = vlevel; } diff --git a/src/rom_Control.h b/src/rom_Control.h index 67cb85a6..8c3b4b2c 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -23,6 +23,7 @@ enum class ROMStage RESTORE, // TODO(kevin): what stage is this? BUILD, TEST_POISSON, + TEST_RHO, UNSUPPORTED }; diff --git a/src/rom_main.cc b/src/rom_main.cc index c54bcd37..fa956221 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -125,6 +125,13 @@ int main(int argc, char** argv) testROMPoissonOperator(mgmol); break; + case (ROMStage::TEST_RHO): + if (ct.isLocMode()) + testROMRhoOperator(mgmol); + else + testROMRhoOperator(mgmol); + break; + default: std::cerr << "rom_main error: Unknown ROM stage" << std::endl; MPI_Abort(MPI_COMM_WORLD, 0); diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index d9629c07..68349e90 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -13,6 +13,42 @@ #include #include +namespace CAROM { + +/* + This is not implemented in libROM yet. + A temporary placeholder until it is merged into libROM. +*/ +void mult(const Matrix &A, const std::vector &sample_row_A, const Matrix &B, Matrix &AB) +{ + CAROM_VERIFY(!B.distributed()); + CAROM_VERIFY(A.distributed() == AB.distributed()); + CAROM_VERIFY(A.numColumns() == B.numRows()); + CAROM_VERIFY(sample_row_A.size() <= A.numRows()); + + const int num_rows = sample_row_A.size(); + const int num_cols = B.numColumns(); + AB.setSize(num_rows, num_cols); + + // Do the multiplication. + const int Acol = A.numColumns(); + for (int r = 0; r < num_rows; ++r) { + double *d_Arow = A.getData() + sample_row_A[r] * Acol; + for (int c = 0; c < num_cols; ++c) { + double *d_Aptr = d_Arow; + double *d_Bptr = B.getData() + c; + double result_val = 0.0; + for (int ac = 0; ac < Acol; ++ac, d_Aptr++) { + result_val += (*d_Aptr) * (*d_Bptr); + d_Bptr += num_cols; + } + AB.item(r, c) = result_val; + } + } +} + +} + template std::string string_format( const std::string& format, Args ... args ) { @@ -107,6 +143,9 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) Control& ct = *(Control::instance()); Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); ROMPrivateOptions rom_options = ct.getROMOptions(); /* type of variable we intend to run POD */ @@ -162,6 +201,18 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) delete col; } // for (int c = 0; c < num_pot_basis; c++) + + /* DEIM hyperreduction */ + CAROM::Matrix pot_rhs_rom(num_pot_basis, num_pot_basis, false); + std::vector global_sampled_row(num_pot_basis), sampled_rows_per_proc(nprocs); + DEIM(pot_basis, num_pot_basis, global_sampled_row, sampled_rows_per_proc, + pot_rhs_rom, rank, nprocs); + + /* ROM rescaleTotalCharge operator */ + CAROM::Vector fom_ones(pot_basis->numRows(), true); + CAROM::Vector rom_ones(num_pot_basis, false); + fom_ones = mymesh->grid().vel(); // volume element + pot_basis->transposeMult(fom_ones, rom_ones); /* Save ROM operator */ // write the file from PE0 only @@ -179,10 +230,20 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) h5_helper.putDoubleArray("potential_rom_inverse", pot_rom.getData(), num_pot_basis * num_pot_basis, false); + /* save right-hand side hyper-reduction operator */ + h5_helper.putDoubleArray("potential_rhs_rom_inverse", pot_rhs_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* save right-hand side rescaling operator */ + h5_helper.putDoubleArray("potential_rhs_rescaler", rom_ones.getData(), + num_pot_basis, false); + h5_helper.close(); } } +/* test routines */ + template void testROMPoissonOperator(MGmolInterface *mgmol_) { @@ -429,6 +490,237 @@ void testROMPoissonOperator(MGmolInterface *mgmol_) } } +template +void testROMRhoOperator(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const int subdivx = mymesh->subdivx(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + // if (ct.isLocMode()) + // printf("LocMode is On!\n"); + // else + // printf("LocMode is Off!\n"); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + + /* Load MGmol pointer and Potentials */ + MGmol *mgmol = static_cast *>(mgmol_); + Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + std::shared_ptr> rho = mgmol->getRho(); + const OrthoType ortho_type = rho->getOrthoType(); + assert(ortho_type == OrthoType::Nonorthogonal); + + /* potential should have the same size as rho */ + const int dim = pot.size(); + + /* number of restart files, start/end indices */ + assert(rom_options.restart_file_minidx >= 0); + assert(rom_options.restart_file_maxidx >= 0); + const int minidx = rom_options.restart_file_minidx; + const int maxidx = rom_options.restart_file_maxidx; + const int num_restart = maxidx - minidx + 1; + + /* Initialize libROM classes */ + /* + In practice, we do not produce rho POD basis. + This rho POD basis is for the sake of verification. + */ + CAROM::Options svd_options(dim, num_restart, 1); + svd_options.static_svd_preserve_snapshot = true; + CAROM::BasisGenerator basis_generator(svd_options, false, "test_rho"); + + /* Collect the restart files */ + std::string filename; + for (int k = minidx; k <= maxidx; k++) + { + filename = string_format(rom_options.restart_file_fmt, k); + mgmol->loadRestartFile(filename); + basis_generator.takeSample(&rho->rho_[0][0]); + } + // basis_generator.writeSnapshot(); + const CAROM::Matrix rho_snapshots(*basis_generator.getSnapshotMatrix()); + basis_generator.endSamples(); + + const CAROM::Matrix *rho_basis = basis_generator.getSpatialBasis(); + CAROM::Matrix *proj_rho = rho_basis->transposeMult(rho_snapshots); + + /* DEIM hyperreduction */ + CAROM::Matrix rho_basis_inv(num_restart, num_restart, false); + std::vector global_sampled_row(num_restart), sampled_rows_per_proc(nprocs); + DEIM(rho_basis, num_restart, global_sampled_row, sampled_rows_per_proc, + rho_basis_inv, rank, nprocs); + if (rank == 0) + { + int num_sample_rows = 0; + for (int k = 0; k < sampled_rows_per_proc.size(); k++) + num_sample_rows += sampled_rows_per_proc[k]; + printf("number of sampled row: %d\n", num_sample_rows); + } + + /* get local sampled row */ + std::vector offsets, sampled_row(sampled_rows_per_proc[rank]); + int num_global_sample = CAROM::get_global_offsets(sampled_rows_per_proc[rank], offsets); + for (int s = 0, gs = offsets[rank]; gs < offsets[rank+1]; gs++, s++) + sampled_row[s] = global_sampled_row[gs]; + + /* load only the first restart file for now */ + const int test_idx = 2; + + filename = string_format(rom_options.restart_file_fmt, test_idx + minidx); + /* + currently, this does not update rho. + computeRhoOnSamplePts computes with the new density matrix, + while mgmol rho remains the same as the initial condition. + Commenting line 516 gives a consistent result, both for the initial condition. + */ + mgmol->loadRestartFile(filename); + + const int nrows = mymesh->locNumpt(); + // printf("mesh::locNumpt: %d\n", nrows); + + OrbitalsType *orbitals = mgmol->getOrbitals(); + // printf("orbitals::locNumpt: %d\n", orbitals->getLocNumpt()); + + /* NOTE(kevin): we assume we only use ProjectedMatrices class */ + // ProjectedMatrices> *proj_matrices = + // static_cast> *>(orbitals->getProjMatrices()); + ProjectedMatricesInterface *proj_matrices = orbitals->getProjMatrices(); + proj_matrices->updateSubMatX(); + SquareLocalMatrices& localX(proj_matrices->getLocalX()); + + // printf("localX nmat: %d\n", localX.nmat()); + // printf("localX n: %d\n", localX.n()); + // printf("localX m: %d\n", localX.m()); + + bool dm_distributed = (localX.nmat() > 1); + assert(!dm_distributed); + + /* copy density matrix */ + CAROM::Matrix dm(localX.getRawPtr(), localX.m(), localX.n(), dm_distributed, true); + + // /* random density matrix */ + // /* NOTE(kevin): Due to rescaleTotalCharge, the result is slightly inconsistent. */ + // CAROM::Matrix dm(localX.m(), localX.n(), dm_distributed, true); + // dm = 0.0; + // dist_matrix::DistMatrix dm_mgmol(proj_matrices->dm()); + // for (int i = 0; i < dm_mgmol.m(); i++) + // { + // for (int j = 0; j < dm_mgmol.m(); j++) + // { + // if (dm_mgmol.getVal(i, j) == 0.0) + // continue; + + // dm(i, j) = dm_mgmol.getVal(i, j) * (0.975 + 0.05 * ran0()); + // dm_mgmol.setVal(i, j, dm(i, j)); + // } + // } + + /* update rho first */ + // rho->computeRho(*orbitals, dm_mgmol); + // rho->update(*orbitals); + + const int chrom_num = orbitals->chromatic_number(); + CAROM::Matrix psi(dim, chrom_num, true); + for (int c = 0; c < chrom_num; c++) + { + ORBDTYPE *d_psi = orbitals->getPsi(c); + for (int d = 0; d < dim ; d++) + psi.item(d, c) = *(d_psi + d); + } + + CAROM::Matrix rom_psi(chrom_num, chrom_num, false); + for (int i = 0; i < chrom_num; i++) + for (int j = 0; j < chrom_num; j++) + rom_psi(i, j) = (i == j) ? 1 : 0; + + /* this will be resized in computeRhoOnSamplePts */ + CAROM::Vector sample_rho(1, true); + + computeRhoOnSamplePts(dm, psi, rom_psi, sampled_row, sample_rho); + + for (int s = 0; s < sampled_row.size(); s++) + { + const double error = abs(rho->rho_[0][sampled_row[s]] - sample_rho(s)); + if (error > 1.0e-4) + printf("rank %d, rho[%d]: %.5e, sample_rho: %.5e, librom_snapshot: %.5e\n", + rank, sampled_row[s], rho->rho_[0][sampled_row[s]], sample_rho(s), rho_snapshots(sampled_row[s], test_idx)); + CAROM_VERIFY(error < 1.0e-4); + } + + sample_rho.gather(); + + CAROM::Vector *rom_rho = rho_basis_inv.mult(sample_rho); + for (int d = 0; d < rom_rho->dim(); d++) + { + if ((rank == 0) && (abs(proj_rho->item(d, test_idx) - rom_rho->item(d)) > 1.0e-3)) + printf("rom_rho error: %.3e\n", abs(proj_rho->item(d, test_idx) - rom_rho->item(d))); + CAROM_VERIFY(abs(proj_rho->item(d, test_idx) - rom_rho->item(d)) < 1.0e-3); + } + + CAROM::Vector *fom_rho = rho_basis->mult(*rom_rho); + + CAROM_VERIFY(fom_rho->dim() == rho->rho_[0].size()); + for (int d = 0; d < fom_rho->dim(); d++) + CAROM_VERIFY(abs(fom_rho->item(d) - rho->rho_[0][d]) < 1.0e-4); + + delete rom_rho; + delete fom_rho; +} + +/* + dm: density matrix converted to CAROM::Matrix + phi_basis: POD basis matrix of orbitals, or orbitals themselves + rom_phi: ROM coefficients of POD Basis. If phi_basis is orbitals themselves, then simply an identity. + local_idx: sampled local grid indices on this processor. + sampled_rho: FOM density on sampled grid points. +*/ +void computeRhoOnSamplePts(const CAROM::Matrix &dm, + const CAROM::Matrix &phi_basis, const CAROM::Matrix &rom_phi, + const std::vector &local_idx, CAROM::Vector &sampled_rho) +{ + assert(!dm.distributed()); + assert(sampled_rho.distributed() == phi_basis.distributed()); + + // this will be resized. + CAROM::Matrix sampled_phi(1, 1, phi_basis.distributed()); + + CAROM::mult(phi_basis, local_idx, rom_phi, sampled_phi); + + /* same product as in computeRhoSubdomainUsingBlas3 */ + CAROM::Matrix product(1, 1, sampled_phi.distributed()); + sampled_phi.mult(dm, product); + + sampled_rho.setSize(sampled_phi.numRows()); + double *d_product = product.getData(); + double *d_phi = sampled_phi.getData(); + for (int d = 0; d < sampled_rho.dim(); d++) + { + double val = 0.0; + /* CAROM Matrices are row-major */ + for (int c = 0; c < sampled_phi.numColumns(); c++, d_product++, d_phi++) + val += (*d_product) * (*d_phi); + + sampled_rho(d) = val; + } + + /* + TODO(kevin): need to figure out what these functions do. + and probably need to make ROM-equivalent functions with another hyper-reduction? + */ + // gatherSpin(); + + /* + rescaleTotalCharge is handled after hyperreduction. + */ + // rescaleTotalCharge(); +} + template void readRestartFiles(MGmolInterface *mgmol_); template void readRestartFiles(MGmolInterface *mgmol_); @@ -436,4 +728,7 @@ template void buildROMPoissonOperator(MGmolInterface *mgmol_); template void buildROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); -template void testROMPoissonOperator(MGmolInterface *mgmol_); \ No newline at end of file +template void testROMPoissonOperator(MGmolInterface *mgmol_); + +template void testROMRhoOperator(MGmolInterface *mgmol_); +template void testROMRhoOperator(MGmolInterface *mgmol_); \ No newline at end of file diff --git a/src/rom_workflows.h b/src/rom_workflows.h index 2399d065..fd00c687 100644 --- a/src/rom_workflows.h +++ b/src/rom_workflows.h @@ -12,6 +12,7 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" +#include "ProjectedMatrices.h" #include "LocGridOrbitals.h" #include "Potentials.h" #include "MGmol.h" @@ -36,6 +37,7 @@ namespace po = boost::program_options; #include "librom.h" #include "utils/HDFDatabase.h" +#include "utils/mpi_utils.h" template void readRestartFiles(MGmolInterface *mgmol_); @@ -46,4 +48,11 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); +template +void testROMRhoOperator(MGmolInterface *mgmol_); + +void computeRhoOnSamplePts(const CAROM::Matrix &dm, + const CAROM::Matrix &phi_basis, const CAROM::Matrix &rom_phi, + const std::vector &local_idx, CAROM::Vector &sampled_rho); + #endif // ROM_WORKFLOWS_H From da33f942676e7a536743fff42573974f0690abb2 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sat, 21 Sep 2024 22:04:48 -0700 Subject: [PATCH 17/33] Fix CMakeList --- src/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 05cc45a1..e602101f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -208,7 +208,6 @@ if(${MGMOL_WITH_LIBXC}) endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-opt DESTINATION bin) -<<<<<<< HEAD # build ROM executable if(USE_LIBROM) @@ -231,5 +230,3 @@ if(USE_LIBROM) endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-rom DESTINATION bin) endif(USE_LIBROM) -======= ->>>>>>> release From f1d42a312a1cde1f69bd766c1fe4c11c79d1d511 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sun, 22 Sep 2024 14:51:42 -0700 Subject: [PATCH 18/33] Fix test --- src/MGmol.cc | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/MGmol.cc b/src/MGmol.cc index 085d952b..8da0d331 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1135,21 +1135,6 @@ void MGmol::dumpRestart() #endif } } -template -void MGmol::cleanup() -{ - closing_tm_.start(); - - Control& ct = *(Control::instance()); - - printTimers(); - - // Save data to restart file - if (!ct.AtomsMove()) - { - dumpRestart(); - } -} template void MGmol::cleanup() From f0e2d22399133c07ffba1895edd9dccbd34c05f2 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sun, 22 Sep 2024 19:49:43 -0700 Subject: [PATCH 19/33] Bypass build error --- src/rom_workflows.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 68349e90..5dc3718e 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -512,7 +512,7 @@ void testROMRhoOperator(MGmolInterface *mgmol_) MGmol *mgmol = static_cast *>(mgmol_); Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); Potentials& pot = mgmol->getHamiltonian()->potential(); - std::shared_ptr> rho = mgmol->getRho(); + std::shared_ptr> rho = NULL; // mgmol->getRho(); const OrthoType ortho_type = rho->getOrthoType(); assert(ortho_type == OrthoType::Nonorthogonal); @@ -731,4 +731,4 @@ template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); -template void testROMRhoOperator(MGmolInterface *mgmol_); \ No newline at end of file +template void testROMRhoOperator(MGmolInterface *mgmol_); From a03c891b19f8f888d89959e45230a4dfcbc61bed Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sun, 22 Sep 2024 20:03:06 -0700 Subject: [PATCH 20/33] Fix merge mistake --- src/MGmol.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/MGmol.h b/src/MGmol.h index 14e661fc..6a4cc830 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -182,10 +182,8 @@ class MGmol : public MGmolInterface /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } - std::shared_ptr> getHamiltonian() - { - return hamiltonian_; - } + std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getRho() { return rho_; } void run() override; From a7478331b9fc73b6d99bb5dc7a0facb3b9a566d3 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sat, 12 Oct 2024 14:25:05 -0700 Subject: [PATCH 21/33] Add verification of orbital projection --- examples/PinnedH2O/job.basis | 8 +++--- examples/PinnedH2O/job.rom | 2 +- examples/PinnedH2O/mgmol_rom.cfg | 2 +- src/MGmol.h | 1 + src/md.cc | 42 ++++++++++++++++++++++++++++++-- src/rom.cc | 34 ++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 8 deletions(-) diff --git a/examples/PinnedH2O/job.basis b/examples/PinnedH2O/job.basis index 8f8c79f6..bf02a621 100644 --- a/examples/PinnedH2O/job.basis +++ b/examples/PinnedH2O/job.basis @@ -1,7 +1,7 @@ #!/bin/tcsh #SBATCH -N 2 #SBATCH -t 0:10:00 -#SBATCH -p pdebug +#SBATCH -p pbatch date @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20240815 +set maindir = /p/lustre2/cheung26/mgmol-20240919 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -18,14 +18,14 @@ set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples set snapshot_files = "" set increment_md_steps = 1 -set num_md_steps = 500 +set num_md_steps = 50 foreach k (`seq $increment_md_steps $increment_md_steps $num_md_steps`) set snapshot_files = "$snapshot_files MD_mdstep${k}_snapshot" end echo "Snapshot files: $snapshot_files" -set basis_file = "PinnedH2O_orbitals_basis" +set basis_file = "PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps}" srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_Pinned_H2O.out diff --git a/examples/PinnedH2O/job.rom b/examples/PinnedH2O/job.rom index 854c56d8..8de3d6b7 100644 --- a/examples/PinnedH2O/job.rom +++ b/examples/PinnedH2O/job.rom @@ -21,7 +21,7 @@ cp $maindir/install_quartz/bin/$exe . set datadir = $maindir/examples/PinnedH2O set cfg_rom = mgmol_rom.cfg -#cp $datadir/$cfg_rom . +cp $datadir/$cfg_rom . cp $datadir/coords.in . diff --git a/examples/PinnedH2O/mgmol_rom.cfg b/examples/PinnedH2O/mgmol_rom.cfg index 332dd77d..1f724546 100644 --- a/examples/PinnedH2O/mgmol_rom.cfg +++ b/examples/PinnedH2O/mgmol_rom.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=MD [MD] -num_steps=500 +num_steps=50 dt=40. thermostat=ON [Thermostat] diff --git a/src/MGmol.h b/src/MGmol.h index 6a4cc830..78ef92d3 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -356,6 +356,7 @@ class MGmol : public MGmolInterface #ifdef MGMOL_HAS_LIBROM int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); + void project_orbital(std::string snapshot_dir, int rdim, OrbitalsType& orbitals); #endif }; // Instantiate static variables here to avoid clang warnings diff --git a/src/md.cc b/src/md.cc index ed1e5969..920ac1f3 100644 --- a/src/md.cc +++ b/src/md.cc @@ -492,6 +492,44 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) // Compute forces force(**orbitals, ions); +#ifdef MGMOL_HAS_LIBROM + // TODO: cleanup + int rdim = 39; + std::string basis_filename = "PinnedH2O_orbitals_basis_1_50"; + bool ROM_md = false; + + if (rdim > 0) + { + if (onpe0) + { + os_ << "Projecting orbitals onto ROM subspaces" << std::endl; + os_ << "Loading ROM basis " << basis_filename << std::endl; + os_ << "ROM basis dimension = " << rdim << std::endl; + } + project_orbital(basis_filename, rdim, **orbitals); + if (ROM_md) + { + force(**orbitals, ions); + } + else + { + double shift[3]; + for (short i = 0; i < 3; i++) shift[i] = 0.; + Ions ROM_ions(ions, shift); + force(**orbitals, ROM_ions); + std::string zero = "0"; + if (ions_->getNumIons() < 256 || ct.verbose > 2) + { + if (ct.verbose > 0) ROM_ions.printForcesGlobal(os_); + } + else if (zero.compare(ct.md_print_filename) == 0) + { + ROM_ions.printForcesLocal(os_); + } + } + } +#endif + // set fion ions.getLocalForces(fion); @@ -633,8 +671,8 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) #ifdef MGMOL_HAS_LIBROM // Save orbital snapshots - if (md_iteration_ % librom_snapshot_freq == 0 - && ct.getROMOptions().save_librom_snapshot > 0) + if (ct.getROMOptions().save_librom_snapshot > 0 && + md_iteration_ % librom_snapshot_freq == 0) { int ierr = save_orbital_snapshot( ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); diff --git a/src/rom.cc b/src/rom.cc index a554d2cf..19ff66a3 100644 --- a/src/rom.cc +++ b/src/rom.cc @@ -57,6 +57,40 @@ int MGmol::save_orbital_snapshot(std::string file_path, OrbitalsTy return 0; } +template +void MGmol::project_orbital(std::string file_path, int rdim, OrbitalsType& orbitals) +{ + const int dim = orbitals.getLocNumpt(); + const int totalSamples = orbitals.chromatic_number(); + + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, "foo"); + + for (int i = 0; i < totalSamples; ++i) + basis_generator.takeSample(orbitals.getPsi(i)); + const CAROM::Matrix* orbital_snapshots = basis_generator.getSnapshotMatrix(); + + CAROM::BasisReader reader(file_path); + CAROM::Matrix* orbital_basis = reader.getSpatialBasis(rdim); + + CAROM::Matrix* proj_orbital_coeff = orbital_basis->transposeMult(orbital_snapshots); + CAROM::Matrix* proj_orbital_snapshots = orbital_basis->mult(proj_orbital_coeff); + + Control& ct = *(Control::instance()); + Mesh* mesh = Mesh::instance(); + pb::GridFunc gf_psi(mesh->grid(), ct.bcWF[0], ct.bcWF[1], ct.bcWF[2]); + CAROM::Vector snapshot, proj_snapshot; + for (int i = 0; i < totalSamples; ++i) + { + orbital_snapshots->getColumn(i, snapshot); + proj_orbital_snapshots->getColumn(i, proj_snapshot); + gf_psi.assign(proj_snapshot.getData()); + orbitals.setPsi(gf_psi, i); + snapshot -= proj_snapshot; + std::cout << "Error for orbital " << i << " = " << snapshot.norm() << std::endl; + } +} + template class MGmol; template class MGmol; From e9c3807c4294486f1b7211b7eb88e3f9a9b54540 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 15 Oct 2024 12:11:15 -0700 Subject: [PATCH 22/33] Merge release (#276) * Reenable testShortSighted test (#248) * Make new driver to check input (#247) * clean up/reorganize main.cc * use shared_ptr in class MGmol * Add possible periodic dimensions to xyz2in.py (#249) * Add possible periodic dimensions to xyz2in.py * Remove unused/untested option extrapolateH (#250) * Exit with failure if density off by more than 2% (#251) * Exit with failure if density off by more than 2% * adapt SiH4 test to catch that * fix bug in DFTsolver that was leading to wrong density * Example driver (#252) * add example driver, showing use of MGmol as a force/energy computational engine * clean up related functions in class Ions * loadOrbitalsFromRestartFile -> loadRestartFile (#253) * Add SG15 PBE potential for N (#258) * Update 2-pyridone example (#259) * Adjust verbosity in some functions (#260) * Add new example: pinned H2O (#261) * Print out eigenvalues out of MVP solver (#262) Previously, the wrong eigenvalues (0) were printed out because eigenvalues outside solver were not up-to-date. * Add Li2 example with local GTH potential (#263) * Fix LBFGS termination when converged (#264) * Remove unused code to extrapolate rho (#265) * Fix and test EnergyAndForces interface (#266) * Atomic potentials were not updated when atomic positions were changed * Added test to make sure energies and forces are the same after positions move by one mesh spacing * Add new functionality to compute energy and forces (#267) * use specified initial conditions for wavefunctions * Add functionality to compute energy and forces (#270) * use specified wavefunctions as solution, with unknown DM * Add ONCV for Sulfur + example (#275) * Fix CMakeList * Fix test * Bypass build error * Fix merge mistake --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Kevin" Seung Whan Chung --- examples/S2/coords.in | 2 + examples/S2/mgmol.cfg | 23 + potentials/pseudo.S_ONCV_PBE_SG15 | 1921 +++++++++++++++++ src/DensityMatrix.cc | 9 +- src/MGmol.cc | 45 +- src/MGmol.h | 25 +- src/MGmolInterface.h | 10 +- src/MVPSolver.cc | 12 + src/ProjectedMatrices.cc | 5 +- src/Rho.cc | 6 +- src/rom_workflows.cc | 4 +- tests/CMakeLists.txt | 11 + tests/DMandEnergyAndForces/coords.in | 2 + tests/DMandEnergyAndForces/lrs.in | 6 + tests/DMandEnergyAndForces/mgmol.cfg | 35 + tests/DMandEnergyAndForces/test.py | 73 + .../testDMandEnergyAndForces.cc | 211 ++ 17 files changed, 2380 insertions(+), 20 deletions(-) create mode 100644 examples/S2/coords.in create mode 100644 examples/S2/mgmol.cfg create mode 100644 potentials/pseudo.S_ONCV_PBE_SG15 create mode 100644 tests/DMandEnergyAndForces/coords.in create mode 100644 tests/DMandEnergyAndForces/lrs.in create mode 100644 tests/DMandEnergyAndForces/mgmol.cfg create mode 100755 tests/DMandEnergyAndForces/test.py create mode 100644 tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc diff --git a/examples/S2/coords.in b/examples/S2/coords.in new file mode 100644 index 00000000..f2db9834 --- /dev/null +++ b/examples/S2/coords.in @@ -0,0 +1,2 @@ +S1 1 0. 0. 0. +S2 1 0. 0. 3.61 diff --git a/examples/S2/mgmol.cfg b/examples/S2/mgmol.cfg new file mode 100644 index 00000000..3e31f57f --- /dev/null +++ b/examples/S2/mgmol.cfg @@ -0,0 +1,23 @@ +verbosity=1 +xcFunctional=PBE +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-8. +oy=-8. +oz=-8. +lx=16. +ly=16. +lz=16. +[Potentials] +pseudopotential=pseudo.S_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=200 +atol=1.e-8 +[Orbitals] +initial_width=2. diff --git a/potentials/pseudo.S_ONCV_PBE_SG15 b/potentials/pseudo.S_ONCV_PBE_SG15 new file mode 100644 index 00000000..93e0e237 --- /dev/null +++ b/potentials/pseudo.S_ONCV_PBE_SG15 @@ -0,0 +1,1921 @@ +# +# This pseudopotential file has been produced using the code +# ONCVPSP (Optimized Norm-Conservinng Vanderbilt PSeudopotential) +# scalar-relativistic version 2.1.1, 03/26/2014 by D. R. Hamann +# The code is available through a link at URL www.mat-simresearch.com. +# Documentation with the package provides a full discription of the +# input data below. +# +# +# While it is not required under the terms of the GNU GPL, it is +# suggested that you cite D. R. Hamann, Phys. Rev. B 88, 085117 (2013) +# in any publication using these pseudopotentials. +# +# +# Copyright 2015 The Regents of the University of California +# +# This work is licensed under the Creative Commons Attribution-ShareAlike +# 4.0 International License. To view a copy of this license, visit +# http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to +# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. +# +# This pseudopotential is part of the Schlipf-Gygi norm-conserving +# pseudopotential library. Its construction parameters were tuned to +# reproduce materials of a training set with very high accuracy and +# should be suitable as a general purpose pseudopotential to treat a +# variety of different compounds. For details of the construction and +# testing of the pseudopotential please refer to: +# +# M. Schlipf, F. Gygi, Comp. Phys. Comm. 196, 36 (2015) +# http://dx.doi.org/10.1016/j.cpc.2015.05.011 +# +# We kindly ask that you include this reference in all publications +# associated to this pseudopotential. +# +# +# Input file for PP generation: +# +# # ATOM AND REFERENCE CONFIGURATION +# # atsym z nc nv iexc psfile +# S 16.00 3 2 4 fpmd +# # +# # n l f energy (Ha) +# 1 0 2.00 +# 2 0 2.00 +# 2 1 6.00 +# 3 0 2.00 +# 3 1 4.00 +# # +# # PSEUDOPOTENTIAL AND OPTIMIZATION +# # lmax +# 1 +# # +# # l, rc, ep, ncon, nbas, qcut +# 0 2.12945 -0.63416 5 8 6.75573 +# 1 1.52480 -0.25757 5 8 8.63513 +# # +# # LOCAL POTENTIAL +# # lloc, lpopt, rc(5), dvloc0 +# 4 5 0.95758 0.00000 +# # +# # VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# # l, nproj, debl +# 0 2 1.22534 +# 1 2 1.65602 +# # +# # MODEL CORE CHARGE +# # icmod, fcfact +# 0 0.00000 +# # +# # LOG DERIVATIVE ANALYSIS +# # epsh1, epsh2, depsh +# -5.00 3.00 0.02 +# # +# # OUTPUT GRID +# # rlmax, drl +# 6.00 0.01 +# # +# # TEST CONFIGURATIONS +# # ncnf +# 0 +# # nvcnf +# # n l f +# +S_ONCV_PBE-1 +# +color +#radii of balls and covalent bonds +-1. -1. +# Nlcc flag +0 +# Atomic number +16 +# Atomic mass +32.06000137 +# Number of valence electrons +6 +#Gaussian core charge parameter rc +1.0 +# Number of potentials +3 +# l-value for state which is local, then type of potential format +2 3 +# Local potential radius +3.2 +# Non-local potential radius +3.2 +# number of points in radial grid +602 +# VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# l, nproj +0 2 0.1035057429E+02 0.6748828187E+00 +1 2 0.9516274860E+01 0.2804708546E+01 +# l= 0 +0.0 3.7717412366 2.9189109498 +0.01 3.7719018776 2.9183509866 +0.02 3.7724026253 2.9166916915 +0.03 3.7732245023 2.9139122053 +0.04 3.774348531 2.9099916687 +0.05 3.7757482954 2.9049011428 +0.06 3.7773900923 2.898603871 +0.07 3.7792331245 2.8910556129 +0.08 3.7812297354 2.8822050481 +0.09 3.7833256829 2.8719942473 +0.1 3.7854604514 2.8603592078 +0.11 3.7875676008 2.8472304493 +0.12 3.7895751502 2.8325336672 +0.13 3.7914059948 2.8161904385 +0.14 3.7929783525 2.7981189755 +0.15 3.7942062396 2.7782349235 +0.16 3.7949999726 2.7564521953 +0.17 3.7952666921 2.7326838392 +0.18 3.7949109084 2.706842933 +0.19 3.7938350635 2.6788434987 +0.2 3.7919401088 2.6486014322 +0.21 3.7891260935 2.6160354399 +0.22 3.7852927623 2.5810679779 +0.23 3.7803401572 2.5436261857 +0.24 3.7741692231 2.5036428099 +0.25 3.766682409 2.4610571015 +0.26 3.7577842707 2.4158157069 +0.27 3.7473820587 2.3678735041 +0.28 3.7353862998 2.3171944159 +0.29 3.7217113638 2.2637521758 +0.3 3.7062760135 2.2075310435 +0.31 3.6890039342 2.1485264641 +0.32 3.6698242411 2.0867456663 +0.33 3.6486719506 2.0222081731 +0.34 3.6254884511 1.9549462973 +0.35 3.6002219022 1.8850054775 +0.36 3.5728276471 1.8124446249 +0.37 3.5432685235 1.7373362626 +0.38 3.5115152339 1.6597667516 +0.39 3.4775465763 1.5798362469 +0.4 3.4413496887 1.4976586593 +0.41 3.4029202406 1.4133615024 +0.42 3.3622625791 1.327085641 +0.43 3.319389829 1.2389849408 +0.44 3.2743239481 1.1492258211 +0.45 3.2270957347 1.0579867109 +0.46 3.1777447898 0.96545740865 +0.47 3.1263194312 0.87183834757 +0.48 3.0728765592 0.77733976783 +0.49 3.0174814494 0.68218077698 +0.5 2.9602075867 0.58658841233 +0.51 2.9011363835 0.49079653457 +0.52 2.8403567906 0.39504462368 +0.53 2.7779650276 0.29957662678 +0.54 2.7140641957 0.2046396408 +0.55 2.6487636899 0.11048255501 +0.56 2.5821789998 0.017354672646 +0.57 2.5144308728 -0.07449568823 +0.58 2.4456451103 -0.1648227841 +0.59 2.3759516826 -0.25338470387 +0.6 2.3054844173 -0.33994520629 +0.61 2.2343801679 -0.42427487726 +0.62 2.1627782898 -0.50615285932 +0.63 2.0908199666 -0.58536839417 +0.64 2.0186475097 -0.66172187533 +0.65 1.9464037206 -0.73502682527 +0.66 1.8742311772 -0.80511101968 +0.67 1.8022715715 -0.87181759713 +0.68 1.7306650412 -0.93500637095 +0.69 1.6595494083 -0.99455553892 +0.7 1.5890595921 -1.0503620586 +0.71 1.5193269353 -1.1023428144 +0.72 1.4504786116 -1.1504352703 +0.73 1.3826369115 -1.1945986146 +0.74 1.31591866 -1.2348142122 +0.75 1.2504347283 -1.2710857281 +0.76 1.1862894709 -1.3034395225 +0.77 1.123580233 -1.3319247288 +0.78 1.0623968961 -1.3566131489 +0.79 1.0028214682 -1.3775989572 +0.8 0.94492772055 -1.3949982017 +0.81 0.88878087471 -1.4089480982 +0.82 0.83443734235 -1.4196061062 +0.83 0.78194454126 -1.4271487482 +0.84 0.73134067628 -1.4317703672 +0.85 0.6826547342 -1.4336813557 +0.86 0.63590645531 -1.4331063392 +0.87 0.5911063603 -1.4302821209 +0.88 0.54825581874 -1.4254553729 +0.89 0.50734727773 -1.4188799828 +0.9 0.46836449306 -1.4108142305 +0.91 0.43128284695 -1.4015176891 +0.92 0.39606973745 -1.3912478723 +0.93 0.36268504025 -1.3802566269 +0.94 0.3310816433 -1.3687862735 +0.95 0.30120608287 -1.3570657607 +0.96 0.27299926125 -1.3453069539 +0.97 0.24639578856 -1.3336862799 +0.98 0.22132168405 -1.3223057818 +0.99 0.19769389367 -1.3111537553 +1.0 0.17542633582 -1.3001149815 +1.01 0.1544351692 -1.2889999625 +1.02 0.13464260071 -1.2775700083 +1.03 0.11597959258 -1.2655663263 +1.04 0.098386926129 -1.2527354371 +1.05 0.081815967788 -1.2388526331 +1.06 0.066227886294 -1.2237437341 +1.07 0.051590732077 -1.2072859225 +1.08 0.037877974845 -1.1894097625 +1.09 0.025066652728 -1.1700936969 +1.1 0.013135956759 -1.1493653828 +1.11 0.0020652968231 -1.1272870244 +1.12 -0.0081664103401 -1.1039449016 +1.13 -0.017581129962 -1.0794433933 +1.14 -0.026202272846 -1.0539001366 +1.15 -0.034054897507 -1.0274377592 +1.16 -0.041165298371 -1.0001795668 +1.17 -0.047560882987 -0.9722482902 +1.18 -0.053270364115 -0.94376212735 +1.19 -0.058323151395 -0.91483359715 +1.2 -0.062749148379 -0.88557010392 +1.21 -0.066578693596 -0.85607305284 +1.22 -0.069842280335 -0.8264354089 +1.23 -0.072570319662 -0.79674499202 +1.24 -0.074793187344 -0.76708482733 +1.25 -0.076540789321 -0.73752894997 +1.26 -0.077842647818 -0.70814740364 +1.27 -0.078728007273 -0.67900641864 +1.28 -0.079225164905 -0.65016418765 +1.29 -0.079361907267 -0.62167582309 +1.3 -0.079165594132 -0.59359326795 +1.31 -0.078662315233 -0.56596149123 +1.32 -0.077877693273 -0.5388234109 +1.33 -0.076836664008 -0.51221851122 +1.34 -0.075562844569 -0.48618085279 +1.35 -0.074079490886 -0.46074313072 +1.36 -0.072408738606 -0.43593375667 +1.37 -0.070571646522 -0.41177753661 +1.38 -0.068589032885 -0.38829837557 +1.39 -0.066479963161 -0.36551457197 +1.4 -0.064263165708 -0.34344348953 +1.41 -0.061956426491 -0.32209959305 +1.42 -0.059576161543 -0.30149349744 +1.43 -0.057138757575 -0.28163569015 +1.44 -0.054658681977 -0.26253144748 +1.45 -0.052150267116 -0.24418575291 +1.46 -0.049626683011 -0.22660053136 +1.47 -0.047100026817 -0.20977509975 +1.48 -0.044582198816 -0.19370822964 +1.49 -0.042083238373 -0.1783943207 +1.5 -0.039613494312 -0.16382848229 +1.51 -0.037181427181 -0.15000157131 +1.52 -0.034795503397 -0.13690452119 +1.53 -0.032462954533 -0.12452560427 +1.54 -0.030190365866 -0.11285187679 +1.55 -0.027983786923 -0.10186926883 +1.56 -0.025848056637 -0.09156155708 +1.57 -0.023788076285 -0.08191254844 +1.58 -0.02180709884 -0.072903372245 +1.59 -0.019908910212 -0.064515938844 +1.6 -0.018095394178 -0.056729413683 +1.61 -0.016369129845 -0.049523900811 +1.62 -0.01473094107 -0.042877342745 +1.63 -0.013182202875 -0.036768329307 +1.64 -0.011722836062 -0.031174028374 +1.65 -0.010353138167 -0.026071977824 +1.66 -0.0090722113073 -0.021438913179 +1.67 -0.0078794281674 -0.017251778077 +1.68 -0.0067731608474 -0.013487222834 +1.69 -0.0057520052889 -0.01012199958 +1.7 -0.0048136986886 -0.0071330256482 +1.71 -0.0039561989961 -0.0044972001989 +1.72 -0.0031767112554 -0.0021920616629 +1.73 -0.0024726810814 -0.0001949267295 +1.74 -0.0018408921881 0.0015156954421 +1.75 -0.0012783877013 0.0029618665928 +1.76 -0.0007816485767 0.0041637712608 +1.77 -0.00034740795298 0.0051427045896 +1.78 2.8026401829e-05 0.0059173024502 +1.79 0.00034816989119 0.0065077964019 +1.8 0.00061675389244 0.0069313251376 +1.81 0.00083746198141 0.0072065643935 +1.82 0.0010139835081 0.0073493701414 +1.83 0.0011500484087 0.0073764932044 +1.84 0.0012492723645 0.0073026182129 +1.85 0.0013152933449 0.0071422877869 +1.86 0.0013516442618 0.0069091520426 +1.87 0.0013617291682 0.0066153224449 +1.88 0.0013490054888 0.0062735408013 +1.89 0.0013165068112 0.0058933579291 +1.9 0.0012676271725 0.0054866792447 +1.91 0.001204999954 0.0050609209086 +1.92 0.0011316946665 0.0046258512343 +1.93 0.0010501549063 0.0041881793203 +1.94 0.00096292345948 0.0037549844277 +1.95 0.00087232492188 0.0033324956288 +1.96 0.0007802401781 0.0029250611317 +1.97 0.00068893170795 0.0025385304964 +1.98 0.00059963649799 0.0021750068783 +1.99 0.00051423875926 0.0018388258619 +2.0 0.00043370922442 0.0015313603799 +2.01 0.0003591837815 0.0012544582322 +2.02 0.0002916241788 0.0010094946409 +2.03 0.00023127892332 0.00079576300695 +2.04 0.0001791120696 0.00061453546295 +2.05 0.00013472582495 0.00046345775503 +2.06 9.8384817291e-05 0.00034188555777 +2.07 6.9768245347e-05 0.00024773278945 +2.08 4.8147423849e-05 0.00017790159289 +2.09 3.3341603917e-05 0.00013060236927 +2.1 2.3868358715e-05 0.00010101589887 +2.11 1.9371649837e-05 8.8269925068e-05 +2.12 1.7922694389e-05 8.4856271419e-05 +2.13 1.6497714121e-05 7.859897815e-05 +2.14 9.7420279224e-06 4.6912008392e-05 +2.15 2.9056005592e-06 1.4094314355e-05 +2.16 -8.0918591293e-07 -3.8523450141e-06 +2.17 -9.9639126312e-07 -4.743585934e-06 +2.18 -9.6753942982e-08 -4.6062290987e-07 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 1 +0.0 0.0 0.0 +0.01 -0.18215119847 -0.026814620552 +0.02 -0.36272317524 -0.052415548675 +0.03 -0.54015344747 -0.07560362298 +0.04 -0.71291283526 -0.095208589256 +0.05 -0.87952167778 -0.11010316826 +0.06 -1.0385655317 -0.11921666472 +0.07 -1.188710187 -0.12154797139 +0.08 -1.3287158419 -0.11617782803 +0.09 -1.4574502862 -0.10228020199 +0.1 -1.5739009532 -0.079132666269 +0.11 -1.67718571 -0.046125659777 +0.12 -1.7665622681 -0.0027705264466 +0.13 -1.8414361117 0.051293759259 +0.14 -1.9013668551 0.11629525692 +0.15 -1.9460729525 0.19232518352 +0.16 -1.9754347089 0.27933553968 +0.17 -1.989495546 0.37713828001 +0.18 -1.9884615105 0.48540603768 +0.19 -1.9726990125 0.6036744131 +0.2 -1.9427308156 0.73134580733 +0.21 -1.8992303133 0.86769476907 +0.22 -1.8430141422 1.0118748079 +0.23 -1.7750332038 1.1629266125 +0.24 -1.6963621806 1.3197875925 +0.25 -1.6081876946 1.4813026155 +0.26 -1.5117951026 1.6462359424 +0.27 -1.4085542777 1.8132840472 +0.28 -1.299904332 1.981089365 +0.29 -1.1873375371 2.1482547351 +0.3 -1.0723826049 2.3133583982 +0.31 -0.95658750373 2.4749693875 +0.32 -0.84150200799 2.6316631397 +0.33 -0.72866032208 2.7820370206 +0.34 -0.6195634148 2.9247260924 +0.35 -0.51566229021 3.0584180243 +0.36 -0.41834109033 3.1818681389 +0.37 -0.32890186775 3.2939129473 +0.38 -0.24854887804 3.3934841005 +0.39 -0.17837537049 3.4796200912 +0.4 -0.11935084639 3.5514775245 +0.41 -0.072309863841 3.6083409945 +0.42 -0.037942402868 3.6496315528 +0.43 -0.016785896057 3.6749136782 +0.44 -0.0092190212914 3.6839006618 +0.45 -0.015457336 3.6764583357 +0.46 -0.035550815344 3.6526070933 +0.47 -0.069383337328 3.6125221611 +0.48 -0.1166741332 3.5565321086 +0.49 -0.1769813265 3.4851154898 +0.5 -0.24970675592 3.3988963203 +0.51 -0.33410335936 3.2986372681 +0.52 -0.42928430785 3.1852312936 +0.53 -0.53423308634 3.0596924101 +0.54 -0.64781595293 2.9231443229 +0.55 -0.76879490854 2.7768086642 +0.56 -0.89584335116 2.6219906967 +0.57 -1.0275600348 2.4600667262 +0.58 -1.1624883636 2.2924665043 +0.59 -1.2991297631 2.1206611932 +0.6 -1.4359660279 1.946143015 +0.61 -1.5714725575 1.7704133204 +0.62 -1.7041389887 1.5949638382 +0.63 -1.83248722 1.4212603213 +0.64 -1.9550843518 1.2507308243 +0.65 -2.0705647191 1.0847456461 +0.66 -2.1776428361 0.9246055585 +0.67 -2.2751261301 0.77153019367 +0.68 -2.3619290324 0.62664516025 +0.69 -2.4370892058 0.4909671653 +0.7 -2.4997720928 0.36539976592 +0.71 -2.5492813476 0.25072369837 +0.72 -2.5850645145 0.1475915076 +0.73 -2.6067209504 0.056520054476 +0.74 -2.6140038528 -0.022111641313 +0.75 -2.6068196146 -0.088062154853 +0.76 -2.5852280088 -0.14122824405 +0.77 -2.5494395231 -0.18164288689 +0.78 -2.4998108504 -0.20947169362 +0.79 -2.4368386188 -0.22500776758 +0.8 -2.361151466 -0.22866511 +0.81 -2.2735005867 -0.22097068355 +0.82 -2.1747489012 -0.20255526698 +0.83 -2.0658589876 -0.17414321461 +0.84 -1.9478801281 -0.13654150961 +0.85 -1.8219341314 -0.090627645356 +0.86 -1.6892009619 -0.03733740677 +0.87 -1.5509038255 0.022347806514 +0.88 -1.4082937146 0.087414096918 +0.89 -1.2626343255 0.15682791127 +0.9 -1.1151869952 0.22954880156 +0.91 -0.96719607558 0.30454179611 +0.92 -0.81987491325 0.38078922285 +0.93 -0.67439262381 0.45730181877 +0.94 -0.53186183868 0.53312897122 +0.95 -0.39332757469 0.60736810359 +0.96 -0.25975734689 0.67917309919 +0.97 -0.13203329184 0.74775321949 +0.98 -0.01094906337 0.81235698313 +0.99 0.10279630832 0.87226236508 +1.0 0.20860467089 0.92679716672 +1.01 0.30597435445 0.97535396608 +1.02 0.39450484154 1.0174098802 +1.03 0.47389811631 1.0525426144 +1.04 0.54395642972 1.080438937 +1.05 0.60458713018 1.1009088448 +1.06 0.65580515498 1.11389353 +1.07 0.69771928596 1.119447206 +1.08 0.7305315527 1.1177346277 +1.09 0.75452833442 1.10901769 +1.1 0.77008388967 1.0936559167 +1.11 0.77764462483 1.0720831454 +1.12 0.77771729738 1.04479204 +1.13 0.77086206847 1.0123250096 +1.14 0.75768669294 0.97526556969 +1.15 0.73883116761 0.93422091892 +1.16 0.71495745184 0.88981251387 +1.17 0.68674624699 0.84267298854 +1.18 0.6548790065 0.79342705498 +1.19 0.62002916582 0.74268516285 +1.2 0.58286154107 0.6910437846 +1.21 0.54402441605 0.63907779944 +1.22 0.50412083068 0.58731389232 +1.23 0.46372675315 0.53624961293 +1.24 0.42339395221 0.48635589533 +1.25 0.38359561591 0.43802682011 +1.26 0.34477020037 0.39162193845 +1.27 0.30732463519 0.34746880189 +1.28 0.27157134852 0.30580606771 +1.29 0.23778707603 0.26683810864 +1.3 0.20621131665 0.23073313381 +1.31 0.17698834696 0.19757219659 +1.32 0.15023637673 0.16741147173 +1.33 0.12602499295 0.14026193142 +1.34 0.10435202907 0.1160705183 +1.35 0.085197164124 0.094766660847 +1.36 0.068484932802 0.076230851442 +1.37 0.054104389019 0.060313067757 +1.38 0.041928184436 0.046847269937 +1.39 0.031785627449 0.035632582704 +1.4 0.023500713689 0.026463074369 +1.41 0.016876824869 0.019116104017 +1.42 0.011714252485 0.013370408212 +1.43 0.0078034446545 0.0089945394479 +1.44 0.0049469134056 0.0057748945206 +1.45 0.0029474249359 0.0034976284065 +1.46 0.0016205346613 0.0019645451715 +1.47 0.00080702693374 0.0010042103105 +1.48 0.00034539687203 0.00044320077201 +1.49 0.00012966994735 0.00016608419693 +1.5 4.0352556958e-05 4.3207922912e-05 +1.51 2.2926656376e-05 1.1362596335e-05 +1.52 1.9476388882e-05 4.6382966744e-06 +1.53 1.5321466077e-05 5.2321743433e-06 +1.54 3.138718166e-06 9.0699570749e-07 +1.55 -1.4215409932e-06 -5.417831369e-07 +1.56 -3.2732566399e-07 -1.247516082e-07 +1.57 0.0 0.0 +1.58 0.0 0.0 +1.59 0.0 0.0 +1.6 0.0 0.0 +1.61 0.0 0.0 +1.62 0.0 0.0 +1.63 0.0 0.0 +1.64 0.0 0.0 +1.65 0.0 0.0 +1.66 0.0 0.0 +1.67 0.0 0.0 +1.68 0.0 0.0 +1.69 0.0 0.0 +1.7 0.0 0.0 +1.71 0.0 0.0 +1.72 0.0 0.0 +1.73 0.0 0.0 +1.74 0.0 0.0 +1.75 0.0 0.0 +1.76 0.0 0.0 +1.77 0.0 0.0 +1.78 0.0 0.0 +1.79 0.0 0.0 +1.8 0.0 0.0 +1.81 0.0 0.0 +1.82 0.0 0.0 +1.83 0.0 0.0 +1.84 0.0 0.0 +1.85 0.0 0.0 +1.86 0.0 0.0 +1.87 0.0 0.0 +1.88 0.0 0.0 +1.89 0.0 0.0 +1.9 0.0 0.0 +1.91 0.0 0.0 +1.92 0.0 0.0 +1.93 0.0 0.0 +1.94 0.0 0.0 +1.95 0.0 0.0 +1.96 0.0 0.0 +1.97 0.0 0.0 +1.98 0.0 0.0 +1.99 0.0 0.0 +2.0 0.0 0.0 +2.01 0.0 0.0 +2.02 0.0 0.0 +2.03 0.0 0.0 +2.04 0.0 0.0 +2.05 0.0 0.0 +2.06 0.0 0.0 +2.07 0.0 0.0 +2.08 0.0 0.0 +2.09 0.0 0.0 +2.1 0.0 0.0 +2.11 0.0 0.0 +2.12 0.0 0.0 +2.13 0.0 0.0 +2.14 0.0 0.0 +2.15 0.0 0.0 +2.16 0.0 0.0 +2.17 0.0 0.0 +2.18 0.0 0.0 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# local +0.0 -2.8265571261E+01 +0.01 -2.2359855375E+01 +0.02 -2.0368958441E+01 +0.03 -2.0339038676E+01 +0.04 -2.0316254296E+01 +0.05 -2.0282114131E+01 +0.06 -2.0240247209E+01 +0.07 -2.0191868895E+01 +0.08 -2.0137404927E+01 +0.09 -2.0077087977E+01 +0.1 -2.0011108927E+01 +0.11 -1.9939651235E+01 +0.12 -1.9862895945E+01 +0.13 -1.9781019764E+01 +0.14 -1.9694191855E+01 +0.15 -1.9602570651E+01 +0.16 -1.9506300948E+01 +0.17 -1.9405511319E+01 +0.18 -1.9300311898E+01 +0.19 -1.9190792636E+01 +0.2 -1.9077022111E+01 +0.21 -1.8959047040E+01 +0.22 -1.8836892547E+01 +0.23 -1.8710563271E+01 +0.24 -1.8580045317E+01 +0.25 -1.8445309034E+01 +0.26 -1.8306312492E+01 +0.27 -1.8163005607E+01 +0.28 -1.8015334685E+01 +0.29 -1.7863247225E+01 +0.3 -1.7706696809E+01 +0.31 -1.7545647845E+01 +0.32 -1.7380080014E+01 +0.33 -1.7209992216E+01 +0.34 -1.7035406021E+01 +0.35 -1.6856368304E+01 +0.36 -1.6672953309E+01 +0.37 -1.6485263714E+01 +0.38 -1.6293431310E+01 +0.39 -1.6097616584E+01 +0.4 -1.5898007851E+01 +0.41 -1.5694819743E+01 +0.42 -1.5488291167E+01 +0.43 -1.5278682835E+01 +0.44 -1.5066274431E+01 +0.45 -1.4851361530E+01 +0.46 -1.4634252359E+01 +0.47 -1.4415264476E+01 +0.48 -1.4194721483E+01 +0.49 -1.3972949858E+01 +0.5 -1.3750275981E+01 +0.51 -1.3527023349E+01 +0.52 -1.3303510245E+01 +0.53 -1.3080047626E+01 +0.54 -1.2856937358E+01 +0.55 -1.2634471213E+01 +0.56 -1.2412929405E+01 +0.57 -1.2192580697E+01 +0.58 -1.1973681335E+01 +0.59 -1.1756475863E+01 +0.6 -1.1541196471E+01 +0.61 -1.1328064042E+01 +0.62 -1.1117288193E+01 +0.63 -1.0909067786E+01 +0.64 -1.0703591786E+01 +0.65 -1.0501039311E+01 +0.66 -1.0301580308E+01 +0.67 -1.0105376044E+01 +0.68 -9.9125793483E+00 +0.69 -9.7233346213E+00 +0.7 -9.5377783000E+00 +0.71 -9.3560388251E+00 +0.72 -9.1782367054E+00 +0.73 -9.0044842839E+00 +0.74 -8.8348856514E+00 +0.75 -8.6695365678E+00 +0.76 -8.5085241813E+00 +0.77 -8.3519267656E+00 +0.78 -8.1998134301E+00 +0.79 -8.0522438094E+00 +0.8 -7.9092677372E+00 +0.81 -7.7709249112E+00 +0.82 -7.6372445518E+00 +0.83 -7.5082450841E+00 +0.84 -7.3839337053E+00 +0.85 -7.2643061416E+00 +0.86 -7.1493463077E+00 +0.87 -7.0390259598E+00 +0.88 -6.9333043128E+00 +0.89 -6.8321277895E+00 +0.9 -6.7354296853E+00 +0.91 -6.6431298585E+00 +0.92 -6.5551344233E+00 +0.93 -6.4713354453E+00 +0.94 -6.3916106378E+00 +0.95 -6.3158232811E+00 +0.96 -6.2438223651E+00 +0.97 -6.1754307543E+00 +0.98 -6.1104172671E+00 +0.99 -6.0484745559E+00 +1.0 -5.9892424903E+00 +1.01 -5.9323378585E+00 +1.02 -5.8773782328E+00 +1.03 -5.8240043388E+00 +1.04 -5.7718954850E+00 +1.05 -5.7207836611E+00 +1.06 -5.6704621661E+00 +1.07 -5.6207763507E+00 +1.08 -5.5716202763E+00 +1.09 -5.5229278277E+00 +1.1 -5.4746697590E+00 +1.11 -5.4268397857E+00 +1.12 -5.3794464898E+00 +1.13 -5.3325087696E+00 +1.14 -5.2860519007E+00 +1.15 -5.2401023226E+00 +1.16 -5.1946856843E+00 +1.17 -5.1498262685E+00 +1.18 -5.1055447926E+00 +1.19 -5.0618585531E+00 +1.2 -5.0187819454E+00 +1.21 -4.9763261333E+00 +1.22 -4.9344983286E+00 +1.23 -4.8933035585E+00 +1.24 -4.8527447656E+00 +1.25 -4.8128214763E+00 +1.26 -4.7735319010E+00 +1.27 -4.7348728924E+00 +1.28 -4.6968388415E+00 +1.29 -4.6594233562E+00 +1.3 -4.6226191303E+00 +1.31 -4.5864172119E+00 +1.32 -4.5508082761E+00 +1.33 -4.5157822571E+00 +1.34 -4.4813282169E+00 +1.35 -4.4474350405E+00 +1.36 -4.4140910352E+00 +1.37 -4.3812842360E+00 +1.38 -4.3490025608E+00 +1.39 -4.3172336495E+00 +1.4 -4.2859651845E+00 +1.41 -4.2551847731E+00 +1.42 -4.2248801719E+00 +1.43 -4.1950391076E+00 +1.44 -4.1656496515E+00 +1.45 -4.1366999524E+00 +1.46 -4.1081784403E+00 +1.47 -4.0800739508E+00 +1.48 -4.0523753529E+00 +1.49 -4.0250723040E+00 +1.5 -3.9981543198E+00 +1.51 -3.9716118268E+00 +1.52 -3.9454352845E+00 +1.53 -3.9196158557E+00 +1.54 -3.8941450333E+00 +1.55 -3.8690146230E+00 +1.56 -3.8442169737E+00 +1.57 -3.8197444086E+00 +1.58 -3.7955900088E+00 +1.59 -3.7717466974E+00 +1.6 -3.7482079063E+00 +1.61 -3.7249668234E+00 +1.62 -3.7020174180E+00 +1.63 -3.6793534228E+00 +1.64 -3.6569691197E+00 +1.65 -3.6348586486E+00 +1.66 -3.6130166420E+00 +1.67 -3.5914376227E+00 +1.68 -3.5701165469E+00 +1.69 -3.5490482769E+00 +1.7 -3.5282280799E+00 +1.71 -3.5076511215E+00 +1.72 -3.4873129676E+00 +1.73 -3.4672090542E+00 +1.74 -3.4473352335E+00 +1.75 -3.4276871831E+00 +1.76 -3.4082610295E+00 +1.77 -3.3890526663E+00 +1.78 -3.3700584716E+00 +1.79 -3.3512745626E+00 +1.8 -3.3326975135E+00 +1.81 -3.3143236922E+00 +1.82 -3.2961498163E+00 +1.83 -3.2781725056E+00 +1.84 -3.2603885991E+00 +1.85 -3.2427949590E+00 +1.86 -3.2253885270E+00 +1.87 -3.2081663955E+00 +1.88 -3.1911255950E+00 +1.89 -3.1742634310E+00 +1.9 -3.1575770162E+00 +1.91 -3.1410638156E+00 +1.92 -3.1247211160E+00 +1.93 -3.1085464256E+00 +1.94 -3.0925372294E+00 +1.95 -3.0766910669E+00 +1.96 -3.0610056048E+00 +1.97 -3.0454784118E+00 +1.98 -3.0301072913E+00 +1.99 -3.0148899109E+00 +2.0 -2.9998241122E+00 +2.01 -2.9849077106E+00 +2.02 -2.9701385466E+00 +2.03 -2.9555145757E+00 +2.04 -2.9410336472E+00 +2.05 -2.9266938006E+00 +2.06 -2.9124929871E+00 +2.07 -2.8984292304E+00 +2.08 -2.8845006052E+00 +2.09 -2.8707051247E+00 +2.1 -2.8570409378E+00 +2.11 -2.8435061046E+00 +2.12 -2.8300987849E+00 +2.13 -2.8168171860E+00 +2.14 -2.8036593975E+00 +2.15 -2.7906238243E+00 +2.16 -2.7777087263E+00 +2.17 -2.7649124420E+00 +2.18 -2.7522333894E+00 +2.19 -2.7396698723E+00 +2.2 -2.7272204068E+00 +2.21 -2.7148834353E+00 +2.22 -2.7026574071E+00 +2.23 -2.6905409040E+00 +2.24 -2.6785324058E+00 +2.25 -2.6666305058E+00 +2.26 -2.6548338281E+00 +2.27 -2.6431409030E+00 +2.28 -2.6315504527E+00 +2.29 -2.6200611214E+00 +2.3 -2.6086715522E+00 +2.31 -2.5973805206E+00 +2.32 -2.5861867125E+00 +2.33 -2.5750888816E+00 +2.34 -2.5640858453E+00 +2.35 -2.5531763351E+00 +2.36 -2.5423592043E+00 +2.37 -2.5316333101E+00 +2.38 -2.5209974313E+00 +2.39 -2.5104505085E+00 +2.4 -2.4999914335E+00 +2.41 -2.4896190490E+00 +2.42 -2.4793323557E+00 +2.43 -2.4691302829E+00 +2.44 -2.4590117456E+00 +2.45 -2.4489757861E+00 +2.46 -2.4390213773E+00 +2.47 -2.4291474977E+00 +2.48 -2.4193532305E+00 +2.49 -2.4096375927E+00 +2.5 -2.3999996175E+00 +2.51 -2.3904384291E+00 +2.52 -2.3809530882E+00 +2.53 -2.3715426750E+00 +2.54 -2.3622063543E+00 +2.55 -2.3529432311E+00 +2.56 -2.3437524248E+00 +2.57 -2.3346331407E+00 +2.58 -2.3255845284E+00 +2.59 -2.3166057390E+00 +2.6 -2.3076960187E+00 +2.61 -2.2988545615E+00 +2.62 -2.2900805438E+00 +2.63 -2.2813732525E+00 +2.64 -2.2727319259E+00 +2.65 -2.2641557604E+00 +2.66 -2.2556440824E+00 +2.67 -2.2471961689E+00 +2.68 -2.2388112499E+00 +2.69 -2.2304886716E+00 +2.7 -2.2222277509E+00 +2.71 -2.2140277620E+00 +2.72 -2.2058880551E+00 +2.73 -2.1978079902E+00 +2.74 -2.1897868865E+00 +2.75 -2.1818240928E+00 +2.76 -2.1739190118E+00 +2.77 -2.1660710082E+00 +2.78 -2.1582794254E+00 +2.79 -2.1505437072E+00 +2.8 -2.1428632558E+00 +2.81 -2.1352374327E+00 +2.82 -2.1276656945E+00 +2.83 -2.1201474798E+00 +2.84 -2.1126821952E+00 +2.85 -2.1052692798E+00 +2.86 -2.0979082160E+00 +2.87 -2.0905984571E+00 +2.88 -2.0833394215E+00 +2.89 -2.0761306334E+00 +2.9 -2.0689715744E+00 +2.91 -2.0618616971E+00 +2.92 -2.0548005091E+00 +2.93 -2.0477875341E+00 +2.94 -2.0408222714E+00 +2.95 -2.0339041981E+00 +2.96 -2.0270328810E+00 +2.97 -2.0202078488E+00 +2.98 -2.0134286031E+00 +2.99 -2.0066946955E+00 +3.0 -2.0000056943E+00 +3.01 -1.9933611482E+00 +3.02 -1.9867605708E+00 +3.03 -1.9802035734E+00 +3.04 -1.9736897277E+00 +3.05 -1.9672185866E+00 +3.06 -1.9607897246E+00 +3.07 -1.9544027573E+00 +3.08 -1.9480572765E+00 +3.09 -1.9417528417E+00 +3.1 -1.9354890815E+00 +3.11 -1.9292656159E+00 +3.12 -1.9230820522E+00 +3.13 -1.9169379695E+00 +3.14 -1.9108330323E+00 +3.15 -1.9047668696E+00 +3.16 -1.8987390964E+00 +3.17 -1.8927493343E+00 +3.18 -1.8867972555E+00 +3.19 -1.8808825061E+00 +3.2 -1.8750047107E+00 +3.21 -1.8691635277E+00 +3.22 -1.8633586373E+00 +3.23 -1.8575897021E+00 +3.24 -1.8518563576E+00 +3.25 -1.8461582932E+00 +3.26 -1.8404951981E+00 +3.27 -1.8348667506E+00 +3.28 -1.8292725984E+00 +3.29 -1.8237124563E+00 +3.3 -1.8181860236E+00 +3.31 -1.8126929936E+00 +3.32 -1.8072330273E+00 +3.33 -1.8018058595E+00 +3.34 -1.7964112011E+00 +3.35 -1.7910487594E+00 +3.36 -1.7857182099E+00 +3.37 -1.7804193024E+00 +3.38 -1.7751517604E+00 +3.39 -1.7699153049E+00 +3.4 -1.7647096262E+00 +3.41 -1.7595344851E+00 +3.42 -1.7543896185E+00 +3.43 -1.7492747607E+00 +3.44 -1.7441896176E+00 +3.45 -1.7391339564E+00 +3.46 -1.7341075289E+00 +3.47 -1.7291100816E+00 +3.48 -1.7241413371E+00 +3.49 -1.7192010651E+00 +3.5 -1.7142890331E+00 +3.51 -1.7094049992E+00 +3.52 -1.7045487037E+00 +3.53 -1.6997199148E+00 +3.54 -1.6949184165E+00 +3.55 -1.6901439783E+00 +3.56 -1.6853963587E+00 +3.57 -1.6806753213E+00 +3.58 -1.6759806670E+00 +3.59 -1.6713121760E+00 +3.6 -1.6666696263E+00 +3.61 -1.6620527736E+00 +3.62 -1.6574614359E+00 +3.63 -1.6528954039E+00 +3.64 -1.6483544681E+00 +3.65 -1.6438383964E+00 +3.66 -1.6393470018E+00 +3.67 -1.6348800920E+00 +3.68 -1.6304374672E+00 +3.69 -1.6260189163E+00 +3.7 -1.6216242366E+00 +3.71 -1.6172532555E+00 +3.72 -1.6129057828E+00 +3.73 -1.6085816279E+00 +3.74 -1.6042805737E+00 +3.75 -1.6000024639E+00 +3.76 -1.5957471180E+00 +3.77 -1.5915143546E+00 +3.78 -1.5873039784E+00 +3.79 -1.5831158111E+00 +3.8 -1.5789496941E+00 +3.81 -1.5748054543E+00 +3.82 -1.5706829186E+00 +3.83 -1.5665818885E+00 +3.84 -1.5625022227E+00 +3.85 -1.5584437575E+00 +3.86 -1.5544063279E+00 +3.87 -1.5503897588E+00 +3.88 -1.5463938812E+00 +3.89 -1.5424185545E+00 +3.9 -1.5384636215E+00 +3.91 -1.5345289249E+00 +3.92 -1.5306142882E+00 +3.93 -1.5267195713E+00 +3.94 -1.5228446314E+00 +3.95 -1.5189893184E+00 +3.96 -1.5151534815E+00 +3.97 -1.5113369477E+00 +3.98 -1.5075395980E+00 +3.99 -1.5037612894E+00 +4.0 -1.5000018789E+00 +4.01 -1.4962612164E+00 +4.02 -1.4925391500E+00 +4.03 -1.4888355611E+00 +4.04 -1.4851503135E+00 +4.05 -1.4814832707E+00 +4.06 -1.4778342848E+00 +4.07 -1.4742032212E+00 +4.08 -1.4705899626E+00 +4.09 -1.4669943793E+00 +4.1 -1.4634163411E+00 +4.11 -1.4598557035E+00 +4.12 -1.4563123453E+00 +4.13 -1.4527861519E+00 +4.14 -1.4492769995E+00 +4.15 -1.4457847640E+00 +4.16 -1.4423093057E+00 +4.17 -1.4388505130E+00 +4.18 -1.4354082750E+00 +4.19 -1.4319824738E+00 +4.2 -1.4285729910E+00 +4.21 -1.4251796927E+00 +4.22 -1.4218024736E+00 +4.23 -1.4184412277E+00 +4.24 -1.4150958425E+00 +4.25 -1.4117662053E+00 +4.26 -1.4084521888E+00 +4.27 -1.4051536908E+00 +4.28 -1.4018706114E+00 +4.29 -1.3986028433E+00 +4.3 -1.3953502791E+00 +4.31 -1.3921127992E+00 +4.32 -1.3888903014E+00 +4.33 -1.3856826929E+00 +4.34 -1.3824898713E+00 +4.35 -1.3793117343E+00 +4.36 -1.3761481711E+00 +4.37 -1.3729990766E+00 +4.38 -1.3698643659E+00 +4.39 -1.3667439414E+00 +4.4 -1.3636377056E+00 +4.41 -1.3605455574E+00 +4.42 -1.3574673862E+00 +4.43 -1.3544031159E+00 +4.44 -1.3513526534E+00 +4.45 -1.3483159058E+00 +4.46 -1.3452927799E+00 +4.47 -1.3422831652E+00 +4.48 -1.3392869871E+00 +4.49 -1.3363041596E+00 +4.5 -1.3333345938E+00 +4.51 -1.3303782011E+00 +4.52 -1.3274348824E+00 +4.53 -1.3245045517E+00 +4.54 -1.3215871341E+00 +4.55 -1.3186825448E+00 +4.56 -1.3157906994E+00 +4.57 -1.3129115112E+00 +4.58 -1.3100448808E+00 +4.59 -1.3071907446E+00 +4.6 -1.3043490220E+00 +4.61 -1.3015196324E+00 +4.62 -1.2987024949E+00 +4.63 -1.2958975175E+00 +4.64 -1.2931046259E+00 +4.65 -1.2903237505E+00 +4.66 -1.2875548145E+00 +4.67 -1.2847977410E+00 +4.68 -1.2820524517E+00 +4.69 -1.2793188544E+00 +4.7 -1.2765968926E+00 +4.71 -1.2738864929E+00 +4.72 -1.2711875820E+00 +4.73 -1.2685000867E+00 +4.74 -1.2658239254E+00 +4.75 -1.2631590246E+00 +4.76 -1.2605053242E+00 +4.77 -1.2578627544E+00 +4.78 -1.2552312452E+00 +4.79 -1.2526107267E+00 +4.8 -1.2500011156E+00 +4.81 -1.2474023538E+00 +4.82 -1.2448143787E+00 +4.83 -1.2422371237E+00 +4.84 -1.2396705221E+00 +4.85 -1.2371145069E+00 +4.86 -1.2345689953E+00 +4.87 -1.2320339404E+00 +4.88 -1.2295092784E+00 +4.89 -1.2269949459E+00 +4.9 -1.2244908794E+00 +4.91 -1.2219970119E+00 +4.92 -1.2195132705E+00 +4.93 -1.2170396079E+00 +4.94 -1.2145759636E+00 +4.95 -1.2121222768E+00 +4.96 -1.2096784872E+00 +4.97 -1.2072445288E+00 +4.98 -1.2048203360E+00 +4.99 -1.2024058622E+00 +5.0 -1.2000010495E+00 +5.01 -1.1976058402E+00 +5.02 -1.1952201766E+00 +5.03 -1.1928439953E+00 +5.04 -1.1904772347E+00 +5.05 -1.1881198501E+00 +5.06 -1.1857717864E+00 +5.07 -1.1834329885E+00 +5.08 -1.1811034014E+00 +5.09 -1.1787829649E+00 +5.1 -1.1764716193E+00 +5.11 -1.1741693226E+00 +5.12 -1.1718760222E+00 +5.13 -1.1695916658E+00 +5.14 -1.1673162007E+00 +5.15 -1.1650495712E+00 +5.16 -1.1627917167E+00 +5.17 -1.1605425990E+00 +5.18 -1.1583021681E+00 +5.19 -1.1560703738E+00 +5.2 -1.1538471662E+00 +5.21 -1.1516324945E+00 +5.22 -1.1494262954E+00 +5.23 -1.1472285351E+00 +5.24 -1.1450391658E+00 +5.25 -1.1428581400E+00 +5.26 -1.1406854098E+00 +5.27 -1.1385209276E+00 +5.28 -1.1363646342E+00 +5.29 -1.1342164918E+00 +5.3 -1.1320764582E+00 +5.31 -1.1299444877E+00 +5.32 -1.1278205350E+00 +5.33 -1.1257045544E+00 +5.34 -1.1235964939E+00 +5.35 -1.1214963079E+00 +5.36 -1.1194039607E+00 +5.37 -1.1173194088E+00 +5.38 -1.1152426088E+00 +5.39 -1.1131735175E+00 +5.4 -1.1111120906E+00 +5.41 -1.1090582727E+00 +5.42 -1.1070120354E+00 +5.43 -1.1049733372E+00 +5.44 -1.1029421369E+00 +5.45 -1.1009183930E+00 +5.46 -1.0989020643E+00 +5.47 -1.0968931015E+00 +5.48 -1.0948914664E+00 +5.49 -1.0928971254E+00 +5.5 -1.0909100390E+00 +5.51 -1.0889301677E+00 +5.52 -1.0869574722E+00 +5.53 -1.0849919127E+00 +5.54 -1.0830334374E+00 +5.55 -1.0810820213E+00 +5.56 -1.0791376269E+00 +5.57 -1.0772002166E+00 +5.58 -1.0752697529E+00 +5.59 -1.0733461980E+00 +5.6 -1.0714295094E+00 +5.61 -1.0695196473E+00 +5.62 -1.0676165836E+00 +5.63 -1.0657202827E+00 +5.64 -1.0638307085E+00 +5.65 -1.0619478254E+00 +5.66 -1.0600715973E+00 +5.67 -1.0582019802E+00 +5.68 -1.0563389434E+00 +5.69 -1.0544824569E+00 +5.7 -1.0526324867E+00 +5.71 -1.0507889984E+00 +5.72 -1.0489519580E+00 +5.73 -1.0471213313E+00 +5.74 -1.0452970734E+00 +5.75 -1.0434791606E+00 +5.76 -1.0416675619E+00 +5.77 -1.0398622448E+00 +5.78 -1.0380631765E+00 +5.79 -1.0362703247E+00 +5.8 -1.0344836566E+00 +5.81 -1.0327031279E+00 +5.82 -1.0309287193E+00 +5.83 -1.0291603996E+00 +5.84 -1.0273981380E+00 +5.85 -1.0256419032E+00 +5.86 -1.0238916644E+00 +5.87 -1.0221473898E+00 +5.88 -1.0204090379E+00 +5.89 -1.0186765901E+00 +5.9 -1.0169500168E+00 +5.91 -1.0152292884E+00 +5.92 -1.0135143752E+00 +5.93 -1.0118052478E+00 +5.94 -1.0101018764E+00 +5.95 -1.0084042202E+00 +5.96 -1.0067122621E+00 +5.97 -1.0050259739E+00 +5.98 -1.0033453274E+00 +5.99 -1.0016702943E+00 +6.0 -1.0000008465E+00 +6.01 -9.9833695567E-01 diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 87b30224..6e6ea652 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -149,14 +149,11 @@ template void DensityMatrix::setUniform( const double nel, const int new_orbitals_index) { -#ifdef PRINT_OPERATIONS - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const double occ = (double)((double)nel / (double)dim_); if (mmpi.instancePE0()) - std::cout << "template " - "DensityMatrix::setUniform()" + std::cout << "DensityMatrix::setUniform(), occupation = " << occ << std::endl; -#endif - const double occ = (double)((double)nel / (double)dim_); assert(occ < 1.01); for (int i = 0; i < dim_; i++) occupation_[i] = occ; diff --git a/src/MGmol.cc b/src/MGmol.cc index 5eb37c2c..8da0d331 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1135,6 +1135,7 @@ void MGmol::dumpRestart() #endif } } + template void MGmol::cleanup() { @@ -1443,7 +1444,7 @@ void MGmol::getAtomicNumbers(std::vector& an) template double MGmol::evaluateEnergyAndForces( - const std::vector& tau, std::vector& atnumbers, + const std::vector& tau, const std::vector& atnumbers, std::vector& forces) { return evaluateEnergyAndForces(current_orbitals_, tau, atnumbers, forces); @@ -1451,7 +1452,7 @@ double MGmol::evaluateEnergyAndForces( template double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, - const std::vector& tau, std::vector& atnumbers, + const std::vector& tau, const std::vector& atnumbers, std::vector& forces) { assert(tau.size() == 3 * atnumbers.size()); @@ -1473,6 +1474,46 @@ double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, return eks; } +template +double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, + const std::vector& tau, const std::vector& atnumbers, + std::vector& forces) +{ + OrbitalsType* dorbitals = dynamic_cast(orbitals); + + ions_->setPositions(tau, atnumbers); + + moveVnuc(*ions_); + + // initialize electronic density + rho_->update(*dorbitals); + + // initialize potential + update_pot(*ions_); + + // initialize projected matrices + updateHmatrix(*dorbitals, *ions_); + proj_matrices_->updateThetaAndHB(); + + // compute DM + std::shared_ptr> dm_strategy( + DMStrategyFactory>::create(comm_, os_, *ions_, + rho_.get(), energy_.get(), electrostat_.get(), this, + proj_matrices_.get(), dorbitals)); + + dm_strategy->update(*dorbitals); + + // evaluate energy and forces + double ts = 0.; + double eks + = energy_->evaluateTotal(ts, proj_matrices_.get(), *dorbitals, 2, os_); + + force(*dorbitals, *ions_); + + return eks; +} + template class MGmol; template class MGmol; template int MGmol::initial(); diff --git a/src/MGmol.h b/src/MGmol.h index ab07703d..6a4cc830 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -187,11 +187,30 @@ class MGmol : public MGmolInterface void run() override; + /* + * Evaluate the energy and forces for an atomic configuration + * specified by tau (input) + */ double evaluateEnergyAndForces(const std::vector& tau, - std::vector& atnumbers, std::vector& forces); + const std::vector& atnumbers, std::vector& forces); - double evaluateEnergyAndForces(Orbitals*, const std::vector& tau, - std::vector& atnumbers, std::vector& forces); + /* + * Evaluate the energy and forces for an atomic configuration + * specified by tau (input), using the input "orbitals" as initial + * guess for the wavefunctions + */ + double evaluateEnergyAndForces(Orbitals* orbitals, + const std::vector& tau, const std::vector& atnumbers, + std::vector& forces); + + /* + * Evaluate the energy and forces for an atomic configuration + * specified by tau (input), using the input "orbitals" as wavefunctions + * (fixed) + */ + double evaluateDMandEnergyAndForces(Orbitals* orbitals, + const std::vector& tau, const std::vector& atnumbers, + std::vector& forces); /* * get internal atomic positions diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index af7be67b..9a9bf8a6 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -25,13 +25,19 @@ class MGmolInterface virtual int setupConstraintsFromInput(const std::string input_file) = 0; virtual void setup() = 0; virtual void run() = 0; + virtual double evaluateEnergyAndForces(const std::vector& tau, - std::vector& atnumbers, std::vector& forces) + const std::vector& atnumbers, std::vector& forces) = 0; virtual double evaluateEnergyAndForces(Orbitals*, - const std::vector& tau, std::vector& atnumbers, + const std::vector& tau, const std::vector& atnumbers, + std::vector& forces) + = 0; + virtual double evaluateDMandEnergyAndForces(Orbitals*, + const std::vector& tau, const std::vector& atnumbers, std::vector& forces) = 0; + virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; virtual std::shared_ptr getProjectedMatrices() diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 7feb564d..6115f933 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -208,6 +208,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) kbpsi.computeHvnlMatrix(&kbpsi, ions_, h11_nl); + const double tol_de0 = 1.e-12; for (int inner_it = 0; inner_it < n_inner_steps_; inner_it++) { if (onpe0 && ct.verbose > 1) @@ -270,6 +271,14 @@ int MVPSolver::solve(OrbitalsType& orbitals) double de0 = evaluateDerivative(dmInit, delta_dm, ts0); + if (std::abs(de0) < tol_de0 && inner_it > 0) + { + if (onpe0 && ct.verbose > 0) + std::cout << "MVP: de0 = " << de0 + << ", convergence achieved" << std::endl; + break; + } + // // evaluate free energy at beta=1 // @@ -306,6 +315,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) // line minimization const double beta = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); + assert(!std::isnan(beta)); if (onpe0 && ct.verbose > 0) { @@ -348,6 +358,8 @@ int MVPSolver::solve(OrbitalsType& orbitals) orbitals.getProjMatrices()); projmatrices->setDM(*work_, orbitals.getIterativeIndex()); projmatrices->setEigenvalues(proj_mat_work_->getEigenvalues()); + projmatrices->assignH(proj_mat_work_->getH()); + projmatrices->setHB2H(); } // Generate new density diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index bdfe7786..e0e679e9 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -650,8 +650,9 @@ double ProjectedMatrices::computeEntropy() else { if (mmpi.PE0() && ct.verbose > 1) - (*MPIdata::sout) - << "occupations uptodate, skip computation..." << std::endl; + (*MPIdata::sout) << "computeEntropy: occupations uptodate, " + "skip computation..." + << std::endl; } entropy = computeEntropy(width_); } diff --git a/src/Rho.cc b/src/Rho.cc index ca0f20ea..45a41f3e 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -90,8 +90,8 @@ void Rho::update(OrbitalsType& current_orbitals) update_tm_.start(); - if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Rho::update()" << std::endl; + if (verbosity_level_ > 1 && onpe0) + (*MPIdata::sout) << "Rho::update()..." << std::endl; const int new_iterative_index = ((1 + current_orbitals.getIterativeIndex()) % 100) @@ -99,7 +99,7 @@ void Rho::update(OrbitalsType& current_orbitals) if (iterative_index_ == new_iterative_index) { - if (onpe0 && verbosity_level_ > 2) + if (onpe0 && verbosity_level_ > 1) (*MPIdata::sout) << "Rho already up to date, iterative_index_=" << iterative_index_ << std::endl; return; diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 68349e90..5dc3718e 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -512,7 +512,7 @@ void testROMRhoOperator(MGmolInterface *mgmol_) MGmol *mgmol = static_cast *>(mgmol_); Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); Potentials& pot = mgmol->getHamiltonian()->potential(); - std::shared_ptr> rho = mgmol->getRho(); + std::shared_ptr> rho = NULL; // mgmol->getRho(); const OrthoType ortho_type = rho->getOrthoType(); assert(ortho_type == OrthoType::Nonorthogonal); @@ -731,4 +731,4 @@ template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); -template void testROMRhoOperator(MGmolInterface *mgmol_); \ No newline at end of file +template void testROMRhoOperator(MGmolInterface *mgmol_); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 11125081..cc205167 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -250,6 +250,8 @@ add_executable(testEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) add_executable(testWFEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/WFEnergyAndForces/testWFEnergyAndForces.cc) +add_executable(testDMandEnergyAndForces + ${CMAKE_SOURCE_DIR}/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload @@ -353,6 +355,14 @@ add_test(NAME testWFEnergyAndForces ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/mgmol.cfg ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/sih4.xyz ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testDMandEnergyAndForces + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testDMandEnergyAndForces + ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/coords.in + ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/lrs.in + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -539,6 +549,7 @@ target_link_libraries(testSuperSampling PRIVATE MPI::MPI_CXX) target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/DMandEnergyAndForces/coords.in b/tests/DMandEnergyAndForces/coords.in new file mode 100644 index 00000000..cda898b9 --- /dev/null +++ b/tests/DMandEnergyAndForces/coords.in @@ -0,0 +1,2 @@ +N1 1 0. 0. -1.0345 +N2 1 0. 0. 1.0345 diff --git a/tests/DMandEnergyAndForces/lrs.in b/tests/DMandEnergyAndForces/lrs.in new file mode 100644 index 00000000..9136a500 --- /dev/null +++ b/tests/DMandEnergyAndForces/lrs.in @@ -0,0 +1,6 @@ +0.00 0.7 1.4 +0.00 -0.7 -1.4 +0.00 0.7 -1.4 +0.00 -0.7 1.4 +0.46 0.0 0.0 +-0.46 0.0 0.0 diff --git a/tests/DMandEnergyAndForces/mgmol.cfg b/tests/DMandEnergyAndForces/mgmol.cfg new file mode 100644 index 00000000..5278f40b --- /dev/null +++ b/tests/DMandEnergyAndForces/mgmol.cfg @@ -0,0 +1,35 @@ +verbosity=2 +xcFunctional=LDA +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.N_ONCVPSP_LDA +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=100 +atol=1.e-8 +step_length=2. +ortho_freq=10 +[Orbitals] +initial_type=Gaussian +initial_width=1.5 +temperature=10. +nempty=2 +[Restart] +output_level=3 +output_filename=WF +[DensityMatrix] +solver=MVP +nb_inner_it=1 diff --git a/tests/DMandEnergyAndForces/test.py b/tests/DMandEnergyAndForces/test.py new file mode 100755 index 00000000..c36f0767 --- /dev/null +++ b/tests/DMandEnergyAndForces/test.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test DMandEnergyAndForces...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-6): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-5] +inp = sys.argv[nargs-4] +coords = sys.argv[nargs-3] +print("coordinates file: %s"%coords) +lrs = sys.argv[-2] + +#create links to potentials files +dst = 'pseudo.N_ONCVPSP_LDA' +src = sys.argv[-1] + '/' + dst + +if not os.path.exists(dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run +command = "{} {} -c {} -i {} -l {}".format(mpicmd,exe,inp,coords,lrs) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#analyse output +energies=[] +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'achieved'): + energies.append(energy) + break + +for line in lines: + if line.count(b'MVP') and line.count(b'iteration'): + print(line) + if line.count(b'Eks2'): + print(line) + words=line.split() + word=words[2] + energy = word.decode() + energies.append(energy) + break + +print("Check energies...") +print( energies ) +if len(energies)<2: + print("Expected two converged energies") + sys.exit(1) + +tol = 1.e-6 +diff=eval(energies[1])-eval(energies[0]) +print(diff) +if abs(diff)>tol: + print("Energies differ: {} vs {} !!!".format(energies[0],energies[1])) + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc b/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc new file mode 100644 index 00000000..1cf1cf88 --- /dev/null +++ b/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc @@ -0,0 +1,211 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // compute energy and forces using all MPI tasks + // expect positions to be replicated on all MPI tasks + std::vector forces; + double eks + = mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + mgmol->dumpRestart(); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks1 : " << eks << std::endl; + std::cout << "Forces2 :" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + // compute energy and forces again using wavefunctions + // from previous call + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + std::shared_ptr projmatrices + = mgmol->getProjectedMatrices(); + + ExtendedGridOrbitals orbitals("new_orbitals", mygrid, mymesh->subdivx(), + ct.numst, ct.bcWF, projmatrices.get(), nullptr, nullptr, nullptr, + nullptr); + + const pb::PEenv& myPEenv = mymesh->peenv(); + HDFrestart h5file("WF", myPEenv, ct.out_restart_file_type); + orbitals.read_hdf5(h5file); + + // + // evaluate energy and forces again, with wavefunctions + // frozen to solution of previous problem + // + + // reset initial DM to test iterative solve for it + projmatrices->setDMuniform(ct.getNelSpin(), 0); + ct.dm_inner_steps = 50; + eks = mgmol->evaluateDMandEnergyAndForces( + &orbitals, positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks2 : " << eks << std::endl; + std::cout << "Forces2 :" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} From b32d6868f87179afd825a9d2450262ddba3b7f9e Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Tue, 15 Oct 2024 12:42:50 -0700 Subject: [PATCH 23/33] Merge release 2 (#283) * Reenable testShortSighted test (#248) * Make new driver to check input (#247) * clean up/reorganize main.cc * use shared_ptr in class MGmol * Add possible periodic dimensions to xyz2in.py (#249) * Add possible periodic dimensions to xyz2in.py * Remove unused/untested option extrapolateH (#250) * Exit with failure if density off by more than 2% (#251) * Exit with failure if density off by more than 2% * adapt SiH4 test to catch that * fix bug in DFTsolver that was leading to wrong density * Example driver (#252) * add example driver, showing use of MGmol as a force/energy computational engine * clean up related functions in class Ions * loadOrbitalsFromRestartFile -> loadRestartFile (#253) * Add SG15 PBE potential for N (#258) * Update 2-pyridone example (#259) * Adjust verbosity in some functions (#260) * Add new example: pinned H2O (#261) * Print out eigenvalues out of MVP solver (#262) Previously, the wrong eigenvalues (0) were printed out because eigenvalues outside solver were not up-to-date. * Add Li2 example with local GTH potential (#263) * Fix LBFGS termination when converged (#264) * Remove unused code to extrapolate rho (#265) * Fix and test EnergyAndForces interface (#266) * Atomic potentials were not updated when atomic positions were changed * Added test to make sure energies and forces are the same after positions move by one mesh spacing * Add new functionality to compute energy and forces (#267) * use specified initial conditions for wavefunctions * Add functionality to compute energy and forces (#270) * use specified wavefunctions as solution, with unknown DM * Add ONCV for Sulfur + example (#275) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) --------- Co-authored-by: Jean-Luc Fattebert --- src/Control.cc | 1 + src/DensityMatrix.cc | 7 +- src/DensityMatrix.h | 13 +- src/Electrostatic.cc | 200 +----------------- src/Electrostatic.h | 1 + src/Hartree.cc | 9 +- src/Hartree.h | 2 - src/Hartree_CG.cc | 1 + src/Ions.cc | 324 ++++++++++++------------------ src/Ions.h | 23 ++- src/KBprojectorSparse.cc | 4 +- src/MGmol.cc | 2 +- src/MGmol.h | 5 +- src/PCGSolver.cc | 59 +++--- src/PCGSolver.h | 52 +++-- src/PCGSolver_Diel.cc | 68 +++---- src/PCGSolver_Diel.h | 30 +-- src/PoissonSolverFactory.h | 228 +++++++++++++++++++++ src/Potentials.cc | 24 +-- src/Potentials.h | 56 ++++-- src/ProjectedMatrices.cc | 1 + src/pb/Mgm.h | 3 - src/pb/Solver.h | 7 +- src/pb/SolverLap.cc | 77 +------ src/pb/SolverLap.h | 2 +- src/pb/SolverPB.cc | 5 +- src/pb/SolverPB.h | 4 +- src/tools.cc | 17 +- src/tools.h | 1 + tests/CMakeLists.txt | 8 + tests/WFEnergyAndForces/mgmol.cfg | 2 +- tests/testIons.cc | 104 ++++++++++ 32 files changed, 704 insertions(+), 636 deletions(-) create mode 100644 src/PoissonSolverFactory.h create mode 100644 tests/testIons.cc diff --git a/src/Control.cc b/src/Control.cc index 2231813e..f1dcef23 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -70,6 +70,7 @@ Control::Control() dm_approx_ndigits = 1; dm_approx_power_maxits = 100; wf_extrapolation_ = 1; + verbose = 0; // undefined values dm_algo_ = -1; diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 6e6ea652..e2c7b473 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -28,7 +28,7 @@ const double factor_kernel4dot = 10.; template DensityMatrix::DensityMatrix(const int ndim) { - assert(ndim >= 0); + assert(ndim > 0); dim_ = ndim; @@ -45,6 +45,7 @@ DensityMatrix::DensityMatrix(const int ndim) kernel4dot_ = new MatrixType("K4dot", ndim, ndim); work_ = new MatrixType("work", ndim, ndim); occupation_.resize(dim_); + setDummyOcc(); } template @@ -109,6 +110,7 @@ void DensityMatrix::build( const std::vector& occ, const int new_orbitals_index) { assert(dm_ != nullptr); + assert(!occ.empty()); setOccupations(occ); @@ -149,6 +151,8 @@ template void DensityMatrix::setUniform( const double nel, const int new_orbitals_index) { + assert(!occupation_.empty()); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); const double occ = (double)((double)nel / (double)dim_); if (mmpi.instancePE0()) @@ -314,6 +318,7 @@ void DensityMatrix::computeOccupations(const MatrixType& ls) template void DensityMatrix::setOccupations(const std::vector& occ) { + assert(!occ.empty()); #ifdef PRINT_OPERATIONS MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (mmpi.instancePE0()) diff --git a/src/DensityMatrix.h b/src/DensityMatrix.h index 960ad18e..84804a86 100644 --- a/src/DensityMatrix.h +++ b/src/DensityMatrix.h @@ -84,17 +84,24 @@ class DensityMatrix *dm_ = mat; orbitals_index_ = orbitals_index; - occupation_.clear(); + setDummyOcc(); occ_uptodate_ = false; uniform_occ_ = false; stripped_ = false; } + // set occupations to meaningless values to catch uninitialized use + void setDummyOcc() + { + for (auto& occ : occupation_) + occ = -1.; + } + void initMatrix(const double* const val) { dm_->init(val, dim_); - occupation_.clear(); + setDummyOcc(); occ_uptodate_ = false; uniform_occ_ = false; @@ -105,8 +112,10 @@ class DensityMatrix void getOccupations(std::vector& occ) const { + assert(!occupation_.empty()); assert(occ_uptodate_); assert((int)occ.size() == dim_); + memcpy(&occ[0], &occupation_[0], dim_ * sizeof(double)); } diff --git a/src/Electrostatic.cc b/src/Electrostatic.cc index 80cb754b..6ac4fef3 100644 --- a/src/Electrostatic.cc +++ b/src/Electrostatic.cc @@ -11,6 +11,7 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" #include "GridFactory.h" +#include "GridFunc.h" #include "Hartree.h" #include "Hartree_CG.h" #include "Ions.h" @@ -23,15 +24,6 @@ #include "ShiftedHartree.h" #include "mputils.h" -#include "GridFunc.h" -#include "Laph2.h" -#include "Laph4.h" -#include "Laph4M.h" -#include "Laph4MP.h" -#include "Laph6.h" -#include "Laph8.h" -#include "ShiftedLaph4M.h" - Timer Electrostatic::solve_tm_("Electrostatic::solve"); Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], @@ -49,109 +41,9 @@ Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], Mesh* mymesh = Mesh::instance(); const pb::Grid& myGrid = mymesh->grid(); - Control& ct = *(Control::instance()); - if (ct.MGPoissonSolver()) // use MG for Poisson Solver - { - if (screening_const > 0.) - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new ShiftedHartree>( - myGrid, bc_, screening_const); - break; - default: - (*MPIdata::sout) - << "Electrostatic, shifted, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - else - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h2: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h4: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h6: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h8: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h4MP: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - default: - (*MPIdata::sout) << "Electrostatic, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - } - else // use PCG for Poisson Solver - { - if (screening_const > 0.) - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new ShiftedHartree>( - myGrid, bc_, screening_const); - break; - default: - (*MPIdata::sout) - << "PCG Electrostatic, shifted, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - else - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h2: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h4: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h6: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h8: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h4MP: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - default: - (*MPIdata::sout) << "PCG Electrostatic, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - } + // create Poisson solver + poisson_solver_ = PoissonSolverFactory::create( + myGrid, lap_type, bcPoisson, screening_const); grhoc_ = nullptr; diel_flag_ = false; @@ -244,73 +136,8 @@ void Electrostatic::setupPB( ngpts, origin, cell, static_cast(laptype_), true, myPEenv); if (poisson_solver_ != nullptr) delete poisson_solver_; - Control& ct = *(Control::instance()); - if (ct.MGPoissonSolver()) // use MG for Poisson Solver - { - switch (laptype_) - { - case PoissonFDtype::h4M: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h2: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h6: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h8: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4MP: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - default: - (*MPIdata::sout) - << "Electrostatic, Undefined option" << std::endl; - } - } - else // use PCG for Poisson Solver - { - switch (laptype_) - { - case PoissonFDtype::h4M: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h2: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h6: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h8: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4MP: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - default: - (*MPIdata::sout) - << "Electrostatic, Undefined option" << std::endl; - } - } + poisson_solver_ = PoissonSolverFactory::createDiel( + *pbGrid_, laptype_, bc_, e0, rho0, drho0); if (grhoc_ != nullptr) { @@ -330,6 +157,7 @@ void Electrostatic::setupPB( poisson_solver_->set_vh(gf_vh); } +// This function is only useful for Hartree problem with dielectric continuum void Electrostatic::fillFuncAroundIons(const Ions& ions) { assert(grhod_ != nullptr); @@ -352,7 +180,6 @@ void Electrostatic::fillFuncAroundIons(const Ions& ions) std::vector::const_iterator ion = rc_ions.begin(); while (ion != rc_ions.end()) { - double rc = (*ion)->getRC(); // Special case: silicon if ((*ion)->isMass28()) rc = 2.0; @@ -373,43 +200,32 @@ void Electrostatic::fillFuncAroundIons(const Ions& ions) #endif for (unsigned int ix = 0; ix < pbGrid_->dim(0); ix++) { - xc[1] = pbGrid_->start(1); const int ix1 = (ix + shift) * incx; for (unsigned int iy = 0; iy < pbGrid_->dim(1); iy++) { - xc[2] = pbGrid_->start(2); const int iy1 = ix1 + (iy + shift) * incy; for (unsigned int iz = 0; iz < pbGrid_->dim(2); iz++) { - const double r = (*ion)->minimage(xc, lattice, bc_); if (r < rc) { const double alpha = 0.2 * (1. + cos(r * pi_rc)); - - const int iz1 = iy1 + iz + shift; + const int iz1 = iy1 + iz + shift; vv[iz1] += alpha; } - xc[2] += pbGrid_->hgrid(2); } - xc[1] += pbGrid_->hgrid(1); - } // end for iy - xc[0] += pbGrid_->hgrid(0); - } // end for ix } - ion++; - } // end loop on list of ions return; diff --git a/src/Electrostatic.h b/src/Electrostatic.h index 785bc909..7ae977e1 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -13,6 +13,7 @@ #include "Control.h" #include "GridFunc.h" #include "Poisson.h" +#include "PoissonSolverFactory.h" #include "Rho.h" #include "Timer.h" diff --git a/src/Hartree.cc b/src/Hartree.cc index b3984b87..7d0ba4d8 100644 --- a/src/Hartree.cc +++ b/src/Hartree.cc @@ -7,10 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id$ #include #include -using namespace std; #include "Control.h" #include "Hartree.h" @@ -82,6 +80,8 @@ void Hartree::solve( // { /* solve with POTDTYPE precision */ pb::GridFunc rhs(work_rho); + // Hartree units + rhs *= (4. * M_PI); poisson_solver_->solve(*Poisson::vh_, rhs); // } // else @@ -92,10 +92,11 @@ void Hartree::solve( double final_residual = poisson_solver_->getFinalResidual(); if (onpe0) - (*MPIdata::sout) << setprecision(2) << scientific + (*MPIdata::sout) << std::setprecision(2) << std::scientific << "Hartree: residual reduction = " << residual_reduction - << ", final residual = " << final_residual << endl; + << ", final residual = " << final_residual + << std::endl; Poisson::Int_vhrho_ = vel * Poisson::vh_->gdot(rho); Poisson::Int_vhrhoc_ = vel * Poisson::vh_->gdot(rhoc); diff --git a/src/Hartree.h b/src/Hartree.h index a7050e24..b87ff01e 100644 --- a/src/Hartree.h +++ b/src/Hartree.h @@ -7,14 +7,12 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id$ #ifndef included_Hartree #define included_Hartree #include "Poisson.h" #include "PoissonInterface.h" -// pb #include "SolverLap.h" template diff --git a/src/Hartree_CG.cc b/src/Hartree_CG.cc index f9bf3860..375237f8 100644 --- a/src/Hartree_CG.cc +++ b/src/Hartree_CG.cc @@ -81,6 +81,7 @@ void Hartree_CG::solve( // { /* solve with POTDTYPE precision */ pb::GridFunc rhs(work_rho); + rhs *= (4. * M_PI); poisson_solver_->solve(*Poisson::vh_, rhs); // } // else diff --git a/src/Ions.cc b/src/Ions.cc index 940e12e8..852151c7 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -584,11 +584,9 @@ void Ions::printPositionsLocal(std::ostream& os, const int root) const os.setf(std::ios::right, std::ios::adjustfield); os.setf(std::ios::fixed, std::ios::floatfield); - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->printPosition(os); - ion++; + ion->printPosition(os); } os << std::endl; @@ -627,14 +625,12 @@ void Ions::writeAtomicNumbers(HDFrestart& h5f_file) } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - assert((*ion)->atomic_number() > 0); - assert((*ion)->atomic_number() < 200); + assert(ion->atomic_number() > 0); + assert(ion->atomic_number() < 200); - data.push_back((*ion)->atomic_number()); - ion++; + data.push_back(ion->atomic_number()); } } @@ -685,17 +681,15 @@ void Ions::writeAtomNames(HDFrestart& h5f_file) void Ions::lockAtom(const std::string& name) { - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - std::string name_ion((*ion)->name()); + std::string name_ion(ion->name()); if (name.compare(name_ion) == 0) { - (*ion)->lock(); + ion->lock(); if (onpe0) (*MPIdata::sout) << "Lock atom " << name << std::endl; break; } - ion++; } } @@ -975,12 +969,10 @@ void Ions::readRestartPositions(HDFrestart& h5_file) std::vector data; h5_file.readAtomicPositions(data); - int i = 0; - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1109,14 +1101,12 @@ void Ions::removeMassCenterMotion() #endif } - ion = local_ions_.begin(); - i = 0; - while (ion != local_ions_.end()) + i = 0; + for (auto& ion : local_ions_) { const int threei = 3 * i; - (*ion)->setVelocity(velocities[threei] - tmp[0], + ion->setVelocity(velocities[threei] - tmp[0], velocities[threei + 1] - tmp[1], velocities[threei + 2] - tmp[2]); - ion++; i++; } @@ -1124,16 +1114,15 @@ void Ions::removeMassCenterMotion() mv[0] = 0.; mv[1] = 0.; mv[2] = 0.; - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - velocities[3 * i] = (*ion)->velocity(0); - velocities[3 * i + 1] = (*ion)->velocity(1); - velocities[3 * i + 2] = (*ion)->velocity(2); + velocities[3 * i] = ion->velocity(0); + velocities[3 * i + 1] = ion->velocity(1); + velocities[3 * i + 2] = ion->velocity(2); mv[0] += mass[i] * velocities[3 * i]; mv[1] += mass[i] * velocities[3 * i + 1]; mv[2] += mass[i] * velocities[3 * i + 2]; - ion++; i++; } @@ -1155,12 +1144,10 @@ void Ions::readRestartVelocities(HDFrestart& h5_file) std::vector data; h5_file.readAtomicVelocities(data); - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setVelocity(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setVelocity(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1174,12 +1161,10 @@ void Ions::readRestartRandomStates(HDFrestart& h5f_file) std::vector data; h5f_file.readRestartRandomStates(data); - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setRandomState(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setRandomState(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1202,17 +1187,14 @@ void Ions::writeForces(HDFrestart& h5f_file) } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { // get position of local ion double force[3]; - (*ion)->getForce(&force[0]); + ion->getForce(&force[0]); data.push_back(force[0]); data.push_back(force[1]); data.push_back(force[2]); - - ++ion; } } @@ -1364,64 +1346,54 @@ void Ions::printForcesLocal(std::ostream& os, const int root) const << "FX" << std::setw(10) << "FY" << std::setw(10) << "FZ" << std::endl; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { + ion->printPositionAndForce(os); - (*ion)->printPositionAndForce(os); - - if (!(*ion)->locked()) + if (!ion->locked()) { + avg_forces[0] += fabs(ion->force(0)); + avg_forces[1] += fabs(ion->force(1)); + avg_forces[2] += fabs(ion->force(2)); - avg_forces[0] += fabs((*ion)->force(0)); - avg_forces[1] += fabs((*ion)->force(1)); - avg_forces[2] += fabs((*ion)->force(2)); - - double ff = (*ion)->norm2F(); + double ff = ion->norm2F(); maxf[0] = std::max(maxf[0], ff); - max_forces[0] = std::max(max_forces[0], fabs((*ion)->force(0))); - max_forces[1] = std::max(max_forces[1], fabs((*ion)->force(1))); - max_forces[2] = std::max(max_forces[2], fabs((*ion)->force(2))); + max_forces[0] = std::max(max_forces[0], fabs(ion->force(0))); + max_forces[1] = std::max(max_forces[1], fabs(ion->force(1))); + max_forces[2] = std::max(max_forces[2], fabs(ion->force(2))); num_movable++; } for (short ii = 0; ii < 3; ii++) - sum_forces[ii] += (*ion)->force(ii); + sum_forces[ii] += ion->force(ii); num_atoms++; - - ion++; } } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - - if (!(*ion)->locked()) + if (!ion->locked()) { + avg_forces[0] += fabs(ion->force(0)); + avg_forces[1] += fabs(ion->force(1)); + avg_forces[2] += fabs(ion->force(2)); - avg_forces[0] += fabs((*ion)->force(0)); - avg_forces[1] += fabs((*ion)->force(1)); - avg_forces[2] += fabs((*ion)->force(2)); - - double ff = (*ion)->norm2F(); + double ff = ion->norm2F(); maxf[0] = std::max(maxf[0], ff); - max_forces[0] = std::max(max_forces[0], fabs((*ion)->force(0))); - max_forces[1] = std::max(max_forces[1], fabs((*ion)->force(1))); - max_forces[2] = std::max(max_forces[2], fabs((*ion)->force(2))); + max_forces[0] = std::max(max_forces[0], fabs(ion->force(0))); + max_forces[1] = std::max(max_forces[1], fabs(ion->force(1))); + max_forces[2] = std::max(max_forces[2], fabs(ion->force(2))); num_movable++; } for (short ii = 0; ii < 3; ii++) - sum_forces[ii] += (*ion)->force(ii); + sum_forces[ii] += ion->force(ii); num_atoms++; - - ion++; } } // global statistics @@ -1479,12 +1451,10 @@ int Ions::countIonsHere() const { return (int)local_ions_.size(); } int Ions::countProjectorsHere() const { - int count = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int count = 0; + for (auto& ion : local_ions_) { - count += (*ion)->nProjectors(); - ion++; + count += ion->nProjectors(); } return count; } @@ -1497,12 +1467,10 @@ int Ions::countProjectors() const Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); - int nproj = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int nproj = 0; + for (auto& ion : local_ions_) { - nproj += (*ion)->nProjectors(); - ion++; + nproj += ion->nProjectors(); } int tmp = nproj; MPI_Allreduce(&tmp, &nproj, 1, MPI_INT, MPI_SUM, myPEenv.comm()); @@ -1569,13 +1537,10 @@ void Ions::setLocalPositions(const std::vector& tau) { assert(tau.size() == 3 * local_ions_.size()); - std::vector::iterator ion = local_ions_.begin(); - int ia = 0; - while (ion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition(tau[3 * ia + 0], tau[3 * ia + 1], tau[3 * ia + 2]); - - ion++; + ion->setPosition(tau[3 * ia + 0], tau[3 * ia + 1], tau[3 * ia + 2]); ia++; } @@ -1780,45 +1745,8 @@ int Ions::setAtoms( if (ia < 1000) aname.append("0"); if (ia < 10000) aname.append("0"); aname.append(ss.str()); - Ion* new_ion - = new Ion(species_[isp], aname, &crds[3 * ia], velocity, locked); - new_ion->bcast(mmpi.commGlobal()); - - // Populate list_ions_ list - // std::cout<<"crds: "< 2) - (*MPIdata::sout) - << "Ion " << aname << " at position " << crds[3 * ia + 0] - << "," << crds[3 * ia + 1] << "," << crds[3 * ia + 2] - << " added to the list... on PE" << mmpi.mypeGlobal() - << std::endl; - // populate local_ions_ list - if (inLocalIons( - crds[3 * ia + 0], crds[3 * ia + 1], crds[3 * ia + 2])) - { - (new_ion)->set_here(true); - local_ions_.push_back(new_ion); - if (onpe0 && ct.verbose > 2) - (*MPIdata::sout) - << "Ion " << aname << " at position " - << crds[3 * ia + 0] << "," << crds[3 * ia + 1] << "," - << crds[3 * ia + 2] - << " added to the list of local ions... on PE" - << mmpi.mypeGlobal() << std::endl; - } - else - (new_ion)->set_here(false); - } - else - { - //(*MPIdata::sout)<<"Ion "<set_here(true); + local_ions_.push_back(new_ion); + if (onpe0 && ct.verbose > 2) + (*MPIdata::sout) << "Ion " << name << " at position " << crds[0] + << "," << crds[1] << "," << crds[2] + << " added to the list of local ions... on PE" + << mmpi.mypeGlobal() << std::endl; + } + else + (new_ion)->set_here(false); + } + else + { + // delete Ion if not put in list + delete new_ion; + } +} + int Ions::readNatoms(const std::string& filename, const bool cell_relative) { Control& ct(*(Control::instance())); @@ -2117,9 +2085,8 @@ void Ions::setVelocities(const std::vector& tau0, assert(tau0.size() == 3 * local_ions_.size()); assert(taup.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { double v[3]; for (short i = 0; i < 3; i++) @@ -2128,8 +2095,7 @@ void Ions::setVelocities(const std::vector& tau0, v[i] -= tau0[3 * ia + i]; v[i] /= dt; } - (*iion)->setVelocity(v[0], v[1], v[2]); - iion++; + ion->setVelocity(v[0], v[1], v[2]); ia++; } } @@ -2138,12 +2104,10 @@ void Ions::getLocalPositions(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->getPosition(&tau[3 * ia]); - iion++; + ion->getPosition(&tau[3 * ia]); ia++; } } @@ -2188,12 +2152,10 @@ void Ions::setTau0() { assert(tau0_.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->getPosition(&tau0_[3 * ia]); - iion++; + ion->getPosition(&tau0_[3 * ia]); ia++; } } @@ -2202,13 +2164,11 @@ void Ions::setPositionsToTau0() { assert(tau0_.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->setPosition( + ion->setPosition( tau0_[3 * ia + 0], tau0_[3 * ia + 1], tau0_[3 * ia + 2]); - iion++; ia++; } } @@ -2244,13 +2204,11 @@ void Ions::getLocalForces(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { assert(3 * ia + 2 < (int)tau.size()); - (*iion)->getForce(&tau[3 * ia]); - iion++; + ion->getForce(&tau[3 * ia]); ia++; } } @@ -2351,12 +2309,10 @@ double Ions::computeMaxVlRadius() const { double radius = 0.; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + for (auto& ion : local_ions_) { - double r = (*iion)->computeRadiusVl(); + double r = ion->computeRadiusVl(); radius = r > radius ? r : radius; - iion++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -2393,13 +2349,11 @@ void Ions::gatherNames(std::map& names, const int root, std::vector data; std::vector local_names; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { // get local name and index - local_names.push_back((*ion)->name()); - local_indexes.push_back((*ion)->index()); - ++ion; + local_names.push_back(ion->name()); + local_indexes.push_back(ion->index()); } // gather data to PE root @@ -2695,15 +2649,13 @@ bool Ions::hasLockedAtoms() const return (flag == 1); } -double Ions::getMaxNLradius() const +double Ions::getSpeciesMaxNLradius() const { - double radius = 0; - std::vector::const_iterator spi = species_.begin(); - while (spi != species_.end()) + double radius = 0.; + for (const auto& spi : species_) { - const double nlradius = spi->nlradius(); + const double nlradius = spi.nlradius(); radius = radius > nlradius ? radius : nlradius; - spi++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); mmpi.allreduce(&radius, 1, MPI_MAX); @@ -2711,15 +2663,13 @@ double Ions::getMaxNLradius() const return radius; } -double Ions::getMaxLradius() const +double Ions::getSpeciesMaxLradius() const { - double radius = 0; - std::vector::const_iterator spi = species_.begin(); - while (spi != species_.end()) + double radius = 0.; + for (const auto& spi : species_) { - const double lradius = spi->lradius(); + const double lradius = spi.lradius(); radius = radius > lradius ? radius : lradius; - spi++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); mmpi.allreduce(&radius, 1, MPI_MAX); @@ -2918,8 +2868,8 @@ void Ions::computeNumIons(void) double Ions::getMaxListRadius() const { // get radius of projectors - const double nlradius = getMaxNLradius(); - const double lradius = getMaxLradius(); + const double nlradius = getSpeciesMaxNLradius(); + const double lradius = getSpeciesMaxLradius(); double rmax = nlradius > lradius ? nlradius : lradius; @@ -3289,21 +3239,15 @@ void Ions::updateIons() Control& ct(*(Control::instance())); // update local_ions data - std::vector::iterator ion = local_ions_.begin(); - int ia = 0; - while (ion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition( + ion->setPosition( tau0_[3 * ia + 0], tau0_[3 * ia + 1], tau0_[3 * ia + 2]); - // (*ion)->setForce(fion_[3*ia+0], - // fion_[3*ia+1], - // fion_[3*ia+2]); - - (*ion)->setRandomState(rand_states_[3 * ia + 0], - rand_states_[3 * ia + 1], rand_states_[3 * ia + 2]); + ion->setRandomState(rand_states_[3 * ia + 0], rand_states_[3 * ia + 1], + rand_states_[3 * ia + 2]); - ion++; ia++; } diff --git a/src/Ions.h b/src/Ions.h index 772c51d1..ae4f6adb 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -71,13 +71,14 @@ class Ions void setMapVL(); double computeMaxVlRadius() const; double computeMaxNLprojRadius() const; - double getMaxNLradius() const; - double getMaxLradius() const; - void updateListIons(); - // compute boundary for box containing all atoms to be known on local - // processor that is values for list_boundary_left_, list_boundary_right_ - void setupListIonsBoundaries(const double rmax); + /* + * Evaluate maximum pseudopotential radius among all species in class + */ + double getSpeciesMaxNLradius() const; + double getSpeciesMaxLradius() const; + + void updateListIons(); void augmentIonsData(const int nsteps, const int dir, const int disp, const int locSize, const int maxLocSize, std::vector& data, @@ -163,6 +164,8 @@ class Ions bool hasLockedAtoms() const; void clearLists(); + void rescaleVelocities(const double factor); + public: Ions(const double lat[3], const std::vector& sp); @@ -172,6 +175,10 @@ class Ions void setup(); + // compute boundary for box containing all atoms to be known on local + // processor that is values for list_boundary_left_, list_boundary_right_ + void setupListIonsBoundaries(const double rmax); + std::vector& getLocalNames() { return local_names_; } std::vector& getTau0() { return tau0_; } std::vector& getTaup() { return taup_; } @@ -334,13 +341,15 @@ class Ions void updateForcesInteractingIons(); void updateTaupInteractingIons(); - void rescaleVelocities(const double factor); /*! * Calculate minimum distance between local pairs */ double computeMinLocalSpacing() const; + void addIonToList(const Species& sp, const std::string& name, + const double crds[3], const double velocity[3], const bool lock); + // void checkUnicityLocalIons(); }; diff --git a/src/KBprojectorSparse.cc b/src/KBprojectorSparse.cc index afd8871c..937b95d2 100644 --- a/src/KBprojectorSparse.cc +++ b/src/KBprojectorSparse.cc @@ -103,7 +103,7 @@ void KBprojectorSparse::setNLindex( for (int i = 0; i < size_nl; i++) { - assert(i < lnumpt); + // assert(i < lnumpt); if ((pvec[i] < iloc * lnumpt) || (pvec[i] >= (iloc + 1) * lnumpt)) { (*MPIdata::sout) << " iloc=" << iloc << ", i=" << i @@ -960,7 +960,7 @@ bool KBprojectorSparse::setIndexesAndProjectors() // get "pvec" and "is_in_domain" int icount = get_index_array(pvec, iloc, index_low, index_high); assert(icount <= nl3); - assert(icount <= mymesh->npointsPatch()); + // assert(icount <= mymesh->npointsPatch()); assert(icount > 0); setNLindex(iloc, icount, pvec); diff --git a/src/MGmol.cc b/src/MGmol.cc index 8da0d331..acc2b944 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -837,7 +837,7 @@ void MGmol::initNuc(Ions& ions) pot.initialize(ions); // Check compensating charges - double comp_rho = pot.getCharge(pot.rho_comp()); + double comp_rho = getCharge(pot.rho_comp()); if (onpe0 && ct.verbose > 1) { diff --git a/src/MGmol.h b/src/MGmol.h index 6a4cc830..a9bca4ba 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -182,7 +182,10 @@ class MGmol : public MGmolInterface /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } - std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getHamiltonian() + { + return hamiltonian_; + } std::shared_ptr> getRho() { return rho_; } void run() override; diff --git a/src/PCGSolver.cc b/src/PCGSolver.cc index 12103aaa..b3bd7ec3 100644 --- a/src/PCGSolver.cc +++ b/src/PCGSolver.cc @@ -11,14 +11,13 @@ #include #include -using namespace std; -template -void PCGSolver::clear() +template +void PCGSolver::clear() { - for (short i = 0; i < (short)pc_oper_.size(); i++) + for (short i = 0; i < (short)precond_oper_.size(); i++) { - delete pc_oper_[i]; + delete precond_oper_[i]; } for (short i = 0; i < (short)gf_work_.size(); i++) { @@ -35,26 +34,26 @@ void PCGSolver::clear() assert(gf_newv_[i] != nullptr); delete gf_newv_[i]; } - // delete grids after pb::GridFunc objects since those + // delete grids after pb::GridFunc objects since those // have data members references to grids for (short i = 0; i < (short)grid_.size(); i++) { delete grid_[i]; } - pc_oper_.clear(); + precond_oper_.clear(); grid_.clear(); gf_work_.clear(); gf_rcoarse_.clear(); gf_newv_.clear(); } -template -void PCGSolver::setupPrecon() +template +void PCGSolver::setupPrecon() { // check if precon is already setup // Assumes operator does not change, hence // a single setup is sufficient - if (is_pc_setup_) return; + if (is_precond_setup_) return; // fine level pb::Grid* mygrid = new pb::Grid(oper_.grid()); @@ -63,7 +62,7 @@ void PCGSolver::setupPrecon() pb::Lap* myoper = LapFactory::createLap(*grid_[0], lap_type_); - pc_oper_.push_back(myoper); + precond_oper_.push_back(myoper); pb::GridFunc* gf_work = new pb::GridFunc( @@ -92,7 +91,7 @@ void PCGSolver::setupPrecon() pb::Lap* myoper = LapFactory::createLap(*coarse_grid, 1); - pc_oper_.push_back(myoper); + precond_oper_.push_back(myoper); gf_work = new pb::GridFunc( *coarse_grid, bc_[0], bc_[1], bc_[2]); @@ -109,12 +108,13 @@ void PCGSolver::setupPrecon() mygrid = coarse_grid; } - is_pc_setup_ = true; + is_precond_setup_ = true; } // MG V-cycle with no mask -template -void PCGSolver::preconSolve(pb::GridFunc& gf_v, +template +void PCGSolver::preconSolve( + pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) { //(*MPIdata::sout)<<"Preconditioning::mg() at level "<::preconSolve(pb::GridFunc& gf_v, ncycl = 4 > (nu1_ + nu2_) ? 4 : (nu1_ + nu2_); } - pb::Lap* myoper = pc_oper_[level]; + pb::Lap* myoper = precond_oper_[level]; // SMOOTHING for (short it = 0; it < ncycl; it++) @@ -161,25 +161,23 @@ void PCGSolver::preconSolve(pb::GridFunc& gf_v, } // Left Preconditioned CG -template -bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) +template +bool PCGSolver::solve( + pb::GridFunc& gf_phi, const pb::GridFunc& gf_rhs) { bool converged = false; const pb::Grid& finegrid = gf_phi.grid(); // initial data and residual - We assume a nonzero initial guess - pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); // scale initial guess with epsilon oper_.inv_transform(gf_phi); // compute initial residual: r := b - Ax /* compute Ax */ oper_.apply(gf_phi, lhs); /* set r = b */ - pb::GridFunc res(gf_rhs); + pb::GridFunc res(gf_rhs); oper_.transform(res); - // Hartree units - const double hu = 4. * M_PI; - res *= hu; /* compute r = r - Ax */ res -= lhs; @@ -199,11 +197,11 @@ bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) /* preconditioning step */ prec_z.setValues(0.); preconSolve(prec_z, prec_res, 0); - pb::GridFunc z(prec_z); + pb::GridFunc z(prec_z); // conjugate vectors - pb::GridFunc p(prec_z); - pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc p(prec_z); + pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); double rtz = res.gdot(z); @@ -251,13 +249,14 @@ bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) } // Left Preconditioned CG -template -bool PCGSolver::solve(T2* phi, T2* rhs, const char dis) +template +bool PCGSolver::solve( + ScalarType* phi, ScalarType* rhs, const char dis) { - pb::GridFunc gf_phi(oper_.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc gf_phi(oper_.grid(), bc_[0], bc_[1], bc_[2]); gf_phi.assign(phi, dis); - pb::GridFunc gf_work(oper_.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc gf_work(oper_.grid(), bc_[0], bc_[1], bc_[2]); gf_work.assign(rhs, dis); bool converged = solve(gf_phi, gf_work); diff --git a/src/PCGSolver.h b/src/PCGSolver.h index d38787f1..f6db7cc6 100644 --- a/src/PCGSolver.h +++ b/src/PCGSolver.h @@ -7,8 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef _PCG_SOLVER_H_ -#define _PCG_SOLVER_H_ +#ifndef MGMOL_PCG_SOLVER_H +#define MGMOL_PCG_SOLVER_H #include "Control.h" #include "Lap.h" @@ -17,32 +17,41 @@ #include -template +template class PCGSolver { - private: std::vector grid_; short lap_type_; short bc_[3]; bool fully_periodic_; - // operators + + // operator to solve for T oper_; - std::vector*> pc_oper_; + + // preconditioner operator for each MG level + std::vector*> precond_oper_; std::vector*> gf_work_; std::vector*> gf_rcoarse_; std::vector*> gf_newv_; - // solver params + + // solver parameters int maxiters_; double tol_; double final_residual_; double residual_reduction_; - // precon params + + // preconditioner parameters short nu1_; short nu2_; short max_nlevels_; short nlevels_; - bool is_pc_setup_; + bool is_precond_setup_; + + void preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level = 0); + void setupPrecon(); + void clear(); public: PCGSolver(T& oper, const short px, const short py, const short pz) @@ -61,15 +70,10 @@ class PCGSolver bc_[1] = py; bc_[2] = pz; fully_periodic_ = ((bc_[0] == 1) && (bc_[1] == 1) && (bc_[2] == 1)); - // fine grid info - // pb::Grid* mygrid=new pb::Grid(oper.grid()); - // grid_.push_back(mygrid); - // fine grid operator - Control& ct = *(Control::instance()); - lap_type_ = ct.lap_type; - is_pc_setup_ = false; - // pb::Lap* myoper = LapFactory::createLap(*grid_[0],lap_type_); - // pc_oper_.push_back(myoper); + + Control& ct = *(Control::instance()); + lap_type_ = ct.lap_type; + is_precond_setup_ = false; }; void setup(const short nu1, const short nu2, const short max_sweeps, @@ -83,16 +87,10 @@ class PCGSolver setupPrecon(); } - void clear(); - - void setupPrecon(); - - void preconSolve(pb::GridFunc& gf_v, - const pb::GridFunc& gf_f, const short level = 0); - - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); + bool solve(pb::GridFunc& gf_phi, + const pb::GridFunc& gf_rhs); - bool solve(T2* phi, T2* rhs, const char dis); + bool solve(ScalarType* phi, ScalarType* rhs, const char dis); double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } diff --git a/src/PCGSolver_Diel.cc b/src/PCGSolver_Diel.cc index 0cdc5aed..4e7c88a9 100644 --- a/src/PCGSolver_Diel.cc +++ b/src/PCGSolver_Diel.cc @@ -9,10 +9,8 @@ #include "PCGSolver_Diel.h" -using namespace std; - -template -void PCGSolver_Diel::clear() +template +void PCGSolver_Diel::clear() { for (short i = 0; i < (short)pc_oper_.size(); i++) { @@ -33,7 +31,7 @@ void PCGSolver_Diel::clear() assert(gf_newv_[i] != nullptr); delete gf_newv_[i]; } - // delete grids after pb::GridFunc objects since those + // delete grids after pb::GridFunc objects since those // have data members references to grids for (short i = 0; i < (short)grid_.size(); i++) { @@ -46,8 +44,8 @@ void PCGSolver_Diel::clear() gf_newv_.clear(); } -template -void PCGSolver_Diel::setupPrecon() +template +void PCGSolver_Diel::setupPrecon() { // fine level pb::Grid* mygrid = new pb::Grid(oper_.grid()); @@ -57,8 +55,8 @@ void PCGSolver_Diel::setupPrecon() T* myoper = new T(oper_); pc_oper_.push_back(myoper); - pb::GridFunc* gf_work - = new pb::GridFunc(*grid_[0], bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_work + = new pb::GridFunc(*grid_[0], bc_[0], bc_[1], bc_[2]); gf_work_.push_back(gf_work); // coarse levels @@ -86,24 +84,25 @@ void PCGSolver_Diel::setupPrecon() T* myoper = new T(pc_oper_[ln - 1]->coarseOp(*mygrid)); pc_oper_.push_back(myoper); - gf_work = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + gf_work = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_work_.push_back(gf_work); - pb::GridFunc* gf_rcoarse - = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_rcoarse = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_rcoarse_.push_back(gf_rcoarse); - pb::GridFunc* gf_newv - = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_newv = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_newv_.push_back(gf_newv); mygrid = coarse_grid; } } -template +template // MG V-cycle with no mask -void PCGSolver_Diel::preconSolve( - pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) +void PCGSolver_Diel::preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level) { //(*MPIdata::sout)<<"Preconditioning::mg() at level "<::preconSolve( // COARSE GRID CORRECTION // restrictions - pb::GridFunc* rcoarse = gf_rcoarse_[level]; + pb::GridFunc* rcoarse = gf_rcoarse_[level]; gf_work_[level]->restrict3D(*rcoarse); // storage functions for coarse grid - pb::GridFunc* newv = gf_newv_[level]; + pb::GridFunc* newv = gf_newv_[level]; // call mgrid solver on a coarser level newv->resetData(); @@ -149,15 +148,16 @@ void PCGSolver_Diel::preconSolve( if (bc_[0] != 1 || bc_[2] != 1 || bc_[2] != 1) gf_v.trade_boundaries(); } -template +template // Left Preconditioned CG -bool PCGSolver_Diel::solve( - pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) +bool PCGSolver_Diel::solve( + pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) { if (!oper_.initialized()) { - cout << "Error in PCGSolver_Diel::solve: operator not initialized" - << endl; + std::cout + << "Error in PCGSolver_Diel::solve: operator not initialized" + << std::endl; return 0.; } @@ -165,13 +165,13 @@ bool PCGSolver_Diel::solve( const pb::Grid& finegrid = gf_phi.grid(); // initial data and residual - We assume a nonzero initial guess - pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); - pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); // scale initial guess with epsilon oper_.inv_transform(gf_phi); // compute initial residual oper_.apply(gf_phi, lhs); - pb::GridFunc rhs(gf_rhs); + pb::GridFunc rhs(gf_rhs); oper_.transform(rhs); // Hartree units rhs *= (4. * M_PI); @@ -180,13 +180,13 @@ bool PCGSolver_Diel::solve( double rnorm = init_rnorm; // preconditioned residual - pb::GridFunc z(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc z(finegrid, bc_[0], bc_[1], bc_[2]); // preconditioning step z = 0.; preconSolve(z, res, 0); // conjugate vectors - pb::GridFunc p(z); - pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc p(z); + pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); double rtz = res.gdot(z); @@ -225,11 +225,11 @@ bool PCGSolver_Diel::solve( return converged; } -template +template // Left Preconditioned CG -bool PCGSolver_Diel::solve(pb::GridFunc& gf_phi, - pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, - pb::GridFunc& gf_vks) +bool PCGSolver_Diel::solve(pb::GridFunc& gf_phi, + pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, + pb::GridFunc& gf_vks) { // initialize the linear system operator and the preconditioner oper_.init(gf_rhod); diff --git a/src/PCGSolver_Diel.h b/src/PCGSolver_Diel.h index 9fd97914..f2001a4c 100644 --- a/src/PCGSolver_Diel.h +++ b/src/PCGSolver_Diel.h @@ -7,8 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef _PCG_SOLVER_DIEL_H_ -#define _PCG_SOLVER_DIEL_H_ +#ifndef MGMOL_PCG_SOLVER_DIEL_H_ +#define MGMOL_PCG_SOLVER_DIEL_H_ #include "Control.h" #include "PB.h" @@ -21,7 +21,7 @@ #include -template +template class PCGSolver_Diel { @@ -32,9 +32,9 @@ class PCGSolver_Diel // operators T oper_; std::vector pc_oper_; - std::vector*> gf_work_; - std::vector*> gf_rcoarse_; - std::vector*> gf_newv_; + std::vector*> gf_work_; + std::vector*> gf_rcoarse_; + std::vector*> gf_newv_; // solver params int maxiters_; double tol_; @@ -47,6 +47,11 @@ class PCGSolver_Diel short nlevels_; void setupPrecon(); + void clear(); + + void preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level = 0); + public: PCGSolver_Diel(T& oper, const short px, const short py, const short pz) : oper_(oper) @@ -78,15 +83,12 @@ class PCGSolver_Diel max_nlevels_ = max_nlevels; } - void clear(); - - void preconSolve(pb::GridFunc& gf_v, const pb::GridFunc& gf_f, - const short level = 0); - - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); + bool solve( + pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs, - pb::GridFunc& gf_rhod, pb::GridFunc& gf_vks); + bool solve(pb::GridFunc& gf_phi, + pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, + pb::GridFunc& gf_vks); double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } diff --git a/src/PoissonSolverFactory.h b/src/PoissonSolverFactory.h new file mode 100644 index 00000000..91b99bf2 --- /dev/null +++ b/src/PoissonSolverFactory.h @@ -0,0 +1,228 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE +#ifndef MGMOL_PoissonSolverFactory +#define MGMOL_PoissonSolverFactory + +#include "Control.h" +#include "Hartree.h" +#include "Hartree_CG.h" +#include "Mesh.h" +#include "PBdiel.h" +#include "PBdiel_CG.h" +#include "ShiftedHartree.h" +#include "mputils.h" + +#include "GridFunc.h" +#include "Laph2.h" +#include "Laph4.h" +#include "Laph4M.h" +#include "Laph4MP.h" +#include "Laph6.h" +#include "Laph8.h" +#include "ShiftedLaph4M.h" + +class PoissonSolverFactory +{ + +public: + /*! + * return specific Poisson solver needed to solve Hartree problem + */ + static Poisson* create(const pb::Grid& myGrid, PoissonFDtype lap_type, + const short bc[3], const double screening_const) + { + Poisson* poisson_solver = nullptr; + + Control& ct = *(Control::instance()); + if (ct.MGPoissonSolver()) // use MG for Poisson Solver + { + if (screening_const > 0.) + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new ShiftedHartree>( + myGrid, bc, screening_const); + break; + default: + (*MPIdata::sout) + << "Electrostatic, shifted, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + else + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h2: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h4: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h6: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h8: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h4MP: + poisson_solver + = new Hartree>(myGrid, bc); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + } + else // use PCG for Poisson Solver + { + if (screening_const > 0.) + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new ShiftedHartree>( + myGrid, bc, screening_const); + break; + default: + (*MPIdata::sout) + << "PCG Electrostatic, shifted, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + else + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h2: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h4: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h6: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h8: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h4MP: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + default: + (*MPIdata::sout) + << "PCG Electrostatic, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + } + + return poisson_solver; + } + + static Poisson* createDiel(pb::Grid& pbGrid, PoissonFDtype lap_type, + const short bc[3], const double e0, const double rho0, + const double drho0) + { + Poisson* poisson_solver = nullptr; + + Control& ct = *(Control::instance()); + if (ct.MGPoissonSolver()) // use MG for Poisson Solver + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h2: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h6: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h8: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4MP: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option" << std::endl; + } + } + else // use PCG for Poisson Solver + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h2: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h6: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h8: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4MP: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option" << std::endl; + } + } + return poisson_solver; + } +}; + +#endif diff --git a/src/Potentials.cc b/src/Potentials.cc index ee874f10..380f05a0 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -27,6 +27,7 @@ #include using namespace std; +// unit conversion factor Ha -> Ry const double ha2ry = 2.; Potentials::~Potentials() @@ -57,13 +58,9 @@ Potentials::Potentials(const bool vh_frozen) size_ = dim_[0] * dim_[1] * dim_[2]; - mix_ = 1.; - scf_dvrho_ = 1000.; scf_dv_ = 1000.; - vh_frozen_ = vh_frozen; - vtot_.resize(size_); vtot_old_.resize(size_); @@ -101,7 +98,7 @@ void Potentials::initWithVnuc() double one = 1.; LinearAlgebraUtils::MPaxpy( size_, one, &v_ext_[0], &vtot_[0]); - // factor 2 to get total potential in [Ry] for calculations + // factor ha2ry to get total potential in [Ry] for calculations LinearAlgebraUtils::MPscal(size_, ha2ry, &vtot_[0]); } @@ -568,9 +565,8 @@ template void Potentials::setVxc(const T* const vxc, const int iterativeIndex) { assert(iterativeIndex >= 0); - // int ione=1; + itindex_vxc_ = iterativeIndex; - // Tcopy(&size_, vxc, &ione, &vxc_rho_[0], &ione); MPcpy(&vxc_rho_[0], vxc, size_); } void Potentials::setVh(const POTDTYPE* const vh, const int iterativeIndex) @@ -849,20 +845,6 @@ void Potentials::rescaleRhoComp() if (comp_rho < 0.) mmpi.abort(); } -double Potentials::getCharge(RHODTYPE* rho) -{ - Control& ct = *(Control::instance()); - Mesh* mymesh = Mesh::instance(); - const pb::Grid& mygrid = mymesh->grid(); - - double charge = mygrid.integralOverMesh(rho); - - if (onpe0 && ct.verbose > 0) - cout << setprecision(8) << fixed << "Charge: " << charge << endl; - - return charge; -} - void Potentials::addBackgroundToRhoComp() { if (fabs(background_charge_) > 0.) diff --git a/src/Potentials.h b/src/Potentials.h index 0cd7a8a6..205fb538 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -30,7 +30,6 @@ class Potentials int gdim_[3]; int dim_[3]; bool diel_; - double mix_; double scf_dvrho_; double scf_dv_; @@ -39,8 +38,9 @@ class Potentials double charge_in_cell_; double ionic_charge_; - bool vh_frozen_; - + /*! + * Total KS potential seen by electrons + */ std::vector vtot_; std::vector vtot_old_; @@ -48,16 +48,30 @@ class Potentials std::vector vh_rho_; std::vector vxc_rho_; - // nuclei local potential + /* + * Potential contribution from atomic cores (local pseudopotential) + */ std::vector v_nuc_; - // external potential (read from input) + /*! + * Optional external potential (read from input) + * Used only in special cases. + */ std::vector v_ext_; #ifdef HAVE_TRICUBIC pb::TriCubic* vext_tricubic_; #endif + /*! + * Potential associated with the sum of Gaussian charge distributions + * compensating the Coulomb potential of each atom + */ std::vector v_comp_; + + /*! + * Sum of Gaussian charge distributions compensating the Coulomb potential + * of each atom + */ std::vector rho_comp_; std::vector dv_; @@ -121,14 +135,11 @@ class Potentials void turnOnDiel() { diel_ = true; } int size() const { return size_; } - bool vh_frozen() const { return vh_frozen_; } - void freeze_vh() { vh_frozen_ = true; } double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } POTDTYPE* vtot() { return &vtot_[0]; } POTDTYPE* vh_rho() { return &vh_rho_[0]; } - POTDTYPE* vxc_rho() { return &vxc_rho_[0]; } RHODTYPE* rho_comp() { return &rho_comp_[0]; } const std::vector& vnuc() const { return v_nuc_; } @@ -136,12 +147,6 @@ class Potentials POTDTYPE* vext() { return &v_ext_[0]; } POTDTYPE* vepsilon() { return &vepsilon_[0]; } - void set_vcomp(const POTDTYPE val) - { - const int n = (int)v_comp_.size(); - for (int i = 0; i < n; i++) - v_comp_[i] = val; - } void axpVcompToVh(const double alpha); void axpVcomp(POTDTYPE* v, const double alpha); @@ -154,13 +159,29 @@ class Potentials double getChargeInCell() const { return charge_in_cell_; } + /*! + * initialize total potential as local pseudopotential + */ void initWithVnuc(); void getVofRho(std::vector& vrho) const; - double delta_v(const std::vector>&); - double update(const std::vector>&); - void update(const double); + /*! + * evaluate potential correction associated with a new rho + */ + double delta_v(const std::vector>& rho); + + /*! + * update potentials based on argument rho + */ + double update(const std::vector>& rho); + + /*! + * update potentials based on potential correction delta v and mixing + * parameter + */ + void update(const double mix); + double max() const; double min() const; void readAll(std::vector& sp); @@ -172,7 +193,6 @@ class Potentials void initialize(Ions& ions); void rescaleRhoComp(); - double getCharge(RHODTYPE* rho); void initBackground(Ions& ions); void addBackgroundToRhoComp(); diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index e0e679e9..3606aa40 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -509,6 +509,7 @@ template void ProjectedMatrices::setOccupations( const std::vector& occ) { + assert(!occ.empty()); #ifdef PRINT_OPERATIONS if (mmpi.instancePE0()) (*MPIdata::sout) << "ProjectedMatrices::setOccupations()" diff --git a/src/pb/Mgm.h b/src/pb/Mgm.h index d16f5610..4b5ad68d 100644 --- a/src/pb/Mgm.h +++ b/src/pb/Mgm.h @@ -7,7 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: Mgm.h,v 1.13 2010/01/28 22:56:47 jeanluc Exp $ #ifndef PB_MGM_H #define PB_MGM_H @@ -41,8 +40,6 @@ bool Mgm(T1& A, T2& vh, const GridFunc& rho, const short cogr, A.rhs(res, rhs); // Hartree units - rhs *= (4. * M_PI); - // work GridFunc GridFunc lhs(finegrid, bcx, bcy, bcz); diff --git a/src/pb/Solver.h b/src/pb/Solver.h index 062fbaa2..f82b3b6a 100644 --- a/src/pb/Solver.h +++ b/src/pb/Solver.h @@ -7,9 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: Solver.h,v 1.14 2010/01/28 22:56:47 jeanluc Exp $ -#ifndef SOLVE_H -#define SOLVE_H +#ifndef PB_SOLVER_H +#define PB_SOLVER_H #include "GridFunc.h" @@ -33,7 +32,7 @@ class Solver fully_periodic_ = ((bc_[0] == 1) && (bc_[1] == 1) && (bc_[2] == 1)); } - virtual bool solve(GridFunc&, GridFunc&) = 0; + virtual bool solve(GridFunc&, const GridFunc&) = 0; virtual void setup(const short nu1, const short nu2, const short max_sweeps, const double tol, const short max_nlevels, diff --git a/src/pb/SolverLap.cc b/src/pb/SolverLap.cc index 5a85fefe..062755a8 100644 --- a/src/pb/SolverLap.cc +++ b/src/pb/SolverLap.cc @@ -7,8 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: SolverLap.cc,v 1.15 2010/01/28 22:56:31 jeanluc Exp $ - #include "SolverLap.h" #include "Laph2.h" #include "Laph4.h" @@ -26,9 +24,8 @@ Timer vcycle_repvcycle_tm("vcycle_repvcycle"); namespace pb { - // explicit instantiation declaration -#if 1 + // double template class SolverLap, double>; template class SolverLap, double>; @@ -45,77 +42,7 @@ template class SolverLap, float>; template class SolverLap, float>; template class SolverLap, float>; template class SolverLap, float>; -#else -// double -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); - -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -// float -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -#endif -/* -template bool Mgm(Laph4MP&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph4M&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph2&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph4&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph6&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(ShiftedLaph4M&, GridFunc&, const GridFunc&, -const short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -*/ -/* -template int Vcycle(ShiftedLaph4M&, GridFunc&, const GridFunc -&, const short, const short, const short, const bool); template int -Vcycle(Laph4MP&, GridFunc&, const GridFunc &, const short, -const short, const short, const bool); template int Vcycle(Laph4M&, -GridFunc&, const GridFunc &, const short, const short, const -short, const bool); template int Vcycle(Laph2&, GridFunc&, const -GridFunc &, const short, const short, const short, const bool); -template int Vcycle(Laph4&, GridFunc&, const GridFunc &, const -short, const short, const short, const bool); template int Vcycle(Laph6&, -GridFunc&, const GridFunc &, const short, const short, const -short, const bool); -*/ template bool SolverLap::solve(T2* phi, T2* rhs, const char dis) { @@ -139,7 +66,7 @@ bool SolverLap::solve(T2* phi, T2* rhs, const char dis) } template -bool SolverLap::solve(GridFunc& gf_phi, GridFunc& gf_rhs) +bool SolverLap::solve(GridFunc& gf_phi, const GridFunc& gf_rhs) { bool conv = Mgm(oper_, gf_phi, gf_rhs, max_nlevels_, max_sweeps_, tol_, nu1_, nu2_, gather_coarse_level_, final_residual_, diff --git a/src/pb/SolverLap.h b/src/pb/SolverLap.h index 939e15bc..567bed05 100644 --- a/src/pb/SolverLap.h +++ b/src/pb/SolverLap.h @@ -64,7 +64,7 @@ class SolverLap : public Solver bool solve(T2* phi, T2* rhs, const char dis); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs) override; + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs) override; short getNbSweeps() const override { return nb_sweeps_; } double getFinalResidual() const override { return final_residual_; } diff --git a/src/pb/SolverPB.cc b/src/pb/SolverPB.cc index 53252c26..756b49d8 100644 --- a/src/pb/SolverPB.cc +++ b/src/pb/SolverPB.cc @@ -7,7 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: SolverPB.cc,v 1.13 2010/01/28 22:56:32 jeanluc Exp $ #include "SolverPB.h" #include "Mgm.h" #include "PBh2.h" @@ -85,7 +84,7 @@ bool SolverPB::solve(T2* phi, T2* rhs, T2* rhod, T2* vks, const char dis) } template -bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs, +bool SolverPB::solve(GridFunc& gf_phi, const GridFunc& gf_rhs, GridFunc& gf_rhod, GridFunc& gf_vks) { oper_.init(gf_rhod); @@ -101,7 +100,7 @@ bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs, } template -bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs) +bool SolverPB::solve(GridFunc& gf_phi, const GridFunc& gf_rhs) { if (!oper_.initialized()) { diff --git a/src/pb/SolverPB.h b/src/pb/SolverPB.h index a05e1d49..99eae0e3 100644 --- a/src/pb/SolverPB.h +++ b/src/pb/SolverPB.h @@ -64,9 +64,9 @@ class SolverPB : public Solver } bool solve(T2* phi, T2* rhs, T2* rhod, T2* vks, const char dis); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs, + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs, GridFunc& gf_rhod, GridFunc& gf_vks); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs) override; + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs) override; ~SolverPB() override{}; diff --git a/src/tools.cc b/src/tools.cc index 4e1bd314..95790c59 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -228,7 +228,7 @@ void printWithTimeStamp(const std::string& string2print, std::ostream& os) int mpierr=MPI_Reduce(&s, &r, 1, MPI_INT, MPI_SUM, 0, mmpi.commGlobal()); if( mpierr!=MPI_SUCCESS ) { - cerr << " Error in MPI!!!" << endl; + cerr << " Error in MPI!!!" << std::endl; MPI_Abort(mmpi.commGlobal(),1); } if( r!=mmpi.size()*s && onpe0 ) @@ -388,3 +388,18 @@ void getkvector(const int index, const int kmax, int kvector[3]) // std::cout << " k=(" << kvector[0] << "," << kvector[1] << "," // << kvector[2] << ")" << std::endl; } + +double getCharge(double* rho) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + double charge = mygrid.integralOverMesh(rho); + + if (onpe0 && ct.verbose > 0) + std::cout << std::setprecision(8) << std::fixed << "Charge: " << charge + << std::endl; + + return charge; +} diff --git a/src/tools.h b/src/tools.h index 4e8374da..be3d13e0 100644 --- a/src/tools.h +++ b/src/tools.h @@ -37,5 +37,6 @@ double minQuadPolynomial(const double e0, const double e1, const double de0, double minQuadPolynomialFrom3values(const double e0, const double e1, const double e12, const bool print_flag, std::ostream& os); void getkvector(const int index, const int kmax, int kvector[3]); +double getCharge(double* rho); #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cc205167..6de34971 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -209,6 +209,8 @@ add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) +add_executable(testIons + ${CMAKE_SOURCE_DIR}/tests/testIons.cc) add_executable(testGramMatrix ${CMAKE_SOURCE_DIR}/tests/testGramMatrix.cc ${CMAKE_SOURCE_DIR}/src/GramMatrix.cc @@ -334,6 +336,10 @@ add_test(NAME testBatchLaph4 add_test(NAME testtMGkernels COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testMGkernels) +add_test(NAME testIons + COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testIons + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testGramMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testGramMatrix) @@ -541,6 +547,7 @@ target_include_directories(testPowerDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_link_libraries(testMPI PRIVATE MPI::MPI_CXX) target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} @@ -550,6 +557,7 @@ target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testIons PRIVATE mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/WFEnergyAndForces/mgmol.cfg b/tests/WFEnergyAndForces/mgmol.cfg index d543b626..12002703 100644 --- a/tests/WFEnergyAndForces/mgmol.cfg +++ b/tests/WFEnergyAndForces/mgmol.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H type=QUENCH [Quench] max_steps=50 -atol=1.e-9 +atol=1.e-8 num_lin_iterations=2 [Orbitals] initial_type=Gaussian diff --git a/tests/testIons.cc b/tests/testIons.cc new file mode 100644 index 00000000..30b2ad0c --- /dev/null +++ b/tests/testIons.cc @@ -0,0 +1,104 @@ +#include "Control.h" +#include "Ions.h" +#include "MGmol_MPI.h" +#include "Mesh.h" +#include "Species.h" + +#include + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + + MPI_Comm comm = MPI_COMM_WORLD; + + int myrank; + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + + MGmol_MPI::setup(comm, std::cout); + Control::setup(comm, false, 0.); + + // create a domain [0.10.]^3 + const double origin[3] = { 0., 0., 0. }; + const double ll = 10.; + const double lattice[3] = { ll, ll, ll }; + const unsigned ngpts[3] = { 32, 24, 20 }; + short lap_type = 0; + + Mesh::setup(comm, ngpts, origin, lattice, lap_type); + + const double h[3] = { ll / (double(ngpts[0])), ll / (double(ngpts[1])), + ll / (double(ngpts[2])) }; + + // random number generator + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_real_distribution<> dis(0.0, 1.0); + + // create one species + Species sp(MPI_COMM_WORLD); + + // read species info from pseudopotential file + std::string file_path = argv[1]; + std::string filename(file_path + "/pseudo.C_ONCV_PBE_SG15"); + std::cout << "Potential = " << filename << std::endl; + + sp.read_1species(filename); + sp.set_dim_nl(h[0]); + sp.set_dim_l(h[0]); + sp.initPotentials('f', h[0], true); + + // put species into a vector + std::vector vsp; + vsp.push_back(sp); + + Ions ions(lattice, vsp); + ions.setupListIonsBoundaries(10000.); + + double velocity[3] = { 0., 0., 0. }; + + // set "na" atoms coordinates and add them to "ions" + const int na = 10; + for (int i = 0; i < na; i++) + { + double x[3] = { origin[0] + lattice[0] * dis(gen), + origin[1] + lattice[1] * dis(gen), + origin[2] + lattice[2] * dis(gen) }; + if (myrank == 0) + std::cout << "x,y,z = " << x[0] << ", " << x[1] << ", " << x[2] + << std::endl; + + // set all x to the values of PE0 + MPI_Bcast(&x[0], 3, MPI_DOUBLE, 0, comm); + + // make a name for atom based on species and order of reading in + std::string stri = std::to_string(i); + std::string aname("C" + stri); + + ions.addIonToList(sp, aname, &x[0], velocity, false); + } + + ions.setup(); + + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + return 1; + } + + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + return 1; + } + + return 0; +} From 23db1eeb5e76bdda3cc20b181e6ac5839205d008 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 15 Oct 2024 21:01:23 -0700 Subject: [PATCH 24/33] Make user defined options --- src/Control.cc | 8 ++++++++ src/md.cc | 17 +++++++---------- src/read_config.cc | 4 ++++ src/rom_Control.h | 6 ++++-- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index 2231813e..8e86f47b 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2056,6 +2056,8 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); rom_pri_option.librom_snapshot_freq = vm["ROM.offline.librom_snapshot_freq"].as(); + rom_pri_option.compare_md = vm["ROM.basis.compare_md"].as(); + rom_pri_option.num_orbbasis = vm["ROM.basis.number_of_orbital_basis"].as(); rom_pri_option.num_potbasis = vm["ROM.basis.number_of_potential_basis"].as(); } // onpe0 @@ -2105,6 +2107,12 @@ void Control::syncROMOptions() rom_pri_option.rom_stage = static_cast(rom_stage); rom_pri_option.variable = static_cast(rom_var); + mpirc = MPI_Bcast(&rom_pri_option.compare_md, 1, MPI_C_BOOL, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.num_orbbasis, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + mpirc = MPI_Bcast(&rom_pri_option.num_potbasis, 1, MPI_INT, 0, comm_global_); bcast_check(mpirc); } diff --git a/src/md.cc b/src/md.cc index 920ac1f3..314fd0dc 100644 --- a/src/md.cc +++ b/src/md.cc @@ -494,20 +494,17 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) #ifdef MGMOL_HAS_LIBROM // TODO: cleanup - int rdim = 39; - std::string basis_filename = "PinnedH2O_orbitals_basis_1_50"; - bool ROM_md = false; - - if (rdim > 0) + if (ct.getROMOptions().num_orbbasis > 0) { if (onpe0) { - os_ << "Projecting orbitals onto ROM subspaces" << std::endl; - os_ << "Loading ROM basis " << basis_filename << std::endl; - os_ << "ROM basis dimension = " << rdim << std::endl; + os_ << "Projecting orbitals onto ROM subspaces to compare " + << ((ct.getROMOptions().compare_md) ? "MD dynamics" : "force") << std::endl; + os_ << "Loading ROM basis " << ct.getROMOptions().basis_file << std::endl; + os_ << "ROM basis dimension = " << ct.getROMOptions().num_orbbasis << std::endl; } - project_orbital(basis_filename, rdim, **orbitals); - if (ROM_md) + project_orbital(ct.getROMOptions().basis_file, ct.getROMOptions().num_orbbasis, **orbitals); + if (ct.getROMOptions().compare_md) { force(**orbitals, ions); } diff --git a/src/read_config.cc b/src/read_config.cc index 40194678..e50977fb 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -430,6 +430,10 @@ void setupROMConfigOption(po::options_description &rom_cfg) "Frequency of saving libROM snapshot file at FOM simulation.") ("ROM.offline.variable", po::value()->default_value(""), "FOM variable to perform POD: either orbitals or potential.") + ("ROM.basis.compare_md", po::value()->default_value(false), + "Compare MD or single-step force.") + ("ROM.basis.number_of_orbital_basis", po::value()->default_value(-1), + "Number of orbital POD basis.") ("ROM.basis.number_of_potential_basis", po::value()->default_value(-1), "Number of potential POD basis to build Hartree potential ROM operator."); } diff --git a/src/rom_Control.h b/src/rom_Control.h index 8c3b4b2c..dddbc1b4 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -43,13 +43,15 @@ struct ROMPrivateOptions int restart_file_minidx = -1; int restart_file_maxidx = -1; std::string basis_file = ""; - ROMVariable variable=ROMVariable::NONE; + ROMVariable variable = ROMVariable::NONE; - /* save librom snapshot matrix at FOM simulation. */ + /* save librom orbital snapshot matrix at FOM simulation. */ bool save_librom_snapshot = false; int librom_snapshot_freq = -1; /* options for ROM building */ + bool compare_md = false; + int num_orbbasis = -1; int num_potbasis = -1; }; From a27efbab7c8cf4e0cccb6c837abbe80e93c6f3d9 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 15 Oct 2024 21:07:00 -0700 Subject: [PATCH 25/33] Add scripts --- .../PinnedH2O/{job.basis => job.basis_1_50} | 4 +- examples/PinnedH2O/{job.rom => job.offline} | 8 ++-- examples/PinnedH2O/job.rom_1_50 | 39 ++++++++++++++++++ .../{mgmol_rom.cfg => mgmol_offline.cfg} | 2 +- examples/PinnedH2O/mgmol_rom_1_50.cfg | 40 +++++++++++++++++++ 5 files changed, 86 insertions(+), 7 deletions(-) rename examples/PinnedH2O/{job.basis => job.basis_1_50} (88%) rename examples/PinnedH2O/{job.rom => job.offline} (73%) create mode 100644 examples/PinnedH2O/job.rom_1_50 rename examples/PinnedH2O/{mgmol_rom.cfg => mgmol_offline.cfg} (97%) create mode 100644 examples/PinnedH2O/mgmol_rom_1_50.cfg diff --git a/examples/PinnedH2O/job.basis b/examples/PinnedH2O/job.basis_1_50 similarity index 88% rename from examples/PinnedH2O/job.basis rename to examples/PinnedH2O/job.basis_1_50 index bf02a621..a5069570 100644 --- a/examples/PinnedH2O/job.basis +++ b/examples/PinnedH2O/job.basis_1_50 @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20240919 +set maindir = /p/lustre2/cheung26/mgmol-20241012 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -27,6 +27,6 @@ echo "Snapshot files: $snapshot_files" set basis_file = "PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps}" -srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_Pinned_H2O.out +srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_PinnedH2O.out date diff --git a/examples/PinnedH2O/job.rom b/examples/PinnedH2O/job.offline similarity index 73% rename from examples/PinnedH2O/job.rom rename to examples/PinnedH2O/job.offline index 8de3d6b7..0f8ef90e 100644 --- a/examples/PinnedH2O/job.rom +++ b/examples/PinnedH2O/job.offline @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20240815 +set maindir = /p/lustre2/cheung26/mgmol-20241012 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -20,8 +20,8 @@ cp $maindir/install_quartz/bin/$exe . set datadir = $maindir/examples/PinnedH2O -set cfg_rom = mgmol_rom.cfg -cp $datadir/$cfg_rom . +set cfg_offline = mgmol_offline.cfg +cp $datadir/$cfg_offline . cp $datadir/coords.in . @@ -30,6 +30,6 @@ ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . source $maindir/scripts/modules.quartz -srun -n $ncpus $exe -c $cfg_rom -i coords.in > rom_PinnedH2O.out +srun -n $ncpus $exe -c $cfg_offline -i coords.in > offline_PinnedH2O.out date diff --git a/examples/PinnedH2O/job.rom_1_50 b/examples/PinnedH2O/job.rom_1_50 new file mode 100644 index 00000000..aa6df335 --- /dev/null +++ b/examples/PinnedH2O/job.rom_1_50 @@ -0,0 +1,39 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 1:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20241012 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set datadir = $maindir/examples/PinnedH2O + +set increment_md_steps = 1 +set num_md_steps = 50 +set basis_file = PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps} + +set cfg_rom = mgmol_rom_${increment_md_steps}_${num_md_steps}.cfg +cp $datadir/$cfg_rom . + +cp $datadir/coords.in . + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg_rom -i coords.in > rom_${increment_md_steps}_${num_md_steps}_PinnedH2O.out + +date diff --git a/examples/PinnedH2O/mgmol_rom.cfg b/examples/PinnedH2O/mgmol_offline.cfg similarity index 97% rename from examples/PinnedH2O/mgmol_rom.cfg rename to examples/PinnedH2O/mgmol_offline.cfg index 1f724546..332dd77d 100644 --- a/examples/PinnedH2O/mgmol_rom.cfg +++ b/examples/PinnedH2O/mgmol_offline.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=MD [MD] -num_steps=50 +num_steps=500 dt=40. thermostat=ON [Thermostat] diff --git a/examples/PinnedH2O/mgmol_rom_1_50.cfg b/examples/PinnedH2O/mgmol_rom_1_50.cfg new file mode 100644 index 00000000..337a86eb --- /dev/null +++ b/examples/PinnedH2O/mgmol_rom_1_50.cfg @@ -0,0 +1,40 @@ +verbosity=1 +xcFunctional=PBE +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=1000. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 +[ROM.offline] +basis_file=PinnedH2O_orbitals_basis_1_50 +[ROM.basis] +compare_md=false +number_of_orbital_basis=39 From f2f1821485b731fd1a6daa04c7f00d010c1de4e8 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Tue, 22 Oct 2024 13:54:06 -0700 Subject: [PATCH 26/33] Ion density sampled evaluation (#274) * save/load poisson rom operator. * test routine for ion density * ion density hyper-reduction verified. * added rom ion test in ci workflow. * fixed ci workflow * setting ion configuration using Ions::setPositions. --- .github/workflows/ci.yml | 6 +- src/Control.cc | 6 + src/Electrostatic.h | 3 + src/MGmol.h | 1 + src/Potentials.cc | 91 ++++++++ src/Potentials.h | 7 + src/read_config.cc | 4 +- src/rom_Control.h | 3 + src/rom_main.cc | 14 ++ src/rom_workflows.cc | 216 +++++++++++++++++- src/rom_workflows.h | 6 + tests/ROM/test_rom_poisson/carbyne.ion.cfg | 63 +++++ .../{carbyne.cfg => carbyne.poisson.cfg} | 0 13 files changed, 417 insertions(+), 3 deletions(-) create mode 100644 tests/ROM/test_rom_poisson/carbyne.ion.cfg rename tests/ROM/test_rom_poisson/{carbyne.cfg => carbyne.poisson.cfg} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e94113f..43953585 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,11 @@ jobs: cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson ln -s ${GITHUB_WORKSPACE}/build/src/mgmol-rom . ln -s ${GITHUB_WORKSPACE}/potentials/* . - mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.cfg -i carbyne.in + mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.poisson.cfg -i carbyne.in + - name: test ROM ion density evaluation + run: | + cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson + mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.ion.cfg -i carbyne.in # code-style: # runs-on: ubuntu-latest # needs: [docker-image] diff --git a/src/Control.cc b/src/Control.cc index f1dcef23..bd8f5ced 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2034,10 +2034,14 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.rom_stage = ROMStage::ONLINE; else if (str.compare("build") == 0) rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("online_poisson") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE_POISSON; else if (str.compare("test_poisson") == 0) rom_pri_option.rom_stage = ROMStage::TEST_POISSON; else if (str.compare("test_rho") == 0) rom_pri_option.rom_stage = ROMStage::TEST_RHO; + else if (str.compare("test_ion") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_ION; else if (str.compare("none") == 0) rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; @@ -2058,6 +2062,7 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.librom_snapshot_freq = vm["ROM.offline.librom_snapshot_freq"].as(); rom_pri_option.num_potbasis = vm["ROM.basis.number_of_potential_basis"].as(); + rom_pri_option.pot_rom_file = vm["ROM.potential_rom_file"].as(); } // onpe0 // synchronize all processors @@ -2073,6 +2078,7 @@ void Control::syncROMOptions() mmpi.bcast(rom_pri_option.restart_file_fmt, comm_global_); mmpi.bcast(rom_pri_option.basis_file, comm_global_); + mmpi.bcast(rom_pri_option.pot_rom_file, comm_global_); auto bcast_check = [](int mpirc) { if (mpirc != MPI_SUCCESS) diff --git a/src/Electrostatic.h b/src/Electrostatic.h index 7ae977e1..a8eb4709 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -47,6 +47,9 @@ class Electrostatic ~Electrostatic(); static Timer solve_tm() { return solve_tm_; } + const bool isDielectric() { return diel_flag_; } + pb::GridFunc* getRhoc() { return grhoc_; } + Poisson* getPoissonSolver() { return poisson_solver_; } void setup(const short max_sweeps); diff --git a/src/MGmol.h b/src/MGmol.h index a9bca4ba..a6c383f3 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -187,6 +187,7 @@ class MGmol : public MGmolInterface return hamiltonian_; } std::shared_ptr> getRho() { return rho_; } + std::shared_ptr getIons() { return ions_; } void run() override; diff --git a/src/Potentials.cc b/src/Potentials.cc index 380f05a0..a54f0c47 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -899,6 +899,97 @@ void Potentials::initBackground(Ions& ions) if (fabs(background_charge_) < 1.e-10) background_charge_ = 0.; } +void Potentials::evalIonDensityOnSamplePts( + Ions& ions, const std::vector &local_idx, std::vector &sampled_rhoc) +{ + Mesh* mymesh = Mesh::instance(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const pb::Grid& mygrid = mymesh->grid(); + + const char flag_filter = pot_type(0); + if (flag_filter == 's') + { + if (onpe0) + { + cout << "Potentials::evalIonDensityOnSamplePts - flag_filter s is not supported" + << endl; + } + mmpi.abort(); + } + + // initialize output vector + sampled_rhoc.resize(local_idx.size()); + for (int d = 0; d < sampled_rhoc.size(); d++) + sampled_rhoc[d] = 0.0; + + // Loop over ions + for (auto& ion : ions.overlappingVL_ions()) + { + const Species& sp(ion->getSpecies()); + + Vector3D position(ion->position(0), ion->position(1), ion->position(2)); + + initializeRadialDataOnSampledPts(position, sp, local_idx, sampled_rhoc); + } + + return; +} + +void Potentials::initializeRadialDataOnSampledPts( + const Vector3D& position, const Species& sp, const std::vector &local_idx, std::vector &sampled_rhoc) +{ + assert(local_idx.size() == sampled_rhoc.size()); + + Control& ct = *(Control::instance()); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + const int dim0 = mygrid.dim(0); + const int dim1 = mygrid.dim(1); + const int dim2 = mygrid.dim(2); + + const double start0 = mygrid.start(0); + const double start1 = mygrid.start(1); + const double start2 = mygrid.start(2); + + const double h0 = mygrid.hgrid(0); + const double h1 = mygrid.hgrid(1); + const double h2 = mygrid.hgrid(2); + + Vector3D point(0., 0., 0.); + + const double lrad = sp.lradius(); + + const RadialInter& lpot(sp.local_pot()); + const Vector3D lattice(mygrid.ll(0), mygrid.ll(1), mygrid.ll(2)); + + for(int k = 0; k < local_idx.size(); k++) + { + /* + local_idx provides offset. + offset = iz + iy * dim2 + ix * dim1 * dim2; + evaluate x,y,z indices backward. + */ + int iz = local_idx[k] % dim2; + int ix = local_idx[k] / dim2; + int iy = ix % dim1; + ix /= dim1; + + /* compute grid point position */ + point[0] = start0 + ix * h0; + point[1] = start1 + iy * h1; + point[2] = start2 + iz * h2; + + /* accumulate ion species density */ + const double r = position.minimage(point, lattice, ct.bcPoisson); + if (r < lrad) + sampled_rhoc[k] += sp.getRhoComp(r); + } + + return; +} + template void Potentials::setVxc( const double* const vxc, const int iterativeIndex); template void Potentials::setVxc( diff --git a/src/Potentials.h b/src/Potentials.h index 205fb538..0ddd10fa 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -95,6 +95,9 @@ class Potentials void initializeSupersampledRadialDataOnMesh( const Vector3D& position, const Species& sp); + void initializeRadialDataOnSampledPts( + const Vector3D& position, const Species& sp, const std::vector &local_idx, std::vector &sampled_rhoc); + public: Potentials(const bool vh_frozen = false); @@ -159,6 +162,8 @@ class Potentials double getChargeInCell() const { return charge_in_cell_; } + const double getBackgroundCharge() const { return background_charge_; } + /*! * initialize total potential as local pseudopotential */ @@ -196,6 +201,8 @@ class Potentials void initBackground(Ions& ions); void addBackgroundToRhoComp(); + void evalIonDensityOnSamplePts(Ions& ions, const std::vector &local_idx, std::vector &sampled_rhoc); + #ifdef HAVE_TRICUBIC void readExternalPot(const string filename, const char type); void setupVextTricubic(); diff --git a/src/read_config.cc b/src/read_config.cc index 40194678..4f3b34c1 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -431,6 +431,8 @@ void setupROMConfigOption(po::options_description &rom_cfg) ("ROM.offline.variable", po::value()->default_value(""), "FOM variable to perform POD: either orbitals or potential.") ("ROM.basis.number_of_potential_basis", po::value()->default_value(-1), - "Number of potential POD basis to build Hartree potential ROM operator."); + "Number of potential POD basis to build Hartree potential ROM operator.") + ("ROM.potential_rom_file", po::value()->default_value(""), + "File name to save/load potential ROM operators."); } #endif diff --git a/src/rom_Control.h b/src/rom_Control.h index 8c3b4b2c..8218bbb4 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -22,8 +22,10 @@ enum class ROMStage ONLINE, RESTORE, // TODO(kevin): what stage is this? BUILD, + ONLINE_POISSON, TEST_POISSON, TEST_RHO, + TEST_ION, UNSUPPORTED }; @@ -51,6 +53,7 @@ struct ROMPrivateOptions /* options for ROM building */ int num_potbasis = -1; + std::string pot_rom_file = ""; }; #endif // ROM_CONTROL_H diff --git a/src/rom_main.cc b/src/rom_main.cc index fa956221..7eb323a7 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -118,6 +118,13 @@ int main(int argc, char** argv) buildROMPoissonOperator(mgmol); break; + case (ROMStage::ONLINE_POISSON): + if (ct.isLocMode()) + runPoissonROM(mgmol); + else + runPoissonROM(mgmol); + break; + case (ROMStage::TEST_POISSON): if (ct.isLocMode()) testROMPoissonOperator(mgmol); @@ -130,6 +137,13 @@ int main(int argc, char** argv) testROMRhoOperator(mgmol); else testROMRhoOperator(mgmol); + + case (ROMStage::TEST_ION): + if (ct.isLocMode()) + testROMIonDensity(mgmol); + else + testROMIonDensity(mgmol); + break; default: diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 5dc3718e..52ad6e47 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -218,7 +218,7 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) // write the file from PE0 only if (MPIdata::onpe0) { - std::string rom_oper = "pot_rom_oper.h5"; + std::string rom_oper = rom_options.pot_rom_file; CAROM::HDFDatabase h5_helper; h5_helper.create(rom_oper); h5_helper.putInteger("number_of_potential_basis", num_pot_basis); @@ -242,6 +242,67 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) } } +template +void runPoissonROM(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + /* type of variable we intend to run POD */ + ROMVariable rom_var = rom_options.variable; + if (rom_var != ROMVariable::POTENTIAL) + { + std::cerr << "runPoissonROM error: ROM variable must be POTENTIAL to run this stage!\n" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + /* Load Hartree potential basis matrix */ + std::string basis_file = rom_options.basis_file; + const int num_pot_basis = rom_options.num_potbasis; + CAROM::BasisReader basis_reader(basis_file); + CAROM::Matrix *pot_basis = basis_reader.getSpatialBasis(num_pot_basis); + + /* initialize rom operator variables */ + CAROM::Matrix pot_rom(num_pot_basis, num_pot_basis, false); + CAROM::Matrix pot_rom_inv(num_pot_basis, num_pot_basis, false); + CAROM::Matrix pot_rhs_rom(num_pot_basis, num_pot_basis, false); + CAROM::Vector pot_rhs_rescaler(num_pot_basis, false); + + /* Load ROM operator */ + // read the file from PE0 only + if (MPIdata::onpe0) + { + std::string rom_oper = rom_options.pot_rom_file; + CAROM::HDFDatabase h5_helper; + h5_helper.open(rom_oper, "r"); + int num_pot_basis_file = -1; + h5_helper.getInteger("number_of_potential_basis", num_pot_basis_file); + CAROM_VERIFY(num_pot_basis_file == num_pot_basis); + + h5_helper.getDoubleArray("potential_rom_operator", pot_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* load the inverse as well */ + h5_helper.getDoubleArray("potential_rom_inverse", pot_rom_inv.getData(), + num_pot_basis * num_pot_basis, false); + + /* load right-hand side hyper-reduction operator */ + h5_helper.getDoubleArray("potential_rhs_rom_inverse", pot_rhs_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* load right-hand side rescaling operator */ + h5_helper.getDoubleArray("potential_rhs_rescaler", pot_rhs_rescaler.getData(), + num_pot_basis, false); + + h5_helper.close(); + } +} + /* test routines */ template @@ -721,14 +782,167 @@ void computeRhoOnSamplePts(const CAROM::Matrix &dm, // rescaleTotalCharge(); } +template +void testROMIonDensity(MGmolInterface *mgmol_) +{ + /* random number generator */ + static std::random_device rd; // Will be used to obtain a seed for the random number engine + static std::mt19937 gen(rd()); // Standard mersenne_twister_engine seeded with rd(){} + static std::uniform_real_distribution<> dis(0.0, 1.0); + + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const int subdivx = mymesh->subdivx(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + + /* Load MGmol pointer and Potentials */ + MGmol *mgmol = static_cast *>(mgmol_); + Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + const int dim = pot.size(); + std::shared_ptr ions = mgmol->getIons(); + + /* get the extent of global domain */ + const double origin[3] = { mygrid.origin(0), mygrid.origin(1), mygrid.origin(2) }; + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + if (rank == 0) + { + printf("origin: (%.3e, %.3e, %.3e)\n", origin[0], origin[1], origin[2]); + printf("lattice: (%.3e, %.3e, %.3e)\n", lattice[0], lattice[1], lattice[2]); + } + + /* get global atomic numbers */ + const int num_ions = ions->getNumIons(); + std::vector atnumbers(num_ions); + ions->getAtomicNumbers(atnumbers); + + assert(!(mgmol->electrostat_->isDielectric())); + assert(pot.getBackgroundCharge() <= 0.0); + + const int num_snap = 3; + + /* 3 fictitious ion configurations */ + std::vector> cfgs(num_snap); + for (int idx = 0; idx < num_snap; idx++) + { + cfgs[idx].resize(3 * num_ions); + if (rank == 0) + for (int k = 0; k < num_ions; k++) + for (int d = 0; d < 3; d++) + cfgs[idx][3 * k + d] = origin[d] + lattice[d] * dis(gen); + + mmpi.bcastGlobal(cfgs[idx].data(), 3 * num_ions, 0); + } + + /* Collect fictitious ion density based on each configuration */ + std::vector> fom_rhoc(num_snap); + /* Sanity check for overlappingVL_ions */ + std::vector>> fom_overlap_ions(num_snap); + for (int idx = 0; idx < num_snap; idx++) + { + /* set ion positions */ + ions->setPositions(cfgs[idx], atnumbers); + + /* save overlapping ions for sanity check */ + fom_overlap_ions[idx].resize(ions->overlappingVL_ions().size()); + for (int k = 0; k < ions->overlappingVL_ions().size(); k++) + { + fom_overlap_ions[idx][k].resize(3); + for (int d = 0; d < 3; d++) + fom_overlap_ions[idx][k][d] = ions->overlappingVL_ions()[k]->position(d); + } + + /* compute resulting ion density */ + /* NOTE: we exclude rescaling for the sake of verification */ + pot.initialize(*ions); + + mgmol->electrostat_->setupRhoc(pot.rho_comp()); + fom_rhoc[idx].resize(dim); + mgmol->electrostat_->getRhoc()->init_vect(fom_rhoc[idx].data(), 'd'); + } + + /* Initialize libROM classes */ + /* + In practice, we do not produce rhoc POD basis. + This rhoc POD basis is for the sake of verification. + */ + CAROM::Options svd_options(dim, num_snap, 1); + svd_options.static_svd_preserve_snapshot = true; + CAROM::BasisGenerator basis_generator(svd_options, false, "test_rhoc"); + for (int idx = 0; idx < num_snap; idx++) + basis_generator.takeSample(&fom_rhoc[idx][0]); + + const CAROM::Matrix rhoc_snapshots(*basis_generator.getSnapshotMatrix()); + basis_generator.endSamples(); + + const CAROM::Matrix *rhoc_basis = basis_generator.getSpatialBasis(); + CAROM::Matrix *proj_rhoc = rhoc_basis->transposeMult(rhoc_snapshots); + + /* DEIM hyperreduction */ + CAROM::Matrix rhoc_basis_inv(num_snap, num_snap, false); + std::vector global_sampled_row(num_snap), sampled_rows_per_proc(nprocs); + DEIM(rhoc_basis, num_snap, global_sampled_row, sampled_rows_per_proc, + rhoc_basis_inv, rank, nprocs); + if (rank == 0) + { + int num_sample_rows = 0; + for (int k = 0; k < sampled_rows_per_proc.size(); k++) + num_sample_rows += sampled_rows_per_proc[k]; + printf("number of sampled row: %d\n", num_sample_rows); + } + + /* get local sampled row */ + std::vector offsets, sampled_row(sampled_rows_per_proc[rank]); + int num_global_sample = CAROM::get_global_offsets(sampled_rows_per_proc[rank], offsets); + for (int s = 0, gs = offsets[rank]; gs < offsets[rank+1]; gs++, s++) + sampled_row[s] = global_sampled_row[gs]; + + /* test one solution */ + std::uniform_int_distribution<> distrib(0, num_snap-1); + int test_idx = distrib(gen); + mmpi.bcastGlobal(&test_idx); + if (rank == 0) printf("test index: %d\n", test_idx); + + /* set ion positions */ + ions->setPositions(cfgs[test_idx], atnumbers); + + /* Sanity check for overlapping ions */ + CAROM_VERIFY(fom_overlap_ions[test_idx].size() == ions->overlappingVL_ions().size()); + for (int k = 0; k < ions->overlappingVL_ions().size(); k++) + for (int d = 0; d < 3; d++) + CAROM_VERIFY(abs(fom_overlap_ions[test_idx][k][d] - ions->overlappingVL_ions()[k]->position(d)) < 1.0e-12); + + /* eval ion density on sample grid points */ + std::vector sampled_rhoc(sampled_row.size()); + pot.evalIonDensityOnSamplePts(*ions, sampled_row, sampled_rhoc); + + for (int d = 0; d < sampled_row.size(); d++) + { + printf("rank %d, fom rhoc[%d]: %.3e, rom rhoc: %.3e\n", rank, sampled_row[d], fom_rhoc[test_idx][sampled_row[d]], sampled_rhoc[d]); + CAROM_VERIFY(abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-12); + } +} + template void readRestartFiles(MGmolInterface *mgmol_); template void readRestartFiles(MGmolInterface *mgmol_); template void buildROMPoissonOperator(MGmolInterface *mgmol_); template void buildROMPoissonOperator(MGmolInterface *mgmol_); +template void runPoissonROM(MGmolInterface *mgmol_); +template void runPoissonROM(MGmolInterface *mgmol_); + template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); + +template void testROMIonDensity(MGmolInterface *mgmol_); +template void testROMIonDensity(MGmolInterface *mgmol_); diff --git a/src/rom_workflows.h b/src/rom_workflows.h index fd00c687..9222249f 100644 --- a/src/rom_workflows.h +++ b/src/rom_workflows.h @@ -45,12 +45,18 @@ void readRestartFiles(MGmolInterface *mgmol_); template void buildROMPoissonOperator(MGmolInterface *mgmol_); +template +void runPoissonROM(MGmolInterface *mgmol_); + template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); +template +void testROMIonDensity(MGmolInterface *mgmol_); + void computeRhoOnSamplePts(const CAROM::Matrix &dm, const CAROM::Matrix &phi_basis, const CAROM::Matrix &rom_phi, const std::vector &local_idx, CAROM::Vector &sampled_rho); diff --git a/tests/ROM/test_rom_poisson/carbyne.ion.cfg b/tests/ROM/test_rom_poisson/carbyne.ion.cfg new file mode 100644 index 00000000..d98232ad --- /dev/null +++ b/tests/ROM/test_rom_poisson/carbyne.ion.cfg @@ -0,0 +1,63 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Poisson] +FDtype=4th +#max_steps_initial=99 +#max_steps=99 +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +#type=QUENCH +type=MD +[Quench] +#solver=PR +max_steps=300 +atol=1.e-8 +[MD] +num_steps=3000 +dt=40. +print_interval=1 +[Orbitals] +initial_type=Fourier +nempty=10 +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 + +[Thermostat] +type=Langevin +temperature=300. +relax_time=1000. + +[Restart] +#input_filename=init_cond_144proc +#input_level=4 +output_level=4 +interval=1 + +[ROM] +stage=test_ion + +[ROM.offline] +restart_filefmt=case-300K/snapshot%05d +restart_min_idx=800 +restart_max_idx=1999 +basis_file=basis_300K_2/test_300K +variable=potential + diff --git a/tests/ROM/test_rom_poisson/carbyne.cfg b/tests/ROM/test_rom_poisson/carbyne.poisson.cfg similarity index 100% rename from tests/ROM/test_rom_poisson/carbyne.cfg rename to tests/ROM/test_rom_poisson/carbyne.poisson.cfg From bc5992dedefc33d3e556ec0831deefb4df099f67 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 31 Oct 2024 12:09:42 -0700 Subject: [PATCH 27/33] Add get_result.sh --- examples/PinnedH2O/get_result.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 examples/PinnedH2O/get_result.sh diff --git a/examples/PinnedH2O/get_result.sh b/examples/PinnedH2O/get_result.sh new file mode 100644 index 00000000..443e4cdd --- /dev/null +++ b/examples/PinnedH2O/get_result.sh @@ -0,0 +1,27 @@ +filename="offline_PinnedH2O.out" # FOM +#filename="rom39_PinnedH2O.out" # compare MD +#filename="39_force_PinnedH2O.out" # compare force + +# Extracting H1, H2, F1, F2 from MGmgol output log +# if FOM, these files contain the FOM results +# if compare MD, these files contain the results with projected orbitals +awk '/H1 / {print $3, $4, $5}' $filename > H1_$filename +awk '/H2 / {print $3, $4, $5}' $filename > H2_$filename +awk '/F1 / {print $6, $7, $8}' $filename > F1_$filename +awk '/F2 / {print $6, $7, $8}' $filename > F2_$filename + +# if compare force, files with "_fom" contain the FOM results +# files with "_rom" contain the results with projected orbitals +if [[ "$filename" == *"force_"* ]]; then + sed -n '1~2p' H1_$filename > H1_rom$filename + sed -n '1~2p' H2_$filename > H2_rom$filename + sed -n '1~2p' F1_$filename > F1_rom$filename + sed -n '1~2p' F2_$filename > F2_rom$filename + + sed -n '2~2p' H1_$filename > H1_fom$filename + sed -n '2~2p' H2_$filename > H2_fom$filename + sed -n '2~2p' F1_$filename > F1_fom$filename + sed -n '2~2p' F2_$filename > F2_fom$filename +fi + +rm -rf snapshot0_* From da6e19a03b2b9a860b6adaa7ff1b333efa4696bd Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 31 Oct 2024 12:25:00 -0700 Subject: [PATCH 28/33] Add visualization scripts --- examples/PinnedH2O/plot_PinnedH2O_force.m | 84 +++++++++++++++++++++++ examples/PinnedH2O/plot_PinnedH2O_md.m | 59 ++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 examples/PinnedH2O/plot_PinnedH2O_force.m create mode 100644 examples/PinnedH2O/plot_PinnedH2O_md.m diff --git a/examples/PinnedH2O/plot_PinnedH2O_force.m b/examples/PinnedH2O/plot_PinnedH2O_force.m new file mode 100644 index 00000000..a41457a5 --- /dev/null +++ b/examples/PinnedH2O/plot_PinnedH2O_force.m @@ -0,0 +1,84 @@ +clc; clear all; close all; + +%% +plot_fom = 0; +plot_rom = 0; +rdim = 77; + +%% +load F_fom.mat +fprintf(1, 'Force statistics using FOM orbitals\n'); +fprintf(1, 'Mean of force on H1: %6.4e, %6.4e, %6.4e\n', mean(F1_fom)); +fprintf(1, 'Variance of force on H1: %6.4e, %6.4e, %6.4e\n', var(F1_fom)); +fprintf(1, 'Mean of force on H2: %6.4e, %6.4e, %6.4e\n', mean(F2_fom)); +fprintf(1, 'Variance of force on H2: %6.4e, %6.4e, %6.4e\n', var(F2_fom)); + +if plot_fom + plotForce(F1_fom, 'F_H1_fom'); + plotForce(F2_fom, 'F_H2_fom'); + plotForceHistograms(F1_fom, 'H1_fom'); + plotForceHistograms(F2_fom, 'H2_fom'); +end + +%% +load(['F_rom' int2str(rdim) '.mat']) +fprintf(1, 'Force statistics using projected orbitals\n'); +fprintf(1, 'Mean of force on H1: %6.4e, %6.4e, %6.4e\n', mean(F1_rom)); +fprintf(1, 'Variance of force on H1: %6.4e, %6.4e, %6.4e\n', var(F1_rom)); +%H1_correlation = sum(F1_fom(:) .* F1_rom(:)) / (norm(F1_fom(:)) * norm(F1_rom(:))) +fprintf(1, 'Mean of force on H2: %6.4e, %6.4e, %6.4e\n', mean(F2_rom)); +fprintf(1, 'Variance of force on H2: %6.4e, %6.4e, %6.4e\n', var(F2_rom)); +%H2_correlation = sum(F2_fom(:) .* F2_rom(:)) / (norm(F2_fom(:)) * norm(F2_rom(:))) + +if plot_rom + plotForce(F1_rom, ['F_H1_rom' int2str(rdim)]); + plotForce(F2_rom, ['F_H2_rom' int2str(rdim)]); + plotForceHistograms(F1_rom, ['H1_rom' int2str(rdim)]); + plotForceHistograms(F2_rom, ['H2_rom' int2str(rdim)]); + plotForceHistogram(abs(F1_fom - F1_rom), ['H1_rom' int2str(rdim)], 'Fdiff'); + plotForceHistogram(abs(F2_fom - F2_rom), ['H2_rom' int2str(rdim)], 'Fdiff'); +end + +%% +function plotForce(F, suffix) + figure; + imagesc(F'); + axis tight; + axis equal; + colorbar; + saveas(gcf, suffix, 'jpg'); +end + +function plotForceHistogram(F, suffix, var) + figure; + if strcmp(var,'Fx') + X = F(:,1); + var_name = 'x-directional Force'; + elseif strcmp(var,'Fy') + X = F(:,2); + var_name = 'y-directional Force'; + elseif strcmp(var,'Fz') + X = F(:,3); + var_name = 'z-directional Force'; + elseif strcmp(var,'Fmag') + X = sqrt(sum(F.^2, 2)); + var_name = 'Force Magitude'; + elseif strcmp(var,'Fdiff') + X = sqrt(sum(F.^2, 2)); + var_name = 'Magitude of Difference in Force'; + else + error('Invalid type'); + end + histogram(X, 20); + xlabel(var_name); + ylabel('Frequency'); + title(['Histogram of ' var_name]); + saveas(gcf, [var '_' suffix], 'jpg'); +end + +function plotForceHistograms(F, suffix) + plotForceHistogram(F, suffix, 'Fx'); + plotForceHistogram(F, suffix, 'Fy'); + plotForceHistogram(F, suffix, 'Fz'); + plotForceHistogram(F, suffix, 'Fmag'); +end diff --git a/examples/PinnedH2O/plot_PinnedH2O_md.m b/examples/PinnedH2O/plot_PinnedH2O_md.m new file mode 100644 index 00000000..47c309b2 --- /dev/null +++ b/examples/PinnedH2O/plot_PinnedH2O_md.m @@ -0,0 +1,59 @@ +clc; clear all; close all; + +%% +plot_fom = 0; +rdims = [77, 39]; + +%% +load md_fom.mat +if plot_fom + plotAngleHistogram(H1_fom, H2_fom, 'fom'); +end + +%% + +all_H1_rom = zeros(length(rdims), size(H1_fom, 1), 3); +all_H2_rom = zeros(length(rdims), size(H2_fom, 1), 3); +k = 0; + +for rdim = rdims + k = k + 1; + load(['md_rom' int2str(rdim) '.mat']) + plotAngleHistogram(H1_rom, H2_rom, ['rom' int2str(rdim)]); + all_H1_rom(k, :, :) = H1_rom; + all_H2_rom(k, :, :) = H2_rom; +end + +plotAtomTrajectory(H1_fom(:,1), all_H1_rom(:,:,1), rdims, 'x', 1) +plotAtomTrajectory(H1_fom(:,2), all_H1_rom(:,:,2), rdims, 'y', 1) +plotAtomTrajectory(H1_fom(:,3), all_H1_rom(:,:,3), rdims, 'z', 1) + +plotAtomTrajectory(H2_fom(:,1), all_H2_rom(:,:,1), rdims, 'x', 2) +plotAtomTrajectory(H2_fom(:,2), all_H2_rom(:,:,2), rdims, 'y', 2) +plotAtomTrajectory(H2_fom(:,3), all_H2_rom(:,:,3), rdims, 'z', 2) + +%% +function plotAtomTrajectory(X_fom, all_X_rom, rdims, var, idx) + figure; + hold on; + plot(X_fom, 'Linewidth', 2, 'DisplayName', 'FOM'); + k = 0; + for rdim = rdims + k = k + 1; + X_rom = all_X_rom(k, :); + plot(X_rom, 'Linewidth', 2, 'DisplayName', ['ROM dim = ' int2str(rdim)]); + end + title([var '-coordinate of H' int2str(idx)]) + legend; + saveas(gcf, [var '_H' int2str(idx)], 'jpg'); +end + +function plotAngleHistogram(X1, X2, suffix) + figure; + A = acosd(sum(X1.*X2,2) ./ sqrt(sum(X1.^2,2)) ./ sqrt(sum(X2.^2,2))); + histogram(A, 20); + xlabel('Angle'); + ylabel('Frequency'); + title('Histogram of angle'); + saveas(gcf, ['angle_' suffix], 'jpg'); +end \ No newline at end of file From bcdd6a11269c427feeb6f4c328c89cd66ffefd35 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 31 Oct 2024 12:27:13 -0700 Subject: [PATCH 29/33] Add Python script for plane action --- examples/PinnedH2O/generate_coord.py | 63 ++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 examples/PinnedH2O/generate_coord.py diff --git a/examples/PinnedH2O/generate_coord.py b/examples/PinnedH2O/generate_coord.py new file mode 100644 index 00000000..c239eb87 --- /dev/null +++ b/examples/PinnedH2O/generate_coord.py @@ -0,0 +1,63 @@ +import numpy as np +import os + +# coords.in +O1 = np.array([0.00, 0.00, 0.00]) +ref_H1 = np.array([-0.45, 1.42, -1.07]) +ref_H2 = np.array([-0.45, -1.48, -0.97]) + +# factors and increments for bond lengths and bond angle +bondlength1_factor = np.linspace(0.95, 1.05, 11) +bondlength2_factor = np.linspace(0.95, 1.05, 11) +bondangle_increment = np.linspace(-5, 5, 11) + +# output directory +output_dir = "PinnedH2O_3dof_coords" + +# utilities +def calculate_bondlength(atom1, atom2): + return np.linalg.norm(atom1 - atom2) + +def calculate_bondangle(atom1, atom2, atom3): + vector1 = atom1 - atom2 + vector2 = atom3 - atom2 + dot_product = np.dot(vector1, vector2) + magnitude_product = np.linalg.norm(vector1) * np.linalg.norm(vector2) + angle_rad = np.arccos(dot_product / magnitude_product) + angle_deg = np.degrees(angle_rad) + return angle_deg + +# Rodrigues' rotation formula +def rotation_matrix(axis, angle_degrees): + angle = np.radians(angle_degrees) + cos_theta = np.cos(angle) + sin_theta = np.sin(angle) + ux, uy, uz = axis + return np.array([ + [cos_theta + ux**2 * (1 - cos_theta), ux * uy * (1 - cos_theta) - uz * sin_theta, ux * uz * (1 - cos_theta) + uy * sin_theta], + [uy * ux * (1 - cos_theta) + uz * sin_theta, cos_theta + uy**2 * (1 - cos_theta), uy * uz * (1 - cos_theta) - ux * sin_theta], + [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)] + ]) + +# generation +os.makedirs(output_dir, exist_ok=True) + +ref_bondlength1 = calculate_bondlength(ref_H1, O1) +ref_bondlength2 = calculate_bondlength(ref_H2, O1) +ref_bondangle = calculate_bondangle(ref_H1, O1, ref_H2) + +normal_vector = np.cross(ref_H1, ref_H2) +normal_unit_vector = normal_vector / np.linalg.norm(normal_vector) + +for d_bondangle in bondangle_increment: + Q = rotation_matrix(normal_unit_vector, d_bondangle) + Q_ref_H2 = np.dot(Q, ref_H2) + for f_bondlength1 in bondlength1_factor: + for f_bondlength2 in bondlength2_factor: + H1 = f_bondlength1 * ref_H1 + H2 = f_bondlength2 * Q_ref_H2 + filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in" + with open(filename, "w") as file: + file.write(f"O1 1 {O1[0]:.2f} {O1[1]:.2f} {O1[2]:.2f} 0\n") + file.write(f"H1 2 {H1[0]:.2f} {H1[1]:.2f} {H1[2]:.2f} 1\n") + file.write(f"H2 2 {H2[0]:.2f} {H2[1]:.2f} {H2[2]:.2f} 1\n") From a733973bc917d0fd02a6d0d56dee9179e039374f Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 5 Nov 2024 10:14:51 -0800 Subject: [PATCH 30/33] Add new files --- examples/PinnedH2O/generate_coord_simple.py | 32 +++++++++++ examples/PinnedH2O/rotation_test.py | 62 +++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 examples/PinnedH2O/generate_coord_simple.py create mode 100644 examples/PinnedH2O/rotation_test.py diff --git a/examples/PinnedH2O/generate_coord_simple.py b/examples/PinnedH2O/generate_coord_simple.py new file mode 100644 index 00000000..181691a5 --- /dev/null +++ b/examples/PinnedH2O/generate_coord_simple.py @@ -0,0 +1,32 @@ +import numpy as np +import os + +O1 = np.array([0.00, 0.00, 0.00]) + +ref_bondlength = 1.83 +ref_bondangle = 104.5 + +# factors and increments for bond lengths and bond angle +bondlength1_factor = np.linspace(0.95, 1.05, 11) +bondlength2_factor = np.linspace(0.95, 1.05, 11) +bondangle_increment = np.linspace(-5, 5, 11) + +# output directory +output_dir = "PinnedH2O_3dof_coords" + +# generation +os.makedirs(output_dir, exist_ok=True) + +for d_bondangle in bondangle_increment: + bondangle = ref_bondangle + d_bondangle + x = ref_bondlength * np.cos(np.radians(bondangle / 2)) + y = ref_bondlength * np.cos(np.radians(bondangle / 2)) + for f_bondlength1 in bondlength1_factor: + for f_bondlength2 in bondlength2_factor: + H1 = np.array([f_bondlength1*x, f_bondlength1*y, 0.0]) + H2 = np.array([f_bondlength2*x, -f_bondlength2*y, 0.0]) + filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in" + with open(filename, "w") as file: + file.write(f"O1 1 {O1[0]:.2f} {O1[1]:.2f} {O1[2]:.2f} 0\n") + file.write(f"H1 2 {H1[0]:.2f} {H1[1]:.2f} {H1[2]:.2f} 1\n") + file.write(f"H2 2 {H2[0]:.2f} {H2[1]:.2f} {H2[2]:.2f} 1\n") diff --git a/examples/PinnedH2O/rotation_test.py b/examples/PinnedH2O/rotation_test.py new file mode 100644 index 00000000..7f74c55b --- /dev/null +++ b/examples/PinnedH2O/rotation_test.py @@ -0,0 +1,62 @@ +import numpy as np + +O1 = np.array([0.00, 0.00, 0.00]) +H1 = np.array([-0.45, 1.42, -1.07]) +H2 = np.array([-0.45, -1.48, -0.97]) + +def calculate_bondlength(atom1, atom2): + return np.linalg.norm(atom1 - atom2) + +def calculate_bondangle(atom1, atom2, atom3, radian): + vector1 = atom1 - atom2 + vector2 = atom3 - atom2 + dot_product = np.dot(vector1, vector2) + magnitude_product = np.linalg.norm(vector1) * np.linalg.norm(vector2) + angle = np.arccos(dot_product / magnitude_product) + if not radian: + angle = np.degrees(angle) + return angle + +def rotation_matrix(axis, angle): + cos_theta = np.cos(angle) + sin_theta = np.sin(angle) + ux, uy, uz = axis + return np.array([ + [cos_theta + ux**2 * (1 - cos_theta), ux * uy * (1 - cos_theta) - uz * sin_theta, ux * uz * (1 - cos_theta) + uy * sin_theta], + [uy * ux * (1 - cos_theta) + uz * sin_theta, cos_theta + uy**2 * (1 - cos_theta), uy * uz * (1 - cos_theta) - ux * sin_theta], + [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)] + ]) + +plane_normal = np.cross(H1, H2) +plane_normal = plane_normal / np.linalg.norm(plane_normal) + +target_plane_normal = np.array([0, 0, 1]) +axis_to_align = np.cross(plane_normal, target_plane_normal) +axis_to_align /= np.linalg.norm(axis_to_align) +angle_to_align = np.arccos(np.clip(np.dot(plane_normal, target_plane_normal), -1.0, 1.0)) + +rot_matrix_align_plane = rotation_matrix(axis_to_align, angle_to_align) +H1_rotated = np.dot(rot_matrix_align_plane, H1) +H2_rotated = np.dot(rot_matrix_align_plane, H2) + +bondlength1 = calculate_bondlength(H1, O1) +bondlength2 = calculate_bondlength(H2, O1) +bondangle = calculate_bondangle(H1, O1, H2, False) + +print('Original system') +print(f'H1 = {H1}') +print(f'H2 = {H2}') +print(f'Bondlength of O1-H1 = {bondlength1}') +print(f'Bondlength of O1-H2 = {bondlength2}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle}') + +bondlength1 = calculate_bondlength(H1_rotated, O1) +bondlength2 = calculate_bondlength(H2_rotated, O1) +bondangle = calculate_bondangle(H1_rotated, O1, H2_rotated, False) + +print('Aligned system') +print(f'H1 = {H1_rotated}') +print(f'H2 = {H2_rotated}') +print(f'Bondlength of O1-H1 = {bondlength1}') +print(f'Bondlength of O1-H2 = {bondlength2}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle}') From 3e5cf8fd8c405a25bce149e1cbdccfdcb9c88d6b Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 5 Nov 2024 13:18:46 -0800 Subject: [PATCH 31/33] Fix mistake --- examples/PinnedH2O/generate_coord_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/PinnedH2O/generate_coord_simple.py b/examples/PinnedH2O/generate_coord_simple.py index 181691a5..9c1ef06a 100644 --- a/examples/PinnedH2O/generate_coord_simple.py +++ b/examples/PinnedH2O/generate_coord_simple.py @@ -20,7 +20,7 @@ for d_bondangle in bondangle_increment: bondangle = ref_bondangle + d_bondangle x = ref_bondlength * np.cos(np.radians(bondangle / 2)) - y = ref_bondlength * np.cos(np.radians(bondangle / 2)) + y = ref_bondlength * np.sin(np.radians(bondangle / 2)) for f_bondlength1 in bondlength1_factor: for f_bondlength2 in bondlength2_factor: H1 = np.array([f_bondlength1*x, f_bondlength1*y, 0.0]) From 7982658b4e091366d68635383ed0faaeb617d6a5 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 5 Nov 2024 13:28:40 -0800 Subject: [PATCH 32/33] Minor fix to rotation orientation --- examples/PinnedH2O/rotation_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/PinnedH2O/rotation_test.py b/examples/PinnedH2O/rotation_test.py index 7f74c55b..63b81d4e 100644 --- a/examples/PinnedH2O/rotation_test.py +++ b/examples/PinnedH2O/rotation_test.py @@ -27,7 +27,7 @@ def rotation_matrix(axis, angle): [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)] ]) -plane_normal = np.cross(H1, H2) +plane_normal = np.cross(H2, H1) plane_normal = plane_normal / np.linalg.norm(plane_normal) target_plane_normal = np.array([0, 0, 1]) From 5ff4a66b9e161019fb284c69cbc83d9f0b73f7ec Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 14 Nov 2024 09:02:18 -0800 Subject: [PATCH 33/33] Make system unique --- examples/PinnedH2O/generate_coord_simple.py | 7 +++---- examples/PinnedH2O/rotation_test.py | 4 +++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/PinnedH2O/generate_coord_simple.py b/examples/PinnedH2O/generate_coord_simple.py index 9c1ef06a..7e799bdd 100644 --- a/examples/PinnedH2O/generate_coord_simple.py +++ b/examples/PinnedH2O/generate_coord_simple.py @@ -7,8 +7,7 @@ ref_bondangle = 104.5 # factors and increments for bond lengths and bond angle -bondlength1_factor = np.linspace(0.95, 1.05, 11) -bondlength2_factor = np.linspace(0.95, 1.05, 11) +bondlength_factor = np.linspace(0.95, 1.05, 11) bondangle_increment = np.linspace(-5, 5, 11) # output directory @@ -21,8 +20,8 @@ bondangle = ref_bondangle + d_bondangle x = ref_bondlength * np.cos(np.radians(bondangle / 2)) y = ref_bondlength * np.sin(np.radians(bondangle / 2)) - for f_bondlength1 in bondlength1_factor: - for f_bondlength2 in bondlength2_factor: + for i, f_bondlength1 in enumerate(bondlength_factor): + for f_bondlength2 in bondlength_factor[:(i+1)]: H1 = np.array([f_bondlength1*x, f_bondlength1*y, 0.0]) H2 = np.array([f_bondlength2*x, -f_bondlength2*y, 0.0]) filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in" diff --git a/examples/PinnedH2O/rotation_test.py b/examples/PinnedH2O/rotation_test.py index 63b81d4e..79f1aaef 100644 --- a/examples/PinnedH2O/rotation_test.py +++ b/examples/PinnedH2O/rotation_test.py @@ -53,8 +53,10 @@ def rotation_matrix(axis, angle): bondlength1 = calculate_bondlength(H1_rotated, O1) bondlength2 = calculate_bondlength(H2_rotated, O1) bondangle = calculate_bondangle(H1_rotated, O1, H2_rotated, False) +if bondlength1 < bondlength2: + H1_rotated, H2_rotated, bondlength1, bondlength2 = H2_rotated, H1_rotated, bondlength2, bondlength1 -print('Aligned system') +print('Rotated system in z=0 plane about x=0 axis, with longer bondlength in H1') print(f'H1 = {H1_rotated}') print(f'H2 = {H2_rotated}') print(f'Bondlength of O1-H1 = {bondlength1}')