Merge branch 'ROMFPMD' into merge-release

.github/workflows/ci.yml

@@ -0,0 +1,82 @@
+name: CI
+on:
+  workflow_dispatch: {}
+  pull_request:
+    types: [opened, labeled, synchronize]
+    branches:
+      - 'ROMFPMD'
+  # push:
+  #   branches:
+  #     - release
+
+jobs:
+  docker-image:
+    uses: ./.github/workflows/docker_image.yml
+  build:
+    runs-on: ubuntu-latest
+    needs: [docker-image]
+    container:
+      image: ghcr.io/llnl/mgmol/mgmol_env:latest
+      options: --user 1001 --privileged
+      volumes:
+        - /mnt:/mnt
+    steps:
+      - name: Cancel previous runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      # - name: Set Swap Space
+      #   uses: pierotofy/set-swap-space@master
+      #   with:
+      #     swap-size-gb: 10
+      - name: Check out mgmol
+        uses: actions/checkout@v1
+        with:
+            submodules: 'true'
+      - name: cmake
+        run: |
+            mkdir ${GITHUB_WORKSPACE}/build
+            cd ${GITHUB_WORKSPACE}/build
+            cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" -DUSE_LIBROM=On -DLIBROM_PATH=/env/dependencies/libROM
+      - name: make
+        run: |
+            cd ${GITHUB_WORKSPACE}/build && make -j 4
+      - name: test
+        run: |
+            cd ${GITHUB_WORKSPACE}/build && ctest --no-compress-output -V -T Test -I 1,20,1
+      - name: test ROM Poisson operator
+        run: |
+            cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson
+            ln -s ${GITHUB_WORKSPACE}/build/src/mgmol-rom .
+            ln -s ${GITHUB_WORKSPACE}/potentials/* .
+            mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.poisson.cfg -i carbyne.in
+      - name: test ROM ion density evaluation
+        run: |
+            cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson
+            mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.ion.cfg -i carbyne.in
+  # code-style:
+  #   runs-on: ubuntu-latest
+  #   needs: [docker-image]
+  #   container:
+  #     image: ghcr.io/llnl/mgmol/mgmol_env:latest
+  #     options: --user 1001 --privileged
+  #     volumes:
+  #       - /mnt:/mnt
+  #   steps:
+  #     - name: Cancel previous runs
+  #       uses: styfle/[email protected]
+  #       with:
+  #         access_token: ${{ github.token }}
+  #     - name: Check out mgmol
+  #       uses: actions/checkout@v1
+  #       with:
+  #           submodules: 'true'
+  #     - name: cmake
+  #       run: |
+  #           mkdir build
+  #           cd build
+  #           cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON
+  #     - name: make
+  #       run: |
+  #           cd build && make format
+

.github/workflows/docker_image.yml

@@ -0,0 +1,59 @@
+name: docker-image
+on:
+  workflow_call:
+
+env:
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: llnl/mgmol/mgmol_env
+  DOCKERPATH: docker
+
+jobs:
+  docker-ci:
+    runs-on: ubuntu-latest
+    name: "docker env"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: Ana06/[email protected]       
+        id: files
+      - name: DockerPATH configuration
+        run: echo "DOCKERPATH=$DOCKERPATH"
+      - name: DockerPATH - check if files in docker path changed
+        if: contains(steps.files.outputs.all,env.DOCKERPATH) || contains(steps.files.outputs.all,'docker_image.yml')
+        run: |
+          echo "CI container needs rebuilding..."
+          echo "CI_NEEDS_REBUILD=true" >> $GITHUB_ENV
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: env.CI_NEEDS_REBUILD
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        if: env.CI_NEEDS_REBUILD
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: type=sha
+          flavor: latest=true
+      - name: Build Container motd
+        if: env.CI_NEEDS_REBUILD
+        run: |
+          echo "#!/bin/bash" > ${{env.DOCKERPATH}}/motd.sh
+          echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh
+          echo "echo mgmol_env/CI Development Container"  >> ${{env.DOCKERPATH}}/motd.sh
+          echo "echo \"Revision: `echo ${GITHUB_SHA} | cut -c1-8`\"" >> ${{env.DOCKERPATH}}/motd.sh
+          echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh
+          chmod 755 ${{env.DOCKERPATH}}/motd.sh
+          cat ${{env.DOCKERPATH}}/motd.sh
+      - name: Docker Image - Build and push
+        if: env.CI_NEEDS_REBUILD
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ${{ env.DOCKERPATH }}
+          tags: ${{ steps.meta.outputs.tags }}

.gitignore

@@ -0,0 +1 @@
+src/mgmol_config.h

CMakeLists.txt

@@ -126,6 +126,22 @@ if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
   endif(${SCALAPACK_FOUND})  
 endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
 
+# libROM (optional)
+set(USE_LIBROM False CACHE BOOL "Build with libROM")
+set(LIBROM_PATH "" CACHE STRING "Path of libROM")
+if(USE_LIBROM)
+  message(STATUS "LIBROM_PATH: ${LIBROM_PATH}")
+  if(NOT LIBROM_PATH)
+    message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.")
+  endif(NOT LIBROM_PATH)
+
+  find_package(libROM REQUIRED)
+
+  if(libROM_FOUND)
+    set(MGMOL_HAS_LIBROM 1)
+  endif(libROM_FOUND)
+endif(USE_LIBROM)
+
 # ARPACK (optional)
 set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package")
 if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT)
@@ -249,6 +265,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra")
 include_directories("${PROJECT_SOURCE_DIR}/src/tools")
 include_directories("${PROJECT_SOURCE_DIR}/src")
 
+include_directories("${LIBROM_PATH}/lib")
+link_libraries(${LIBROM_LIB})
+
 # add subdirectories for source files, tests
 add_subdirectory(src)
 

cmake_modules/FindlibROM.cmake

@@ -0,0 +1,11 @@
+if(NOT LIBROM_PATH)
+    message(FATAL_ERROR "LIBROM_PATH not specified.")
+endif(NOT LIBROM_PATH)
+
+find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib")
+find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib")
+
+mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES)

cmake_toolchains/quartz.default.cmake

@@ -0,0 +1,6 @@
+set(CMAKE_C_COMPILER mpicc)
+set(CMAKE_CXX_COMPILER mpicxx)
+set(CMAKE_Fortran_COMPILER mpif90)
+
+set(SCALAPACK_ROOT $ENV{MKLROOT})
+set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so)

docker/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:22.04
+
+ENV ENVDIR=env
+
+# install sudo
+RUN apt-get -yq update && apt-get -yq install sudo
+
+WORKDIR /$ENVDIR
+
+# install packages
+RUN sudo apt-get install -yq git
+RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
+RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev
+RUN sudo apt-get install -yq libboost-all-dev
+RUN sudo apt-get install -yq vim
+RUN sudo apt-get install -yq git-lfs
+RUN sudo apt-get install -yq valgrind hdf5-tools
+RUN sudo apt-get install -yq wget
+### clang-format seems to be updated to 14.0. Not using it for now.
+# RUN sudo apt-get install -yq clang-format
+
+# install lldb and gdb for debugging
+RUN sudo apt-get install -yq lldb gdb
+
+RUN sudo apt-get clean -q
+
+ENV LIB_DIR=/$ENVDIR/dependencies
+WORKDIR $LIB_DIR
+
+# cmake toolchain file for librom
+RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\
+set(CMAKE_CXX_COMPILER mpicxx)\n\
+set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake
+ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake
+
+# install libROM for scaleupROM
+RUN sudo git clone https://github.com/LLNL/libROM.git
+WORKDIR ./libROM/build
+# libROM without MFEM.
+RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF
+RUN sudo make -j 16
+
+# create and switch to a user
+ENV USERNAME=test
+RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME
+RUN adduser $USERNAME sudo
+USER $USERNAME
+WORKDIR /home/$USERNAME
+

examples/Carbyne/carbyne.rom.cfg

@@ -0,0 +1,34 @@
+verbosity=2
+xcFunctional=PBE
+FDtype=4th
+[Mesh]
+nx= 96
+ny= 96
+nz= 192
+[Domain]
+ox= -10.
+oy= -10.
+oz= -20.
+lx= 20.
+ly= 20.
+lz= 40.
+[Potentials]
+pseudopotential=pseudo.H_ONCV_PBE_SG15
+pseudopotential=pseudo.C_ONCV_PBE_SG15
+[Run]
+type=QUENCH
+[Quench]
+max_steps=5
+atol=1.e-8
+[Orbitals]
+initial_type=Fourier
+[Restart]
+output_level=4
+input_level=4
+input_filename=snapshot0_000
+
+[ROM.offline]
+restart_filefmt=snapshot0_%03d
+restart_min_idx=0
+restart_max_idx=1
+basis_file=carom

examples/PinnedH2O/generate_coord.py

@@ -0,0 +1,63 @@
+import numpy as np
+import os
+
+# coords.in
+O1 = np.array([0.00, 0.00, 0.00])
+ref_H1 = np.array([-0.45, 1.42, -1.07])
+ref_H2 = np.array([-0.45, -1.48, -0.97])
+
+# factors and increments for bond lengths and bond angle
+bondlength1_factor = np.linspace(0.95, 1.05, 11)
+bondlength2_factor = np.linspace(0.95, 1.05, 11)
+bondangle_increment = np.linspace(-5, 5, 11)
+
+# output directory
+output_dir = "PinnedH2O_3dof_coords"
+
+# utilities
+def calculate_bondlength(atom1, atom2):
+    return np.linalg.norm(atom1 - atom2)
+
+def calculate_bondangle(atom1, atom2, atom3):
+    vector1 = atom1 - atom2
+    vector2 = atom3 - atom2
+    dot_product = np.dot(vector1, vector2)
+    magnitude_product = np.linalg.norm(vector1) * np.linalg.norm(vector2)
+    angle_rad = np.arccos(dot_product / magnitude_product)
+    angle_deg = np.degrees(angle_rad)
+    return angle_deg
+
+# Rodrigues' rotation formula
+def rotation_matrix(axis, angle_degrees):
+    angle = np.radians(angle_degrees)
+    cos_theta = np.cos(angle)
+    sin_theta = np.sin(angle)
+    ux, uy, uz = axis
+    return np.array([
+        [cos_theta + ux**2 * (1 - cos_theta), ux * uy * (1 - cos_theta) - uz * sin_theta, ux * uz * (1 - cos_theta) + uy * sin_theta],
+        [uy * ux * (1 - cos_theta) + uz * sin_theta, cos_theta + uy**2 * (1 - cos_theta), uy * uz * (1 - cos_theta) - ux * sin_theta],
+        [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)]
+    ])
+
+# generation
+os.makedirs(output_dir, exist_ok=True)
+
+ref_bondlength1 = calculate_bondlength(ref_H1, O1)
+ref_bondlength2 = calculate_bondlength(ref_H2, O1)
+ref_bondangle = calculate_bondangle(ref_H1, O1, ref_H2)
+
+normal_vector = np.cross(ref_H1, ref_H2)
+normal_unit_vector = normal_vector / np.linalg.norm(normal_vector)
+
+for d_bondangle in bondangle_increment:
+    Q = rotation_matrix(normal_unit_vector, d_bondangle)
+    Q_ref_H2 = np.dot(Q, ref_H2)
+    for f_bondlength1 in bondlength1_factor:
+        for f_bondlength2 in bondlength2_factor:
+            H1 = f_bondlength1 * ref_H1
+            H2 = f_bondlength2 * Q_ref_H2
+            filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in"
+            with open(filename, "w") as file:
+                file.write(f"O1  1   {O1[0]:.2f}    {O1[1]:.2f}    {O1[2]:.2f}  0\n")
+                file.write(f"H1  2   {H1[0]:.2f}    {H1[1]:.2f}    {H1[2]:.2f}  1\n")
+                file.write(f"H2  2   {H2[0]:.2f}    {H2[1]:.2f}    {H2[2]:.2f}  1\n")

examples/PinnedH2O/generate_coord_simple.py

@@ -0,0 +1,31 @@
+import numpy as np
+import os
+
+O1 = np.array([0.00, 0.00, 0.00])
+
+ref_bondlength = 1.83
+ref_bondangle = 104.5
+
+# factors and increments for bond lengths and bond angle
+bondlength_factor = np.linspace(0.95, 1.05, 11)
+bondangle_increment = np.linspace(-5, 5, 11)
+
+# output directory
+output_dir = "PinnedH2O_3dof_coords"
+
+# generation
+os.makedirs(output_dir, exist_ok=True)
+
+for d_bondangle in bondangle_increment:
+    bondangle = ref_bondangle + d_bondangle
+    x = ref_bondlength * np.cos(np.radians(bondangle / 2))
+    y = ref_bondlength * np.sin(np.radians(bondangle / 2))
+    for i, f_bondlength1 in enumerate(bondlength_factor):
+        for f_bondlength2 in bondlength_factor[:(i+1)]:
+            H1 = np.array([f_bondlength1*x, f_bondlength1*y, 0.0])
+            H2 = np.array([f_bondlength2*x, -f_bondlength2*y, 0.0])
+            filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in"
+            with open(filename, "w") as file:
+                file.write(f"O1  1   {O1[0]:.2f}    {O1[1]:.2f}    {O1[2]:.2f}  0\n")
+                file.write(f"H1  2   {H1[0]:.2f}    {H1[1]:.2f}    {H1[2]:.2f}  1\n")
+                file.write(f"H2  2   {H2[0]:.2f}    {H2[1]:.2f}    {H2[2]:.2f}  1\n")