MERGE CONFLICT

.gitattributes

@@ -1 +1,2 @@
 /tests/**/* linguist-generated=true
+/toolchain/mechanisms/* linguist-generated=true

.github/CODEOWNERS

@@ -13,3 +13,5 @@ CMakeLists.txt     @sbryngelson @henryleberre
 .vscode/           @sbryngelson @henryleberre
 .github/workflows/ @sbryngelson @henryleberre
 
+# Spencer H. Bryngelson & Ben Wilfong
+.typos.toml        @sbryngelson @wilfonba

.github/file-filter.yml

@@ -7,7 +7,7 @@ fortran_src: &fortran_src
 
 python_src: &python_src
   - '**/*.py'
-  - 'toolchain/requirements.txt'
+  - 'toolchain/pyproject.toml'
 
 cmakelist: &cmakelist
   - 'CMakeLists.txt'

.github/workflows/cleanliness.yml

@@ -0,0 +1,110 @@
+name: Cleanliness
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+    cleanliness:
+        name: Code Cleanliness Test
+        runs-on: "ubuntu-latest"
+        env:
+            pr_everything: 0
+            master_everything: 0
+        steps:
+            - name: Clone - PR
+              uses: actions/checkout@v3
+              with:
+                path: pr
+            - name: Clone - Master
+              uses: actions/checkout@v3
+              with:
+                repository: MFlowCode/MFC
+                ref: master
+                path: master
+
+            - name: Setup Ubuntu
+              run: |
+                sudo apt update -y
+                sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev
+            
+            - name: Build
+              run: |
+                (cd pr && /bin/bash mfc.sh build -j $(nproc) --debug 2> ../pr.txt) 
+                (cd master && /bin/bash mfc.sh build -j $(nproc) --debug 2> ../master.txt)
+                sed -i '/\/pr\//d' pr.txt
+                sed -i '/\/master\//d' master.txt
+            
+            - name: Unused Variables Diff
+              continue-on-error: true
+              run: |
+                grep -F 'Wunused-variable' master.txt > mUnused.txt
+                grep -F 'Wunused-variable' pr.txt > prUnused.txt
+                diff prUnused.txt mUnused.txt
+            
+            - name: Unused Dummy Arguments Diff
+              continue-on-error: true
+              run: |
+                grep -F 'Wunused-dummy-argument' pr.txt > prDummy.txt 
+                grep -F 'Wunused-dummy-argument' master.txt > mDummy.txt
+                diff prDummy.txt mDummy.txt
+
+            - name: Unused Value Diff
+              continue-on-error: true
+              run: |
+                grep -F 'Wunused-value' pr.txt > prUnused_val.txt
+                grep -F 'Wunused-value' master.txt > mUnused_val.txt
+                diff prUnused_val.txt mUnused_val.txt
+
+            - name: Maybe Uninitialized Variables Diff
+              continue-on-error: true
+              run: |
+                grep -F 'Wmaybe-uninitialized' pr.txt > prMaybe.txt
+                grep -F 'Wmaybe-uninitialized' master.txt > mMaybe.txt
+                diff prMaybe.txt mMaybe.txt
+
+
+            - name: Everything Diff
+              continue-on-error: true
+              run: |
+                grep '\-W' pr.txt > pr_every.txt
+                grep '\-W' master.txt > m_every.txt
+                diff pr_every.txt m_every.txt
+
+            - name: List of Warnings
+              run: |
+                cat pr_every.txt
+                
+
+            - name: Summary
+              continue-on-error: true
+              run: |  
+                pr_variable=$(grep -c -F 'Wunused-variable' pr.txt)
+                pr_argument=$(grep -c -F 'Wunused-dummy-argument' pr.txt)
+                pr_value=$(grep -c -F 'Wunused-value' pr.txt)
+                pr_uninit=$(grep -c -F 'Wmaybe-uninitialized' pr.txt)
+                pr_everything=$(grep -c '\-W' pr.txt)
+
+                master_variable=$(grep -c -F 'Wunused-variable' master.txt)
+                master_argument=$(grep -c -F 'Wunused-dummy-argument' master.txt)
+                master_value=$(grep -c -F 'Wunused-value' master.txt)
+                master_uninit=$(grep -c -F 'Wmaybe-uninitialized' master.txt)
+                master_everything=$(grep -c '\-W' master.txt  )    
+
+                echo "pr_everything=$pr_everything" >> $GITHUB_ENV
+                echo "master_everything=$master_everything" >> $GITHUB_ENV
+
+                echo "Difference is how many warnings were added or removed from master to PR."
+                echo "Negative numbers are better since you are removing warnings." 
+                echo " "
+                echo "Unused Variable Count: $pr_variable, Difference: $((pr_variable - master_variable))"
+                echo "Unused Dummy Argument: $pr_argument, Difference: $((pr_argument - master_argument))"
+                echo "Unused Value: $pr_value, Difference: $((pr_value - master_value))"
+                echo "Maybe Uninitialized: $pr_uninit, Difference: $((pr_uninit - master_uninit))"
+                echo "Everything: $pr_everything, Difference: $((pr_everything - master_everything))"
+            
+
+            - name: Check Differences
+              if: env.pr_everything > env.master_everything 
+              run: |
+                echo "Difference between warning count in PR is greater than in master."
+
+

.github/workflows/docs.yml

@@ -1,18 +1,12 @@
 name: Documentation
 
-on:
-  push:
-    branches:
-      - master
-
-  workflow_dispatch:
+on: [push, pull_request, workflow_dispatch]
 
 jobs:
   docs:
     name:    Build & Publish
     runs-on: ubuntu-latest
 
-    if: github.repository == 'MFlowCode/MFC'
     concurrency:
       group: docs-publish
       cancel-in-progress: true
@@ -53,6 +47,7 @@ jobs:
         echo "excluded-count = ${{ steps.sitemap.outputs.excluded-count }}"
 
     - name: Publish Documentation
+      if:   github.repository == 'MFlowCode/MFC' && github.ref == 'refs/heads/master' && github.event_name == 'push'
       run:  |
         set +e
         git ls-remote "${{ secrets.DOC_PUSH_URL }}" -q

.github/workflows/frontier/build.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 . ./mfc.sh load -c f -m g
-./mfc.sh build -j 8 --gpu --sys-hdf5 --sys-fftw
+./mfc.sh build -j 8 --gpu

.github/workflows/frontier/test.sh

@@ -3,5 +3,5 @@
 gpus=`rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' '`
 ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
 
-./mfc.sh test --max-attempts 3 -j $ngpus --sys-hdf5 --sys-fftw -- -c frontier
+./mfc.sh test --max-attempts 3 -j $ngpus -- -c frontier
 

.github/workflows/test.yml

@@ -48,23 +48,18 @@ jobs:
 
       - name: Setup MacOS
         if:   matrix.os == 'macos'
-        run: |
-          brew install wget make python make cmake coreutils gcc@14
-          echo "CC=gcc-14"      >> $GITHUB_ENV
-          echo "CXX=g++-14"     >> $GITHUB_ENV
+        run:  |
+          brew install coreutils python cmake fftw hdf5 gcc@14 boost open-mpi
           echo "FC=gfortran-14" >> $GITHUB_ENV
-          
-      - name: (MacOS) Build OpenMPI
-        if:   matrix.os == 'macos' && matrix.mpi == 'mpi'
-        run: |
-          brew install mpich
+          echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV
 
       - name: Setup Ubuntu
         if:   matrix.os == 'ubuntu' && matrix.intel == false
         run: |
            sudo apt update -y
-           sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev
-     
+           sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \
+                    libfftw3-dev libhdf5-dev openmpi-bin libopenmpi-dev
+
       - name: Setup Ubuntu (Intel)
         if:   matrix.os == 'ubuntu' && matrix.intel == true
         run: |
@@ -83,34 +78,22 @@ jobs:
           echo "OMPI_FC=$(which ifort)"     >> $GITHUB_ENV
           echo "OMPI_CXX=$(which icpc)"     >> $GITHUB_ENV
           echo "OMPI_MPICC=$(which icc)"    >> $GITHUB_ENV
-          echo "MPI_HOME=/opt/intel/oneapi/mpi/2021.7.1/"    >> $GITHUB_ENV
-          echo "I_MPI_ROOT=/opt/intel/oneapi/mpi/2021.7.1/"    >> $GITHUB_ENV
+          echo "MPI_HOME=/opt/intel/oneapi/mpi/2021.7.1/"   >> $GITHUB_ENV
+          echo "I_MPI_ROOT=/opt/intel/oneapi/mpi/2021.7.1/" >> $GITHUB_ENV
 
       - name: Build
         run:  |
-          if [ '${{ matrix.intel }}' == 'true' ]; then source /opt/intel/oneapi/setvars.sh; fi
+          if [ '${{ matrix.intel }}' == 'true' ]; then . /opt/intel/oneapi/setvars.sh; fi
           /bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} 
 
       - name: Test
         run:  |
-          if [ '${{ matrix.intel }}' == 'true' ]; then source /opt/intel/oneapi/setvars.sh; fi
+          if [ '${{ matrix.intel }}' == 'true' ]; then . /opt/intel/oneapi/setvars.sh; fi
           /bin/bash mfc.sh test --max-attempts 3 -j $(nproc) $OPT1 $OPT2
         env:
           OPT1: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
           OPT2: ${{ matrix.debug == 'debug' && '-% 20' || '' }}
 
-  docker:
-    name: Github | Docker
-    if: needs.file-changes.outputs.checkall == 'true'
-    needs: file-changes
-    runs-on: ubuntu-latest
-    steps:
-    - name: Clone
-      uses: actions/checkout@v4
-
-    - name: Test
-      run:  sudo ./mfc.sh docker ./mfc.sh test --max-attempts 3 -j $(nproc) -a
-
   self:
     name: Self Hosted
     if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true'
@@ -156,4 +139,3 @@ jobs:
         with:
           name: logs
           path: test-${{ matrix.device }}.out
-

.gitignore

@@ -1,7 +1,6 @@
 node_modules/
 package.json
 yarn.lock
-docker-compose.yml
 
 .venv/
 /build/

CMakeLists.txt

@@ -69,8 +69,8 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
         message(FATAL_ERROR "ERROR: When using NVHPC, v21.7 or newer is required to build MFC.\n${__err_msg}")
     endif()
 
-    if ((CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 23.11) AND (CMAKE_BUILD_TYPE STREQUAL "Debug") AND MFC_OpenACC)
-        message(FATAL_ERROR "ERROR: When using NVHPC, MFC with Debug and GPU options requires NVHPC v23.11 or newer.\n${__err_msg}")
+    if ((CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 24.5) AND (CMAKE_BUILD_TYPE STREQUAL "Debug") AND MFC_OpenACC)
+        message(FATAL_ERROR "ERROR: When using NVHPC, MFC with Debug and GPU options requires NVHPC v24.5 or newer.\n${__err_msg}")
     endif()
 elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_C_COMPILER_ID STREQUAL "AppleClang")
     message(FATAL_ERROR "ERROR: MFC does not support the Apple Clang compilers. Please consult the documentation.\n${__err_msg}")
@@ -89,7 +89,10 @@ find_program(FYPP_EXE fypp REQUIRED)
 
 link_libraries("${CMAKE_DL_LIBS}")
 set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS ON)
-list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cmake")
+list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cmake/regular")
+if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
+    list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cmake/cce")
+endif()
 
 
 # Compiler Flags: Here, we specify our own compiler flags for both release and
@@ -130,6 +133,8 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
             -fimplicit-none
             #-ffpe-trap=invalid,zero,denormal,overflow
             -fsignaling-nans
+            -finit-real=snan
+            -finit-integer=-99999999
 	    )
     endif()
 
@@ -211,7 +216,9 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
     # Enable LTO/IPO if supported
     if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
         if (MFC_Unified)
-            message(STATUS "IPO is not available with NVHPC using Unified Memory")
+            message(STATUS "LTO/IPO is not available with NVHPC using Unified Memory")
+        elseif(CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "23.11")
+            message(STATUS "LTO/IPO is not supported in NVHPC Version < 23.11. Use a newer version of NVHPC for best performance.")
         else()
             message(STATUS "Performing IPO using -Mextract followed by -Minline")
             set(NVHPC_USE_TWO_PASS_IPO TRUE)
@@ -337,6 +344,7 @@ macro(HANDLE_SOURCES target useCommon)
                                  -D MFC_${${target}_UPPER}
                                  -D MFC_COMPILER="${CMAKE_Fortran_COMPILER_ID}"
                                  -D MFC_CASE_OPTIMIZATION=False
+                                 -D chemistry=False
                                  --line-numbering
                                  --no-folding
                                  "${fpp}" "${f90}"
@@ -526,15 +534,6 @@ if (MFC_SIMULATION)
     MFC_SETUP_TARGET(TARGET  simulation
                      SOURCES "${simulation_SRCs}"
                      MPI OpenACC FFTW)
-
-    if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray" AND MFC_OpenACC)
-        add_custom_command(TARGET simulation POST_BUILD
-            COMMAND           "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cce_simulation_workgroup_256.sh"
-                              "${CMAKE_CURRENT_BINARY_DIR}"
-            WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
-            COMMENT           "Patching & Rebuilding with Cray hacks"
-        )
-    endif()
 endif()
 
 if (MFC_POST_PROCESS)

README.md

@@ -58,18 +58,19 @@ It's rather straightforward.
 We'll give a brief intro. here for MacOS.
 Using [brew](https://brew.sh), install MFC's dependencies:
 ```shell
-brew install wget python cmake gcc@14 mpich
+brew install coreutils python cmake fftw hdf5 gcc boost open-mpi
 ```
 You're now ready to build and test MFC!
 Put it to a convenient directory via
 ```shell
 git clone https://github.com/MFlowCode/MFC
 cd MFC
 ```
-and be sure MFC knows what compilers to use by appending and sourcing your `~/.profile` file via this command
+and be sure MFC knows where to find Boost by appending to your dotfiles and sourcing them again
 ```shell
-echo -e "export CC=gcc-14 \nexport CXX=g++-14 \nexport FC=gfortran-14" >> ~/.profile
-source ~/.profile
+echo -e "export BOOST_INCLUDE='$(brew --prefix --installed boost)/include'" | tee -a ~/.bash_profile ~/.zshrc
+. ~/.bash_profile 2>/dev/null || . ~/.zshrc 2>/dev/null
+! [ -z "${BOOST_INCLUDE+x}" ] && echo 'Environment is ready!' || echo 'Error: $BOOST_INCLUDE is unset. Please adjust the previous commands to fit with your environment.'
 ```
 then you can build MFC and run the test suite!
 ```shell
@@ -131,7 +132,7 @@ They are organized below. Just click the drop-downs!
 
 * Shock and interface capturing schemes
 	* First-order upwinding
- 	* WENO reconstructions of order 3 and 5
+ 	* WENO reconstructions of order 3, 5, and 7
   	* WENO variants: WENO-JS, WENO-M, WENO-Z, TENO
    	* Monotonicity-preserving reconstructions
 	* Reliable handling of high density ratios

benchmarks/5eq_rk3_weno3_hllc/case.py

@@ -14,11 +14,13 @@
     description="This MFC case was created for the purposes of benchmarking MFC.",
     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-parser.add_argument("dict", type=str, metavar="DICT", help=argparse.SUPPRESS)
-parser.add_argument("gbpp", type=int, metavar="MEM", default=16, help="Adjusts the problem size per rank to fit into [MEM] GB of GPU memory per GPU.")
+parser.add_argument("--mfc", type=json.loads, default='{}', metavar="DICT",
+                    help="MFC's toolchain's internal state.")
+parser.add_argument("--gbpp", type=int, metavar="MEM", default=16,
+                    help="Adjusts the problem size per rank to fit into [MEM] GB of GPU memory per GPU.")
 
 ARGS = vars(parser.parse_args())
-DICT = json.loads(ARGS["dict"])
+DICT = ARGS["mfc"]
 
 size = 1 if DICT["gpu"] else 0