diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index da5b2afcf1..a6c019f427 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Login to Docker Hub uses: docker/login-action@v2 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5f7c4c2d5b..6d5ae21807 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,7 +18,7 @@ jobs: cancel-in-progress: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # We build doxygen from source because of # https://github.com/doxygen/doxygen/issues/9016 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a2f219485f..ca22f9705f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Lint the toolchain run: ./mfc.sh lint diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh index 78df3dd139..a2ef778cd6 100644 --- a/.github/workflows/phoenix/bench.sh +++ b/.github/workflows/phoenix/bench.sh @@ -8,4 +8,8 @@ if [ "$job_device" == "gpu" ]; then device_opts="--gpu -g $gpu_ids" fi -./mfc.sh bench -j $(nproc) -o "$job_slug.yaml" -- -c phoenix $device_opts -n $n_ranks +if ["$job_device" == "gpu"]; then + ./mfc.sh bench --mem 8 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix $device_opts -n $n_ranks +else + ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix $device_opts -n $n_ranks +fi \ No newline at end of file diff --git a/.github/workflows/pretty.yml b/.github/workflows/pretty.yml index 4c37aeb599..9181addf8a 100644 --- a/.github/workflows/pretty.yml +++ b/.github/workflows/pretty.yml @@ -13,9 +13,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check formatting run: | - ./mfc.sh format + ./mfc.sh format -j $(nproc) git diff --exit-code diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index a170f4e52e..63c3666948 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Spell Check uses: crate-ci/typos@master diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b8813ef40..d4e14c519d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: runs-on: ${{ matrix.os }}-latest steps: - name: Clone - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup MacOS if: matrix.os == 'macos' @@ -46,10 +46,7 @@ jobs: - name: (MacOS) Build OpenMPI if: matrix.os == 'macos' && matrix.mpi == 'mpi' run: | - echo "OMPI_FC=gfortran-13" >> $GITHUB_ENV - echo "OMPI_CXX=g++-13" >> $GITHUB_ENV - echo "OMPI_MPICC=gcc-13" >> $GITHUB_ENV - HOMEBREW_MAKE_JOBS=$(nproc) brew install --cc=gcc-13 --verbose --build-from-source open-mpi + brew install mpich - name: Setup Ubuntu if: matrix.os == 'ubuntu' && matrix.intel == false @@ -96,7 +93,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Clone - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Test run: sudo ./mfc.sh docker ./mfc.sh test -j $(nproc) -a diff --git a/.gitignore b/.gitignore index 6d57fb3afd..c7a2ed8fee 100644 --- a/.gitignore +++ b/.gitignore @@ -52,4 +52,30 @@ examples/*/*.err examples/*/viz/ examples/*.jpg examples/*.png + +benchmarks/*batch/*/ +benchmarks/*/D/* +benchmarks/*/p* +benchmarks/*/D_* +benchmarks/*/*.inf +benchmarks/*/*.inp +benchmarks/*/*.dat +benchmarks/*/*.o* +benchmarks/*/silo* +benchmarks/*/restart_data* +benchmarks/*/*.out +benchmarks/*/binary +benchmarks/*/fort.1 +benchmarks/*/*.sh +benchmarks/*/*.err +benchmarks/*/viz/ +benchmarks/*.jpg +benchmarks/*.png + *.mod + +# Video Files +*.mp4 +*.mov +*.mkv +*.avi diff --git a/.vscode/settings.json b/.vscode/settings.json index 10fc8c664d..f18256f1f8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,7 +11,8 @@ "cmake.configureOnOpen": false, + "python.defaultInterpreterPath": "${workspaceFolder}/build/venv/bin/python3", + "fortran.preferredCase": "lowercase", "fortran.linter.includePaths": [ "${workspacefolder}/src/**" ], - "fortran.linter.fypp.enabled": true } diff --git a/CMakeLists.txt b/CMakeLists.txt index 0553f38a04..fd15f0cf60 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,8 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") -fcheck=all,no-array-temps -fbacktrace -fimplicit-none - ) + #-ffpe-trap=invalid,zero,denormal,overflow + ) endif() if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER 10) diff --git a/README.md b/README.md index a5dbea6ad4..248129816a 100644 --- a/README.md +++ b/README.md @@ -39,24 +39,37 @@ MFC can execute high-fidelity simulations of shock-droplet interaction (see `exa Shock Droplet Example

+Another example is the high-Mach flow over an airfoil, shown below. + +

+ Airfoil Example
+

+ + ## Getting started You can navigate [to this webpage](https://mflowcode.github.io/documentation/md_getting-started.html) to get started using MFC! It's rather straightforward. - We'll give a brief intro. here for MacOS. Using [brew](https://brew.sh), install MFC's modest set of dependencies: ```console -brew install wget make python make cmake coreutils gcc openmpi +brew install wget python cmake gcc@13 mpich ``` You're now ready to build and test MFC! -Clone it to a convenient directory via +Put it to a convenient directory via ```console git clone https://github.com/mflowcode/MFC.git cd MFC ``` -then build and test! +and make sure MFC knows what compilers to use by putting the following in your `~/.profile` +```console +export CC=gcc-13 +export CXX=g++-13 +export FC=gfortran-13 +``` +and source that file, build, and test! ```console +source ~/.profile ./mfc.sh build -j 8 ./mfc.sh test -j 8 ``` diff --git a/benchmarks/3D_shockdroplet/case.py b/benchmarks/5eq_rk3_weno3_hllc/case.py similarity index 84% rename from benchmarks/3D_shockdroplet/case.py rename to benchmarks/5eq_rk3_weno3_hllc/case.py index 650bdf89df..ee8950d4e0 100644 --- a/benchmarks/3D_shockdroplet/case.py +++ b/benchmarks/5eq_rk3_weno3_hllc/case.py @@ -1,7 +1,32 @@ #!/usr/bin/env python3 -import math -import json +# Benchmark model_equations_2_time_stepper_3_weno_order_3_riemann_solver_2 +# Additional Benchmarked Features +# - model_equations : 2 +# - time_stepper : 3 +# - weno_order : 3 +# - riemann_solver : 2 + +import json, math, argparse + +parser = argparse.ArgumentParser( + prog="Benchmarking Case 1", + description="This MFC case was created for the purposes of benchmarking MFC.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + +parser.add_argument("dict", type=str, metavar="DICT", help=argparse.SUPPRESS) +parser.add_argument("gbpp", type=int, metavar="MEM", default=16, help="Adjusts the problem size per rank to fit into [MEM] GB of GPU memory per GPU.") + +ARGS = vars(parser.parse_args()) +DICT = json.loads(ARGS["dict"]) + +size = 1 if DICT["gpu"] else 0 + +ppg = 8000000 / 16.0 +procs = DICT["nodes"] * DICT["tasks_per_node"] +ncells = math.floor(ppg * procs * ARGS["gbpp"]) +s = math.floor((ncells / 2.0) ** (1/3)) +Nx, Ny, Nz = 2*s, s, s # athmospheric pressure - Pa (used as reference value) patm = 101325 @@ -13,10 +38,9 @@ CtD = 0.06 # cavity relative eccentricity (distance between radii) -ecc = 0.564 +ecc = 0.564 -# initial shock distance from the y axis. Note that the droplet center is located at y = 0. -# Thus, the distance from the shock to +# initial shock distance from the y axis. Note that the droplet center is located at y = 0. Thus, the distance from the shock to # the droplet is about D0/8 ISD = 5.0/8 * D0 @@ -26,10 +50,10 @@ p0a = patm # density - kg/m3 -rho0a = 1.204 +rho0a = 1.204 # gamma -gama = 1.40 +gama = 1.40 # pi infinity - Pa pia = 0 @@ -52,7 +76,7 @@ rho0w = 1000 # gama -gamw = 6.12 +gamw = 6.12 # pi infty - Pa piw = 3.43E+08 @@ -60,7 +84,7 @@ # speed of sound - m/s c_w = math.sqrt( gamw * ( p0w + piw ) / rho0w ) -# Shock Mach number of interest. Note that the post-shock properties can be defined in terms of either +# Shock Mach number of interest. Note that the post-shock properties can be defined in terms of either # Min or psOp0a. Just comment/uncomment appropriatelly Min = 2.4 @@ -71,7 +95,7 @@ # psOp0a = 4.5 # density -rhosOrho0a = ( 1 + ( gama + 1 ) / ( gama - 1) * psOp0a ) / ( ( gama + 1 ) / ( gama - 1) + psOp0a ) +rhosOrho0a = ( 1 + ( gama + 1 ) / ( gama - 1) * psOp0a ) / ( ( gama + 1 ) / ( gama - 1) + psOp0a ) # Mach number of the shocked region - just a checker, as it must return "Min" Ms = math.sqrt( ( gama + 1. ) / ( 2. * gama ) * ( psOp0a - 1. ) * ( p0a / ( p0a + pia ) ) + 1. ) @@ -88,7 +112,7 @@ rhos = rhosOrho0a * rho0a # post shock speed of sound - m/s -c_s = math.sqrt( gama * (ps + pia) / rhos ) +c_s = math.sqrt( gama * (ps + pia) / rhos ) # velocity at the post shock - m/s vel = c_a/gama * (psOp0a - 1.) * p0a / ( p0a + pia ) / Ms @@ -111,21 +135,12 @@ ze = 10 * D0 # Stretching factor, to make sure the domaing is sufficiently large after the mesh stretch -StF = 4.0 - -# number of elements into y direction -Ny = 100 - -# number of elements into z direction -Nz = 100 - -# number of elements into x direction -Nx = Ny * 2 +StF = 4.0 # grid delta x if mesh were uniform in x direction - m. Note that I do not need a measure for dy dx = ( xe - xb ) / Nx -# I calculating tend twice; first is an estimate, second is +# I calculate tend twice; first is an estimate, second is # the actual value used. This is because I am getting errors in the # post process part every time I approximate the actual Nt by an integer # number (think of a smarter way). @@ -137,12 +152,10 @@ # mismatches in simulation and post_process parts. Note that I wrote it this way so I have better control over the # of autosaves tendA = ttilde * D0 / vel -# "CFL" number that I use to control both temporal and spatial discretizations, such that the ratio dx/dt remains constant for a given -# simulation -cfl = 0.05 +cfl = 0.1 # time-step - s -dt = cfl * dx/ ss +dt = dx * cfl/ ss # Save Frequency. Note that the number of autosaves will be SF + 1, as th IC (0.dat) is also saved SF = 400 @@ -173,26 +186,14 @@ 'y_domain%end' : ye, 'z_domain%beg' : zb, 'z_domain%end' : ze, - 'stretch_x' : 'T', - 'a_x' : 20, - 'x_a' : -1.2 * D0, - 'x_b' : 1.2 * D0, - 'stretch_y' : 'T', - 'a_y' : 20, - 'y_a' : -0.0 * D0, - 'y_b' : 1.2 * D0, - 'stretch_z' : 'T', - 'a_z' : 20, - 'z_a' : -0.0 * D0, - 'z_b' : 1.2 * D0, 'm' : Nx, 'n' : Ny, 'p' : Nz, 'cyl_coord' : 'F', 'dt' : dt, 't_step_start' : 0, - 't_step_stop' : 100, - 't_step_save' : 100, + 't_step_stop' : int(60*(95*size + 5)), + 't_step_save' : int(12*(95*size + 5)), # ========================================================== # Simulation Algorithm Parameters ========================== @@ -211,7 +212,7 @@ 'mapped_weno' : 'T', 'riemann_solver' : 2, 'wave_speeds' : 1, - 'avg_state' : 2, + 'avg_state' : 2, 'bc_x%beg' : -6, 'bc_x%end' : -6, 'bc_y%beg' : -2, @@ -226,7 +227,7 @@ 'prim_vars_wrt' :'T', 'parallel_io' :'T', # ========================================================== - # I will use 1 for WATER properties, and 2 for AIR properties + # I will use 1 for WATER properties, and 2 for AIR properties # Patch 1: Background (AIR - 2) ============================= 'patch_icpp(1)%geometry' : 9, 'patch_icpp(1)%x_centroid' : (xb+xe) / 2 * StF, @@ -280,7 +281,6 @@ 'patch_icpp(3)%alpha(1)' : 1.0E+00, 'patch_icpp(3)%alpha(2)' : 0.0E+00, # ========================================================== - # Fluids Physical Parameters =============================== 'fluid_pp(1)%gamma' : 1.0E+00/(gamw-1), @@ -291,3 +291,4 @@ })) # ============================================================================== + diff --git a/benchmarks/hypo_hll/case.py b/benchmarks/hypo_hll/case.py new file mode 100644 index 0000000000..f064a15a6b --- /dev/null +++ b/benchmarks/hypo_hll/case.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 + +# Benchmark hypoelasticity_T_riemann_solver_1 +# Additional Benchmarked Features +# - hypoelasticity : T +# - riemann_solver : 1 + +import json, math, argparse + +parser = argparse.ArgumentParser( + prog="Benchmarkin Case 3", + description="This MFC case was created for the purposes of benchmarking MFC.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + +parser.add_argument("dict", type=str, metavar="DICT", help=argparse.SUPPRESS) +parser.add_argument("gbpp", type=int, metavar="MEM", default=16, help="Adjusts the problem size per rank to fit into [MEM] GB of GPU memory per GPU.") + +ARGS = vars(parser.parse_args()) +DICT = json.loads(ARGS["dict"]) + +size = 1 if DICT["gpu"] else 0 + +ppg = 8000000 / 16.0 +procs = DICT["nodes"] * DICT["tasks_per_node"] +ncells = math.floor(ppg * procs * ARGS["gbpp"]) +s = math.floor((ncells / 2.0) ** (1/3)) +Nx, Ny, Nz = 2*s, s, s + +Mu = 1.84E-05 +gam_a = 1.4 + +D = 0.1 + +# Configuring case dictionary +print(json.dumps({ + # Logistics ================================================================ + 'run_time_info' : 'T', + # ========================================================================== + + # Computational Domain Parameters ========================================== + # x direction + 'x_domain%beg' : -5*D, + 'x_domain%end' : 5.0*D, + # y direction + 'y_domain%beg' : -2.5*D, + 'y_domain%end' : 2.5*D, + # z direction + 'z_domain%beg' : -2.5*D, + 'z_domain%end' : 2.5*D, + + 'cyl_coord' : 'F', + 'm' : Nx, + 'n' : Ny, + 'p' : Nz, + 'dt' : 1.0E-7, + 't_step_start' : 0, + 't_step_stop' : int(20*(45*size + 5)), + 't_step_save' : int(4*(45*size + 5)), + # ========================================================================== + + # Simulation Algorithm Parameters ========================================== + # Only one patches are necessary, the air tube + 'num_patches' : 1, + # Use the 5 equation model + 'model_eqns' : 2, + # 6 equations model does not need the K \div(u) term + 'alt_soundspeed' : 'F', + # One fluids: air + 'num_fluids' : 1, + # Advect both volume fractions + 'adv_alphan' : 'T', + # No need to ensure the volume fractions sum to unity at the end of each + # time step + 'mpp_lim' : 'F', + # Correct errors when computing speed of sound + 'mixture_err' : 'T', + # Use TVD RK3 for time marching + 'time_stepper' : 3, + # Reconstruct the primitive variables to minimize spurious + # Use WENO5 + 'weno_order' : 5, + 'weno_eps' : 1.E-16, + 'weno_Re_flux' : 'T', + 'weno_avg' : 'T', + 'avg_state' : 2, + 'mapped_weno' : 'T', + 'null_weights' : 'F', + 'mp_weno' : 'T', + 'riemann_solver' : 2, + 'wave_speeds' : 1, + # We use ghost-cell extrapolation + 'bc_x%beg' : -3, + 'bc_x%end' : -3, + 'bc_y%beg' : -3, + 'bc_y%end' : -3, + 'bc_z%beg' : -3, + 'bc_z%end' : -3, + # Set IB to True and add 1 patch + 'ib' : 'T', + 'num_ibs' : 1, + # ========================================================================== + + # Formatted Database Files Structure Parameters ============================ + 'format' : 1, + 'precision' : 2, + 'prim_vars_wrt' :'T', + 'E_wrt' :'T', + 'parallel_io' :'T', + # ========================================================================== + + # Patch: Constant Tube filled with air ===================================== + # Specify the cylindrical air tube grid geometry + 'patch_icpp(1)%geometry' : 9, + 'patch_icpp(1)%x_centroid' : 0.0, + # Uniform medium density, centroid is at the center of the domain + 'patch_icpp(1)%y_centroid' : 0.0, + 'patch_icpp(1)%z_centroid' : 0.0, + 'patch_icpp(1)%length_x' : 10*D, + 'patch_icpp(1)%length_y' : 5*D, + 'patch_icpp(1)%length_z' : 5*D, + # Specify the patch primitive variables + 'patch_icpp(1)%vel(1)' : 527.2E+00, + 'patch_icpp(1)%vel(2)' : 0.0E+00, + 'patch_icpp(1)%vel(3)' : 0.0E+00, + 'patch_icpp(1)%pres' : 10918.2549, + 'patch_icpp(1)%alpha_rho(1)' : 0.2199, + 'patch_icpp(1)%alpha(1)' : 1.E+00, + # # ======================================================================== + + # Patch: Sphere Immersed Boundary ======================================== + 'patch_ib(1)%geometry' : 8, + 'patch_ib(1)%x_centroid' : -3.0E-3, + 'patch_ib(1)%y_centroid' : 0.0, + 'patch_ib(1)%z_centroid' : 0.0, + 'patch_ib(1)%radius' : D/2, + 'patch_ib(1)%slip' : 'T', + # ========================================================================== + + # Fluids Physical Parameters =============================================== + 'fluid_pp(1)%gamma' : 1.E+00/(gam_a-1.E+00), # 2.50(Not 1.40) + 'fluid_pp(1)%pi_inf' : 0, + 'fluid_pp(1)%Re(1)' : 7535533.2, + # ========================================================================== +})) diff --git a/benchmarks/ibm/case.py b/benchmarks/ibm/case.py new file mode 100644 index 0000000000..9b127d69d0 --- /dev/null +++ b/benchmarks/ibm/case.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +# Benchmark ibm_T +# Additional Benchmarked Features +# - ibm : T + +import json, math, argparse + +parser = argparse.ArgumentParser( + prog="Benchmarking Case 4", + description="This MFC case was created for the purposes of benchmarking MFC.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + +parser.add_argument("dict", type=str, metavar="DICT", help=argparse.SUPPRESS) +parser.add_argument("gbpp", type=int, metavar="MEM", default=16, help="Adjusts the problem size per rank to fit into [MEM] GB of GPU memory per GPU.") + +ARGS = vars(parser.parse_args()) +DICT = json.loads(ARGS["dict"]) + +size = 1 if DICT["gpu"] else 0 + +ppg = 8000000 / 16.0 +procs = DICT["nodes"] * DICT["tasks_per_node"] +ncells = math.floor(ppg * procs * ARGS["gbpp"]) +s = math.floor((ncells / 2.0) ** (1/3)) +Nx, Ny, Nz = 2*s, s, s + +dx = 1./(1.*(Nx+1)) + +Tend = 64E-06 +Nt = 200 +mydt = Tend/(1.*Nt) + +# Configuring case dictionary +print(json.dumps({ + # Logistics ================================================ + 'run_time_info' : 'F', + # ========================================================== + + # Computational Domain Parameters ========================== + 'x_domain%beg' : 0.E+00, + 'x_domain%end' : 1.E+00, + 'y_domain%beg' : 0, + 'y_domain%end' : 0.5, + 'z_domain%beg' : 0., + 'z_domain%end' : 0.5, + 'm' : Nx, + 'n' : Ny, + 'p' : Nz, + 'dt' : mydt, + 't_step_start' : 0, + 't_step_stop' : int(40*(95*size + 5)), + 't_step_save' : int(8*(95*size + 5)), + # ========================================================== + + # Simulation Algorithm Parameters ========================== + 'num_patches' : 2, + 'model_eqns' : 2, + 'alt_soundspeed' : 'F', + 'num_fluids' : 1, + 'adv_alphan' : 'T', + 'mpp_lim' : 'F', + 'mixture_err' : 'F', + 'time_stepper' : 3, + 'weno_order' : 5, + 'weno_eps' : 1.E-16, + 'weno_Re_flux' : 'F', + 'weno_avg' : 'F', + 'mapped_weno' : 'F', + 'null_weights' : 'F', + 'mp_weno' : 'F', + 'riemann_solver' : 1, + 'wave_speeds' : 1, + 'avg_state' : 2, + 'bc_x%beg' : -3, + 'bc_x%end' : -3, + 'bc_y%beg' : -3, + 'bc_y%end' : -3, + 'bc_z%beg' : -3, + 'bc_z%end' : -3, + # ========================================================== + + # Turning on Hypoelasticity ================================ + 'hypoelasticity' : 'T', + # ========================================================== + + # Formatted Database Files Structure Parameters ============ + 'format' : 1, + 'precision' : 2, + 'prim_vars_wrt' :'T', + 'parallel_io' :'T', + # ========================================================== + + # Patch 1 L ================================================ + 'patch_icpp(1)%geometry' : 9, + 'patch_icpp(1)%x_centroid' : 0.25, + 'patch_icpp(1)%y_centroid' : 0.25, + 'patch_icpp(1)%z_centroid' : 0.25, + 'patch_icpp(1)%length_x' : 0.5, + 'patch_icpp(1)%length_y' : 0.5, + 'patch_icpp(1)%length_z' : 0.5, + 'patch_icpp(1)%vel(1)' : 0.0, + 'patch_icpp(1)%vel(2)' : 0, + 'patch_icpp(1)%vel(3)' : 0, + 'patch_icpp(1)%pres' : 1.E+8, + 'patch_icpp(1)%alpha_rho(1)' : 1000, + 'patch_icpp(1)%alpha(1)' : 1., + 'patch_icpp(1)%tau_e(1)' : 0.0, + # ========================================================== + + # Patch 2 R ================================================ + 'patch_icpp(2)%geometry' : 9, + 'patch_icpp(2)%x_centroid' : 0.75, + 'patch_icpp(2)%y_centroid' : 0.25, + 'patch_icpp(2)%z_centroid' : 0.25, + 'patch_icpp(2)%length_x' : 0.5, + 'patch_icpp(2)%length_y' : 0.5, + 'patch_icpp(2)%length_z' : 0.5, + 'patch_icpp(2)%vel(1)' : 0, + 'patch_icpp(2)%vel(2)' : 0, + 'patch_icpp(2)%vel(3)' : 0, + 'patch_icpp(2)%pres' : 1.E+05, + 'patch_icpp(2)%alpha_rho(1)' : 1000, + 'patch_icpp(2)%alpha(1)' : 1., + 'patch_icpp(2)%tau_e(1)' : 0.0, + # ========================================================== + + # Fluids Physical Parameters =============================== + 'fluid_pp(1)%gamma' : 1.E+00/(4.4E+00-1.E+00), + 'fluid_pp(1)%pi_inf' : 4.4E+00*6.E+08/(4.4E+00 - 1.E+00), + 'fluid_pp(1)%G' : 10E+09, + # ========================================================== +})) +# ============================================================================== diff --git a/benchmarks/viscous_weno5_sgb_mono/case.py b/benchmarks/viscous_weno5_sgb_mono/case.py new file mode 100644 index 0000000000..e9b5e913d4 --- /dev/null +++ b/benchmarks/viscous_weno5_sgb_mono/case.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 + +# Benchmark viscosity_weno_Re_flux_T_weno_order_5_bubbles_T_bubble_mode_3_monopole_T +# Additional Benchmarked Features +# - viscosity enabled +# - weno_Re_flux : T +# - weno_order : 5 +# - bubbles : T +# - bubble_model : 3 +# - monopole : T + +import json, math, argparse + +parser = argparse.ArgumentParser( + prog="Benchmarking Case 2", + description="This MFC case was created for the purposes of benchmarking MFC.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + +parser.add_argument("dict", type=str, metavar="DICT", help=argparse.SUPPRESS) +parser.add_argument("gbpp", type=int, metavar="MEM", default=16, help="Adjusts the problem size per rank to fit into [MEM] GB of GPU memory per GPU.") + +ARGS = vars(parser.parse_args()) +DICT = json.loads(ARGS["dict"]) + +size = 1 if DICT["gpu"] else 0 + +ppg = 8000000 / 16.0 +procs = DICT["nodes"] * DICT["tasks_per_node"] +ncells = math.floor(ppg * procs * ARGS["gbpp"]) +s = math.floor((ncells / 2.0) ** (1/3)) +Nx, Ny, Nz = 2*s, s, s + +x0 = 10.E-04 +y0 = 10.E-04 +z0 = 10.E-04 +p0 = 1. +rho0 = 1.E+03 +c0 = math.sqrt( p0/rho0 ) +patm = 1. + +#water props +n_tait = 7.1 +B_tait = 306.E+06 / p0 +mul0 = 1.002E-03 #viscosity +ss = 0.07275 #surface tension +pv = 2.3388E+03 #vapor pressure + +gamma_v = 1.33 +M_v = 18.02 +mu_v = 0.8816E-05 +k_v = 0.019426 + +#air props +gamma_n = 1.4 +M_n = 28.97 +mu_n = 1.8E-05 +k_n = 0.02556 + +#air props +gamma_gas = gamma_n + +#reference bubble size +R0ref = 10.E-06 + +pa = 0.1 * 1.E+06 / 101325. + +#Characteristic velocity +uu = math.sqrt( p0/rho0 ) +#Cavitation number +Ca = (p0 - pv)/(rho0*(uu**2.)) +#Weber number +We = rho0*(uu**2.)*R0ref/ss +#Inv. bubble Reynolds number +Re_inv = mul0/(rho0*uu*R0ref) + +#IC setup +vf0 = 0.00004 +n0 = vf0/(math.pi*4.E+00/3.E+00) + +cact = 1475. +t0 = x0/c0 + +nbubbles = 1 +myr0 = R0ref + +cfl = 0.01 +Ldomain = 20.E-03 +L = Ldomain/x0 +dx = L/float(Nx) +dt = cfl*dx*c0/cact +Lpulse = 0.3*Ldomain +Tpulse = Lpulse/cact + +# Configuring case dictionary +print(json.dumps({ + # Logistics ================================================ + 'run_time_info' : 'T', + # ========================================================== + + # Computational Domain Parameters ========================== + 'x_domain%beg' : -10.E-03/x0, + 'x_domain%end' : 10.E-03/x0, + 'y_domain%beg' : -5.E-03/y0, + 'y_domain%end' : 5.E-03/y0, + 'z_domain%beg' : -5.E-03/z0, + 'z_domain%end' : 5.E-03/z0, + 'stretch_x' : 'F', + 'cyl_coord' : 'F', + 'm' : Nx, + 'n' : Ny, + 'p' : Nz, + 'dt' : dt, + 't_step_start' : 0, + 't_step_stop' : int(30*(25*size + 5)), + 't_step_save' : int(6*(25*size + 5)), + # ========================================================== + + # Simulation Algorithm Parameters ========================== + 'num_patches' : 2, + 'model_eqns' : 2, + 'alt_soundspeed' : 'F', + 'num_fluids' : 1, + 'adv_alphan' : 'T', + 'mpp_lim' : 'F', + 'mixture_err' : 'F', + 'time_stepper' : 3, + 'weno_order' : 5, + 'weno_eps' : 1.E-16, + 'weno_Re_flux' : 'F', + 'weno_avg' : 'F', + 'mapped_weno' : 'T', + 'null_weights' : 'F', + 'mp_weno' : 'T', + 'riemann_solver' : 2, + 'wave_speeds' : 1, + 'avg_state' : 2, + 'bc_x%beg' : -3, + 'bc_x%end' : -3, + 'bc_y%beg' : -3, + 'bc_y%end' : -3, + 'bc_z%beg' : -3, + 'bc_z%end' : -3, + # ========================================================== + + # Formatted Database Files Structure Parameters ============ + 'format' : 1, + 'precision' : 2, + 'prim_vars_wrt' :'T', + 'parallel_io' :'T', + # ========================================================== + + # Patch 1 _ Background ===================================== + 'patch_icpp(1)%geometry' : 9, + 'patch_icpp(1)%x_centroid' : 0., + 'patch_icpp(1)%y_centroid' : 0., + 'patch_icpp(1)%z_centroid' : 0., + 'patch_icpp(1)%length_x' : 20.E-03/x0, + 'patch_icpp(1)%length_y' : 10.E-03/y0, + 'patch_icpp(1)%length_z' : 10.E-03/z0, + 'patch_icpp(1)%vel(1)' : 0.0, + 'patch_icpp(1)%vel(2)' : 0.0, + 'patch_icpp(1)%vel(3)' : 0.0, + 'patch_icpp(1)%pres' : patm, + 'patch_icpp(1)%alpha_rho(1)' : (1.-1.E-12)*1.E+03/rho0, + 'patch_icpp(1)%alpha(1)' : 1.E-12, + 'patch_icpp(1)%r0' : 1., + 'patch_icpp(1)%v0' : 0.0E+00, + # ========================================================== + + # Patch 2 Screen =========================================== + 'patch_icpp(2)%geometry' : 9, + 'patch_icpp(2)%x_centroid' : 0., + 'patch_icpp(2)%y_centroid' : 0., + 'patch_icpp(2)%z_centroid' : 0., + 'patch_icpp(2)%length_x' : 5.E-03/x0, + 'patch_icpp(2)%length_y' : 10.E-03/y0, + 'patch_icpp(2)%length_z' : 10.E-03/z0, + 'patch_icpp(2)%alter_patch(1)' : 'T', + 'patch_icpp(2)%vel(1)' : 0.0, + 'patch_icpp(2)%vel(2)' : 0.0, + 'patch_icpp(2)%vel(3)' : 0.0, + 'patch_icpp(2)%pres' : patm, + 'patch_icpp(2)%alpha_rho(1)' : (1.-vf0)*1.E+03/rho0, + 'patch_icpp(2)%alpha(1)' : vf0, + 'patch_icpp(2)%r0' : 1., + 'patch_icpp(2)%v0' : 0.0E+00, + # ========================================================== + + # Fluids Physical Parameters =============================== + # Surrounding liquid + 'fluid_pp(1)%gamma' : 1.E+00/(n_tait-1.E+00), + 'fluid_pp(1)%pi_inf' : n_tait*B_tait/(n_tait-1.), + 'fluid_pp(1)%mul0' : mul0, + 'fluid_pp(1)%ss' : ss, + 'fluid_pp(1)%pv' : pv, + 'fluid_pp(1)%gamma_v' : gamma_v, + 'fluid_pp(1)%M_v' : M_v, + 'fluid_pp(1)%mu_v' : mu_v, + 'fluid_pp(1)%k_v' : k_v, + 'fluid_pp(1)%Re(1)' : 1e3, + # Last fluid_pp is always reserved for bubble gas state === + # if applicable ========================================== + 'fluid_pp(2)%gamma' : 1./(gamma_gas-1.), + 'fluid_pp(2)%pi_inf' : 0.0E+00, + 'fluid_pp(2)%gamma_v' : gamma_n, + 'fluid_pp(2)%M_v' : M_n, + 'fluid_pp(2)%mu_v' : mu_n, + 'fluid_pp(2)%k_v' : k_n, + # ========================================================== + + # Non-polytropic gas compression model AND/OR Tait EOS ===== + 'pref' : p0, + 'rhoref' : rho0, + # ========================================================== + + # Bubbles ================================================== + 'bubbles' : 'T', + 'bubble_model' : 3, + 'polytropic' : 'T', + 'polydisperse' : 'F', + # 'poly_sigma' : 0.3, + 'thermal' : 3, + 'R0ref' : myr0, + 'nb' : 1, + 'Ca' : Ca, + 'Web' : We, + 'Re_inv' : Re_inv, + # ========================================================== + + # Acoustic source ========================================== + 'Monopole' : 'T', + 'num_mono' : 1, + 'Mono(1)%loc(1)' : -5.E-03/x0, + 'Mono(1)%npulse' : 1, + 'Mono(1)%dir' : 1., + 'Mono(1)%pulse' : 1, + 'Mono(1)%mag' : pa, + 'Mono(1)%length' : (1./(300000.))*cact/x0, + # ========================================================== +})) + +# ============================================================================== + diff --git a/docs/documentation/case.md b/docs/documentation/case.md index e76e82d5b5..91117e6cee 100644 --- a/docs/documentation/case.md +++ b/docs/documentation/case.md @@ -109,6 +109,7 @@ Definition of the parameters is described in the following subsections. | `t_step_start` | Integer | Simulation starting time step | | `t_step_stop` | Integer | Simulation stopping time step | | `t_step_save` | Integer | Frequency to output data | +| `t_step_print` | Integer | Frequency to print the current step number to standard output (default 1) | The parameters define the boundaries of the spatial and temporal domains, and their discretization that are used in simulation. @@ -148,8 +149,9 @@ The value of `dt` needs to be sufficiently small such that the Courant-Friedrich - `t_step_start` and `t_step_end` define the time steps at which simulation starts and ends, respectively. `t_step_save` is the time step interval for data output during simulation. -To newly start simulation, set `t_step_start`=0. -To restart simulation from $k$-th time step, set `t_step_start`=k. +To newly start the simulation, set `t_step_start = 0`. +To restart simulation from $k$-th time step, set `t_step_start = k`, do not run `pre_process`, and run `simulation` directly (`./mfc.sh run [...] -t simulation`). +Ensure the data for the $k$-th time step is stored in the `restart_data/` directory within the case repository. ### 3. Patches diff --git a/docs/documentation/expectedPerformance.md b/docs/documentation/expectedPerformance.md index 5e5550da00..3c3501b72b 100644 --- a/docs/documentation/expectedPerformance.md +++ b/docs/documentation/expectedPerformance.md @@ -5,31 +5,21 @@ This page shows a summary of these results. ## Expected time-steps/hour -The following table outlines expected performance in terms of the number of time steps per hour, rounded to the nearest hundred (higher is better). -A 3D inviscid, 6-equation problem is solved for various problem sizes (grid cells) and hardware. -A 3rd order (3-stage) Runge-Kutta time-stepper is used. -CPU results utilize an entire processor die. +The following table outlines observed performance as nanoseconds per grid point (ns/GP) per right-hand side evaluation (lower is better). +We solve an example 3D, inviscid, 5-equation model problem with two advected species (a total of 8 PDEs). +The numerics are WENO5 and the HLLC approximate Riemann solver. +We report results for various numbers of grid points per CPU die (or GPU device) and hardware. -| Hardware | # Cores | Steps/Hr (1M pts) | Steps/Hr (4M pts) | Steps/Hr (8M pts) | Compiler | Computer | +| Hardware | | 1M GPs | 4M GPs | 8M GPs | Compiler | Computer | | ---: | :----: | :----: | :---: | :---: | :----: | :--- | -| NVIDIA V100 | 1 (device) | 88.5k | 18.7k | N/A | NVHPC 22.11 | PACE Phoenix | -| NVIDIA V100 | 1 (device) | 78.8k | 18.8k | N/A | NVHPC 22.11 | OLCF Summit | -| NVIDIA A100 | 1 (device) | 114.4k | 34.6k | 16.5k | NVHPC 23.5 | Wingtip | -| AMD MI250X | 1 (GCD) | 77.5k | 22.3k | 11.2k | CCE 16.0.1 | OLCF Frontier | -| Intel Xeon Gold 6226 | 12 (cores) | 2.5k | 0.7k | 0.4k | GNU 10.3.0 | PACE Phoenix | -| Apple Silicon M2 | 6 (cores) | 2.8k | 0.6k | 0.2k | GNU 13.2.0 | N/A | - -We also show the expected performance of MFC for the same problem as above, except for the 5-equation model used, in the table below. -It is presented in the same manner as the one above. - -| Hardware | # Cores | Steps/Hr (1M pts) | Steps/Hr (4M pts) | Steps/Hr (8M pts) | Compiler | Computer | -| ---: | :----: | :----: | :---: | :---: | :----: | :--- | -| NVIDIA V100 | 1 (device) | 113.4k | 26.2k | 13.0k | NVHPC 22.11 | PACE Phoenix | -| NVIDIA V100 | 1 (device) | 107.7k | 26.3k | 13.1k | NVHPC 22.11 | OLCF Summit | -| NVIDIA A100 | 1 (device) | 153.5k | 48.0k | 22.5k | NVHPC 23.5 | Wingtip | -| AMD MI250X | 1 (GCD) | 104.2k | 31.0k | 14.8k | CCE 16.0.1 | OLCF Frontier | -| Intel Xeon Gold 6226 | 12 (cores) | 5.4k | 1.6k | 0.8k | GNU 10.3.0 | PACE Phoenix | -| Apple Silicon M2 | 6 (cores) | 3.7k | 11.0k | 0.3k | GNU 13.2.0 | N/A | +| NVIDIA V100 | 1 device | 96 | 104 | 104 | NVHPC 22.11 | PACE Phoenix | +| NVIDIA V100 | 1 device | 101 | 104 | 104 | NVHPC 22.11 | OLCF Summit | +| NVIDIA A100 | 1 device | 71 | 56 | 59 | NVHPC 23.5 | Wingtip | +| AMD MI250X | 1 GCD | 108 | 90 | 96 | CCE 16.0.1 | OLCF Frontier | +| Intel Xeon Gold 6226 | 12 cores | 1963 | 1688 | 1686 | GNU 10.3.0 | PACE Phoenix | +| Apple M2 | 6 cores | 2919 | 245 | 4500 | GNU 13.2.0 | N/A | + +__All results are in nanoseconds (ns) per grid point (gp) per right-hand side (rhs) evaluation. Lower is better.__ ## Weak scaling diff --git a/docs/documentation/getting-started.md b/docs/documentation/getting-started.md index ab97efae3d..ee7c6f0aa0 100644 --- a/docs/documentation/getting-started.md +++ b/docs/documentation/getting-started.md @@ -30,7 +30,8 @@ sudo apt update sudo apt upgrade sudo apt install tar wget make cmake gcc g++ \ python3 python3-dev \ - "openmpi-*" libopenmpi-dev + "openmpi-*" libopenmpi-dev \ + python3-venv ``` - **Via [Pacman](https://wiki.archlinux.org/title/pacman):** @@ -111,26 +112,19 @@ Further reading on `open-mpi` incompatibility with `clang`-based `gcc` on macOS: We do *not* support `clang` due to conflicts with the Silo dependency. ```console -# === MFC MPI Installation === export MFC_GCC_VER=13 -export OMPI_MPICC=gcc-$MFC_GCC_VER -export OMPI_CXX=g++-$MFC_GCC_VER -export OMPI_FC=gfortran-$MFC_GCC_VER export CC=gcc-$MFC_GCC_VER export CXX=g++-$MFC_GCC_VER export FC=gfortran-$MFC_GCC_VER -# === MFC MPI Installation === ``` **Close the open editor and terminal window**. Open a **new terminal** window before executing the commands below. ```console -brew install wget make python make cmake coreutils gcc@$MFC_GCC_VER -HOMEBREW_MAKE_JOBS=$(nproc) brew install --cc=gcc-$MFC_GCC_VER --verbose --build-from-source open-mpi +brew install wget python cmake gcc@$MFC_GCC_VER mpich ``` -They will download the dependencies MFC requires to build itself. `open-mpi` will be compiled from source, using the version of GCC we specified above with the environment variables `HOMEBREW_CC` and `HOMEBREW_CXX`. -Building this package might take a while. +They will download the dependencies MFC requires to build itself. diff --git a/docs/index.html b/docs/index.html index 5d2242926c..ed8a2dfbde 100644 --- a/docs/index.html +++ b/docs/index.html @@ -72,11 +72,12 @@ }; const sims = [ - new FS("Shedding water droplet - Vorticity", "res/simulations/a.png", "Summit", "2016 V100s", "1h", "https://doi.org/10.48550/arXiv.2305.09163"), - new FS("Cavitating bubble cloud - Wall pressure", "res/simulations/b.png", "Summit", "216 V100s", "3h", "https://doi.org/10.48550/arXiv.2305.09163"), - new FS("Cavitating bubble cloud - Streamlines", "res/simulations/c.png", "Summit", "216 V100s", "3h", "https://doi.org/10.48550/arXiv.2305.09163"), - new FS("Kidney stone near a collapsing bubble cloud - Maximum principal stresses", "res/simulations/d.png", "Summit", "576 V100s", "30 min", "https://doi.org/10.48550/arXiv.2305.09163"), - new FS("Breakup of vibrated interface", "res/simulations/f.png", "Summit", "128 V100s", "4h","https://youtu.be/qQV2ZRDpf2M") + new FS("Shedding water droplet", "res/simulations/a.png", "Summit", "960 V100s", "4h", "https://player.vimeo.com/video/905208069"), + new FS("Flow over an airfoil (vorticity)", "res/simulations/g.png", "Delta", "128 A100s", "19h", "https://vimeo.com/917305340/c05fd414c8?share=copy"), + new FS("Cavitation fragments kidney stone", "res/simulations/d.png", "Summit", "576 V100s", "30 min", "https://doi.org/10.48550/arXiv.2305.09163"), + new FS("Breakup of vibrated interface", "res/simulations/f.png", "Summit", "128 V100s", "4h","https://player.vimeo.com/video/922022757"), + new FS("Collapsing bubbles (pressure)", "res/simulations/b.png", "Summit", "216 V100s", "3h", "https://doi.org/10.48550/arXiv.2305.09163"), + new FS("Collapsing bubbles (streamlines)", "res/simulations/c.png", "Summit", "216 V100s", "3h", "https://doi.org/10.48550/arXiv.2305.09163"), ]; /* @@ -198,8 +199,8 @@ }; scalings = [ - new FScale("res/weakScaling/frontier.svg", "Frontier"), - new FScale("res/weakScaling/summit.svg", "Summit") + new FScale("res/weakScaling/frontier.svg", "Oak Ridge Frontier"), + new FScale("res/weakScaling/summit.svg", "Oak Ridge Summit") ] @@ -257,8 +258,7 @@
- MFC simulates compressible multi-component and multi-phase flows, amongst other things. - It scales ideally to tens of thousands of GPUs. + An exascale compressible multiphase and multiphysics flow code.
@@ -276,7 +276,7 @@ - Start + Quick Start diff --git a/docs/res/airfoil.png b/docs/res/airfoil.png new file mode 100644 index 0000000000..d9bba2e6c6 Binary files /dev/null and b/docs/res/airfoil.png differ diff --git a/docs/res/simulations/f.png b/docs/res/simulations/f.png index 13fd3abc60..452be43a48 100644 Binary files a/docs/res/simulations/f.png and b/docs/res/simulations/f.png differ diff --git a/docs/res/simulations/g.png b/docs/res/simulations/g.png new file mode 100644 index 0000000000..7406009109 Binary files /dev/null and b/docs/res/simulations/g.png differ diff --git a/mfc.sh b/mfc.sh index bd6d8b919f..3bc5a2bd55 100755 --- a/mfc.sh +++ b/mfc.sh @@ -34,6 +34,8 @@ elif [ "$1" == "format" ]; then shift; . "$(pwd)/toolchain/bootstrap/format.sh" $@; exit 0 elif [ "$1" == "docker" ]; then shift; . "$(pwd)/toolchain/bootstrap/docker.sh" $@; exit 0 +elif [ "$1" == "venv" ]; then + shift; . "$(pwd)/toolchain/bootstrap/python.sh" $@; return fi mkdir -p "$(pwd)/build" diff --git a/src/post_process/m_start_up.f90 b/src/post_process/m_start_up.f90 index 5217d520d0..db871c9559 100644 --- a/src/post_process/m_start_up.f90 +++ b/src/post_process/m_start_up.f90 @@ -89,7 +89,7 @@ subroutine s_read_input_file() ! --------------------------------------- backspace (1) read (1, fmt='(A)') line print *, 'Invalid line in namelist: '//trim(line) - call s_mpi_abort('Invalid line in pre_process.inp. It is '// & + call s_mpi_abort('Invalid line in post_process.inp. It is '// & 'likely due to a datatype mismatch. Exiting ...') end if diff --git a/src/pre_process/m_data_output.fpp b/src/pre_process/m_data_output.fpp index bdc5192f19..001e5b598c 100644 --- a/src/pre_process/m_data_output.fpp +++ b/src/pre_process/m_data_output.fpp @@ -759,6 +759,18 @@ contains end if + open (1, FILE='equations.dat', STATUS='unknown') + + write (1, '(A)') "Equations: " + if (momxb /= 0) write (1, '(A,I3,I3)') " * Momentum: ", momxb, momxe + if (advxb /= 0) write (1, '(A,I3,I3)') " * Advection: ", advxb, advxe + if (contxb /= 0) write (1, '(A,I3,I3)') " * Continuity: ", contxb, contxe + if (bubxb /= 0) write (1, '(A,I3,I3)') " * Bubbles: ", bubxb, bubxe + if (strxb /= 0) write (1, '(A,I3,I3)') " * Stress: ", strxb, strxe + if (intxb /= 0) write (1, '(A,I3,I3)') " * Internal Energies: ", intxb, intxe + + close (1) + end subroutine s_initialize_data_output_module ! -------------------------- !> Resets s_write_data_files pointer diff --git a/src/simulation/m_global_parameters.fpp b/src/simulation/m_global_parameters.fpp index 27fae930bf..7f96868276 100644 --- a/src/simulation/m_global_parameters.fpp +++ b/src/simulation/m_global_parameters.fpp @@ -100,6 +100,8 @@ module m_global_parameters integer :: t_step_start, t_step_stop, t_step_save !> @} + integer :: t_step_print !< Number of time-steps between printouts + ! ========================================================================== ! Simulation Algorithm Parameters ========================================== @@ -452,6 +454,7 @@ contains t_step_start = dflt_int t_step_stop = dflt_int t_step_save = dflt_int + t_step_print = 1 ! Simulation algorithm parameters model_eqns = dflt_int diff --git a/src/simulation/m_mpi_proxy.fpp b/src/simulation/m_mpi_proxy.fpp index baaf45d751..3a8f0684e9 100644 --- a/src/simulation/m_mpi_proxy.fpp +++ b/src/simulation/m_mpi_proxy.fpp @@ -140,8 +140,8 @@ contains call MPI_BCAST(case_dir, len(case_dir), MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) #:for VAR in ['t_step_old', 'm', 'n', 'p', 'm_glb', 'n_glb', 'p_glb', & - & 't_step_start','t_step_stop','t_step_save','model_eqns', & - & 'num_fluids','time_stepper', 'riemann_solver', & + & 't_step_start','t_step_stop','t_step_save','t_step_print', & + & 'model_eqns','num_fluids','time_stepper', 'riemann_solver', & & 'wave_speeds', 'avg_state', 'precision', 'bc_x%beg', 'bc_x%end', & & 'bc_y%beg', 'bc_y%end', 'bc_z%beg', 'bc_z%end', 'fd_order', & & 'num_probes', 'num_integrals', 'bubble_model', 'thermal', & diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index e366fb7869..a8541b9da7 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -127,7 +127,7 @@ contains ! Namelist of the global parameters which may be specified by user namelist /user_inputs/ case_dir, run_time_info, m, n, p, dt, & - t_step_start, t_step_stop, t_step_save, & + t_step_start, t_step_stop, t_step_save, t_step_print, & model_eqns, num_fluids, adv_alphan, & mpp_lim, time_stepper, weno_eps, weno_flat, & riemann_flat, cu_mpi, cu_tensor, & @@ -169,7 +169,7 @@ contains backspace (1) read (1, fmt='(A)') line print *, 'Invalid line in namelist: '//trim(line) - call s_mpi_abort('Invalid line in pre_process.inp. It is '// & + call s_mpi_abort('Invalid line in simulation.inp. It is '// & 'likely due to a datatype mismatch. Exiting ...') end if @@ -329,39 +329,23 @@ contains end if ! ================================================================== - ! Cell-average Conservative Variables ============================== - if ((bubbles .neqv. .true.) .and. (hypoelasticity .neqv. .true.)) then - do i = 1, adv_idx%end - write (file_path, '(A,I0,A)') & - trim(t_step_dir)//'/q_cons_vf', i, '.dat' - inquire (FILE=trim(file_path), EXIST=file_exist) - if (file_exist) then - open (2, FILE=trim(file_path), & - FORM='unformatted', & - ACTION='read', & - STATUS='old') - read (2) q_cons_vf(i)%sf(0:m, 0:n, 0:p); close (2) - else - call s_mpi_abort(trim(file_path)//' is missing. Exiting ...') - end if - end do - else - !make sure to read bubble variables - do i = 1, sys_size - write (file_path, '(A,I0,A)') & - trim(t_step_dir)//'/q_cons_vf', i, '.dat' - inquire (FILE=trim(file_path), EXIST=file_exist) - if (file_exist) then - open (2, FILE=trim(file_path), & - FORM='unformatted', & - ACTION='read', & - STATUS='old') - read (2) q_cons_vf(i)%sf(0:m, 0:n, 0:p); close (2) - else - call s_mpi_abort(trim(file_path)//' is missing. Exiting ...') - end if - end do - !Read pb and mv for non-polytropic qbmm + do i = 1, sys_size + write (file_path, '(A,I0,A)') & + trim(t_step_dir)//'/q_cons_vf', i, '.dat' + inquire (FILE=trim(file_path), EXIST=file_exist) + if (file_exist) then + open (2, FILE=trim(file_path), & + FORM='unformatted', & + ACTION='read', & + STATUS='old') + read (2) q_cons_vf(i)%sf(0:m, 0:n, 0:p); close (2) + else + call s_mpi_abort(trim(file_path)//' is missing. Exiting ...') + end if + end do + + if ((bubbles .eqv. .true.) .or. (hypoelasticity .eqv. .true.)) then + ! Read pb and mv for non-polytropic qbmm if (qbmm .and. .not. polytropic) then do i = 1, nb do r = 1, nnode @@ -1080,7 +1064,7 @@ contains integer :: i, j, k, l - if (proc_rank == 0) then + if (proc_rank == 0 .and. mod(t_step - t_step_start, t_step_print) == 0) then print '(" ["I3"%] Time step "I8" of "I0" @ t_step = "I0"")', & int(ceiling(100d0*(real(t_step - t_step_start)/(t_step_stop - t_step_start + 1)))), & t_step - t_step_start + 1, & diff --git a/toolchain/bench.yaml b/toolchain/bench.yaml index 50705dd6ec..bd682819cd 100644 --- a/toolchain/bench.yaml +++ b/toolchain/bench.yaml @@ -1,3 +1,38 @@ -- slug: 3D_shockdroplet - path: benchmarks/3D_shockdroplet/case.py +# Benchmark model_equations_2_time_stepper_3_weno_order_3_riemann_solver_2 +# Additional Benchmarked Features +# - model_equations : 2 +# - time_stepper : 3 +# - weno_order : 3 +# - riemann_solver : 2 +- slug: 5eq_rk3_weno3_hllc + path: benchmarks/5eq_rk3_weno3_hllc/case.py args: [] + +# Benchmark viscosity_weno_Re_flux_T_weno_order_5_bubbles_T_bubble_mode_3_monopole_T +# Additional Benchmarked Features +# - viscosity enabled +# - weno_Re_flux : T +# - weno_order : 5 +# - bubbles : T +# - bubble_model : 3 +# - monopole : T +- slug: viscous_weno5_sgb_mono + path: benchmarks/viscous_weno5_sgb_mono/case.py + args: [] + +# Benchmark ibm_T +# Additional Benchmarked Features +# - ibm : T +- slug: ibm + path: benchmarks/ibm/case.py + args: [] + +# Benchmark hypoelasticity_T_riemann_solver_1 +# Additional Benchmarked Features +# - hypoelasticity : T +# - riemann_solver : 1 +- slug: hypo_hll + path: benchmarks/hypo_hll/case.py + args: [] + + diff --git a/toolchain/bootstrap/format.sh b/toolchain/bootstrap/format.sh index 77ad2abf62..0a2ec24a1a 100644 --- a/toolchain/bootstrap/format.sh +++ b/toolchain/bootstrap/format.sh @@ -1,39 +1,33 @@ #!/bin/bash -log "Formatting MFC:" - -fortran_files=$(find ${@:-src} -type f | grep -Ev 'src/.+/autogen/') - -longest=0 -for filepath in $fortran_files; do - if [ "${#filepath}" -gt "$longest" ]; then - longest="${#filepath}" - fi +while [[ $# -gt 0 ]]; do + case "$1" in + -j|--jobs) + JOBS="$2" + shift + ;; + *) + echo "Format, unknown argument: $1." + exit 1 + ;; + esac + + shift done -for filepath in $fortran_files; do - echo -n " > $filepath $(printf '%*s' "$((longest - ${#filepath}))" '')" - - before=$(sha256sum "$filepath" | cut -d' ' -f1) - - python3 toolchain/indenter.py "$filepath" - - if ! fprettify "$filepath" \ - --silent --indent 4 --c-relations --enable-replacements --enable-decl \ - --whitespace-comma 1 --whitespace-multdiv 0 --whitespace-plusminus 1 \ - --case 1 1 1 1 --strict-indent --line-length 1000; then - error "failed to execute fprettify." - error "MFC has not been fprettify'ied." - exit 1 - fi +log "Formatting MFC:" - after=$(sha256sum "$filepath" | cut -d' ' -f1) - - if [ "$before" != "$after" ]; then - echo -e "$YELLOW[formatted]$COLOR_RESET" - else - echo -e "$GREEN[unchanged]$COLOR_RESET" - fi -done +if ! find ${@:-src} -type f | grep -Ev 'autogen' | grep -E '\.(f90|fpp)$' | \ + parallel --jobs ${JOBS:-1} -- \ + echo "\> {}" \&\& \ + python3 toolchain/indenter.py "{}" \&\& \ + fprettify "{}" --silent --indent 4 --c-relations --enable-replacements \ + --enable-decl --whitespace-comma 1 --whitespace-multdiv 0 \ + --whitespace-plusminus 1 --case 1 1 1 1 --strict-indent \ + --line-length 1000\;; then + error "Formatting MFC failed." + exit 1 +fi ok "Done. MFC has been formatted." + diff --git a/toolchain/mfc/args.py b/toolchain/mfc/args.py index 9dc2da9c8c..428bd0fa34 100644 --- a/toolchain/mfc/args.py +++ b/toolchain/mfc/args.py @@ -128,6 +128,7 @@ def add_common_arguments(p, mask = None): # === BENCH === add_common_arguments(bench) bench.add_argument("-o", "--output", metavar="OUTPUT", default=None, type=str, required="True", help="Path to the YAML output file to write the results to.") + bench.add_argument("-m", "--mem", metavar="MEM", default=1, type=int, help="Memory per task for benchmarking cases") bench.add_argument(metavar="FORWARDED", default=[], dest='--', nargs="*", help="Arguments to forward to the ./mfc.sh run invocations.") # === BENCH_DIFF === diff --git a/toolchain/mfc/bench.py b/toolchain/mfc/bench.py index 8843de3507..928953e70e 100644 --- a/toolchain/mfc/bench.py +++ b/toolchain/mfc/bench.py @@ -56,7 +56,7 @@ def bench(targets = None): with open(log_filepath, "w") as log_file: system( - ["./mfc.sh", "run", case.path, "--case-optimization"] + + ["./mfc.sh", "run", case.path, ARG('mem'), "--case-optimization"] + ["--targets"] + [t.name for t in targets] + ["--output-summary", summary_filepath] + case.args, @@ -81,15 +81,24 @@ def diff(): cons.print(f"[bold]Comparing Bencharks: [magenta]{os.path.relpath(ARG('lhs'))}[/magenta] is x times slower than [magenta]{os.path.relpath(ARG('rhs'))}[/magenta].[/bold]") if lhs["metadata"] != rhs["metadata"]: - cons.print(f"[bold yellow]Warning[/bold yellow]: Metadata of lhs and rhs are not equal.") - quit(1) + def _lock_to_str(lock): + return ' '.join([f"{k}={v}" for k, v in lock.items()]) + + cons.print(f"[bold yellow]Warning[/bold yellow]: Metadata in lhs and rhs are not equal.") + cons.print(f" This could mean that the benchmarks are not comparable (e.g. one was run on CPUs and the other on GPUs).") + cons.print(f" lhs:") + cons.print(f" * Invocation: [magenta]{' '.join(lhs['metadata']['invocation'])}[/magenta]") + cons.print(f" * Modes: {_lock_to_str(lhs['metadata']['lock'])}") + cons.print(f" rhs:") + cons.print(f" * Invocation: {' '.join(rhs['metadata']['invocation'])}") + cons.print(f" * Modes: [magenta]{_lock_to_str(rhs['metadata']['lock'])}[/magenta]") slugs = set(lhs["cases"].keys()) & set(rhs["cases"].keys()) if len(slugs) not in [len(lhs["cases"]), len(rhs["cases"])]: - cons.print(f"[bold yellow]Warning[/bold yellow]: Cases of lhs and rhs are not equal.[/bold yellow]") - cons.print(f"[bold yellow]lhs: {set(lhs['cases'].keys()) - slugs}[/bold yellow]") - cons.print(f"[bold yellow]rhs: {set(rhs['cases'].keys()) - slugs}[/bold yellow]") - cons.print(f"[bold yellow]Using intersection: {slugs}[/bold yellow]") + cons.print(f"[bold yellow]Warning[/bold yellow]: Cases in lhs and rhs are not equal.") + cons.print(f" * rhs cases: {', '.join(set(rhs['cases'].keys()) - slugs)}.") + cons.print(f" * lhs cases: {', '.join(set(lhs['cases'].keys()) - slugs)}.") + cons.print(f" Using intersection: {slugs} with {len(slugs)} elements.") table = rich.table.Table(show_header=True, box=rich.table.box.SIMPLE) table.add_column("[bold]Case[/bold]", justify="left") diff --git a/toolchain/mfc/build.py b/toolchain/mfc/build.py index 9bd1486b4f..bf681b4f2c 100644 --- a/toolchain/mfc/build.py +++ b/toolchain/mfc/build.py @@ -38,7 +38,7 @@ def get_slug(self) -> str: m = hashlib.sha256() m.update(self.name.encode()) m.update(CFG().make_slug().encode()) - m.update(input.load({}).get_fpp(self).encode()) + m.update(input.load({}).get_fpp(self, False).encode()) return m.hexdigest()[:10] diff --git a/toolchain/mfc/common.py b/toolchain/mfc/common.py index d3e713412e..c0c5185940 100644 --- a/toolchain/mfc/common.py +++ b/toolchain/mfc/common.py @@ -38,8 +38,13 @@ def system(command: typing.List[str], print_cmd = None, **kwargs) -> subprocess. return subprocess.run(cmd, **kwargs, check=False) -def file_write(filepath: str, content: str): +def file_write(filepath: str, content: str, if_different: bool = False): try: + if if_different and os.path.isfile(filepath): + with open(filepath, "r") as f: + if f.read() == content: + return + with open(filepath, "w") as f: f.write(content) except IOError as exc: diff --git a/toolchain/mfc/run/case_dicts.py b/toolchain/mfc/run/case_dicts.py index 305a90bd6a..ae661cf6d7 100644 --- a/toolchain/mfc/run/case_dicts.py +++ b/toolchain/mfc/run/case_dicts.py @@ -83,7 +83,7 @@ SIMULATION = COMMON + [ 'run_time_info', 't_step_old', 't_tol', 'dt', 't_step_start', - 't_step_stop', 't_step_save', 'time_stepper', 'weno_eps', + 't_step_stop', 't_step_save', 't_step_print', 'time_stepper', 'weno_eps', 'mapped_weno', 'mp_weno', 'weno_avg', 'weno_Re_flux', 'riemann_solver', 'wave_speeds', 'avg_state', 'prim_vars_wrt', 'alt_crv', 'alt_soundspeed', 'regularization', 'null_weights', diff --git a/toolchain/mfc/run/input.py b/toolchain/mfc/run/input.py index c66b27fb84..751beedcd3 100644 --- a/toolchain/mfc/run/input.py +++ b/toolchain/mfc/run/input.py @@ -68,27 +68,16 @@ def generate_inp(self, target) -> None: cons.unindent() def __save_fpp(self, target, contents: str) -> None: - def __contents_equal(str_a: str, str_b: str) -> bool: - lhs = [ l.strip() for l in str_a.splitlines() if not common.isspace(l) ] - rhs = [ l.strip() for l in str_b.splitlines() if not common.isspace(l) ] - - return lhs == rhs - inc_dir = os.path.join(target.get_staging_dirpath(), "include", target.name) common.create_directory(inc_dir) fpp_path = os.path.join(inc_dir, "case.fpp") - opt_fpp = common.file_read(fpp_path) if os.path.exists(fpp_path) else "" - - if __contents_equal(contents, opt_fpp): - cons.print("[yellow]INFO:[/yellow] Custom case.fpp file is up to date.") - return - cons.print("[yellow]INFO:[/yellow] Writing a custom case.fpp file: --case-optimization configuration has changed.") - common.file_write(fpp_path, contents) + cons.print("Writing a (new) custom case.fpp file.") + common.file_write(fpp_path, contents, True) # pylint: disable=too-many-locals - def __get_pre_fpp(self) -> str: + def __get_pre_fpp(self, print: bool) -> str: DATA = { 1: {'ptypes': [1, 15, 16], 'sf_idx': 'i, 0, 0'}, 2: {'ptypes': [2, 3, 4, 5, 6, 7, 17, 18, 21], 'sf_idx': 'i, j, 0'}, @@ -131,20 +120,17 @@ def rhs_replace(match): lines = [] for attribute, expr in items: - varname = re.findall(r"[a-zA-Z][a-zA-Z0-9_]*", attribute)[1] - qpvf_idx_var = QPVF_IDX_VARS[varname] - qpvf_idx_offset = "" + if print: + cons.print(f"* Codegen: {attribute} = {expr}") + + varname = re.findall(r"[a-zA-Z][a-zA-Z0-9_]*", attribute)[1] + qpvf_idx = QPVF_IDX_VARS[varname][:] if len(re.findall(r"[0-9]+", attribute)) == 2: idx = int(re.findall(r'[0-9]+', attribute)[1]) - 1 - if idx != 0: - qpvf_idx_offset = f" + {idx}" - - sf_idx = DATA['sf_idx'] + qpvf_idx = f"{qpvf_idx} + {idx}" - cons.print(f"[yellow]INFO:[/yellow] {self.__get_ndims()}D Analytical Patch #{pid}: Code generation for [magenta]{varname}[/magenta]...") - - lhs = f"q_prim_vf({qpvf_idx_var}{qpvf_idx_offset})%sf({sf_idx})" + lhs = f"q_prim_vf({qpvf_idx})%sf({DATA['sf_idx']})" rhs = re.sub(r"[a-zA-Z]+", rhs_replace, expr) lines.append(f" {lhs} = {rhs}") @@ -167,9 +153,10 @@ def rhs_replace(match): return content - def __get_sim_fpp(self) -> str: + def __get_sim_fpp(self, print: bool) -> str: if ARG("case_optimization"): - cons.print("[yellow]INFO:[/yellow] Case optimization is enabled.") + if print: + cons.print("Case optimization is enabled.") nterms = 1 @@ -194,21 +181,21 @@ def __get_sim_fpp(self) -> str: ! of --case-optimization. """ - def __get_post_fpp(self) -> str: + def __get_post_fpp(self, _) -> str: return """\ ! This file is purposefully empty for all post-process builds. """ - def get_fpp(self, target) -> str: - def _default() -> str: + def get_fpp(self, target, print = True) -> str: + def _default(_) -> str: return "" result = { "pre_process" : self.__get_pre_fpp, "simulation" : self.__get_sim_fpp, "post_process" : self.__get_post_fpp, - }.get(build.get_target(target).name, _default)() - + }.get(build.get_target(target).name, _default)(print) + return result def generate_fpp(self, target) -> None: