Skip to content

Commit

Permalink
CarpetX: New parameters max_grid_sizes_[xyz]
Browse files Browse the repository at this point in the history
  • Loading branch information
@eschnett
eschnett committed Apr 29, 2024
1 parent 07c446f commit ab4e8af
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 105 deletions.
2 changes: 2 additions & 0 deletions CarpetX/interface.ccl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ USES INCLUDE HEADER: loop_device.hxx

INCLUDES HEADER: reduction.hxx IN reduction.hxx



# Number of patches
CCTK_INT FUNCTION MultiPatch_GetSystemSpecification( \
CCTK_INT OUT npatches)
Expand Down
31 changes: 23 additions & 8 deletions CarpetX/param.ccl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ STRING amrex_parameters[100] "Additional AMReX parameters"
} ""


RESTRICTED:

CCTK_REAL xmin "Domain size (location of vertices)"
{
Expand Down Expand Up @@ -70,7 +69,6 @@ CCTK_INT ncells_z "Number of grid cells"




BOOLEAN periodic_x "Periodic"
{
} no
Expand Down Expand Up @@ -295,7 +293,7 @@ STRING robin_upper_z_vars "Override boundary condition at upper z boundary"
".*" :: ""
} ""

RESTRICTED:


CCTK_INT blocking_factor_x "Blocking factor" STEERABLE=recover
{
Expand All @@ -317,23 +315,40 @@ CCTK_REAL grid_efficiency "Minimum AMR grid efficiency" STEERABLE=recover
0.0:* :: ""
} 0.7

#The values of max_grid_size_x[n] correspond to max_grid_size_x for level n. Currently, maximum 10 levels can be set. If number of levels is greater than 10, then max_grid_size for 10th level is used for higher levels as well.
CCTK_INT max_grid_size_x[10] "Maximum grid size" STEERABLE=always
CCTK_INT max_grid_size_x "Maximum grid size" STEERABLE=always
{
1:* :: "must be a multiple of the blocking factor"
} 32

CCTK_INT max_grid_size_y[10] "Maximum grid size" STEERABLE=always
CCTK_INT max_grid_size_y "Maximum grid size" STEERABLE=always
{
1:* :: "must be a multiple of the blocking factor"
} 32

CCTK_INT max_grid_size_z[10] "Maximum grid size" STEERABLE=always
CCTK_INT max_grid_size_z "Maximum grid size" STEERABLE=always
{
1:* :: "must be a multiple of the blocking factor"
} 32

PRIVATE:
CCTK_INT max_grid_sizes_x[20] "Maximum grid size" STEERABLE=always
{
-1 :: "use value from max_grid_size_x"
1:* :: "must be a multiple of the blocking factor"
} -1

CCTK_INT max_grid_sizes_y[20] "Maximum grid size" STEERABLE=always
{
-1 :: "use value from max_grid_size_y"
1:* :: "must be a multiple of the blocking factor"
} -1

CCTK_INT max_grid_sizes_z[20] "Maximum grid size" STEERABLE=always
{
-1 :: "use value from max_grid_size_z"
1:* :: "must be a multiple of the blocking factor"
} -1



BOOLEAN refine_grid_layout "Ensure that each MPI process has at least one grid" STEERABLE=recover
{
Expand Down
58 changes: 28 additions & 30 deletions CarpetX/src/driver.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1924,7 +1924,7 @@ void CactusAmrCore::RemakeLevel(const int level, const amrex::Real time,
check_valid_gf(active_levels, gi, vi, tl, nan_handling,
[]() { return "RemakeLevel before prolongation"; });
} // for vi
} // for tl
} // for tl

} // for gi

Expand Down Expand Up @@ -2363,42 +2363,40 @@ void *SetupGH(tFleshConfig *fc, int convLevel, cGH *restrict cctkGH) {
// we get core files.
pp.add("amrex.throw_exception", 1);

// Enable managed memory (i.e. copy data automatically between accelerator and
// host)
#ifndef AMREX_USE_HIP
// This option would render the code inefficient on AMD GPUs.
// However, it is required to enable some thorns (TOVSolverX, TwoPuncturesX,
// ...) to work on NVIDIA GPUs.
// We might need to update those thorns to git rid of this option completely
// at some point.
pp.add("amrex.the_arena_is_managed", 1);
#endif
// Enable managed memory (i.e. copy data automatically between
// accelerator and host)
#ifndef AMREX_USE_HIP
// This option would render the code inefficient on AMD GPUs.
// However, it is required to enable some thorns (TOVSolverX,
// TwoPuncturesX, ...) to work on NVIDIA GPUs.
// We might need to update those thorns to git rid of this option
// completely at some point.
pp.add("amrex.the_arena_is_managed", 1);
#endif

// Set blocking factors via parameter table since AmrMesh needs to
// know them when its constructor is running, but there are no
// constructor arguments for them
//pp.add("amr.max_grid_size_x", max_grid_size_x);
//pp.add("amr.max_grid_size_y", max_grid_size_y);
//pp.add("amr.max_grid_size_z", max_grid_size_z);

//Set max_grid_size for each level
amrex::Vector<int> max_grid_size_x_vec =
{max_grid_size_x[0], max_grid_size_x[1], max_grid_size_x[2], max_grid_size_x[3],
max_grid_size_x[4], max_grid_size_x[5], max_grid_size_x[6], max_grid_size_x[7],
max_grid_size_x[8], max_grid_size_x[9]};
amrex::Vector<int> max_grid_size_y_vec =
{max_grid_size_y[0], max_grid_size_y[1], max_grid_size_y[2], max_grid_size_y[3],
max_grid_size_y[4], max_grid_size_y[5], max_grid_size_y[6], max_grid_size_y[7],
max_grid_size_y[8], max_grid_size_y[9]};
amrex::Vector<int> max_grid_size_z_vec =
{max_grid_size_z[0], max_grid_size_z[1], max_grid_size_z[2], max_grid_size_z[3],
max_grid_size_z[4], max_grid_size_z[5], max_grid_size_z[6], max_grid_size_z[7],
max_grid_size_z[8], max_grid_size_z[9]};

amrex::Vector<int> max_grid_size_x_vec(20, -1), max_grid_size_y_vec(20, -1),
max_grid_size_z_vec(20, -1);
for (int level = 0; level < 20; ++level) {
int size_x = max_grid_sizes_x[level];
int size_y = max_grid_sizes_y[level];
int size_z = max_grid_sizes_z[level];
if (size_x == -1)
size_x = max_grid_size_x;
if (size_y == -1)
size_y = max_grid_size_y;
if (size_z == -1)
size_z = max_grid_size_z;
max_grid_size_x_vec.at(level) = size_x;
max_grid_size_y_vec.at(level) = size_y;
max_grid_size_z_vec.at(level) = size_z;
}
pp.addarr("amr.max_grid_size_x", max_grid_size_x_vec);
pp.addarr("amr.max_grid_size_y", max_grid_size_y_vec);
pp.addarr("amr.max_grid_size_z", max_grid_size_z_vec);

pp.add("amr.refine_grid_layout", refine_grid_layout);
pp.add("amr.blocking_factor_x", blocking_factor_x);
pp.add("amr.blocking_factor_y", blocking_factor_y);
Expand Down
113 changes: 46 additions & 67 deletions CarpetX/src/schedule.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1067,37 +1067,26 @@ int Initialise(tFleshConfig *config) {
CCTKi_FinaliseParamWarn();

active_levels = optional<active_levels_t>();

//-------------------------------------------------------------------------------------
//Set the value of max_grid_size initially.
CCTK_VINFO("Setting max_grid_size values (for all levels) initially, in schedule...");
for (const auto &patchdata : ghext->patchdata){
amrex::Vector<int> mgs0_vec = {max_grid_size_x[0], max_grid_size_y[0], max_grid_size_z[0]};
amrex::IntVect mgs0(mgs0_vec); //This won't be steered here, but keeping for scaffolding.
amrex::Vector<int> mgs1_vec = {max_grid_size_x[1], max_grid_size_y[1], max_grid_size_z[1]};
amrex::IntVect mgs1(mgs1_vec);
amrex::Vector<int> mgs2_vec = {max_grid_size_x[2], max_grid_size_y[2], max_grid_size_z[2]};
amrex::IntVect mgs2(mgs2_vec);
amrex::Vector<int> mgs3_vec = {max_grid_size_x[3], max_grid_size_y[3], max_grid_size_z[3]};
amrex::IntVect mgs3(mgs3_vec);
amrex::Vector<int> mgs4_vec = {max_grid_size_x[4], max_grid_size_y[4], max_grid_size_z[4]};
amrex::IntVect mgs4(mgs4_vec);
amrex::Vector<int> mgs5_vec = {max_grid_size_x[5], max_grid_size_y[5], max_grid_size_z[5]};
amrex::IntVect mgs5(mgs5_vec);
amrex::Vector<int> mgs6_vec = {max_grid_size_x[6], max_grid_size_y[6], max_grid_size_z[6]};
amrex::IntVect mgs6(mgs6_vec);
amrex::Vector<int> mgs7_vec = {max_grid_size_x[7], max_grid_size_y[7], max_grid_size_z[7]};
amrex::IntVect mgs7(mgs7_vec);
amrex::Vector<int> mgs8_vec = {max_grid_size_x[8], max_grid_size_y[8], max_grid_size_z[8]};
amrex::IntVect mgs8(mgs8_vec);
amrex::Vector<int> mgs9_vec = {max_grid_size_x[9], max_grid_size_y[9], max_grid_size_z[9]};
amrex::IntVect mgs9(mgs9_vec);

amrex::Vector<amrex::IntVect> mgs_all = {mgs0, mgs1, mgs2, mgs3, mgs4, mgs5, mgs6, mgs7, mgs8, mgs9};
patchdata.amrcore->SetMaxGridSize(mgs_all);
}
//-------------------------------------------------------------------------------------

// Set the initial value of max_grid_size for all levels
CCTK_VINFO("Setting initial values for max_grid_size values for all levels");
for (const auto &patchdata : ghext->patchdata) {
amrex::Vector<amrex::IntVect> max_grid_sizes_vec(20);
for (int level = 0; level < 20; ++level) {
int size_x = max_grid_sizes_x[level];
if (size_x == -1)
size_x = max_grid_size_x;
int size_y = max_grid_sizes_y[level];
if (size_y == -1)
size_y = max_grid_size_y;
int size_z = max_grid_sizes_z[level];
if (size_z == -1)
size_z = max_grid_size_z;
const amrex::IntVect max_grid_size_vec{size_x, size_y, size_z};
max_grid_sizes_vec.at(level) = max_grid_size_vec;
}
patchdata.amrcore->SetMaxGridSize(max_grid_sizes_vec);
}

if (config->recovered) {
// Recover
Expand Down Expand Up @@ -1304,7 +1293,7 @@ int Initialise(tFleshConfig *config) {
}
}
} // omp critical
} // for patchdata
} // for patchdata

int first_modified_level = INT_MAX;
int last_modified_level = -1;
Expand Down Expand Up @@ -1619,34 +1608,24 @@ int Evolve(tFleshConfig *config) {
patchdata.amrcore->level_modified.resize(old_numlevels, false);
const CCTK_REAL time = 0; // dummy time

//-------------------------------------------------------------------------------------
//Set the value of max_grid_size before regridding.
CCTK_VINFO("Setting max_grid_size values (for level > 0) before regridding, in schedule...");
amrex::Vector<int> mgs0_vec = {max_grid_size_x[0], max_grid_size_y[0], max_grid_size_z[0]};
amrex::IntVect mgs0(mgs0_vec); //This won't be steered here, but keeping for scaffolding.
amrex::Vector<int> mgs1_vec = {max_grid_size_x[1], max_grid_size_y[1], max_grid_size_z[1]};
amrex::IntVect mgs1(mgs1_vec);
amrex::Vector<int> mgs2_vec = {max_grid_size_x[2], max_grid_size_y[2], max_grid_size_z[2]};
amrex::IntVect mgs2(mgs2_vec);
amrex::Vector<int> mgs3_vec = {max_grid_size_x[3], max_grid_size_y[3], max_grid_size_z[3]};
amrex::IntVect mgs3(mgs3_vec);
amrex::Vector<int> mgs4_vec = {max_grid_size_x[4], max_grid_size_y[4], max_grid_size_z[4]};
amrex::IntVect mgs4(mgs4_vec);
amrex::Vector<int> mgs5_vec = {max_grid_size_x[5], max_grid_size_y[5], max_grid_size_z[5]};
amrex::IntVect mgs5(mgs5_vec);
amrex::Vector<int> mgs6_vec = {max_grid_size_x[6], max_grid_size_y[6], max_grid_size_z[6]};
amrex::IntVect mgs6(mgs6_vec);
amrex::Vector<int> mgs7_vec = {max_grid_size_x[7], max_grid_size_y[7], max_grid_size_z[7]};
amrex::IntVect mgs7(mgs7_vec);
amrex::Vector<int> mgs8_vec = {max_grid_size_x[8], max_grid_size_y[8], max_grid_size_z[8]};
amrex::IntVect mgs8(mgs8_vec);
amrex::Vector<int> mgs9_vec = {max_grid_size_x[9], max_grid_size_y[9], max_grid_size_z[9]};
amrex::IntVect mgs9(mgs9_vec);

amrex::Vector<amrex::IntVect> mgs_all = {mgs0, mgs1, mgs2, mgs3, mgs4, mgs5, mgs6, mgs7, mgs8, mgs9};
patchdata.amrcore->SetMaxGridSize(mgs_all);
//-------------------------------------------------------------------------------------

// Set the value of max_grid_size before regridding
CCTK_VINFO(
"Setting max_grid_size values for all levels before regridding");
amrex::Vector<amrex::IntVect> max_grid_sizes_vec(20);
for (int level = 0; level < 20; ++level) {
int size_x = max_grid_sizes_x[level];
if (size_x == -1)
size_x = max_grid_size_x;
int size_y = max_grid_sizes_y[level];
if (size_y == -1)
size_y = max_grid_size_y;
int size_z = max_grid_sizes_z[level];
if (size_z == -1)
size_z = max_grid_size_z;
const amrex::IntVect max_grid_size_vec{size_x, size_y, size_z};
max_grid_sizes_vec.at(level) = max_grid_size_vec;
}
patchdata.amrcore->SetMaxGridSize(max_grid_sizes_vec);

patchdata.amrcore->regrid(0, time);

Expand Down Expand Up @@ -1682,7 +1661,7 @@ int Evolve(tFleshConfig *config) {
}
}
} // omp critical
} // for patchdata
} // for patchdata

int first_modified_level = INT_MAX;
int last_modified_level = -1;
Expand Down Expand Up @@ -2422,7 +2401,7 @@ int SyncGroupsByDirI(const cGH *restrict cctkGH, int numgroups,
[]() { return "SyncGroupsByDirI before syncing"; });
}
} // for tl
} // for gi
} // for gi

// We need to loop over groups, patches, and levels in a definite
// order so that AMReX's communication pattern does not get
Expand Down Expand Up @@ -2542,7 +2521,7 @@ int SyncGroupsByDirI(const cGH *restrict cctkGH, int numgroups,
[]() { return "SyncGroupsByDirI after syncing"; });
}
} // for tl
} // for gi
} // for gi

if (have_multipatch_boundaries) {
std::vector<CCTK_INT> cactusvarinds;
Expand Down Expand Up @@ -2667,8 +2646,8 @@ void Reflux(const cGH *cctkGH, int level) {
});
}
} // for gi
} // if level exists
} // for patchdata
} // if level exists
} // for patchdata
}

void Restrict(const cGH *cctkGH, int level, const vector<int> &groups) {
Expand Down Expand Up @@ -2788,9 +2767,9 @@ void Restrict(const cGH *cctkGH, int level, const vector<int> &groups) {
}

} // for tl
} // for gi
} // if level exists
} // for patchdata
} // for gi
} // if level exists
} // for patchdata
}

void Restrict(const cGH *cctkGH, int level) {
Expand Down

0 comments on commit ab4e8af

Please sign in to comment.