diff --git a/docs/conf.py b/docs/conf.py index 16bdffd62..0514bbb2d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -68,9 +68,9 @@ intersphinx_mapping = { 'geometric_features': - ('https://mpas-dev.github.io/geometric_features/stable', None), + ('https://mpas-dev.github.io/geometric_features/main', None), 'matplotlib': ('https://matplotlib.org/stable', None), - 'mpas_tools': ('https://mpas-dev.github.io/MPAS-Tools/stable', None), + 'mpas_tools': ('https://mpas-dev.github.io/MPAS-Tools/master', None), 'numpy': ('https://numpy.org/doc/stable', None), 'python': ('https://docs.python.org', None), 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), diff --git a/docs/developers_guide/ocean/api.md b/docs/developers_guide/ocean/api.md index e35e5b04d..07b1494cb 100644 --- a/docs/developers_guide/ocean/api.md +++ b/docs/developers_guide/ocean/api.md @@ -109,6 +109,8 @@ viz.Viz viz.Viz.run + decomp.Decomp + restart.Restart restart.RestartStep @@ -320,8 +322,8 @@ .. autosummary:: :toctree: generated/ - get_resolution_for_tasks - get_timestep_for_tasks + get_resolution_for_task + get_timestep_for_task forward.ConvergenceForward forward.ConvergenceForward.compute_cell_count diff --git a/docs/developers_guide/ocean/tasks/barotropic_gyre.md b/docs/developers_guide/ocean/tasks/barotropic_gyre.md index b1f28a4c1..a2e1b20a7 100644 --- a/docs/developers_guide/ocean/tasks/barotropic_gyre.md +++ b/docs/developers_guide/ocean/tasks/barotropic_gyre.md @@ -34,13 +34,13 @@ the `init` step. Namelist and streams files are updated in with time steps determined algorithmically based on config options. The number of cells is approximated from config options in {py:meth}`polaris.ocean.tasks.barotropic_gyre.forward.Forward.compute_cell_count()` -so that this can be used to constrain the number of MPI tasks that Polaris +so that this can be used to constrain the number of MPI tasks that Polaris tasks have as their target and minimum (if the resources are not explicitly prescribed). For MPAS-Ocean, PIO namelist options are modified and a -graph partition is generated as part of `runtime_setup()`. Next, the ocean +graph partition is generated as part of `runtime_setup()`. Next, the ocean model is run. If `run_time_steps` is provided then this determines the run duration, otherwise the duration is 3 years. Finally, validation of -`layerThickness` and `normalVelocity` in the `output.nc` file are performed +`layerThickness` and `normalVelocity` in the `output.nc` file are performed against a baseline if one is provided when calling {ref}`dev-polaris-setup`. ### analysis @@ -51,9 +51,9 @@ the analytical solution for the linearized dynamics. This step also produces a figure with the model solution, the analytical solution, and the difference between the two. -(dev-ocean-baroclinic-channel-default)= +(dev-ocean-baroclinic-gyre-default)= ## default The {py:class}`polaris.ocean.tasks.baroclinic_channel.default.Default` -test performs a test of the linearized dynamics. +test performs a test of the linearized dynamics. diff --git a/docs/developers_guide/ocean/tasks/cosine_bell.md b/docs/developers_guide/ocean/tasks/cosine_bell.md index 10e4e653c..b92592803 100644 --- a/docs/developers_guide/ocean/tasks/cosine_bell.md +++ b/docs/developers_guide/ocean/tasks/cosine_bell.md @@ -86,6 +86,13 @@ colorbar_limits = 0.0, 1.0 See {ref}`dev-visualization-global` for more details. +## decomp + +The class {py:class}`polaris.ocean.tasks.cosine_bell.decomp.Decomp` defines +a decomposition test across core counts. It runs Cosine Bell test at coarse +resolution once each on 12 and 24 cores to verify bit-for-bit reproducibility +for tracer advection across different core counts. + ## restart The {py:class}`polaris.ocean.tasks.cosine_bell.restart.Restart` class defines diff --git a/docs/tutorials/dev_add_test_group.md b/docs/tutorials/dev_add_test_group.md index 961663cdb..a1bc4a80a 100644 --- a/docs/tutorials/dev_add_test_group.md +++ b/docs/tutorials/dev_add_test_group.md @@ -1011,7 +1011,7 @@ constructor: $ vi ${POLARIS_HEAD}/polaris/ocean/tasks/yet_another_channel/init.py ``` ```{code-block} python -:emphasize-lines: 12-14 +:emphasize-lines: 11-13 ... @@ -1270,7 +1270,7 @@ Next, we add inputs that are outputs from the `init` task: $ vi ${POLARIS_HEAD}/polaris/ocean/tasks/yet_another_channel/forward.py ``` ```{code-block} python -:emphasize-lines: 12-17 +:emphasize-lines: 10-15 ... diff --git a/docs/users_guide/ocean/suites.md b/docs/users_guide/ocean/suites.md index a8c003190..14f864b07 100644 --- a/docs/users_guide/ocean/suites.md +++ b/docs/users_guide/ocean/suites.md @@ -8,25 +8,30 @@ run of the same tasks. (ocean-suite-cosine-bell)= -## cosine_bell and cosine_bell_cached_init suite +## cosine_bell suite ```bash polaris suite -c ocean -t cosine_bell ... ``` -```bash -polaris suite -c ocean -t cosine_bell_cached_init ... -``` - -Both `cosine_bell` suites include the following tasks: +The `cosine_bell` suite includes the following tasks: ```none -ocean/global_convergence/icos/cosine_bell -ocean/global_convergence/qu/cosine_bell +ocean/spherical/icos/cosine_bell/convergence_space +ocean/spherical/icos/cosine_bell/convergence_space/with_viz +ocean/spherical/icos/cosine_bell/convergence_time +ocean/spherical/icos/cosine_bell/convergence_time/with_viz +ocean/spherical/icos/cosine_bell/convergence_both +ocean/spherical/icos/cosine_bell/convergence_both/with_viz +ocean/spherical/icos/cosine_bell/decomp +ocean/spherical/icos/cosine_bell/restart +ocean/spherical/qu/cosine_bell/convergence_space +ocean/spherical/qu/cosine_bell/convergence_space/with_viz +ocean/spherical/qu/cosine_bell/convergence_time +ocean/spherical/qu/cosine_bell/convergence_time/with_viz +ocean/spherical/qu/cosine_bell/convergence_both +ocean/spherical/qu/cosine_bell/convergence_both/with_viz +ocean/spherical/qu/cosine_bell/decomp +ocean/spherical/qu/cosine_bell/restart ``` -The `cosine_bell` suite runs both tasks from -{ref}`ocean-cosine-bell` in full, whereas the -`cosine_bell_cached_init` suite only runs the ocean simulations from as set -of meshes and initial conditions that have been saved from a previous run -(see {ref}`dev-cache`). diff --git a/docs/users_guide/ocean/tasks/cosine_bell.md b/docs/users_guide/ocean/tasks/cosine_bell.md index df9db26b5..3dd6a08cc 100644 --- a/docs/users_guide/ocean/tasks/cosine_bell.md +++ b/docs/users_guide/ocean/tasks/cosine_bell.md @@ -28,7 +28,12 @@ and final state on a lat-lon grid for each resolution. The visualization is not included in the other versions of the task in order to not slow down regression testing. -Another task, `cosine_bell/restart`, performs two time steps of the Cosine Bell +Another task, `cosine_bell/decomp`, performs two runs of the Cosine Bell +test at coarse resolution, once with 12 and once with 24 cores, to verify the +bit-for-bit identical results for tracer advection across different core +counts. + +A final task, `cosine_bell/restart`, performs two time steps of the Cosine Bell test at coarse resolution, then performs reruns the second time step, as a restart run to verify the bit-for-bit restart capability for tracer advection. @@ -51,6 +56,7 @@ ocean/spherical/icos/cosine_bell/convergence_time ocean/spherical/icos/cosine_bell/convergence_time/with_viz ocean/spherical/icos/cosine_bell/convergence_both ocean/spherical/icos/cosine_bell/convergence_both/with_viz +ocean/spherical/icos/cosine_bell/decomp ocean/spherical/icos/cosine_bell/restart ocean/spherical/qu/cosine_bell/convergence_space ocean/spherical/qu/cosine_bell/convergence_space/with_viz @@ -58,6 +64,7 @@ ocean/spherical/qu/cosine_bell/convergence_time ocean/spherical/qu/cosine_bell/convergence_time/with_viz ocean/spherical/qu/cosine_bell/convergence_both ocean/spherical/qu/cosine_bell/convergence_both/with_viz +ocean/spherical/icos/cosine_bell/decomp ocean/spherical/qu/cosine_bell/restart ``` The default resolutions used in the task depends on the mesh type. @@ -99,7 +106,7 @@ your own config file (or add a `spherical_convergence` section to a config file if you're already using one). The resolutions are a comma-separated list of the resolution of the mesh in km. If you specify a different list before setting up `cosine_bell`, steps will be generated with the requested -resolutions. (If you alter `icos_resolutions` or `qu_resolutions`) in the +resolutions. (If you alter `icos_resolutions` or `qu_resolutions` in the task's config file in the work directory, nothing will happen.) For `icos` meshes, make sure you use a resolution close to those listed in {ref}`dev-spherical-meshes`. Each resolution will be rounded to the nearest diff --git a/polaris/ocean/suites/cosine_bell.txt b/polaris/ocean/suites/cosine_bell.txt index 81ea683d1..6ab418228 100644 --- a/polaris/ocean/suites/cosine_bell.txt +++ b/polaris/ocean/suites/cosine_bell.txt @@ -4,6 +4,7 @@ ocean/spherical/icos/cosine_bell/convergence_time ocean/spherical/icos/cosine_bell/convergence_time/with_viz ocean/spherical/icos/cosine_bell/convergence_both ocean/spherical/icos/cosine_bell/convergence_both/with_viz +ocean/spherical/icos/cosine_bell/decomp ocean/spherical/icos/cosine_bell/restart ocean/spherical/qu/cosine_bell/convergence_space ocean/spherical/qu/cosine_bell/convergence_space/with_viz @@ -11,4 +12,5 @@ ocean/spherical/qu/cosine_bell/convergence_time ocean/spherical/qu/cosine_bell/convergence_time/with_viz ocean/spherical/qu/cosine_bell/convergence_both ocean/spherical/qu/cosine_bell/convergence_both/with_viz +ocean/spherical/qu/cosine_bell/decomp ocean/spherical/qu/cosine_bell/restart diff --git a/polaris/ocean/suites/nightly.txt b/polaris/ocean/suites/nightly.txt index e1760b595..972aa7637 100644 --- a/polaris/ocean/suites/nightly.txt +++ b/polaris/ocean/suites/nightly.txt @@ -5,4 +5,5 @@ ocean/planar/ice_shelf_2d/5km/z-star/default/with_restart ocean/planar/ice_shelf_2d/5km/z-level/default/with_restart ocean/planar/inertial_gravity_wave/convergence_both # ocean/planar/manufactured_solution +ocean/spherical/icos/cosine_bell/decomp ocean/spherical/icos/cosine_bell/restart diff --git a/polaris/ocean/suites/omega_pr.txt b/polaris/ocean/suites/omega_pr.txt new file mode 100644 index 000000000..831d57f96 --- /dev/null +++ b/polaris/ocean/suites/omega_pr.txt @@ -0,0 +1,3 @@ +ocean/planar/manufactured_solution/convergence_both +ocean/spherical/icos/cosine_bell/decomp +ocean/spherical/icos/cosine_bell/restart diff --git a/polaris/ocean/suites/pr.txt b/polaris/ocean/suites/pr.txt index c7fc1b175..1f2ac7b56 100644 --- a/polaris/ocean/suites/pr.txt +++ b/polaris/ocean/suites/pr.txt @@ -6,7 +6,8 @@ ocean/planar/ice_shelf_2d/5km/z-level/default/with_restart ocean/planar/inertial_gravity_wave/convergence_both ocean/planar/internal_wave/standard/default ocean/planar/internal_wave/vlr/default -# ocean/planar/manufactured_solution +# ocean/planar/manufactured_solution/convergence_both ocean/single_column/cvmix ocean/single_column/ideal_age +ocean/spherical/icos/cosine_bell/decomp ocean/spherical/icos/cosine_bell/restart diff --git a/polaris/ocean/tasks/cosine_bell/__init__.py b/polaris/ocean/tasks/cosine_bell/__init__.py index 5cf365a79..2749e09d2 100644 --- a/polaris/ocean/tasks/cosine_bell/__init__.py +++ b/polaris/ocean/tasks/cosine_bell/__init__.py @@ -9,6 +9,7 @@ ) from polaris.ocean.mesh.spherical import add_spherical_base_mesh_step from polaris.ocean.tasks.cosine_bell.analysis import Analysis +from polaris.ocean.tasks.cosine_bell.decomp import Decomp from polaris.ocean.tasks.cosine_bell.forward import Forward from polaris.ocean.tasks.cosine_bell.init import Init from polaris.ocean.tasks.cosine_bell.restart import Restart @@ -23,8 +24,8 @@ def add_cosine_bell_tasks(component): the ocean component that the tasks will be added to """ - for icosahedral, prefix, restart_refinement in [(True, 'icos', 8.0), - (False, 'qu', 2.0)]: + for icosahedral, prefix, single_refinement in [(True, 'icos', 8.0), + (False, 'qu', 2.0)]: filepath = f'spherical/{prefix}/cosine_bell/cosine_bell.cfg' config = PolarisConfigParser(filepath=filepath) @@ -47,9 +48,16 @@ def add_cosine_bell_tasks(component): component.add_task(Restart(component=component, config=config, icosahedral=icosahedral, - refinement_factor=restart_refinement, + refinement_factor=single_refinement, refinement='both')) + component.add_task(Decomp(component=component, + config=config, + icosahedral=icosahedral, + refinement_factor=single_refinement, + refinement='both', + proc_counts=[12, 24])) + class CosineBell(Task): """ diff --git a/polaris/ocean/tasks/cosine_bell/decomp.py b/polaris/ocean/tasks/cosine_bell/decomp.py new file mode 100644 index 000000000..ebba0b073 --- /dev/null +++ b/polaris/ocean/tasks/cosine_bell/decomp.py @@ -0,0 +1,93 @@ +from polaris import Task +from polaris.ocean.convergence import get_resolution_for_task +from polaris.ocean.mesh.spherical import add_spherical_base_mesh_step +from polaris.ocean.tasks.cosine_bell.forward import Forward +from polaris.ocean.tasks.cosine_bell.init import Init +from polaris.ocean.tasks.cosine_bell.validate import Validate + + +class Decomp(Task): + """ + A cosine bell decomposition task, which makes sure the model produces + identical results on different numbers of cores. + """ + + def __init__(self, component, config, icosahedral, refinement_factor, + refinement, proc_counts): + """ + Create the convergence test + + Parameters + ---------- + component : polaris.ocean.Ocean + The ocean component that this task belongs to + + config : polaris.config.PolarisConfigParser + A shared config parser + + icosahedral : bool + Whether to use icosahedral, as opposed to less regular, JIGSAW + meshes + + refinement_factor : float + The factor by which to scale space, time or both + + refinement : str + Refinement type. One of 'space', 'time' or 'both' indicating both + space and time + + proc_counts : list of int + The number of processors to run each step on + """ + + if icosahedral: + prefix = 'icos' + else: + prefix = 'qu' + + task_subdir = f'spherical/{prefix}/cosine_bell/decomp' + name = f'{prefix}_cosine_bell_decomp' + config_filename = 'cosine_bell.cfg' + + super().__init__(component=component, name=name, subdir=task_subdir) + + self.set_shared_config(config, link=config_filename) + + resolution = get_resolution_for_task( + config, refinement_factor, refinement=refinement) + + base_mesh_step, mesh_name = add_spherical_base_mesh_step( + component, resolution, icosahedral) + + name = f'{prefix}_init_{mesh_name}' + init_subdir = f'spherical/{prefix}/cosine_bell/init/{mesh_name}' + if init_subdir in component.steps: + init_step = component.steps[init_subdir] + else: + init_step = Init(component=component, name=name, + subdir=init_subdir, base_mesh=base_mesh_step) + init_step.set_shared_config(config, link=config_filename) + + self.add_step(base_mesh_step, symlink=f'base_mesh/{mesh_name}') + self.add_step(init_step, symlink=f'init/{mesh_name}') + + step_names = [] + for procs in proc_counts: + name = f'{procs}proc' + step_names.append(name) + subdir = f'{task_subdir}/{name}' + step = Forward( + component=component, name=name, subdir=subdir, + mesh=base_mesh_step, init=init_step, + refinement_factor=refinement_factor, + refinement=refinement) + step.dynamic_ntasks = False + step.ntasks = procs + step.min_tasks = procs + step.set_shared_config( + config, link=config_filename) + self.add_step(step) + + self.add_step(Validate(component=component, + step_subdirs=step_names, + indir=task_subdir))