Merge pull request #253 from xylar/cosine-bell-decomp

Add a cosine-bell decomposition test

docs/conf.py

@@ -68,9 +68,9 @@
 
 intersphinx_mapping = {
     'geometric_features':
-        ('https://mpas-dev.github.io/geometric_features/stable', None),
+        ('https://mpas-dev.github.io/geometric_features/main', None),
     'matplotlib': ('https://matplotlib.org/stable', None),
-    'mpas_tools': ('https://mpas-dev.github.io/MPAS-Tools/stable', None),
+    'mpas_tools': ('https://mpas-dev.github.io/MPAS-Tools/master', None),
     'numpy': ('https://numpy.org/doc/stable', None),
     'python': ('https://docs.python.org', None),
     'scipy': ('https://docs.scipy.org/doc/scipy/reference', None),

docs/developers_guide/ocean/api.md

@@ -109,6 +109,8 @@
    viz.Viz
    viz.Viz.run
 
+   decomp.Decomp
+
    restart.Restart
 
    restart.RestartStep
@@ -320,8 +322,8 @@
 .. autosummary::
    :toctree: generated/
 
-   get_resolution_for_tasks
-   get_timestep_for_tasks
+   get_resolution_for_task
+   get_timestep_for_task
 
    forward.ConvergenceForward
    forward.ConvergenceForward.compute_cell_count

docs/developers_guide/ocean/tasks/barotropic_gyre.md

@@ -34,13 +34,13 @@ the `init` step. Namelist and streams files are updated in
 with time steps determined algorithmically based on config options.  The
 number of cells is approximated from config options in
 {py:meth}`polaris.ocean.tasks.barotropic_gyre.forward.Forward.compute_cell_count()`
-so that this can be used to constrain the number of MPI tasks that Polaris 
+so that this can be used to constrain the number of MPI tasks that Polaris
 tasks have as their target and minimum (if the resources are not explicitly
 prescribed).  For MPAS-Ocean, PIO namelist options are modified and a
-graph partition is generated as part of `runtime_setup()`.  Next, the ocean 
+graph partition is generated as part of `runtime_setup()`.  Next, the ocean
 model is run. If `run_time_steps` is provided then this determines the run
 duration, otherwise the duration is 3 years. Finally, validation of
-`layerThickness` and `normalVelocity` in the `output.nc` file are performed 
+`layerThickness` and `normalVelocity` in the `output.nc` file are performed
 against a baseline if one is provided when calling {ref}`dev-polaris-setup`.
 
 ### analysis
@@ -51,9 +51,9 @@ the analytical solution for the linearized dynamics. This step also produces a
 figure with the model solution, the analytical solution, and the difference
 between the two.
 
-(dev-ocean-baroclinic-channel-default)=
+(dev-ocean-baroclinic-gyre-default)=
 
 ## default
 
 The {py:class}`polaris.ocean.tasks.baroclinic_channel.default.Default`
-test performs a test of the linearized dynamics. 
+test performs a test of the linearized dynamics.

docs/developers_guide/ocean/tasks/cosine_bell.md

@@ -86,6 +86,13 @@ colorbar_limits = 0.0, 1.0
 
 See {ref}`dev-visualization-global` for more details.
 
+## decomp
+
+The class {py:class}`polaris.ocean.tasks.cosine_bell.decomp.Decomp` defines
+a decomposition test across core counts.  It runs Cosine Bell test at coarse
+resolution once each on 12 and 24 cores to verify bit-for-bit reproducibility
+for tracer advection across different core counts.
+
 ## restart
 
 The {py:class}`polaris.ocean.tasks.cosine_bell.restart.Restart` class defines

docs/tutorials/dev_add_test_group.md

@@ -1011,7 +1011,7 @@ constructor:
 $ vi ${POLARIS_HEAD}/polaris/ocean/tasks/yet_another_channel/init.py
 ```
 ```{code-block} python
-:emphasize-lines: 12-14
+:emphasize-lines: 11-13
 
 ...
 
@@ -1270,7 +1270,7 @@ Next, we add inputs that are outputs from the `init` task:
 $ vi ${POLARIS_HEAD}/polaris/ocean/tasks/yet_another_channel/forward.py
 ```
 ```{code-block} python
-:emphasize-lines: 12-17
+:emphasize-lines: 10-15
 
 ...
 

docs/users_guide/ocean/suites.md

@@ -8,25 +8,30 @@ run of the same tasks.
 
 (ocean-suite-cosine-bell)=
 
-## cosine_bell and cosine_bell_cached_init suite
+## cosine_bell suite
 
 ```bash
 polaris suite -c ocean -t cosine_bell ...
 ```
 
-```bash
-polaris suite -c ocean -t cosine_bell_cached_init ...
-```
-
-Both `cosine_bell` suites include the following tasks:
+The `cosine_bell` suite includes the following tasks:
 
 ```none
-ocean/global_convergence/icos/cosine_bell
-ocean/global_convergence/qu/cosine_bell
+ocean/spherical/icos/cosine_bell/convergence_space
+ocean/spherical/icos/cosine_bell/convergence_space/with_viz
+ocean/spherical/icos/cosine_bell/convergence_time
+ocean/spherical/icos/cosine_bell/convergence_time/with_viz
+ocean/spherical/icos/cosine_bell/convergence_both
+ocean/spherical/icos/cosine_bell/convergence_both/with_viz
+ocean/spherical/icos/cosine_bell/decomp
+ocean/spherical/icos/cosine_bell/restart
+ocean/spherical/qu/cosine_bell/convergence_space
+ocean/spherical/qu/cosine_bell/convergence_space/with_viz
+ocean/spherical/qu/cosine_bell/convergence_time
+ocean/spherical/qu/cosine_bell/convergence_time/with_viz
+ocean/spherical/qu/cosine_bell/convergence_both
+ocean/spherical/qu/cosine_bell/convergence_both/with_viz
+ocean/spherical/qu/cosine_bell/decomp
+ocean/spherical/qu/cosine_bell/restart
 ```
 
-The `cosine_bell` suite runs both tasks from 
-{ref}`ocean-cosine-bell` in full, whereas the
-`cosine_bell_cached_init` suite only runs the ocean simulations from as set
-of meshes and initial conditions that have been saved from a previous run
-(see {ref}`dev-cache`).

docs/users_guide/ocean/tasks/cosine_bell.md

@@ -28,7 +28,12 @@ and final state on a lat-lon grid for each resolution.  The visualization is
 not included in the other versions of the task in order to not slow down
 regression testing.
 
-Another task, `cosine_bell/restart`, performs two time steps of the Cosine Bell
+Another task, `cosine_bell/decomp`, performs two runs of the Cosine Bell
+test at coarse resolution, once with 12 and once with 24 cores, to verify the
+bit-for-bit identical results for tracer advection across different core
+counts.
+
+A final task, `cosine_bell/restart`, performs two time steps of the Cosine Bell
 test at coarse resolution, then performs reruns the second time step,
 as a restart run to verify the bit-for-bit restart capability for tracer
 advection.
@@ -51,13 +56,15 @@ ocean/spherical/icos/cosine_bell/convergence_time
 ocean/spherical/icos/cosine_bell/convergence_time/with_viz
 ocean/spherical/icos/cosine_bell/convergence_both
 ocean/spherical/icos/cosine_bell/convergence_both/with_viz
+ocean/spherical/icos/cosine_bell/decomp
 ocean/spherical/icos/cosine_bell/restart
 ocean/spherical/qu/cosine_bell/convergence_space
 ocean/spherical/qu/cosine_bell/convergence_space/with_viz
 ocean/spherical/qu/cosine_bell/convergence_time
 ocean/spherical/qu/cosine_bell/convergence_time/with_viz
 ocean/spherical/qu/cosine_bell/convergence_both
 ocean/spherical/qu/cosine_bell/convergence_both/with_viz
+ocean/spherical/icos/cosine_bell/decomp
 ocean/spherical/qu/cosine_bell/restart
 ```
 The default resolutions used in the task depends on the mesh type.
@@ -99,7 +106,7 @@ your own config file (or add a `spherical_convergence` section to a config file
 if you're already using one).  The resolutions are a comma-separated list of
 the resolution of the mesh in km.  If you specify a different list
 before setting up `cosine_bell`, steps will be generated with the requested
-resolutions.  (If you alter `icos_resolutions` or `qu_resolutions`) in the
+resolutions.  (If you alter `icos_resolutions` or `qu_resolutions` in the
 task's config file in the work directory, nothing will happen.)  For `icos`
 meshes, make sure you use a resolution close to those listed in
 {ref}`dev-spherical-meshes`.  Each resolution will be rounded to the nearest

polaris/ocean/suites/cosine_bell.txt

@@ -4,11 +4,13 @@ ocean/spherical/icos/cosine_bell/convergence_time
 ocean/spherical/icos/cosine_bell/convergence_time/with_viz
 ocean/spherical/icos/cosine_bell/convergence_both
 ocean/spherical/icos/cosine_bell/convergence_both/with_viz
+ocean/spherical/icos/cosine_bell/decomp
 ocean/spherical/icos/cosine_bell/restart
 ocean/spherical/qu/cosine_bell/convergence_space
 ocean/spherical/qu/cosine_bell/convergence_space/with_viz
 ocean/spherical/qu/cosine_bell/convergence_time
 ocean/spherical/qu/cosine_bell/convergence_time/with_viz
 ocean/spherical/qu/cosine_bell/convergence_both
 ocean/spherical/qu/cosine_bell/convergence_both/with_viz
+ocean/spherical/qu/cosine_bell/decomp
 ocean/spherical/qu/cosine_bell/restart

polaris/ocean/suites/nightly.txt

@@ -5,4 +5,5 @@ ocean/planar/ice_shelf_2d/5km/z-star/default/with_restart
 ocean/planar/ice_shelf_2d/5km/z-level/default/with_restart
 ocean/planar/inertial_gravity_wave/convergence_both
 # ocean/planar/manufactured_solution
+ocean/spherical/icos/cosine_bell/decomp
 ocean/spherical/icos/cosine_bell/restart

polaris/ocean/suites/omega_pr.txt

@@ -0,0 +1,3 @@
+ocean/planar/manufactured_solution/convergence_both
+ocean/spherical/icos/cosine_bell/decomp
+ocean/spherical/icos/cosine_bell/restart

polaris/ocean/suites/pr.txt

@@ -6,7 +6,8 @@ ocean/planar/ice_shelf_2d/5km/z-level/default/with_restart
 ocean/planar/inertial_gravity_wave/convergence_both
 ocean/planar/internal_wave/standard/default
 ocean/planar/internal_wave/vlr/default
-# ocean/planar/manufactured_solution
+# ocean/planar/manufactured_solution/convergence_both
 ocean/single_column/cvmix
 ocean/single_column/ideal_age
+ocean/spherical/icos/cosine_bell/decomp
 ocean/spherical/icos/cosine_bell/restart

polaris/ocean/tasks/cosine_bell/__init__.py

@@ -9,6 +9,7 @@
 )
 from polaris.ocean.mesh.spherical import add_spherical_base_mesh_step
 from polaris.ocean.tasks.cosine_bell.analysis import Analysis
+from polaris.ocean.tasks.cosine_bell.decomp import Decomp
 from polaris.ocean.tasks.cosine_bell.forward import Forward
 from polaris.ocean.tasks.cosine_bell.init import Init
 from polaris.ocean.tasks.cosine_bell.restart import Restart
@@ -23,8 +24,8 @@ def add_cosine_bell_tasks(component):
         the ocean component that the tasks will be added to
     """
 
-    for icosahedral, prefix, restart_refinement in [(True, 'icos', 8.0),
-                                                    (False, 'qu', 2.0)]:
+    for icosahedral, prefix, single_refinement in [(True, 'icos', 8.0),
+                                                   (False, 'qu', 2.0)]:
 
         filepath = f'spherical/{prefix}/cosine_bell/cosine_bell.cfg'
         config = PolarisConfigParser(filepath=filepath)
@@ -47,9 +48,16 @@ def add_cosine_bell_tasks(component):
         component.add_task(Restart(component=component,
                                    config=config,
                                    icosahedral=icosahedral,
-                                   refinement_factor=restart_refinement,
+                                   refinement_factor=single_refinement,
                                    refinement='both'))
 
+        component.add_task(Decomp(component=component,
+                                  config=config,
+                                  icosahedral=icosahedral,
+                                  refinement_factor=single_refinement,
+                                  refinement='both',
+                                  proc_counts=[12, 24]))
+
 
 class CosineBell(Task):
     """

polaris/ocean/tasks/cosine_bell/decomp.py

@@ -0,0 +1,93 @@
+from polaris import Task
+from polaris.ocean.convergence import get_resolution_for_task
+from polaris.ocean.mesh.spherical import add_spherical_base_mesh_step
+from polaris.ocean.tasks.cosine_bell.forward import Forward
+from polaris.ocean.tasks.cosine_bell.init import Init
+from polaris.ocean.tasks.cosine_bell.validate import Validate
+
+
+class Decomp(Task):
+    """
+    A cosine bell decomposition task, which makes sure the model produces
+    identical results on different numbers of cores.
+    """
+
+    def __init__(self, component, config, icosahedral, refinement_factor,
+                 refinement, proc_counts):
+        """
+        Create the convergence test
+
+        Parameters
+        ----------
+        component : polaris.ocean.Ocean
+            The ocean component that this task belongs to
+
+        config : polaris.config.PolarisConfigParser
+            A shared config parser
+
+        icosahedral : bool
+            Whether to use icosahedral, as opposed to less regular, JIGSAW
+            meshes
+
+        refinement_factor : float
+            The factor by which to scale space, time or both
+
+        refinement : str
+            Refinement type. One of 'space', 'time' or 'both' indicating both
+            space and time
+
+        proc_counts : list of int
+            The number of processors to run each step on
+        """
+
+        if icosahedral:
+            prefix = 'icos'
+        else:
+            prefix = 'qu'
+
+        task_subdir = f'spherical/{prefix}/cosine_bell/decomp'
+        name = f'{prefix}_cosine_bell_decomp'
+        config_filename = 'cosine_bell.cfg'
+
+        super().__init__(component=component, name=name, subdir=task_subdir)
+
+        self.set_shared_config(config, link=config_filename)
+
+        resolution = get_resolution_for_task(
+            config, refinement_factor, refinement=refinement)
+
+        base_mesh_step, mesh_name = add_spherical_base_mesh_step(
+            component, resolution, icosahedral)
+
+        name = f'{prefix}_init_{mesh_name}'
+        init_subdir = f'spherical/{prefix}/cosine_bell/init/{mesh_name}'
+        if init_subdir in component.steps:
+            init_step = component.steps[init_subdir]
+        else:
+            init_step = Init(component=component, name=name,
+                             subdir=init_subdir, base_mesh=base_mesh_step)
+            init_step.set_shared_config(config, link=config_filename)
+
+        self.add_step(base_mesh_step, symlink=f'base_mesh/{mesh_name}')
+        self.add_step(init_step, symlink=f'init/{mesh_name}')
+
+        step_names = []
+        for procs in proc_counts:
+            name = f'{procs}proc'
+            step_names.append(name)
+            subdir = f'{task_subdir}/{name}'
+            step = Forward(
+                component=component, name=name, subdir=subdir,
+                mesh=base_mesh_step, init=init_step,
+                refinement_factor=refinement_factor,
+                refinement=refinement)
+            step.dynamic_ntasks = False
+            step.ntasks = procs
+            step.min_tasks = procs
+            step.set_shared_config(
+                config, link=config_filename)
+            self.add_step(step)
+
+        self.add_step(Validate(component=component,
+                               step_subdirs=step_names,
+                               indir=task_subdir))