Add optional preCICE export output for adaptivity computation CPU time

micro_manager/config.py

@@ -49,6 +49,8 @@ def __init__(self, config_file_name):
         self._adaptivity_refining_constant = 0.5
         self._adaptivity_every_implicit_iteration = False
         self._adaptivity_similarity_measure = "L1"
+        self._adaptivity_output_n = 1
+        self._adaptivity_output_cpu_time = False
 
         # Snapshot information
         self._parameter_file_name = None
@@ -199,12 +201,21 @@ def read_json_micro_manager(self):
                 self._data_for_adaptivity[dname] = exchange_data[dname]
 
             if self._data_for_adaptivity.keys() == self._write_data_names.keys():
-                warn(
+                self._logger.log_info_one_rank(
                     "Only micro simulation data is used for similarity computation in adaptivity. This would lead to the"
                     " same set of active and inactive simulations for the entire simulation time. If this is not intended,"
                     " please include macro simulation data as well."
                 )
 
+            try:
+                self._adaptivity_output_n = self._data["simulation_params"][
+                    "adaptivity_settings"
+                ]["output_n"]
+            except BaseException:
+                self._logger.log_info_one_rank(
+                    "No output interval for adaptivity provided. Adaptivity metrics will be output every time window."
+                )
+
             self._adaptivity_history_param = self._data["simulation_params"][
                 "adaptivity_settings"
             ]["history_param"]
@@ -245,6 +256,17 @@ def read_json_micro_manager(self):
             self._write_data_names["active_state"] = False
             self._write_data_names["active_steps"] = False
 
+            try:
+                if self.data["simulation_params"]["adaptivity_settings"][
+                    "output_cpu_time"
+                ]:
+                    self._adaptivity_output_cpu_time = True
+                    self._write_data_names["adaptivity_cpu_time"] = False
+            except BaseException:
+                self._logger.log_info_one_rank(
+                    "Micro Manager will not output CPU time of the adaptivity computation."
+                )
+
         if "interpolate_crash" in self._data["simulation_params"]:
             if self._data["simulation_params"]["interpolate_crash"] == "True":
                 self._interpolate_crash = True
@@ -466,6 +488,17 @@ def get_data_for_adaptivity(self):
         """
         return self._data_for_adaptivity
 
+    def get_adaptivity_output_n(self):
+        """
+        Get the output frequency of adaptivity metrics.
+
+        Returns
+        -------
+        adaptivity_output_n : int
+            Output frequency of adaptivity metrics, so output every N timesteps
+        """
+        return self._adaptivity_output_n
+
     def get_adaptivity_hist_param(self):
         """
         Get adaptivity history parameter.
@@ -525,6 +558,17 @@ def is_adaptivity_required_in_every_implicit_iteration(self):
         """
         return self._adaptivity_every_implicit_iteration
 
+    def output_adaptivity_cpu_time(self):
+        """
+        Check if CPU time of the adaptivity computation needs to be output.
+
+        Returns
+        -------
+        adaptivity_cpu_time : bool
+            True if CPU time of the adaptivity computation needs to be output, False otherwise.
+        """
+        return self._adaptivity_output_cpu_time
+
     def get_micro_dt(self):
         """
         Get the size of the micro time window.

micro_manager/domain_decomposition.py

@@ -6,22 +6,19 @@
 
 
 class DomainDecomposer:
-    def __init__(self, logger, dims, rank, size) -> None:
+    def __init__(self, dims, rank, size) -> None:
         """
         Class constructor.
 
         Parameters
         ----------
-        logger : object of logging
-            Logger defined from the standard package logging.
         dims : int
             Dimensions of the problem.
         rank : int
             MPI rank.
         size : int
             Total number of MPI processes.
         """
-        self._logger = logger
         self._rank = rank
         self._size = size
         self._dims = dims
@@ -98,6 +95,4 @@ def decompose_macro_domain(self, macro_bounds: list, ranks_per_axis: list) -> li
             if rank_in_axis[d] + 1 == ranks_per_axis[d]:
                 mesh_bounds[d * 2 + 1] = macro_bounds[d * 2 + 1]
 
-        self._logger.log_info_any_rank("Bounding box limits are {}".format(mesh_bounds))
-
         return mesh_bounds

micro_manager/micro_manager.py

@@ -20,8 +20,9 @@
 from copy import deepcopy
 from typing import Dict
 from warnings import warn
-
 import numpy as np
+import time
+
 import precice
 
 from .micro_manager_base import MicroManager
@@ -110,6 +111,8 @@ def __init__(self, config_file: str) -> None:
 
             self._adaptivity_data_names = self._config.get_data_for_adaptivity()
 
+            self._adaptivity_output_n = self._config.get_adaptivity_output_n()
+
             # Names of macro data to be used for adaptivity computation
             self._adaptivity_macro_data_names = dict()
 
@@ -125,6 +128,7 @@ def __init__(self, config_file: str) -> None:
                 self._config.is_adaptivity_required_in_every_implicit_iteration()
             )
             self._micro_sims_active_steps = None
+            self._output_adaptivity_cpu_time = self._config.output_adaptivity_cpu_time()
 
         # Define the preCICE Participant
         self._participant = precice.Participant(
@@ -190,6 +194,8 @@ def solve(self) -> None:
 
         while self._participant.is_coupling_ongoing():
 
+            adaptivity_cpu_time = 0.0
+
             dt = min(self._participant.get_max_time_step_size(), self._micro_dt)
 
             # Write a checkpoint
@@ -201,6 +207,7 @@ def solve(self) -> None:
 
                 if self._is_adaptivity_on:
                     if not self._adaptivity_in_every_implicit_step:
+                        start_time = time.process_time()
                         (
                             similarity_dists,
                             is_sim_active,
@@ -213,6 +220,9 @@ def solve(self) -> None:
                             sim_is_associated_to,
                             self._data_for_adaptivity,
                         )
+                        end_time = time.process_time()
+
+                        adaptivity_cpu_time = end_time - start_time
 
                         # Only checkpoint the adaptivity configuration if adaptivity is computed
                         # once in every time window
@@ -239,6 +249,7 @@ def solve(self) -> None:
 
             if self._is_adaptivity_on:
                 if self._adaptivity_in_every_implicit_step:
+                    start_time = time.process_time()
                     (
                         similarity_dists,
                         is_sim_active,
@@ -251,6 +262,8 @@ def solve(self) -> None:
                         sim_is_associated_to,
                         self._data_for_adaptivity,
                     )
+                    end_time = time.process_time()
+                    adaptivity_cpu_time = end_time - start_time
 
                     if self._adaptivity_type == "local":
                         active_sim_ids = np.where(is_sim_active)[0]
@@ -270,6 +283,12 @@ def solve(self) -> None:
                 micro_sims_output = self._solve_micro_simulations_with_adaptivity(
                     micro_sims_input, is_sim_active, sim_is_associated_to, dt
                 )
+
+                if self._output_adaptivity_cpu_time:
+                    for i in range(self._local_number_of_sims):
+                        micro_sims_output[i][
+                            "adaptivity_cpu_time"
+                        ] = adaptivity_cpu_time
             else:
                 micro_sims_output = self._solve_micro_simulations(micro_sims_input, dt)
 
@@ -339,7 +358,7 @@ def solve(self) -> None:
                         global_active_sims = np.count_nonzero(is_sim_active)
                         global_inactive_sims = np.count_nonzero(is_sim_active == False)
 
-                    if self._rank == 0:
+                    if n % self._adaptivity_output_n == 0 and self._rank == 0:
                         self._adaptivity_logger.log_info_one_rank(
                             "{},{},{},{},{}".format(
                                 n,
@@ -368,7 +387,6 @@ def initialize(self) -> None:
         ), "Provided macro mesh bounds are of incorrect dimension"
         if self._is_parallel:
             domain_decomposer = DomainDecomposer(
-                self._logger,
                 self._participant.get_mesh_dimensions(self._macro_mesh_name),
                 self._rank,
                 self._size,
@@ -411,6 +429,29 @@ def initialize(self) -> None:
         # the correct global IDs
         self._comm.Allgatherv(np.array(self._local_number_of_sims), nms_all_ranks)
 
+        max_nms = np.max(nms_all_ranks)
+        min_nms = np.min(nms_all_ranks)
+
+        if (
+            max_nms != min_nms
+        ):  # if the number of maximum and minimum micro simulations per rank are different
+            self._logger.log_info_one_rank(
+                "The following ranks have the maximum number of micro simulations ({}): {}".format(
+                    max_nms, np.where(nms_all_ranks == max_nms)[0]
+                )
+            )
+            self._logger.log_info_one_rank(
+                "The following ranks have the minimum number of micro simulations ({}): {}".format(
+                    min_nms, np.where(nms_all_ranks == min_nms)[0]
+                )
+            )
+        else:  # if the number of maximum and minimum micro simulations per rank are the same
+            self._logger.log_info_one_rank(
+                "All ranks have the same number of micro simulations: {}".format(
+                    max_nms
+                )
+            )
+
         # Get global number of micro simulations
         self._global_number_of_sims = np.sum(nms_all_ranks)
 
@@ -811,11 +852,11 @@ def _solve_micro_simulations_with_adaptivity(
             if not self._has_sim_crashed[active_id]:
                 # Attempt to solve the micro simulation
                 try:
-                    start_time = time.time()
+                    start_time = time.process_time()
                     micro_sims_output[active_id] = self._micro_sims[active_id].solve(
                         micro_sims_input[active_id], dt
                     )
-                    end_time = time.time()
+                    end_time = time.process_time()
                     # Write solve time of the macro simulation if required and the simulation has not crashed
                     if self._is_micro_solve_time_required:
                         micro_sims_output[active_id]["micro_sim_time"] = (
@@ -850,6 +891,7 @@ def _solve_micro_simulations_with_adaptivity(
                     "Exiting simulation after micro simulation crash."
                 )
                 sys.exit()
+
         # Interpolate result for crashed simulation
         unset_sims = []
         for active_id in active_sim_ids: