From e32a82517a5dde41714d98ad4589842e53ca28e5 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Tue, 12 Mar 2024 16:30:10 +0200 Subject: [PATCH 01/23] Add first draft of simulation crash handling --- examples/python-dummy/micro_dummy.py | 7 +++++++ micro_manager/micro_manager.py | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/examples/python-dummy/micro_dummy.py b/examples/python-dummy/micro_dummy.py index 638e2051..0c2b4810 100644 --- a/examples/python-dummy/micro_dummy.py +++ b/examples/python-dummy/micro_dummy.py @@ -21,6 +21,13 @@ def solve(self, macro_data, dt): self._micro_scalar_data = macro_data["macro-scalar-data"] + 1 for d in range(self._dims): self._micro_vector_data.append(macro_data["macro-vector-data"][d] + 1) + + # random simulation crash + import numpy as np + if np.random.rand() < 0.01: + print("Micro simulation {} crashed!".format(self._sim_id)) + self._micro_scalar_data = 1 / 0 + return { "micro-scalar-data": self._micro_scalar_data.copy(), diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index d40745f9..28bf35a8 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -552,12 +552,23 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: for count, sim in enumerate(self._micro_sims): start_time = time.time() - micro_sims_output[count] = sim.solve(micro_sims_input[count], self._dt) + try: + micro_sims_output[count] = sim.solve(micro_sims_input[count], self._dt) + except Exception as e: + _, mesh_vertex_coords = self._participant.get_mesh_vertex_ids_and_coordinates( + self._macro_mesh_name) + self._logger.error( + "Micro simulation with global ID {} at macro coordinates {} has experienced an error. Exiting simulation.".format( + sim.get_global_id(), mesh_vertex_coords[count])) + self._logger.error(e) + # set the micro simulation value to old value and keep it constant + micro_sims_output[count] = self._old_micro_sims_output[count] end_time = time.time() if self._is_micro_solve_time_required: micro_sims_output[count]["micro_sim_time"] = end_time - start_time - + self._old_micro_sims_output = micro_sims_output + return micro_sims_output def _solve_micro_simulations_with_adaptivity( From d459714637eab2c7ed36cfe0cce6da7b5ec39a0e Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Tue, 19 Mar 2024 11:07:51 +0200 Subject: [PATCH 02/23] Extend handling of crashing simulations to adaptivity and corner cases --- examples/python-dummy/micro_dummy.py | 7 -- micro_manager/micro_manager.py | 101 ++++++++++++++++++++++----- tests/unit/test_micro_manager.py | 55 ++++++++++++--- 3 files changed, 129 insertions(+), 34 deletions(-) diff --git a/examples/python-dummy/micro_dummy.py b/examples/python-dummy/micro_dummy.py index 0c2b4810..638e2051 100644 --- a/examples/python-dummy/micro_dummy.py +++ b/examples/python-dummy/micro_dummy.py @@ -21,13 +21,6 @@ def solve(self, macro_data, dt): self._micro_scalar_data = macro_data["macro-scalar-data"] + 1 for d in range(self._dims): self._micro_vector_data.append(macro_data["macro-vector-data"][d] + 1) - - # random simulation crash - import numpy as np - if np.random.rand() < 0.01: - print("Micro simulation {} crashed!".format(self._sim_id)) - self._micro_scalar_data = 1 / 0 - return { "micro-scalar-data": self._micro_scalar_data.copy(), diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 28bf35a8..fbfe47c9 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -389,6 +389,9 @@ def _initialize(self) -> None: sim_id += 1 self._micro_sims = [None] * self._local_number_of_sims # DECLARATION + + self._crashed_sims = [False] * self._local_number_of_sims + self._old_micro_sims_output = [None] * self._local_number_of_sims micro_problem = getattr( importlib.import_module( @@ -551,22 +554,46 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: micro_sims_output = [None] * self._local_number_of_sims for count, sim in enumerate(self._micro_sims): - start_time = time.time() - try: - micro_sims_output[count] = sim.solve(micro_sims_input[count], self._dt) - except Exception as e: - _, mesh_vertex_coords = self._participant.get_mesh_vertex_ids_and_coordinates( - self._macro_mesh_name) - self._logger.error( - "Micro simulation with global ID {} at macro coordinates {} has experienced an error. Exiting simulation.".format( - sim.get_global_id(), mesh_vertex_coords[count])) - self._logger.error(e) - # set the micro simulation value to old value and keep it constant + + if not self._crashed_sims[count]: + try: + start_time = time.time() + micro_sims_output[count] = sim.solve(micro_sims_input[count], self._dt) + end_time = time.time() + except Exception as error_message: + _, mesh_vertex_coords = self._participant.get_mesh_vertex_ids_and_coordinates( + self._macro_mesh_name) + self._logger.error( + "Micro simulation at macro coordinates {} has experienced an error. " + "See next entry for error message. " + "Keeping values constant at results of previous iteration".format( + mesh_vertex_coords[count])) + self._logger.error(error_message) + micro_sims_output[count] = self._old_micro_sims_output[count] + self._crashed_sims[count] = True + else: micro_sims_output[count] = self._old_micro_sims_output[count] - end_time = time.time() + + - if self._is_micro_solve_time_required: + if self._is_micro_solve_time_required and not self._crashed_sims[count]: micro_sims_output[count]["micro_sim_time"] = end_time - start_time + + crash_ratio = np.sum(self._crashed_sims) / len(self._crashed_sims) + if crash_ratio > 0.2: + self._logger.info("More than 20% of the micro simulations on rank {} have crashed. " + "Exiting simulation.".format(self._rank)) + sys.exit() + + set_sims = np.where(micro_sims_output) + none_mask = np.array([item is None for item in micro_sims_output]) + unset_sims = np.where(none_mask)[0] + + for unset_sims in unset_sims: + self._logger.info("Micro Sim {} has previously not run. " + "It will be replace with the output of the first " + "micro sim that ran {}".format(unset_sims, set_sims[0][0])) + micro_sims_output[unset_sims] = micro_sims_output[set_sims[0][0]] self._old_micro_sims_output = micro_sims_output return micro_sims_output @@ -622,11 +649,28 @@ def _solve_micro_simulations_with_adaptivity( # Solve all active micro simulations for active_id in active_sim_ids: - start_time = time.time() - micro_sims_output[active_id] = self._micro_sims[active_id].solve( - micro_sims_input[active_id], self._dt - ) - end_time = time.time() + + if not self._crashed_sims[active_id]: + try: + start_time = time.time() + micro_sims_output[active_id] = self._micro_sims[active_id].solve( + micro_sims_input[active_id], self._dt + ) + end_time = time.time() + except Exception as error_message: + _, mesh_vertex_coords = self._participant.get_mesh_vertex_ids_and_coordinates( + self._macro_mesh_name) + self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " + "See next entry for error message. " + "Keeping values constant at results of previous iteration".format( + mesh_vertex_coords[active_id])) # Access the correct coordinates + self._logger.error(error_message) + # set the micro simulation value to old value and keep it constant if simulation crashes + micro_sims_output[active_id] = self._old_micro_sims_output[active_id] + self._crashed_sims[active_id] = True + else: + micro_sims_output[active_id] = self._old_micro_sims_output[active_id] + # Mark the micro sim as active for export micro_sims_output[active_id]["active_state"] = 1 @@ -634,9 +678,27 @@ def _solve_micro_simulations_with_adaptivity( "active_steps" ] = self._micro_sims_active_steps[active_id] - if self._is_micro_solve_time_required: + if self._is_micro_solve_time_required and not self._crashed_sims[active_id]: micro_sims_output[active_id]["micro_sim_time"] = end_time - start_time + crash_ratio = np.sum(self._crashed_sims) / len(self._crashed_sims) + if crash_ratio > 0.2: + self._logger.info("More than 20% of the micro simulations on rank {} have crashed. " + "Exiting simulation.".format(self._rank)) + sys.exit() + + set_sims = np.where(micro_sims_output) + unset_sims = [] + for active_id in active_sim_ids: + if micro_sims_output[active_id] is None: + unset_sims.append(active_id) + + for unset_sims in unset_sims: + self._logger.info("Micro Sim {} has previously not run. " + "It will be replace with the output of the first " + "micro sim that ran {}".format(unset_sims, set_sims[0][0])) + micro_sims_output[unset_sims] = micro_sims_output[set_sims[0][0]] + # For each inactive simulation, copy data from most similar active simulation if self._adaptivity_type == "global": self._adaptivity_controller.communicate_micro_output( @@ -662,6 +724,7 @@ def _solve_micro_simulations_with_adaptivity( for i in range(self._local_number_of_sims): for name in self._adaptivity_micro_data_names: self._data_for_adaptivity[name][i] = micro_sims_output[i][name] + self._old_micro_sims_output = micro_sims_output return micro_sims_output diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index 062f81e2..d538f03b 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -7,20 +7,33 @@ class MicroSimulation: - def __init__(self, sim_id): + def __init__(self, sim_id, crashing=False): self.very_important_value = 0 + self.sim_id = sim_id + self.crashing = crashing + self.current_time = 0 + def initialize(self): pass def solve(self, macro_data, dt): - assert macro_data["macro-scalar-data"] == 1 - assert macro_data["macro-vector-data"].tolist() == [0, 1, 2] - return { - "micro-scalar-data": macro_data["macro-scalar-data"] + 1, - "micro-vector-data": macro_data["macro-vector-data"] + 1, - } - + if not self.crashing: + assert macro_data["macro-scalar-data"] == 1 + assert macro_data["macro-vector-data"].tolist() == [0, 1, 2] + return { + "micro-scalar-data": macro_data["macro-scalar-data"] + 1, + "micro-vector-data": macro_data["macro-vector-data"] + 1 + } + else: + if self.sim_id == 0: + self.current_time += dt + if self.current_time > dt: + raise Exception("Micro Simulation has crashed") + return { + "micro-scalar-data": macro_data["macro-scalar-data"] + 1, + "micro-vector-data": macro_data["macro-vector-data"] + 1 + } class TestFunctioncalls(TestCase): def setUp(self): @@ -106,11 +119,37 @@ def test_solve_micro_sims(self): for data, fake_data in zip(micro_sims_output, self.fake_write_data): self.assertEqual(data["micro-scalar-data"], 2) + self.assertListEqual( data["micro-vector-data"].tolist(), (fake_data["micro-vector-data"] + 1).tolist(), ) + def test_crash_handling(self): + """ + Test if the micro manager catches a simulation crash and handles it adequately. + """ + manager = micro_manager.MicroManager('micro-manager-config.json') + manager._local_number_of_sims = 4 + manager._micro_sims = [MicroSimulation(i, crashing = True) for i in range(4)] + manager._micro_sims_active_steps = np.zeros(4, dtype=np.int32) + # Momentarily, a simulation crash during the first step is not handled + micro_sims_output = manager._solve_micro_simulations(self.fake_read_data) + for i, data in enumerate(micro_sims_output): + self.fake_read_data[i]["macro-scalar-data"] = data["micro-scalar-data"] + self.fake_read_data[i]["macro-vector-data"] = data["micro-vector-data"] + micro_sims_output = manager._solve_micro_simulations(self.fake_read_data) + # The crashed simulation should have the same data as the previous step + data_crashed = micro_sims_output[0] + self.assertEqual(data_crashed["micro-scalar-data"], 2) + self.assertListEqual(data_crashed["micro-vector-data"].tolist(), + (self.fake_write_data[0]["micro-vector-data"] + 1).tolist()) + # Non-crashed simulations should have updated data + data_normal = micro_sims_output[1] + self.assertEqual(data_normal["micro-scalar-data"], 3) + self.assertListEqual(data_normal["micro-vector-data"].tolist(), + (self.fake_write_data[1]["micro-vector-data"] + 2).tolist()) + def test_config(self): """ Test if the functions in the Config class work. From 493aa9959aff39abf643b7f5df4268739da8dc3f Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Wed, 20 Mar 2024 11:27:42 +0200 Subject: [PATCH 03/23] Adapt format in crash handling to pep8 --- micro_manager/micro_manager.py | 52 ++++++++++++++++------------------ 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index fbfe47c9..3ac1eb5e 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -70,8 +70,10 @@ def __init__(self, config_file: str) -> None: # Define the preCICE Participant self._participant = precice.Participant( - "Micro-Manager", self._config.get_config_file_name(), self._rank, self._size - ) + "Micro-Manager", + self._config.get_config_file_name(), + self._rank, + self._size) self._macro_mesh_name = self._config.get_macro_mesh_name() @@ -389,7 +391,7 @@ def _initialize(self) -> None: sim_id += 1 self._micro_sims = [None] * self._local_number_of_sims # DECLARATION - + self._crashed_sims = [False] * self._local_number_of_sims self._old_micro_sims_output = [None] * self._local_number_of_sims @@ -554,48 +556,45 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: micro_sims_output = [None] * self._local_number_of_sims for count, sim in enumerate(self._micro_sims): - + if not self._crashed_sims[count]: try: start_time = time.time() - micro_sims_output[count] = sim.solve(micro_sims_input[count], self._dt) + micro_sims_output[count] = sim.solve( + micro_sims_input[count], self._dt) end_time = time.time() except Exception as error_message: _, mesh_vertex_coords = self._participant.get_mesh_vertex_ids_and_coordinates( - self._macro_mesh_name) - self._logger.error( - "Micro simulation at macro coordinates {} has experienced an error. " - "See next entry for error message. " - "Keeping values constant at results of previous iteration".format( - mesh_vertex_coords[count])) + self._macro_mesh_name) + self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " + "See next entry for error message. " + "Keeping values constant at results of previous iteration".format( + mesh_vertex_coords[count])) self._logger.error(error_message) micro_sims_output[count] = self._old_micro_sims_output[count] self._crashed_sims[count] = True else: micro_sims_output[count] = self._old_micro_sims_output[count] - - - if self._is_micro_solve_time_required and not self._crashed_sims[count]: micro_sims_output[count]["micro_sim_time"] = end_time - start_time - + crash_ratio = np.sum(self._crashed_sims) / len(self._crashed_sims) if crash_ratio > 0.2: self._logger.info("More than 20% of the micro simulations on rank {} have crashed. " "Exiting simulation.".format(self._rank)) sys.exit() - + set_sims = np.where(micro_sims_output) none_mask = np.array([item is None for item in micro_sims_output]) unset_sims = np.where(none_mask)[0] - + for unset_sims in unset_sims: self._logger.info("Micro Sim {} has previously not run. " "It will be replace with the output of the first " "micro sim that ran {}".format(unset_sims, set_sims[0][0])) micro_sims_output[unset_sims] = micro_sims_output[set_sims[0][0]] self._old_micro_sims_output = micro_sims_output - + return micro_sims_output def _solve_micro_simulations_with_adaptivity( @@ -649,7 +648,7 @@ def _solve_micro_simulations_with_adaptivity( # Solve all active micro simulations for active_id in active_sim_ids: - + if not self._crashed_sims[active_id]: try: start_time = time.time() @@ -659,18 +658,17 @@ def _solve_micro_simulations_with_adaptivity( end_time = time.time() except Exception as error_message: _, mesh_vertex_coords = self._participant.get_mesh_vertex_ids_and_coordinates( - self._macro_mesh_name) - self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " - "See next entry for error message. " - "Keeping values constant at results of previous iteration".format( - mesh_vertex_coords[active_id])) # Access the correct coordinates + self._macro_mesh_name) + self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " + "See next entry for error message. " + "Keeping values constant at results of previous iteration".format( + mesh_vertex_coords[active_id])) self._logger.error(error_message) # set the micro simulation value to old value and keep it constant if simulation crashes micro_sims_output[active_id] = self._old_micro_sims_output[active_id] self._crashed_sims[active_id] = True else: micro_sims_output[active_id] = self._old_micro_sims_output[active_id] - # Mark the micro sim as active for export micro_sims_output[active_id]["active_state"] = 1 @@ -686,13 +684,13 @@ def _solve_micro_simulations_with_adaptivity( self._logger.info("More than 20% of the micro simulations on rank {} have crashed. " "Exiting simulation.".format(self._rank)) sys.exit() - + set_sims = np.where(micro_sims_output) unset_sims = [] for active_id in active_sim_ids: if micro_sims_output[active_id] is None: unset_sims.append(active_id) - + for unset_sims in unset_sims: self._logger.info("Micro Sim {} has previously not run. " "It will be replace with the output of the first " From 4916f1346d4e6ae3f45e13b71ed3e255fbb11a19 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Thu, 21 Mar 2024 11:21:58 +0200 Subject: [PATCH 04/23] Add tests for simulation crashes --- tests/unit/micro-manager-config_crash.json | 25 +++++ tests/unit/test_micro_manager.py | 13 ++- .../test_micro_simulation_crash_handling.py | 103 ++++++++++++++++++ 3 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 tests/unit/micro-manager-config_crash.json create mode 100644 tests/unit/test_micro_simulation_crash_handling.py diff --git a/tests/unit/micro-manager-config_crash.json b/tests/unit/micro-manager-config_crash.json new file mode 100644 index 00000000..9ff06771 --- /dev/null +++ b/tests/unit/micro-manager-config_crash.json @@ -0,0 +1,25 @@ +{ + "micro_file_name": "test_micro_simulation_crash_handling", + "coupling_params": { + "config_file_name": "dummy-config.xml", + "macro_mesh_name": "dummy-macro-mesh", + "read_data_names": {"macro-scalar-data": "scalar", "macro-vector-data": "vector"}, + "write_data_names": {"micro-scalar-data": "scalar", "micro-vector-data": "vector"} + }, + "simulation_params": { + "macro_domain_bounds": [0.0, 25.0, 0.0, 25.0, 0.0, 25.0], + "adaptivity": { + "type": "local", + "data": ["macro-scalar-data", "macro-vector-data"], + "history_param": 0.5, + "coarsening_constant": 0.3, + "refining_constant": 0.4, + "every_implicit_iteration": "False", + "similarity_measure": "L1" + } + }, + "diagnostics": { + "output_micro_sim_solve_time": "True", + "micro_output_n": 10 + } +} diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index d538f03b..f9e9acdf 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -7,13 +7,9 @@ class MicroSimulation: - def __init__(self, sim_id, crashing=False): + def __init__(self, sim_id): self.very_important_value = 0 - self.sim_id = sim_id - self.crashing = crashing - self.current_time = 0 - def initialize(self): pass @@ -106,7 +102,7 @@ def test_read_write_data_from_precice(self): fake_data["macro-vector-data"].tolist(), ) - def test_solve_micro_sims(self): + def test_solve_mico_sims(self): """ Test if the internal function _solve_micro_simulations works as expected. """ @@ -119,6 +115,7 @@ def test_solve_micro_sims(self): for data, fake_data in zip(micro_sims_output, self.fake_write_data): self.assertEqual(data["micro-scalar-data"], 2) +<<<<<<< HEAD self.assertListEqual( data["micro-vector-data"].tolist(), @@ -149,6 +146,10 @@ def test_crash_handling(self): self.assertEqual(data_normal["micro-scalar-data"], 3) self.assertListEqual(data_normal["micro-vector-data"].tolist(), (self.fake_write_data[1]["micro-vector-data"] + 2).tolist()) +======= + self.assertListEqual(data["micro-vector-data"].tolist(), + (fake_data["micro-vector-data"] + 1).tolist()) +>>>>>>> 12a60fc (Add tests for simulation crashes) def test_config(self): """ diff --git a/tests/unit/test_micro_simulation_crash_handling.py b/tests/unit/test_micro_simulation_crash_handling.py new file mode 100644 index 00000000..7f99379d --- /dev/null +++ b/tests/unit/test_micro_simulation_crash_handling.py @@ -0,0 +1,103 @@ +import numpy as np +from unittest import TestCase +import micro_manager + + +class MicroSimulation: + def __init__(self, sim_id): + self.very_important_value = 0 + self.sim_id = sim_id + self.current_time = 0 + + def initialize(self): + pass + + def solve(self, macro_data, dt): + if self.sim_id == 0: + self.current_time += dt + if self.current_time > dt: + raise Exception("Crash") + + return {"micro-scalar-data": macro_data["macro-scalar-data"] + 1, + "micro-vector-data": macro_data["macro-vector-data"] + 1} + + +class TestSimulationCrashHandling(TestCase): + def setUp(self): + self.fake_read_data_names = { + "macro-scalar-data": False, "macro-vector-data": True} + self.fake_read_data = [{"macro-scalar-data": 1, + "macro-vector-data": np.array([0, 1, 2])}] * 10 + self.fake_write_data = [{"micro-scalar-data": 1, + "micro-vector-data": np.array([0, 1, 2]), + "micro_sim_time": 0, + "active_state": 0, + "active_steps": 0}] * 10 + + def test_crash_handling(self): + """ + Test if the micro manager catches a simulation crash and handles it adequately. + """ + manager = micro_manager.MicroManager('micro-manager-config_crash.json') + + manager._local_number_of_sims = 10 + manager._crashed_sims = [False] * 10 + manager._micro_sims = [MicroSimulation(i) for i in range(10)] + manager._micro_sims_active_steps = np.zeros(10, dtype=np.int32) + # Crash during first time step has to be handled differently + + micro_sims_output = manager._solve_micro_simulations( + self.fake_read_data) + for i, data in enumerate(micro_sims_output): + self.fake_read_data[i]["macro-scalar-data"] = data["micro-scalar-data"] + self.fake_read_data[i]["macro-vector-data"] = data["micro-vector-data"] + micro_sims_output = manager._solve_micro_simulations( + self.fake_read_data) + # The crashed simulation should have the same data as the previous step + data_crashed = micro_sims_output[0] + self.assertEqual(data_crashed["micro-scalar-data"], 2) + self.assertListEqual(data_crashed["micro-vector-data"].tolist(), + (self.fake_write_data[0]["micro-vector-data"] + 1).tolist()) + # Non-crashed simulations should have updated data + data_normal = micro_sims_output[1] + self.assertEqual(data_normal["micro-scalar-data"], 3) + self.assertListEqual(data_normal["micro-vector-data"].tolist(), + (self.fake_write_data[1]["micro-vector-data"] + 2).tolist()) + + def test_crash_handling_with_adaptivity(self): + """ + Test if the micro manager catches a simulation crash and handles it adequately with adaptivity. + """ + manager = micro_manager.MicroManager('micro-manager-config_crash.json') + + manager._local_number_of_sims = 10 + manager._crashed_sims = [False] * 10 + manager._micro_sims = [MicroSimulation(i) for i in range(10)] + manager._micro_sims_active_steps = np.zeros(10, dtype=np.int32) + is_sim_active = np.array( + [True, True, False, True, False, False, False, True, True, False,]) + sim_is_associated_to = np.array([-2, -2, 1, -2, 3, 3, 0, -2, -2, 8]) + # Crash in the first time step is handled differently + + micro_sims_output = manager._solve_micro_simulations_with_adaptivity( + self.fake_read_data, is_sim_active, sim_is_associated_to) + for i, data in enumerate(micro_sims_output): + self.fake_read_data[i]["macro-scalar-data"] = data["micro-scalar-data"] + self.fake_read_data[i]["macro-vector-data"] = data["micro-vector-data"] + micro_sims_output = manager._solve_micro_simulations_with_adaptivity( + self.fake_read_data, is_sim_active, sim_is_associated_to) + # The crashed simulation should have the same data as the previous step + data_crashed = micro_sims_output[0] + self.assertEqual(data_crashed["micro-scalar-data"], 2) + self.assertListEqual(data_crashed["micro-vector-data"].tolist(), + (self.fake_write_data[0]["micro-vector-data"] + 1).tolist()) + # Non-crashed simulations should have updated data + data_normal = micro_sims_output[1] + self.assertEqual(data_normal["micro-scalar-data"], 3) + self.assertListEqual(data_normal["micro-vector-data"].tolist(), + (self.fake_write_data[1]["micro-vector-data"] + 2).tolist()) + + +if __name__ == '__main__': + import unittest + unittest.main() From e0b3409fd3cfb65a681d98fa619351483c8251fb Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Thu, 21 Mar 2024 11:25:07 +0200 Subject: [PATCH 05/23] Adapt formatting of tests --- tests/unit/test_micro_manager.py | 2 +- tests/unit/test_micro_simulation_crash_handling.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index f9e9acdf..ff67e025 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -102,7 +102,7 @@ def test_read_write_data_from_precice(self): fake_data["macro-vector-data"].tolist(), ) - def test_solve_mico_sims(self): + def test_solve_micro_sims(self): """ Test if the internal function _solve_micro_simulations works as expected. """ diff --git a/tests/unit/test_micro_simulation_crash_handling.py b/tests/unit/test_micro_simulation_crash_handling.py index 7f99379d..308e315b 100644 --- a/tests/unit/test_micro_simulation_crash_handling.py +++ b/tests/unit/test_micro_simulation_crash_handling.py @@ -75,7 +75,7 @@ def test_crash_handling_with_adaptivity(self): manager._micro_sims = [MicroSimulation(i) for i in range(10)] manager._micro_sims_active_steps = np.zeros(10, dtype=np.int32) is_sim_active = np.array( - [True, True, False, True, False, False, False, True, True, False,]) + [True, True, False, True, False, False, False, True, True, False]) sim_is_associated_to = np.array([-2, -2, 1, -2, 3, 3, 0, -2, -2, 8]) # Crash in the first time step is handled differently From 57cea770f7e79a57976e13ba75f7f63289247456 Mon Sep 17 00:00:00 2001 From: Torben Schiz <49746900+tjwsch@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:28:13 +0100 Subject: [PATCH 06/23] Update logger message for crashing simulation in the first run Co-authored-by: Ishaan Desai --- micro_manager/micro_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 3ac1eb5e..7a664283 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -589,9 +589,8 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: unset_sims = np.where(none_mask)[0] for unset_sims in unset_sims: - self._logger.info("Micro Sim {} has previously not run. " - "It will be replace with the output of the first " - "micro sim that ran {}".format(unset_sims, set_sims[0][0])) + self._logger.info("Micro simulation {} has has crashed in the very first run attempt. " + "The output of the first micro sim that ran ({}) will be used as its output.".format(unset_sims, set_sims[0][0])) micro_sims_output[unset_sims] = micro_sims_output[set_sims[0][0]] self._old_micro_sims_output = micro_sims_output From 4ac424884160952466c8725f52c4fb016a997bd0 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Thu, 28 Mar 2024 15:41:49 +0100 Subject: [PATCH 07/23] Restructure simulation crash handling and add global condition for exit --- micro_manager/micro_manager.py | 57 +++++++++++++++++++------------- tests/unit/test_micro_manager.py | 2 +- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 7a664283..93e979c5 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -74,6 +74,8 @@ def __init__(self, config_file: str) -> None: self._config.get_config_file_name(), self._rank, self._size) + + micro_file_name = self._config.get_micro_file_name() self._macro_mesh_name = self._config.get_macro_mesh_name() @@ -89,6 +91,8 @@ def __init__(self, config_file: str) -> None: self._ranks_per_axis = self._config.get_ranks_per_axis() self._is_micro_solve_time_required = self._config.write_micro_solve_time() + + self._crash_threshold = 0.2 self._local_number_of_sims = 0 self._global_number_of_sims = 0 @@ -261,6 +265,19 @@ def solve(self) -> None: else: micro_sims_output = self._solve_micro_simulations(micro_sims_input) + # Check if more than a certain percentage of the micro simulations have crashed and terminate if threshold is exceeded + crashed_sims_on_all_ranks = np.zeros(self._size, dtype=np.int64) + self._comm.Allgather(np.sum(self._crashed_sims), crashed_sims_on_all_ranks) + + if self._is_parallel: + crash_ratio = np.sum(crashed_sims_on_all_ranks) / self._global_number_of_sims + else: + crash_ratio = np.sum(self._crashed_sims) / len(self._crashed_sims) + if crash_ratio > self._crash_threshold: + self._logger.info("{:.1%} of the micro simulations have crashed exceeding the threshold of {:.1%}. " + "Exiting simulation.".format(crash_ratio, self._crash_threshold)) + sys.exit() + self._write_data_to_precice(micro_sims_output) t += self._dt # increase internal time when time step is done. @@ -338,11 +355,11 @@ def _initialize(self) -> None: ( self._mesh_vertex_ids, - mesh_vertex_coords, + self._mesh_vertex_coords, ) = self._participant.get_mesh_vertex_ids_and_coordinates(self._macro_mesh_name) - assert mesh_vertex_coords.size != 0, "Macro mesh has no vertices." + assert self._mesh_vertex_coords.size != 0, "Macro mesh has no vertices." - self._local_number_of_sims, _ = mesh_vertex_coords.shape + self._local_number_of_sims, _ = self._mesh_vertex_coords.shape self._logger.info( "Number of local micro simulations = {}".format(self._local_number_of_sims) ) @@ -556,34 +573,31 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: micro_sims_output = [None] * self._local_number_of_sims for count, sim in enumerate(self._micro_sims): - + # If micro simulation has not crashed in a previous iteration, attempt to solve it if not self._crashed_sims[count]: + # Attempt to solve the micro simulation try: start_time = time.time() micro_sims_output[count] = sim.solve( micro_sims_input[count], self._dt) end_time = time.time() + # If simulation crashes, log the error and keep the output constant at the previous iteration's output except Exception as error_message: - _, mesh_vertex_coords = self._participant.get_mesh_vertex_ids_and_coordinates( - self._macro_mesh_name) self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " "See next entry for error message. " "Keeping values constant at results of previous iteration".format( - mesh_vertex_coords[count])) + self._mesh_vertex_coords[count])) self._logger.error(error_message) micro_sims_output[count] = self._old_micro_sims_output[count] self._crashed_sims[count] = True + # If simulation has crashed in a previous iteration, keep the output constant else: micro_sims_output[count] = self._old_micro_sims_output[count] + # Write solve time of the macro simulation if required and the simulation has not crashed if self._is_micro_solve_time_required and not self._crashed_sims[count]: micro_sims_output[count]["micro_sim_time"] = end_time - start_time - crash_ratio = np.sum(self._crashed_sims) / len(self._crashed_sims) - if crash_ratio > 0.2: - self._logger.info("More than 20% of the micro simulations on rank {} have crashed. " - "Exiting simulation.".format(self._rank)) - sys.exit() - + # If a simulation crashes in the first iteration it is replaced with the output of the first simulation that ran set_sims = np.where(micro_sims_output) none_mask = np.array([item is None for item in micro_sims_output]) unset_sims = np.where(none_mask)[0] @@ -647,25 +661,26 @@ def _solve_micro_simulations_with_adaptivity( # Solve all active micro simulations for active_id in active_sim_ids: - + # If micro simulation has not crashed in a previous iteration, attempt to solve it if not self._crashed_sims[active_id]: + # Attempt to solve the micro simulation try: start_time = time.time() micro_sims_output[active_id] = self._micro_sims[active_id].solve( micro_sims_input[active_id], self._dt ) end_time = time.time() + # If simulation crashes, log the error and keep the output constant at the previous iteration's output except Exception as error_message: - _, mesh_vertex_coords = self._participant.get_mesh_vertex_ids_and_coordinates( - self._macro_mesh_name) self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " "See next entry for error message. " "Keeping values constant at results of previous iteration".format( - mesh_vertex_coords[active_id])) + self._mesh_vertex_coords[active_id])) self._logger.error(error_message) # set the micro simulation value to old value and keep it constant if simulation crashes micro_sims_output[active_id] = self._old_micro_sims_output[active_id] self._crashed_sims[active_id] = True + # If simulation has crashed in a previous iteration, keep the output constant else: micro_sims_output[active_id] = self._old_micro_sims_output[active_id] @@ -675,15 +690,11 @@ def _solve_micro_simulations_with_adaptivity( "active_steps" ] = self._micro_sims_active_steps[active_id] + # Write solve time of the macro simulation if required and the simulation has not crashed if self._is_micro_solve_time_required and not self._crashed_sims[active_id]: micro_sims_output[active_id]["micro_sim_time"] = end_time - start_time - crash_ratio = np.sum(self._crashed_sims) / len(self._crashed_sims) - if crash_ratio > 0.2: - self._logger.info("More than 20% of the micro simulations on rank {} have crashed. " - "Exiting simulation.".format(self._rank)) - sys.exit() - + # If a simulation crashes in the first iteration it is replaced with the output of the first simulation that ran set_sims = np.where(micro_sims_output) unset_sims = [] for active_id in active_sim_ids: diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index ff67e025..f9e9acdf 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -102,7 +102,7 @@ def test_read_write_data_from_precice(self): fake_data["macro-vector-data"].tolist(), ) - def test_solve_micro_sims(self): + def test_solve_mico_sims(self): """ Test if the internal function _solve_micro_simulations works as expected. """ From 10de7ab1fe8c41ec77b2c991842594a7b08fb916 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Tue, 12 Mar 2024 16:30:10 +0200 Subject: [PATCH 08/23] Add first draft of simulation crash handling --- examples/python-dummy/micro_dummy.py | 7 +++++++ tests/unit/test_micro_manager.py | 5 ----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/python-dummy/micro_dummy.py b/examples/python-dummy/micro_dummy.py index 638e2051..0c2b4810 100644 --- a/examples/python-dummy/micro_dummy.py +++ b/examples/python-dummy/micro_dummy.py @@ -21,6 +21,13 @@ def solve(self, macro_data, dt): self._micro_scalar_data = macro_data["macro-scalar-data"] + 1 for d in range(self._dims): self._micro_vector_data.append(macro_data["macro-vector-data"][d] + 1) + + # random simulation crash + import numpy as np + if np.random.rand() < 0.01: + print("Micro simulation {} crashed!".format(self._sim_id)) + self._micro_scalar_data = 1 / 0 + return { "micro-scalar-data": self._micro_scalar_data.copy(), diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index f9e9acdf..de153473 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -115,7 +115,6 @@ def test_solve_mico_sims(self): for data, fake_data in zip(micro_sims_output, self.fake_write_data): self.assertEqual(data["micro-scalar-data"], 2) -<<<<<<< HEAD self.assertListEqual( data["micro-vector-data"].tolist(), @@ -146,10 +145,6 @@ def test_crash_handling(self): self.assertEqual(data_normal["micro-scalar-data"], 3) self.assertListEqual(data_normal["micro-vector-data"].tolist(), (self.fake_write_data[1]["micro-vector-data"] + 2).tolist()) -======= - self.assertListEqual(data["micro-vector-data"].tolist(), - (fake_data["micro-vector-data"] + 1).tolist()) ->>>>>>> 12a60fc (Add tests for simulation crashes) def test_config(self): """ From 29c824e037941c30cc0e34f5d1844ea2e8a8ca50 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Tue, 19 Mar 2024 11:07:51 +0200 Subject: [PATCH 09/23] Extend handling of crashing simulations to adaptivity and corner cases --- examples/python-dummy/micro_dummy.py | 7 ---- micro_manager/micro_manager.py | 4 ++- tests/unit/test_micro_manager.py | 53 ++++++---------------------- 3 files changed, 14 insertions(+), 50 deletions(-) diff --git a/examples/python-dummy/micro_dummy.py b/examples/python-dummy/micro_dummy.py index 0c2b4810..638e2051 100644 --- a/examples/python-dummy/micro_dummy.py +++ b/examples/python-dummy/micro_dummy.py @@ -21,13 +21,6 @@ def solve(self, macro_data, dt): self._micro_scalar_data = macro_data["macro-scalar-data"] + 1 for d in range(self._dims): self._micro_vector_data.append(macro_data["macro-vector-data"][d] + 1) - - # random simulation crash - import numpy as np - if np.random.rand() < 0.01: - print("Micro simulation {} crashed!".format(self._sim_id)) - self._micro_scalar_data = 1 / 0 - return { "micro-scalar-data": self._micro_scalar_data.copy(), diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 93e979c5..88c25d12 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -408,6 +408,9 @@ def _initialize(self) -> None: sim_id += 1 self._micro_sims = [None] * self._local_number_of_sims # DECLARATION + + self._crashed_sims = [False] * self._local_number_of_sims + self._old_micro_sims_output = [None] * self._local_number_of_sims self._crashed_sims = [False] * self._local_number_of_sims self._old_micro_sims_output = [None] * self._local_number_of_sims @@ -700,7 +703,6 @@ def _solve_micro_simulations_with_adaptivity( for active_id in active_sim_ids: if micro_sims_output[active_id] is None: unset_sims.append(active_id) - for unset_sims in unset_sims: self._logger.info("Micro Sim {} has previously not run. " "It will be replace with the output of the first " diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index de153473..38716464 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -7,28 +7,22 @@ class MicroSimulation: - def __init__(self, sim_id): + def __init__(self, sim_id, crashing=False): self.very_important_value = 0 + self.sim_id = sim_id + self.crashing = crashing + self.current_time = 0 + def initialize(self): pass def solve(self, macro_data, dt): - if not self.crashing: - assert macro_data["macro-scalar-data"] == 1 - assert macro_data["macro-vector-data"].tolist() == [0, 1, 2] - return { - "micro-scalar-data": macro_data["macro-scalar-data"] + 1, - "micro-vector-data": macro_data["macro-vector-data"] + 1 - } - else: - if self.sim_id == 0: - self.current_time += dt - if self.current_time > dt: - raise Exception("Micro Simulation has crashed") - return { - "micro-scalar-data": macro_data["macro-scalar-data"] + 1, - "micro-vector-data": macro_data["macro-vector-data"] + 1 + assert macro_data["macro-scalar-data"] == 1 + assert macro_data["macro-vector-data"].tolist() == [0, 1, 2] + return { + "micro-scalar-data": macro_data["macro-scalar-data"] + 1, + "micro-vector-data": macro_data["macro-vector-data"] + 1 } class TestFunctioncalls(TestCase): @@ -102,7 +96,7 @@ def test_read_write_data_from_precice(self): fake_data["macro-vector-data"].tolist(), ) - def test_solve_mico_sims(self): + def test_solve_micro_sims(self): """ Test if the internal function _solve_micro_simulations works as expected. """ @@ -121,31 +115,6 @@ def test_solve_mico_sims(self): (fake_data["micro-vector-data"] + 1).tolist(), ) - def test_crash_handling(self): - """ - Test if the micro manager catches a simulation crash and handles it adequately. - """ - manager = micro_manager.MicroManager('micro-manager-config.json') - manager._local_number_of_sims = 4 - manager._micro_sims = [MicroSimulation(i, crashing = True) for i in range(4)] - manager._micro_sims_active_steps = np.zeros(4, dtype=np.int32) - # Momentarily, a simulation crash during the first step is not handled - micro_sims_output = manager._solve_micro_simulations(self.fake_read_data) - for i, data in enumerate(micro_sims_output): - self.fake_read_data[i]["macro-scalar-data"] = data["micro-scalar-data"] - self.fake_read_data[i]["macro-vector-data"] = data["micro-vector-data"] - micro_sims_output = manager._solve_micro_simulations(self.fake_read_data) - # The crashed simulation should have the same data as the previous step - data_crashed = micro_sims_output[0] - self.assertEqual(data_crashed["micro-scalar-data"], 2) - self.assertListEqual(data_crashed["micro-vector-data"].tolist(), - (self.fake_write_data[0]["micro-vector-data"] + 1).tolist()) - # Non-crashed simulations should have updated data - data_normal = micro_sims_output[1] - self.assertEqual(data_normal["micro-scalar-data"], 3) - self.assertListEqual(data_normal["micro-vector-data"].tolist(), - (self.fake_write_data[1]["micro-vector-data"] + 2).tolist()) - def test_config(self): """ Test if the functions in the Config class work. From 389fb62de909a817e479c2dd5d0155d3a490fb7b Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Wed, 20 Mar 2024 11:27:42 +0200 Subject: [PATCH 10/23] Adapt format in crash handling to pep8 --- micro_manager/micro_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 88c25d12..bca69e90 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -74,8 +74,6 @@ def __init__(self, config_file: str) -> None: self._config.get_config_file_name(), self._rank, self._size) - - micro_file_name = self._config.get_micro_file_name() self._macro_mesh_name = self._config.get_macro_mesh_name() @@ -408,7 +406,7 @@ def _initialize(self) -> None: sim_id += 1 self._micro_sims = [None] * self._local_number_of_sims # DECLARATION - + self._crashed_sims = [False] * self._local_number_of_sims self._old_micro_sims_output = [None] * self._local_number_of_sims From 6178c7f94ac7845a1577fdd5c00bf8b5c3a12a75 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Thu, 21 Mar 2024 11:21:58 +0200 Subject: [PATCH 11/23] Add tests for simulation crashes --- tests/unit/test_micro_manager.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index 38716464..107f17da 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -7,13 +7,9 @@ class MicroSimulation: - def __init__(self, sim_id, crashing=False): + def __init__(self, sim_id): self.very_important_value = 0 - self.sim_id = sim_id - self.crashing = crashing - self.current_time = 0 - def initialize(self): pass @@ -25,6 +21,7 @@ def solve(self, macro_data, dt): "micro-vector-data": macro_data["macro-vector-data"] + 1 } + class TestFunctioncalls(TestCase): def setUp(self): self.fake_read_data_names = { @@ -96,7 +93,7 @@ def test_read_write_data_from_precice(self): fake_data["macro-vector-data"].tolist(), ) - def test_solve_micro_sims(self): + def test_solve_mico_sims(self): """ Test if the internal function _solve_micro_simulations works as expected. """ From dc6b7718759760ae1c4c064687929a7b2fee1c0d Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Thu, 21 Mar 2024 11:25:07 +0200 Subject: [PATCH 12/23] Adapt formatting of tests --- tests/unit/test_micro_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index 107f17da..66471fb1 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -93,7 +93,7 @@ def test_read_write_data_from_precice(self): fake_data["macro-vector-data"].tolist(), ) - def test_solve_mico_sims(self): + def test_solve_micro_sims(self): """ Test if the internal function _solve_micro_simulations works as expected. """ From 5d07316438ce9f7b68d74e5201f84a223b667939 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Tue, 9 Apr 2024 10:36:56 +0200 Subject: [PATCH 13/23] Add interpolation to crash handling --- micro_manager/interpolation.py | 43 ++++++++++++++++++ micro_manager/micro_manager.py | 76 ++++++++++++++++++++++---------- tests/unit/test_interpolation.py | 33 ++++++++++++++ 3 files changed, 128 insertions(+), 24 deletions(-) create mode 100644 micro_manager/interpolation.py create mode 100644 tests/unit/test_interpolation.py diff --git a/micro_manager/interpolation.py b/micro_manager/interpolation.py new file mode 100644 index 00000000..f19511be --- /dev/null +++ b/micro_manager/interpolation.py @@ -0,0 +1,43 @@ +import numpy as np +from scipy.interpolate import griddata as gd + + +def interpolate(logger, known_coords: list, inter_coord: list , data: list)-> dict: + """ + Interpolate parameters at a given vertex + + Args: + logger : logger object + Logger of the micro manager + known_coords : list + List of vertices where the data is known + inter_coord : list + Vertex where the data is to be interpolated + data : list + List of dicts in which keys are names of data and the values are the data. + + Returns: + result: dict + Interpolated data at inter_coord + """ + result = dict() + assert len(known_coords) == len(data), "Number of known coordinates and data points do not match" + + for params in data[0].keys(): + # Attempt to interpolate the data + try: + result[params] = gd(known_coords, [d[params] for d in data], inter_coord, method='linear').tolist() + # Extrapolation is replaced by taking a nearest neighbor + if np.isnan(result[params]).any(): + nearest_neighbor_pos = np.argmin(np.linalg.norm(np.array(known_coords) - np.array(inter_coord), axis=1)) + nearest_neighbor = known_coords[nearest_neighbor_pos] + logger.info("Interpolation failed at macro vertex {}. Taking value of closest neighbor at macro vertex {}".format(params, inter_coord, nearest_neighbor)) + return data[nearest_neighbor_pos] + return result + # If interpolation fails, take the value of the nearest neighbor + except Exception: + nearest_neighbor_pos = np.argmin(np.linalg.norm(np.array(known_coords) - np.array(inter_coord), axis=1)) + nearest_neighbor = known_coords[nearest_neighbor_pos] + logger.info("Interpolation failed at macro vertex {}. Taking value of closest neighbor at macro vertex {}".format(params, inter_coord, nearest_neighbor)) + return data[nearest_neighbor_pos] + \ No newline at end of file diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index bca69e90..cbfaf398 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -30,6 +30,7 @@ from .config import Config from .domain_decomposition import DomainDecomposer from .micro_simulation import create_simulation_class +from .interpolation import interpolate sys.path.append(os.getcwd()) @@ -582,6 +583,10 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: micro_sims_output[count] = sim.solve( micro_sims_input[count], self._dt) end_time = time.time() + # Write solve time of the macro simulation if required and the simulation has not crashed + if self._is_micro_solve_time_required: + micro_sims_output[count]["micro_sim_time"] = end_time - start_time + # If simulation crashes, log the error and keep the output constant at the previous iteration's output except Exception as error_message: self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " @@ -594,19 +599,30 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: # If simulation has crashed in a previous iteration, keep the output constant else: micro_sims_output[count] = self._old_micro_sims_output[count] - # Write solve time of the macro simulation if required and the simulation has not crashed - if self._is_micro_solve_time_required and not self._crashed_sims[count]: - micro_sims_output[count]["micro_sim_time"] = end_time - start_time - - # If a simulation crashes in the first iteration it is replaced with the output of the first simulation that ran - set_sims = np.where(micro_sims_output) - none_mask = np.array([item is None for item in micro_sims_output]) - unset_sims = np.where(none_mask)[0] - - for unset_sims in unset_sims: - self._logger.info("Micro simulation {} has has crashed in the very first run attempt. " - "The output of the first micro sim that ran ({}) will be used as its output.".format(unset_sims, set_sims[0][0])) - micro_sims_output[unset_sims] = micro_sims_output[set_sims[0][0]] + + # If a simulation crashes in the first iteration its result is interpolated + set_sims = [] + set_coords = [] + unset_sims = [] + for count in range(self._local_number_of_sims): + if micro_sims_output[count] is not None: + set_sims.append(micro_sims_output[count].copy()) + set_sims[-1].pop("micro_sim_time", None) + set_coords.append(self._mesh_vertex_coords[count]) + else: + unset_sims.append(count) + + # Interpolate if no data is available + for unset_sim in unset_sims: + micro_sims_output[unset_sim] = interpolate( + self._logger, + set_coords, + self._mesh_vertex_coords[unset_sim], + set_sims + ) + if self._is_micro_solve_time_required: + micro_sims_output[unset_sim]["micro_sim_time"] = 0 + self._old_micro_sims_output = micro_sims_output return micro_sims_output @@ -671,6 +687,9 @@ def _solve_micro_simulations_with_adaptivity( micro_sims_input[active_id], self._dt ) end_time = time.time() + # Write solve time of the macro simulation if required and the simulation has not crashed + if self._is_micro_solve_time_required: + micro_sims_output[active_id]["micro_sim_time"] = end_time - start_time # If simulation crashes, log the error and keep the output constant at the previous iteration's output except Exception as error_message: self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " @@ -691,21 +710,30 @@ def _solve_micro_simulations_with_adaptivity( "active_steps" ] = self._micro_sims_active_steps[active_id] - # Write solve time of the macro simulation if required and the simulation has not crashed - if self._is_micro_solve_time_required and not self._crashed_sims[active_id]: - micro_sims_output[active_id]["micro_sim_time"] = end_time - start_time + - # If a simulation crashes in the first iteration it is replaced with the output of the first simulation that ran - set_sims = np.where(micro_sims_output) + # If a simulation crashes in the first iteration its result is interpolated + set_sims = [] + set_coords = [] unset_sims = [] for active_id in active_sim_ids: - if micro_sims_output[active_id] is None: + if micro_sims_output[active_id] is not None: + set_sims.append(micro_sims_output[active_id].copy()) + set_sims[-1].pop("micro_sim_time", None) + set_coords.append(self._mesh_vertex_coords[active_id]) + else: unset_sims.append(active_id) - for unset_sims in unset_sims: - self._logger.info("Micro Sim {} has previously not run. " - "It will be replace with the output of the first " - "micro sim that ran {}".format(unset_sims, set_sims[0][0])) - micro_sims_output[unset_sims] = micro_sims_output[set_sims[0][0]] + + # Interpolate if no data is available + for unset_sim in unset_sims: + micro_sims_output[unset_sim] = interpolate( + self._logger, + set_coords, + self._mesh_vertex_coords[unset_sim], + set_sims + ) + if self._is_micro_solve_time_required: + micro_sims_output[unset_sim]["micro_sim_time"] = 0 # For each inactive simulation, copy data from most similar active simulation if self._adaptivity_type == "global": diff --git a/tests/unit/test_interpolation.py b/tests/unit/test_interpolation.py new file mode 100644 index 00000000..62c72279 --- /dev/null +++ b/tests/unit/test_interpolation.py @@ -0,0 +1,33 @@ +import numpy as np +from unittest import TestCase +from unittest.mock import MagicMock +from micro_manager.interpolation import interpolate + + +class TestInterpolation(TestCase): + + def test_local_interpolation(self): + """ + Test if local interpolation works as expected + """ + micro_output = [{"vector data": [1,1,1], "scalar data": [1]}] * 4 + micro_output.append({"vector data": [3,3,3], "scalar data": [0]}) + micro_output.append({"vector data": [0,0,0], "scalar data": [3]}) + coords_known = [[2,0,0],[-2,0,0], [0,2,0], [0,-2,0], [0,0,2], [0,0,-1]] + unknown_coord = [0,0,0] + expected_interpolation_result = {"vector data": np.array([1,1,1]), "scalar data": np.array([2])} + interpolation_result = interpolate(MagicMock(), coords_known, unknown_coord , micro_output) + for key in interpolation_result.keys(): + self.assertTrue(np.allclose(interpolation_result[key], expected_interpolation_result[key])) + + def test_local_extrapolation(self): + micro_output = [{"vector data": [3,2,1], "scalar data": [0]}, + {"vector data": [3,2,1], "scalar data": [0]}, + {"vector data": [3,2,1], "scalar data": [0]}, + {"vector data": [4,3,2], "scalar data": [4]}, + {"vector data": [3,2,1], "scalar data": [2]}] + coords_known = [[0,-2,0],[0,-4,0],[0,0,2], [2,0,0],[1,0,0]] + unknown_coord = [0,0,0] + expected_interpolation_result = {"vector data": [3,2,1], "scalar data": [2]} + interpolation_result = interpolate(MagicMock(), coords_known, unknown_coord , micro_output) + self.assertDictEqual(interpolation_result, expected_interpolation_result) \ No newline at end of file From 564edf4ae09099ddaa38cf46a9c9a7b13fc00809 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Thu, 11 Apr 2024 16:41:52 +0200 Subject: [PATCH 14/23] Add Inverse Distance Weighting and improve crash handling --- micro_manager/interpolation.py | 124 ++++++---- micro_manager/micro_manager.py | 231 +++++++++++------- setup.py | 2 +- tests/unit/test_interpolation.py | 93 +++++-- .../test_micro_simulation_crash_handling.py | 156 ++++++------ 5 files changed, 382 insertions(+), 224 deletions(-) diff --git a/micro_manager/interpolation.py b/micro_manager/interpolation.py index f19511be..412d3ea9 100644 --- a/micro_manager/interpolation.py +++ b/micro_manager/interpolation.py @@ -1,43 +1,83 @@ import numpy as np -from scipy.interpolate import griddata as gd - - -def interpolate(logger, known_coords: list, inter_coord: list , data: list)-> dict: - """ - Interpolate parameters at a given vertex - - Args: - logger : logger object - Logger of the micro manager - known_coords : list - List of vertices where the data is known - inter_coord : list - Vertex where the data is to be interpolated - data : list - List of dicts in which keys are names of data and the values are the data. - - Returns: - result: dict - Interpolated data at inter_coord - """ - result = dict() - assert len(known_coords) == len(data), "Number of known coordinates and data points do not match" - - for params in data[0].keys(): - # Attempt to interpolate the data - try: - result[params] = gd(known_coords, [d[params] for d in data], inter_coord, method='linear').tolist() - # Extrapolation is replaced by taking a nearest neighbor - if np.isnan(result[params]).any(): - nearest_neighbor_pos = np.argmin(np.linalg.norm(np.array(known_coords) - np.array(inter_coord), axis=1)) - nearest_neighbor = known_coords[nearest_neighbor_pos] - logger.info("Interpolation failed at macro vertex {}. Taking value of closest neighbor at macro vertex {}".format(params, inter_coord, nearest_neighbor)) - return data[nearest_neighbor_pos] - return result - # If interpolation fails, take the value of the nearest neighbor - except Exception: - nearest_neighbor_pos = np.argmin(np.linalg.norm(np.array(known_coords) - np.array(inter_coord), axis=1)) - nearest_neighbor = known_coords[nearest_neighbor_pos] - logger.info("Interpolation failed at macro vertex {}. Taking value of closest neighbor at macro vertex {}".format(params, inter_coord, nearest_neighbor)) - return data[nearest_neighbor_pos] - \ No newline at end of file +from sklearn.neighbors import NearestNeighbors + + +class Interpolation: + def __init__(self, logger): + + self._logger = logger + + def get_nearest_neighbor_indices_local( + self, + all_local_coords, + inter_point, + k: int, + inter_point_is_neighbor: bool = False, + ) -> np.ndarray: + """ + Get the indices of the k nearest neighbors of a point in a list of coordinates. + Note: It can be chosen whether the point itself is considered as a neighbor or not. + Args: + all_local_coords: list + List of coordinates of all points. + inter_point: + Coordinates of the point for which the neighbors are to be found. + k: int + inter_point_is_neighbor: bool, optional + Decide whether the interpolation point is considered as its own neighbor. + Defaults to False. + + Returns: np.ndarray + of indices of the k nearest neighbors. + """ + assert ( + len(all_local_coords) > k + ), "Number of neighbors must be less than the number of all available neighbors." + if not inter_point_is_neighbor: + neighbors = NearestNeighbors(n_neighbors=k + 1).fit(all_local_coords) + + dists, neighbor_indices = neighbors.kneighbors( + [inter_point], return_distance=True + ) + + if np.min(dists) < 1e-10: + argmin = np.argmin(dists) + neighbor_indices = np.delete(neighbor_indices, argmin) + else: + argmax = np.argmax(dists) + neighbor_indices = np.delete(neighbor_indices, argmax) + else: + neighbors = NearestNeighbors(n_neighbors=k).fit(all_local_coords) + neighbor_indices = neighbors.kneighbors( + [inter_point], return_distance=False + ) + + return neighbor_indices + + def inv_dist_weighted_interp( + self, neighbors: list, point, values: list + ) -> np.ndarray: + """ + Interpolate a value at a point using inverse distance weighting. + + Args: + neighbors: list + Coordinates at which the values are known. + point: + Coordinates at which the value is to be interpolated. + values: list + Values at the known coordinates. + + Returns: nd.array + Value at interpolation point. + """ + interpol_val = 0 + summed_weights = 0 + for inx in range(len(neighbors)): + norm = np.linalg.norm(np.array(neighbors[inx]) - np.array(point)) ** 2 + if norm < 1e-10: + return values[inx] + interpol_val += values[inx] / norm + summed_weights += 1 / norm + + return interpol_val / summed_weights diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index cbfaf398..48260b0e 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -30,7 +30,7 @@ from .config import Config from .domain_decomposition import DomainDecomposer from .micro_simulation import create_simulation_class -from .interpolation import interpolate +from .interpolation import Interpolation sys.path.append(os.getcwd()) @@ -71,10 +71,8 @@ def __init__(self, config_file: str) -> None: # Define the preCICE Participant self._participant = precice.Participant( - "Micro-Manager", - self._config.get_config_file_name(), - self._rank, - self._size) + "Micro-Manager", self._config.get_config_file_name(), self._rank, self._size + ) self._macro_mesh_name = self._config.get_macro_mesh_name() @@ -90,7 +88,7 @@ def __init__(self, config_file: str) -> None: self._ranks_per_axis = self._config.get_ranks_per_axis() self._is_micro_solve_time_required = self._config.write_micro_solve_time() - + self._crash_threshold = 0.2 self._local_number_of_sims = 0 @@ -267,16 +265,20 @@ def solve(self) -> None: # Check if more than a certain percentage of the micro simulations have crashed and terminate if threshold is exceeded crashed_sims_on_all_ranks = np.zeros(self._size, dtype=np.int64) self._comm.Allgather(np.sum(self._crashed_sims), crashed_sims_on_all_ranks) - + if self._is_parallel: - crash_ratio = np.sum(crashed_sims_on_all_ranks) / self._global_number_of_sims + crash_ratio = ( + np.sum(crashed_sims_on_all_ranks) / self._global_number_of_sims + ) else: crash_ratio = np.sum(self._crashed_sims) / len(self._crashed_sims) if crash_ratio > self._crash_threshold: - self._logger.info("{:.1%} of the micro simulations have crashed exceeding the threshold of {:.1%}. " - "Exiting simulation.".format(crash_ratio, self._crash_threshold)) + self._logger.info( + "{:.1%} of the micro simulations have crashed exceeding the threshold of {:.1%}. " + "Exiting simulation.".format(crash_ratio, self._crash_threshold) + ) sys.exit() - + self._write_data_to_precice(micro_sims_output) t += self._dt # increase internal time when time step is done. @@ -408,11 +410,19 @@ def _initialize(self) -> None: self._micro_sims = [None] * self._local_number_of_sims # DECLARATION + # setup for simulation crashes self._crashed_sims = [False] * self._local_number_of_sims - self._old_micro_sims_output = [None] * self._local_number_of_sims - - self._crashed_sims = [False] * self._local_number_of_sims - self._old_micro_sims_output = [None] * self._local_number_of_sims + self._neighbor_list = [None] * self._local_number_of_sims + number_of_nearest_neighbors = int(self._local_number_of_sims * 0.25) + self._interpolation = Interpolation(self._logger) + for count in range(self._local_number_of_sims): + self._neighbor_list[ + count + ] = self._interpolation.get_nearest_neighbor_indices_local( + self._mesh_vertex_coords, + self._mesh_vertex_coords[count], + number_of_nearest_neighbors, + ) micro_problem = getattr( importlib.import_module( @@ -581,49 +591,68 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: try: start_time = time.time() micro_sims_output[count] = sim.solve( - micro_sims_input[count], self._dt) + micro_sims_input[count], self._dt + ) end_time = time.time() # Write solve time of the macro simulation if required and the simulation has not crashed if self._is_micro_solve_time_required: - micro_sims_output[count]["micro_sim_time"] = end_time - start_time - + micro_sims_output[count]["micro_sim_time"] = ( + end_time - start_time + ) + # If simulation crashes, log the error and keep the output constant at the previous iteration's output except Exception as error_message: - self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " - "See next entry for error message. " - "Keeping values constant at results of previous iteration".format( - self._mesh_vertex_coords[count])) + self._logger.error( + "Micro simulation at macro coordinates {} has experienced an error. " + "See next entry for error message.".format( + self._mesh_vertex_coords[count] + ) + ) self._logger.error(error_message) - micro_sims_output[count] = self._old_micro_sims_output[count] self._crashed_sims[count] = True - # If simulation has crashed in a previous iteration, keep the output constant - else: - micro_sims_output[count] = self._old_micro_sims_output[count] - # If a simulation crashes in the first iteration its result is interpolated - set_sims = [] - set_coords = [] - unset_sims = [] - for count in range(self._local_number_of_sims): - if micro_sims_output[count] is not None: - set_sims.append(micro_sims_output[count].copy()) - set_sims[-1].pop("micro_sim_time", None) - set_coords.append(self._mesh_vertex_coords[count]) - else: - unset_sims.append(count) + # Interpolate result for crashed simulation + if self._crashed_sims.count(True) > 0: + unset_sims = [ + count for count, value in enumerate(micro_sims_output) if value is None + ] - # Interpolate if no data is available - for unset_sim in unset_sims: - micro_sims_output[unset_sim] = interpolate( - self._logger, - set_coords, - self._mesh_vertex_coords[unset_sim], - set_sims - ) - if self._is_micro_solve_time_required: - micro_sims_output[unset_sim]["micro_sim_time"] = 0 - - self._old_micro_sims_output = micro_sims_output + # Interpolate output for crashed simulations + for unset_sim in unset_sims: + self._logger.info( + "Interpolating output for crashed simulation at macro vertex {}.".format( + self._mesh_vertex_coords[unset_sim] + ) + ) + coord = [] + interpol_values = [] + output_interpol = dict() + # Collect neighbor vertices for interpolation + for neighbor in self._neighbor_list[unset_sim]: + if not self._crashed_sims[neighbor]: + coord.append(self._mesh_vertex_coords[neighbor].copy()) + interpol_values.append(micro_sims_output[neighbor].copy()) + interpol_values[-1].pop("micro_sim_time", None) + if len(coord) == 0: + self._logger.error( + "No neighbors available for interpolation at macro vertex {}.".format( + self._mesh_vertex_coords[unset_sim] + ) + ) + else: + # Interpolate output for each parameter + for key in interpol_values[0].keys(): + key_values = [] + for elems in range(len(interpol_values)): + key_values.append(interpol_values[elems][key]) + output_interpol[ + key + ] = self._interpolation.inv_dist_weighted_interp( + coord, self._mesh_vertex_coords[unset_sim], key_values + ) + micro_sims_output[unset_sim] = output_interpol + if self._is_micro_solve_time_required: + micro_sims_output[unset_sim]["micro_sim_time"] = 0 return micro_sims_output @@ -689,51 +718,76 @@ def _solve_micro_simulations_with_adaptivity( end_time = time.time() # Write solve time of the macro simulation if required and the simulation has not crashed if self._is_micro_solve_time_required: - micro_sims_output[active_id]["micro_sim_time"] = end_time - start_time + micro_sims_output[active_id]["micro_sim_time"] = ( + end_time - start_time + ) + + # Mark the micro sim as active for export + micro_sims_output[active_id]["active_state"] = 1 + micro_sims_output[active_id][ + "active_steps" + ] = self._micro_sims_active_steps[active_id] + # If simulation crashes, log the error and keep the output constant at the previous iteration's output except Exception as error_message: - self._logger.error("Micro simulation at macro coordinates {} has experienced an error. " - "See next entry for error message. " - "Keeping values constant at results of previous iteration".format( - self._mesh_vertex_coords[active_id])) + self._logger.error( + "Micro simulation at macro coordinates {} has experienced an error. " + "See next entry for error message.".format( + self._mesh_vertex_coords[active_id] + ) + ) self._logger.error(error_message) - # set the micro simulation value to old value and keep it constant if simulation crashes - micro_sims_output[active_id] = self._old_micro_sims_output[active_id] self._crashed_sims[active_id] = True - # If simulation has crashed in a previous iteration, keep the output constant - else: - micro_sims_output[active_id] = self._old_micro_sims_output[active_id] - - # Mark the micro sim as active for export - micro_sims_output[active_id]["active_state"] = 1 - micro_sims_output[active_id][ - "active_steps" - ] = self._micro_sims_active_steps[active_id] - - - # If a simulation crashes in the first iteration its result is interpolated - set_sims = [] - set_coords = [] - unset_sims = [] - for active_id in active_sim_ids: - if micro_sims_output[active_id] is not None: - set_sims.append(micro_sims_output[active_id].copy()) - set_sims[-1].pop("micro_sim_time", None) - set_coords.append(self._mesh_vertex_coords[active_id]) - else: - unset_sims.append(active_id) + # Interpolate result for crashed simulation + if self._crashed_sims.count(True) > 0: + unset_sims = [] + for active_id in active_sim_ids: + if micro_sims_output[active_id] is None: + unset_sims.append(active_id) - # Interpolate if no data is available - for unset_sim in unset_sims: - micro_sims_output[unset_sim] = interpolate( - self._logger, - set_coords, - self._mesh_vertex_coords[unset_sim], - set_sims - ) - if self._is_micro_solve_time_required: - micro_sims_output[unset_sim]["micro_sim_time"] = 0 + # Interpolate output for crashed simulations + for unset_sim in unset_sims: + self._logger.info( + "Interpolating output for crashed simulation at macro vertex {}.".format( + self._mesh_vertex_coords[unset_sim] + ) + ) + coord = [] + interpol_values = [] + output_interpol = dict() + # Collect neighbor vertices for interpolation + for neighbor in self._neighbor_list[unset_sim]: + if not self._crashed_sims[neighbor] and neighbor in active_sim_ids: + coord.append(self._mesh_vertex_coords[neighbor].copy()) + interpol_values.append(micro_sims_output[neighbor].copy()) + interpol_values[-1].pop("micro_sim_time", None) + interpol_values[-1].pop("active_state", None) + interpol_values[-1].pop("active_steps", None) + if len(coord) == 0: + self._logger.error( + "No neighbors available for interpolation at macro vertex {}.".format( + self._mesh_vertex_coords[unset_sim] + ) + ) + else: + # Interpolate output for each parameter + for key in interpol_values[0].keys(): + key_values = [] + for elems in range(len(interpol_values)): + key_values.append(interpol_values[elems][key]) + output_interpol[ + key + ] = self._interpolation.inv_dist_weighted_interp( + coord, self._mesh_vertex_coords[unset_sim], key_values + ) + micro_sims_output[unset_sim] = output_interpol + if self._is_micro_solve_time_required: + micro_sims_output[unset_sim]["micro_sim_time"] = 0 + micro_sims_output[unset_sim]["active_state"] = 1 + micro_sims_output[unset_sim][ + "active_steps" + ] = self._micro_sims_active_steps[unset_sim] # For each inactive simulation, copy data from most similar active simulation if self._adaptivity_type == "global": @@ -760,7 +814,6 @@ def _solve_micro_simulations_with_adaptivity( for i in range(self._local_number_of_sims): for name in self._adaptivity_micro_data_names: self._data_for_adaptivity[name][i] = micro_sims_output[i][name] - self._old_micro_sims_output = micro_sims_output return micro_sims_output diff --git a/setup.py b/setup.py index 36e79095..20cf67a4 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ author_email="ishaan.desai@uni-stuttgart.de", license="LGPL-3.0", packages=find_packages(exclude=["examples"]), - install_requires=["pyprecice>=3.0.0.0", "numpy>=1.13.3", "mpi4py"], + install_requires=["pyprecice>=3.0.0.0", "numpy>=1.13.3", "scikit-learn", "mpi4py"], test_suite="tests", zip_safe=False, ) diff --git a/tests/unit/test_interpolation.py b/tests/unit/test_interpolation.py index 62c72279..e680c8db 100644 --- a/tests/unit/test_interpolation.py +++ b/tests/unit/test_interpolation.py @@ -1,33 +1,80 @@ import numpy as np from unittest import TestCase from unittest.mock import MagicMock -from micro_manager.interpolation import interpolate +from micro_manager.interpolation import Interpolation class TestInterpolation(TestCase): - def test_local_interpolation(self): """ Test if local interpolation works as expected """ - micro_output = [{"vector data": [1,1,1], "scalar data": [1]}] * 4 - micro_output.append({"vector data": [3,3,3], "scalar data": [0]}) - micro_output.append({"vector data": [0,0,0], "scalar data": [3]}) - coords_known = [[2,0,0],[-2,0,0], [0,2,0], [0,-2,0], [0,0,2], [0,0,-1]] - unknown_coord = [0,0,0] - expected_interpolation_result = {"vector data": np.array([1,1,1]), "scalar data": np.array([2])} - interpolation_result = interpolate(MagicMock(), coords_known, unknown_coord , micro_output) - for key in interpolation_result.keys(): - self.assertTrue(np.allclose(interpolation_result[key], expected_interpolation_result[key])) - - def test_local_extrapolation(self): - micro_output = [{"vector data": [3,2,1], "scalar data": [0]}, - {"vector data": [3,2,1], "scalar data": [0]}, - {"vector data": [3,2,1], "scalar data": [0]}, - {"vector data": [4,3,2], "scalar data": [4]}, - {"vector data": [3,2,1], "scalar data": [2]}] - coords_known = [[0,-2,0],[0,-4,0],[0,0,2], [2,0,0],[1,0,0]] - unknown_coord = [0,0,0] - expected_interpolation_result = {"vector data": [3,2,1], "scalar data": [2]} - interpolation_result = interpolate(MagicMock(), coords_known, unknown_coord , micro_output) - self.assertDictEqual(interpolation_result, expected_interpolation_result) \ No newline at end of file + coords = [[-2, 0, 0], [-1, 0, 0], [2, 0, 0]] + inter_point = [1, 0, 0] + vector_data = [[-2, -2, -2], [-1, -1, -1], [2, 2, 2]] + expected_vector_data = [55 / 49, 55 / 49, 55 / 49] + scalar_data = [[-2], [-1], [2]] + expected_scalar_data = 55 / 49 + interpolation = Interpolation(MagicMock()) + interpolated_vector_data = interpolation.inv_dist_weighted_interp( + coords, inter_point, vector_data + ) + interpolated_scalar_data = interpolation.inv_dist_weighted_interp( + coords, inter_point, scalar_data + ) + self.assertTrue(np.allclose(interpolated_vector_data, expected_vector_data)) + self.assertAlmostEqual(interpolated_scalar_data, expected_scalar_data) + + def test_nearest_neighbor(self): + """ + Test if finding nearest neighbor works as expected if interpolation point + itself is not part of neighbor coordinates + """ + neighbors = [[0, 2, 0], [0, 3, 0], [0, 0, 4], [-5, 0, 0]] + inter_coord = [0, 0, 0] + expected_nearest_neighbor_index = [0, 1] + k = 2 + interpolation = Interpolation(MagicMock()) + nearest_neighbor_index = interpolation.get_nearest_neighbor_indices_local( + neighbors, inter_coord, k + ) + self.assertListEqual( + nearest_neighbor_index.tolist(), expected_nearest_neighbor_index + ) + + def test_nearest_neighbor_excluding_interpolation_point(self): + """ + Test if finding nearest neighbor works as expected when the + interpolation point is part of the coordinate list but its + distance shall not be considered + """ + neighbors = [[0, 0, 0], [0, 3, 0], [0, 0, 4], [-5, 0, 0]] + inter_coord = [0, 0, 0] + expected_nearest_neighbor_index = [1, 2] + k = 2 + interpolation = Interpolation(MagicMock()) + nearest_neighbor_index = interpolation.get_nearest_neighbor_indices_local( + neighbors, inter_coord, k + ) + self.assertListEqual( + nearest_neighbor_index.tolist(), expected_nearest_neighbor_index + ) + + +def test_nearest_neighbor_including_interpolation_point(self): + """ + Test if finding nearest neighbor works as expected when the + interpolation point is part of the coordinate list and its + distance shall be considered + """ + neighbors = [[0, 0, 0], [0, 3, 0], [0, 0, 4], [-5, 0, 0]] + inter_coord = [0, 0, 0] + expected_nearest_neighbor_index = [0, 1] + k = 2 + interpolation = Interpolation(MagicMock()) + nearest_neighbor_index = interpolation.get_nearest_neighbor_indices_local( + neighbors, inter_coord, k, inter_point_is_neighbor=True + ) + self.assertListEqual( + nearest_neighbor_index.tolist(), expected_nearest_neighbor_index + ) diff --git a/tests/unit/test_micro_simulation_crash_handling.py b/tests/unit/test_micro_simulation_crash_handling.py index 308e315b..37656644 100644 --- a/tests/unit/test_micro_simulation_crash_handling.py +++ b/tests/unit/test_micro_simulation_crash_handling.py @@ -1,103 +1,121 @@ import numpy as np from unittest import TestCase import micro_manager +import micro_manager.interpolation class MicroSimulation: def __init__(self, sim_id): - self.very_important_value = 0 self.sim_id = sim_id - self.current_time = 0 def initialize(self): pass def solve(self, macro_data, dt): - if self.sim_id == 0: - self.current_time += dt - if self.current_time > dt: - raise Exception("Crash") + if self.sim_id == 2: + raise Exception("Simulation experienced a crash") - return {"micro-scalar-data": macro_data["macro-scalar-data"] + 1, - "micro-vector-data": macro_data["macro-vector-data"] + 1} + return { + "micro-vector-data": macro_data["macro-vector-data"], + "micro-scalar-data": macro_data["macro-scalar-data"], + } class TestSimulationCrashHandling(TestCase): - def setUp(self): - self.fake_read_data_names = { - "macro-scalar-data": False, "macro-vector-data": True} - self.fake_read_data = [{"macro-scalar-data": 1, - "macro-vector-data": np.array([0, 1, 2])}] * 10 - self.fake_write_data = [{"micro-scalar-data": 1, - "micro-vector-data": np.array([0, 1, 2]), - "micro_sim_time": 0, - "active_state": 0, - "active_steps": 0}] * 10 - def test_crash_handling(self): """ Test if the micro manager catches a simulation crash and handles it adequately. """ - manager = micro_manager.MicroManager('micro-manager-config_crash.json') - - manager._local_number_of_sims = 10 - manager._crashed_sims = [False] * 10 - manager._micro_sims = [MicroSimulation(i) for i in range(10)] - manager._micro_sims_active_steps = np.zeros(10, dtype=np.int32) - # Crash during first time step has to be handled differently - - micro_sims_output = manager._solve_micro_simulations( - self.fake_read_data) - for i, data in enumerate(micro_sims_output): - self.fake_read_data[i]["macro-scalar-data"] = data["micro-scalar-data"] - self.fake_read_data[i]["macro-vector-data"] = data["micro-vector-data"] - micro_sims_output = manager._solve_micro_simulations( - self.fake_read_data) - # The crashed simulation should have the same data as the previous step - data_crashed = micro_sims_output[0] - self.assertEqual(data_crashed["micro-scalar-data"], 2) - self.assertListEqual(data_crashed["micro-vector-data"].tolist(), - (self.fake_write_data[0]["micro-vector-data"] + 1).tolist()) - # Non-crashed simulations should have updated data + + macro_data = [] + for i in [-2, -1, 1, 2]: + macro_data.append( + {"macro-vector-data": np.array([i, i, i]), "macro-scalar-data": [i]} + ) + expected_crash_vector_data = np.array([55 / 49, 55 / 49, 55 / 49]) + expected_crash_scalar_data = 55 / 49 + + manager = micro_manager.MicroManager("micro-manager-config_crash.json") + + manager._local_number_of_sims = 4 + manager._crashed_sims = [False] * 4 + manager._mesh_vertex_coords = np.array( + [[-2, 0, 0], [-1, 0, 0], [1, 0, 0], [2, 0, 0]] + ) + manager._neighbor_list = np.array([[1, 2, 3], [0, 2, 3], [0, 1, 3], [0, 1, 2]]) + manager._micro_sims = [MicroSimulation(i) for i in range(4)] + + micro_sims_output = manager._solve_micro_simulations(macro_data) + + # Crashed data has interpolated value + data_crashed = micro_sims_output[2] + self.assertEqual(data_crashed["micro-scalar-data"], expected_crash_scalar_data) + self.assertListEqual( + data_crashed["micro-vector-data"].tolist(), + expected_crash_vector_data.tolist(), + ) + # Non-crashed simulations should remain constant data_normal = micro_sims_output[1] - self.assertEqual(data_normal["micro-scalar-data"], 3) - self.assertListEqual(data_normal["micro-vector-data"].tolist(), - (self.fake_write_data[1]["micro-vector-data"] + 2).tolist()) + self.assertEqual( + data_normal["micro-scalar-data"], macro_data[1]["macro-scalar-data"] + ) + self.assertListEqual( + data_normal["micro-vector-data"].tolist(), + macro_data[1]["macro-vector-data"].tolist(), + ) def test_crash_handling_with_adaptivity(self): """ Test if the micro manager catches a simulation crash and handles it adequately with adaptivity. """ - manager = micro_manager.MicroManager('micro-manager-config_crash.json') - manager._local_number_of_sims = 10 - manager._crashed_sims = [False] * 10 - manager._micro_sims = [MicroSimulation(i) for i in range(10)] - manager._micro_sims_active_steps = np.zeros(10, dtype=np.int32) - is_sim_active = np.array( - [True, True, False, True, False, False, False, True, True, False]) - sim_is_associated_to = np.array([-2, -2, 1, -2, 3, 3, 0, -2, -2, 8]) - # Crash in the first time step is handled differently + macro_data = [] + for i in [-2, -1, 1, 2, 10]: + macro_data.append( + {"macro-vector-data": np.array([i, i, i]), "macro-scalar-data": [i]} + ) + expected_crash_vector_data = np.array([55 / 49, 55 / 49, 55 / 49]) + expected_crash_scalar_data = 55 / 49 + manager = micro_manager.MicroManager("micro-manager-config_crash.json") + + manager._local_number_of_sims = 5 + manager._micro_sims_active_steps = np.zeros(5, dtype=np.int32) + manager._crashed_sims = [False] * 5 + manager._mesh_vertex_coords = np.array( + [[-2, 0, 0], [-1, 0, 0], [1, 0, 0], [2, 0, 0], [1, 1, 0]] + ) + manager._neighbor_list = np.array( + [[1, 2, 3, 4], [0, 2, 3, 4], [0, 1, 3, 4], [0, 1, 2, 4], [0, 1, 2, 3]] + ) + manager._micro_sims = [MicroSimulation(i) for i in range(5)] + + is_sim_active = np.array([True, True, True, True, False]) + sim_is_associated_to = np.array([-2, -2, -2, -2, 2]) micro_sims_output = manager._solve_micro_simulations_with_adaptivity( - self.fake_read_data, is_sim_active, sim_is_associated_to) - for i, data in enumerate(micro_sims_output): - self.fake_read_data[i]["macro-scalar-data"] = data["micro-scalar-data"] - self.fake_read_data[i]["macro-vector-data"] = data["micro-vector-data"] - micro_sims_output = manager._solve_micro_simulations_with_adaptivity( - self.fake_read_data, is_sim_active, sim_is_associated_to) - # The crashed simulation should have the same data as the previous step - data_crashed = micro_sims_output[0] - self.assertEqual(data_crashed["micro-scalar-data"], 2) - self.assertListEqual(data_crashed["micro-vector-data"].tolist(), - (self.fake_write_data[0]["micro-vector-data"] + 1).tolist()) - # Non-crashed simulations should have updated data - data_normal = micro_sims_output[1] - self.assertEqual(data_normal["micro-scalar-data"], 3) - self.assertListEqual(data_normal["micro-vector-data"].tolist(), - (self.fake_write_data[1]["micro-vector-data"] + 2).tolist()) + macro_data, is_sim_active, sim_is_associated_to + ) + # Crashed data has interpolated value + data_crashed = micro_sims_output[2] + self.assertEqual(data_crashed["micro-scalar-data"], expected_crash_scalar_data) + self.assertListEqual( + data_crashed["micro-vector-data"].tolist(), + expected_crash_vector_data.tolist(), + ) -if __name__ == '__main__': + # Inactive simulation that is associated with crashed simulation has same value + data_associated = micro_sims_output[4] + self.assertEqual( + data_associated["micro-scalar-data"], expected_crash_scalar_data + ) + self.assertListEqual( + data_associated["micro-vector-data"].tolist(), + expected_crash_vector_data.tolist(), + ) + + +if __name__ == "__main__": import unittest + unittest.main() From 96384cd28787aa2409923992711fe4b89bef90f6 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Thu, 11 Apr 2024 17:02:39 +0200 Subject: [PATCH 15/23] Format with pre-commit --- tests/unit/test_micro_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index 66471fb1..d5db7b7c 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -18,8 +18,8 @@ def solve(self, macro_data, dt): assert macro_data["macro-vector-data"].tolist() == [0, 1, 2] return { "micro-scalar-data": macro_data["macro-scalar-data"] + 1, - "micro-vector-data": macro_data["macro-vector-data"] + 1 - } + "micro-vector-data": macro_data["macro-vector-data"] + 1, + } class TestFunctioncalls(TestCase): From c5b3a6c95c2abd8d74042fbb60305ef3d6c9322f Mon Sep 17 00:00:00 2001 From: Torben Schiz <49746900+tjwsch@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:43:57 +0200 Subject: [PATCH 16/23] Apply suggestions from code review Co-authored-by: Ishaan Desai --- micro_manager/interpolation.py | 2 +- micro_manager/micro_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/micro_manager/interpolation.py b/micro_manager/interpolation.py index 412d3ea9..1cef11e9 100644 --- a/micro_manager/interpolation.py +++ b/micro_manager/interpolation.py @@ -32,7 +32,7 @@ def get_nearest_neighbor_indices_local( """ assert ( len(all_local_coords) > k - ), "Number of neighbors must be less than the number of all available neighbors." + ), "Desired number of neighbors must be less than the number of all available neighbors." if not inter_point_is_neighbor: neighbors = NearestNeighbors(n_neighbors=k + 1).fit(all_local_coords) diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 48260b0e..0f6a590c 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -411,7 +411,7 @@ def _initialize(self) -> None: self._micro_sims = [None] * self._local_number_of_sims # DECLARATION # setup for simulation crashes - self._crashed_sims = [False] * self._local_number_of_sims + self._has_sim_crashed = [False] * self._local_number_of_sims self._neighbor_list = [None] * self._local_number_of_sims number_of_nearest_neighbors = int(self._local_number_of_sims * 0.25) self._interpolation = Interpolation(self._logger) From 1dc33ce73fe46e8b792a8e526ad245e0354baa73 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Tue, 23 Apr 2024 17:37:21 +0200 Subject: [PATCH 17/23] Incoporate review into crash handling --- .github/workflows/run-unit-tests.yml | 10 +- micro_manager/interpolation.py | 100 ++++++----- micro_manager/micro_manager.py | 167 +++++++++--------- tests/unit/micro-manager-config_crash.json | 3 +- tests/unit/test_interpolation.py | 28 +-- .../test_micro_simulation_crash_handling.py | 13 +- 6 files changed, 170 insertions(+), 151 deletions(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 72b04c72..a8fddd62 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -24,6 +24,14 @@ jobs: pip3 install --user . pip3 uninstall -y pyprecice - - name: Run unit tests + - name: Run micro_manager unit test working-directory: micro-manager/tests/unit run: python3 -m unittest test_micro_manager.py + + - name: Run interpolation unit test + working-directory: micro-manager/tests/unit + run: python3 -m unittest test_interpolation.py + + - name: Run micro simulation crash unit test + working-directory: micro-manager/tests/unit + run: python3 -m unittest test_micro_simulation_crash_handling.py diff --git a/micro_manager/interpolation.py b/micro_manager/interpolation.py index 1cef11e9..86187328 100644 --- a/micro_manager/interpolation.py +++ b/micro_manager/interpolation.py @@ -9,75 +9,89 @@ def __init__(self, logger): def get_nearest_neighbor_indices_local( self, - all_local_coords, - inter_point, + all_local_coords: np.ndarray, + inter_point: np.ndarray, k: int, - inter_point_is_neighbor: bool = False, ) -> np.ndarray: """ Get the indices of the k nearest neighbors of a point in a list of coordinates. - Note: It can be chosen whether the point itself is considered as a neighbor or not. - Args: - all_local_coords: list - List of coordinates of all points. - inter_point: - Coordinates of the point for which the neighbors are to be found. - k: int - inter_point_is_neighbor: bool, optional - Decide whether the interpolation point is considered as its own neighbor. - Defaults to False. + If inter_point is part of all_local_coords, it is only considered one time less than it occurs. + inter_point is expected to be in all_local_coords at most once. - Returns: np.ndarray - of indices of the k nearest neighbors. + Parameters + ---------- + all_local_coords : list + List of coordinates of all points. + inter_point : list | np.ndarray + Coordinates of the point for which the neighbors are to be found. + k : int + Number of neighbors to consider. + + Returns + ------ + neighbor_indices : np.ndarray + Indices of the k nearest neighbors in all local points. """ assert ( - len(all_local_coords) > k - ), "Desired number of neighbors must be less than the number of all available neighbors." - if not inter_point_is_neighbor: - neighbors = NearestNeighbors(n_neighbors=k + 1).fit(all_local_coords) + len(all_local_coords) >= k + ), "Desired number of neighbors must be less than or equal to the number of all available neighbors." + # If the number of neighbors is larger than the number of all available neighbors, increase the number of neighbors + # to be able to delete a neighbor if it coincides with the interpolation point. + if len(all_local_coords) > k: + k += 1 + neighbors = NearestNeighbors(n_neighbors=k).fit(all_local_coords) - dists, neighbor_indices = neighbors.kneighbors( - [inter_point], return_distance=True - ) + dists, neighbor_indices = neighbors.kneighbors( + [inter_point], return_distance=True + ) - if np.min(dists) < 1e-10: - argmin = np.argmin(dists) - neighbor_indices = np.delete(neighbor_indices, argmin) - else: - argmax = np.argmax(dists) - neighbor_indices = np.delete(neighbor_indices, argmax) + # Check whether the inter_point is also part of the neighbor list and remove it. + if np.min(dists) < 1e-16: + argmin = np.argmin(dists) + neighbor_indices = np.delete(neighbor_indices, argmin) + # If point itself is not in neighbor list, remove neighbor with largest distance + # to return the desired number of neighbors else: - neighbors = NearestNeighbors(n_neighbors=k).fit(all_local_coords) - neighbor_indices = neighbors.kneighbors( - [inter_point], return_distance=False - ) + argmax = np.argmax(dists) + neighbor_indices = np.delete(neighbor_indices, argmax) return neighbor_indices def inv_dist_weighted_interp( - self, neighbors: list, point, values: list - ) -> np.ndarray: + self, neighbors: np.ndarray, point: np.ndarray, values + ): """ Interpolate a value at a point using inverse distance weighting. + .. math:: + f(x) = (\sum_{i=1}^{n} \frac{f_i}{\Vert x_i - x \Vert^2}) / (\sum_{j=1}^{n} \frac{1}{\Vert x_j - x \Vert^2}) - Args: - neighbors: list - Coordinates at which the values are known. - point: - Coordinates at which the value is to be interpolated. - values: list - Values at the known coordinates. + Parameters + ---------- + neighbors : np.ndarray + Coordinates at which the values are known. + point : np.ndarray + Coordinates at which the value is to be interpolated. + values : + Values at the known coordinates. - Returns: nd.array + Returns + ------- + interpol_val / summed_weights : Value at interpolation point. """ interpol_val = 0 summed_weights = 0 + # iterate over all neighbors for inx in range(len(neighbors)): + # compute the squared norm of the difference between interpolation point and neighbor norm = np.linalg.norm(np.array(neighbors[inx]) - np.array(point)) ** 2 - if norm < 1e-10: + # If interpolation point is already part of the data it is returned as the interpolation result + # This avoids division by zero + if norm < 1e-16: return values[inx] + # update interpolation value interpol_val += values[inx] / norm + # extend normalization factor summed_weights += 1 / norm return interpol_val / summed_weights diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 0f6a590c..9bbfbf8f 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -90,6 +90,7 @@ def __init__(self, config_file: str) -> None: self._is_micro_solve_time_required = self._config.write_micro_solve_time() self._crash_threshold = 0.2 + self._number_of_nearest_neighbors = 4 self._local_number_of_sims = 0 self._global_number_of_sims = 0 @@ -264,14 +265,16 @@ def solve(self) -> None: # Check if more than a certain percentage of the micro simulations have crashed and terminate if threshold is exceeded crashed_sims_on_all_ranks = np.zeros(self._size, dtype=np.int64) - self._comm.Allgather(np.sum(self._crashed_sims), crashed_sims_on_all_ranks) + self._comm.Allgather( + np.sum(self._has_sim_crashed), crashed_sims_on_all_ranks + ) if self._is_parallel: crash_ratio = ( np.sum(crashed_sims_on_all_ranks) / self._global_number_of_sims ) else: - crash_ratio = np.sum(self._crashed_sims) / len(self._crashed_sims) + crash_ratio = np.sum(self._has_sim_crashed) / len(self._has_sim_crashed) if crash_ratio > self._crash_threshold: self._logger.info( "{:.1%} of the micro simulations have crashed exceeding the threshold of {:.1%}. " @@ -412,8 +415,7 @@ def _initialize(self) -> None: # setup for simulation crashes self._has_sim_crashed = [False] * self._local_number_of_sims - self._neighbor_list = [None] * self._local_number_of_sims - number_of_nearest_neighbors = int(self._local_number_of_sims * 0.25) + self._neighbor_list = [None] * self._local_number_of_sims # DECLARATION self._interpolation = Interpolation(self._logger) for count in range(self._local_number_of_sims): self._neighbor_list[ @@ -421,7 +423,7 @@ def _initialize(self) -> None: ] = self._interpolation.get_nearest_neighbor_indices_local( self._mesh_vertex_coords, self._mesh_vertex_coords[count], - number_of_nearest_neighbors, + self._number_of_nearest_neighbors, ) micro_problem = getattr( @@ -586,7 +588,7 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: for count, sim in enumerate(self._micro_sims): # If micro simulation has not crashed in a previous iteration, attempt to solve it - if not self._crashed_sims[count]: + if not self._has_sim_crashed[count]: # Attempt to solve the micro simulation try: start_time = time.time() @@ -604,55 +606,29 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: except Exception as error_message: self._logger.error( "Micro simulation at macro coordinates {} has experienced an error. " - "See next entry for error message.".format( + "See next entry on this rank for error message.".format( self._mesh_vertex_coords[count] ) ) self._logger.error(error_message) - self._crashed_sims[count] = True + self._has_sim_crashed[count] = True # Interpolate result for crashed simulation - if self._crashed_sims.count(True) > 0: + if self._has_sim_crashed.count(True) > 0: unset_sims = [ count for count, value in enumerate(micro_sims_output) if value is None ] - # Interpolate output for crashed simulations + # Iterate over all crashed simulations to interpolate output for unset_sim in unset_sims: self._logger.info( "Interpolating output for crashed simulation at macro vertex {}.".format( self._mesh_vertex_coords[unset_sim] ) ) - coord = [] - interpol_values = [] - output_interpol = dict() - # Collect neighbor vertices for interpolation - for neighbor in self._neighbor_list[unset_sim]: - if not self._crashed_sims[neighbor]: - coord.append(self._mesh_vertex_coords[neighbor].copy()) - interpol_values.append(micro_sims_output[neighbor].copy()) - interpol_values[-1].pop("micro_sim_time", None) - if len(coord) == 0: - self._logger.error( - "No neighbors available for interpolation at macro vertex {}.".format( - self._mesh_vertex_coords[unset_sim] - ) - ) - else: - # Interpolate output for each parameter - for key in interpol_values[0].keys(): - key_values = [] - for elems in range(len(interpol_values)): - key_values.append(interpol_values[elems][key]) - output_interpol[ - key - ] = self._interpolation.inv_dist_weighted_interp( - coord, self._mesh_vertex_coords[unset_sim], key_values - ) - micro_sims_output[unset_sim] = output_interpol - if self._is_micro_solve_time_required: - micro_sims_output[unset_sim]["micro_sim_time"] = 0 + micro_sims_output[unset_sim] = self._interpolate_output_for_crashed_sim( + micro_sims_output, unset_sim + ) return micro_sims_output @@ -708,7 +684,7 @@ def _solve_micro_simulations_with_adaptivity( # Solve all active micro simulations for active_id in active_sim_ids: # If micro simulation has not crashed in a previous iteration, attempt to solve it - if not self._crashed_sims[active_id]: + if not self._has_sim_crashed[active_id]: # Attempt to solve the micro simulation try: start_time = time.time() @@ -732,62 +708,31 @@ def _solve_micro_simulations_with_adaptivity( except Exception as error_message: self._logger.error( "Micro simulation at macro coordinates {} has experienced an error. " - "See next entry for error message.".format( + "See next entry on this rank for error message.".format( self._mesh_vertex_coords[active_id] ) ) self._logger.error(error_message) - self._crashed_sims[active_id] = True + self._has_sim_crashed[active_id] = True # Interpolate result for crashed simulation - if self._crashed_sims.count(True) > 0: + if self._has_sim_crashed.count(True) > 0: unset_sims = [] for active_id in active_sim_ids: if micro_sims_output[active_id] is None: unset_sims.append(active_id) - # Interpolate output for crashed simulations + # Iterate over all crashed simulations to interpolate output for unset_sim in unset_sims: self._logger.info( "Interpolating output for crashed simulation at macro vertex {}.".format( self._mesh_vertex_coords[unset_sim] ) ) - coord = [] - interpol_values = [] - output_interpol = dict() - # Collect neighbor vertices for interpolation - for neighbor in self._neighbor_list[unset_sim]: - if not self._crashed_sims[neighbor] and neighbor in active_sim_ids: - coord.append(self._mesh_vertex_coords[neighbor].copy()) - interpol_values.append(micro_sims_output[neighbor].copy()) - interpol_values[-1].pop("micro_sim_time", None) - interpol_values[-1].pop("active_state", None) - interpol_values[-1].pop("active_steps", None) - if len(coord) == 0: - self._logger.error( - "No neighbors available for interpolation at macro vertex {}.".format( - self._mesh_vertex_coords[unset_sim] - ) - ) - else: - # Interpolate output for each parameter - for key in interpol_values[0].keys(): - key_values = [] - for elems in range(len(interpol_values)): - key_values.append(interpol_values[elems][key]) - output_interpol[ - key - ] = self._interpolation.inv_dist_weighted_interp( - coord, self._mesh_vertex_coords[unset_sim], key_values - ) - micro_sims_output[unset_sim] = output_interpol - if self._is_micro_solve_time_required: - micro_sims_output[unset_sim]["micro_sim_time"] = 0 - micro_sims_output[unset_sim]["active_state"] = 1 - micro_sims_output[unset_sim][ - "active_steps" - ] = self._micro_sims_active_steps[unset_sim] + + micro_sims_output[unset_sim] = self._interpolate_output_for_crashed_sim( + micro_sims_output, unset_sim, active_sim_ids + ) # For each inactive simulation, copy data from most similar active simulation if self._adaptivity_type == "global": @@ -817,6 +762,70 @@ def _solve_micro_simulations_with_adaptivity( return micro_sims_output + def _interpolate_output_for_crashed_sim( + self, micro_sims_output: list, unset_sim: int, active_sim_ids: np.ndarray = None + ) -> dict: + """ + Using the output of neighboring simulations, interpolate the output for a crashed simulation. + + Parameters + ---------- + micro_sims_output : list + Output of local micro simulations. + unset_sim : int + Index of the crashed simulation in the list of all local simulations currently interpolating. + active_sim_ids : numpy.ndarray, optional + Array of active simulation IDs. + + Returns + ------- + output_interpol : dict + Result of the interpolation in which keys are names of data and the values are the data. + """ + output_interpol = dict() + coord = [] # DECLARATION + interpol_values = [] # DECLARATION + # Collect neighbor vertices for interpolation + for neighbor in self._neighbor_list[unset_sim]: + # Only include neighbors that have not crashed and remove data not required for interpolation + if not self._has_sim_crashed[neighbor]: + if self._is_adaptivity_on: + if neighbor in active_sim_ids: + coord.append(self._mesh_vertex_coords[neighbor].copy()) + interpol_values.append(micro_sims_output[neighbor].copy()) + interpol_values[-1].pop("micro_sim_time", None) + interpol_values[-1].pop("active_state", None) + interpol_values[-1].pop("active_steps", None) + else: + coord.append(self._mesh_vertex_coords[neighbor].copy()) + interpol_values.append(micro_sims_output[neighbor].copy()) + interpol_values[-1].pop("micro_sim_time", None) + if len(coord) == 0: + self._logger.error( + "No neighbors available for interpolation at macro vertex {}.".format( + self._mesh_vertex_coords[unset_sim] + ) + ) + else: + # Interpolate for each parameter + for key in interpol_values[0].keys(): + key_values = [] # DECLARATION + # Collect values of current parameter from neighboring simulations + for elems in range(len(interpol_values)): + key_values.append(interpol_values[elems][key]) + output_interpol[key] = self._interpolation.inv_dist_weighted_interp( + coord, self._mesh_vertex_coords[unset_sim], key_values + ) + # Reintroduce removed information + if self._is_micro_solve_time_required: + output_interpol["micro_sim_time"] = 0 + if self._is_adaptivity_on: + output_interpol["active_state"] = 1 + output_interpol["active_steps"] = self._micro_sims_active_steps[ + unset_sim + ] + return output_interpol + def main(): parser = argparse.ArgumentParser(description=".") diff --git a/tests/unit/micro-manager-config_crash.json b/tests/unit/micro-manager-config_crash.json index 9ff06771..e5b8ac32 100644 --- a/tests/unit/micro-manager-config_crash.json +++ b/tests/unit/micro-manager-config_crash.json @@ -8,7 +8,8 @@ }, "simulation_params": { "macro_domain_bounds": [0.0, 25.0, 0.0, 25.0, 0.0, 25.0], - "adaptivity": { + "adaptivity": "True", + "adaptivity_settings": { "type": "local", "data": ["macro-scalar-data", "macro-vector-data"], "history_param": 0.5, diff --git a/tests/unit/test_interpolation.py b/tests/unit/test_interpolation.py index e680c8db..cbaf5409 100644 --- a/tests/unit/test_interpolation.py +++ b/tests/unit/test_interpolation.py @@ -7,7 +7,7 @@ class TestInterpolation(TestCase): def test_local_interpolation(self): """ - Test if local interpolation works as expected + Test if local interpolation works as expected. """ coords = [[-2, 0, 0], [-1, 0, 0], [2, 0, 0]] inter_point = [1, 0, 0] @@ -28,7 +28,7 @@ def test_local_interpolation(self): def test_nearest_neighbor(self): """ Test if finding nearest neighbor works as expected if interpolation point - itself is not part of neighbor coordinates + itself is not part of neighbor coordinates. """ neighbors = [[0, 2, 0], [0, 3, 0], [0, 0, 4], [-5, 0, 0]] inter_coord = [0, 0, 0] @@ -42,11 +42,10 @@ def test_nearest_neighbor(self): nearest_neighbor_index.tolist(), expected_nearest_neighbor_index ) - def test_nearest_neighbor_excluding_interpolation_point(self): + def test_nearest_neighbor_with_point_its_own_neighbor(self): """ Test if finding nearest neighbor works as expected when the - interpolation point is part of the coordinate list but its - distance shall not be considered + interpolation point is part of the coordinate list. """ neighbors = [[0, 0, 0], [0, 3, 0], [0, 0, 4], [-5, 0, 0]] inter_coord = [0, 0, 0] @@ -59,22 +58,3 @@ def test_nearest_neighbor_excluding_interpolation_point(self): self.assertListEqual( nearest_neighbor_index.tolist(), expected_nearest_neighbor_index ) - - -def test_nearest_neighbor_including_interpolation_point(self): - """ - Test if finding nearest neighbor works as expected when the - interpolation point is part of the coordinate list and its - distance shall be considered - """ - neighbors = [[0, 0, 0], [0, 3, 0], [0, 0, 4], [-5, 0, 0]] - inter_coord = [0, 0, 0] - expected_nearest_neighbor_index = [0, 1] - k = 2 - interpolation = Interpolation(MagicMock()) - nearest_neighbor_index = interpolation.get_nearest_neighbor_indices_local( - neighbors, inter_coord, k, inter_point_is_neighbor=True - ) - self.assertListEqual( - nearest_neighbor_index.tolist(), expected_nearest_neighbor_index - ) diff --git a/tests/unit/test_micro_simulation_crash_handling.py b/tests/unit/test_micro_simulation_crash_handling.py index 37656644..3e4b33ca 100644 --- a/tests/unit/test_micro_simulation_crash_handling.py +++ b/tests/unit/test_micro_simulation_crash_handling.py @@ -1,5 +1,7 @@ -import numpy as np from unittest import TestCase + +import numpy as np + import micro_manager import micro_manager.interpolation @@ -25,6 +27,7 @@ class TestSimulationCrashHandling(TestCase): def test_crash_handling(self): """ Test if the micro manager catches a simulation crash and handles it adequately. + A crash if caught by interpolation within _solve_micro_simulations. """ macro_data = [] @@ -38,11 +41,14 @@ def test_crash_handling(self): manager = micro_manager.MicroManager("micro-manager-config_crash.json") manager._local_number_of_sims = 4 - manager._crashed_sims = [False] * 4 + manager._has_sim_crashed = [False] * 4 manager._mesh_vertex_coords = np.array( [[-2, 0, 0], [-1, 0, 0], [1, 0, 0], [2, 0, 0]] ) manager._neighbor_list = np.array([[1, 2, 3], [0, 2, 3], [0, 1, 3], [0, 1, 2]]) + manager._is_adaptivity_on = ( + False # make sure adaptivity is off overriding config + ) manager._micro_sims = [MicroSimulation(i) for i in range(4)] micro_sims_output = manager._solve_micro_simulations(macro_data) @@ -67,6 +73,7 @@ def test_crash_handling(self): def test_crash_handling_with_adaptivity(self): """ Test if the micro manager catches a simulation crash and handles it adequately with adaptivity. + A crash if caught by interpolation within _solve_micro_simulations_with_adaptivity. """ macro_data = [] @@ -81,7 +88,7 @@ def test_crash_handling_with_adaptivity(self): manager._local_number_of_sims = 5 manager._micro_sims_active_steps = np.zeros(5, dtype=np.int32) - manager._crashed_sims = [False] * 5 + manager._has_sim_crashed = [False] * 5 manager._mesh_vertex_coords = np.array( [[-2, 0, 0], [-1, 0, 0], [1, 0, 0], [2, 0, 0], [1, 1, 0]] ) From b1464e5cc4a72a06f2eedd21cf4933ba02880b08 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Fri, 26 Apr 2024 17:17:36 +0200 Subject: [PATCH 18/23] Base crash interpolation on macro parameter --- micro_manager/interpolation.py | 33 +++---- micro_manager/micro_manager.py | 90 +++++++++++++------ tests/unit/test_interpolation.py | 23 +---- .../test_micro_simulation_crash_handling.py | 3 +- 4 files changed, 78 insertions(+), 71 deletions(-) diff --git a/micro_manager/interpolation.py b/micro_manager/interpolation.py index 86187328..13b6b55a 100644 --- a/micro_manager/interpolation.py +++ b/micro_manager/interpolation.py @@ -15,8 +15,6 @@ def get_nearest_neighbor_indices_local( ) -> np.ndarray: """ Get the indices of the k nearest neighbors of a point in a list of coordinates. - If inter_point is part of all_local_coords, it is only considered one time less than it occurs. - inter_point is expected to be in all_local_coords at most once. Parameters ---------- @@ -32,28 +30,19 @@ def get_nearest_neighbor_indices_local( neighbor_indices : np.ndarray Indices of the k nearest neighbors in all local points. """ - assert ( - len(all_local_coords) >= k - ), "Desired number of neighbors must be less than or equal to the number of all available neighbors." - # If the number of neighbors is larger than the number of all available neighbors, increase the number of neighbors - # to be able to delete a neighbor if it coincides with the interpolation point. - if len(all_local_coords) > k: - k += 1 + assert len(all_local_coords) > 0, "No local coordinates provided." + if len(all_local_coords) < k: + self._logger.info( + "Number of neighbors {} is larger than the number of neighbors {}. Setting k to {}.".format( + k, len(all_local_coords), len(all_local_coords) + ) + ) + k = len(all_local_coords) neighbors = NearestNeighbors(n_neighbors=k).fit(all_local_coords) - dists, neighbor_indices = neighbors.kneighbors( - [inter_point], return_distance=True - ) - - # Check whether the inter_point is also part of the neighbor list and remove it. - if np.min(dists) < 1e-16: - argmin = np.argmin(dists) - neighbor_indices = np.delete(neighbor_indices, argmin) - # If point itself is not in neighbor list, remove neighbor with largest distance - # to return the desired number of neighbors - else: - argmax = np.argmax(dists) - neighbor_indices = np.delete(neighbor_indices, argmax) + neighbor_indices = neighbors.kneighbors( + [inter_point], return_distance=False + ).flatten() return neighbor_indices diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 9bbfbf8f..dc38a3a3 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -20,6 +20,7 @@ from copy import deepcopy from typing import Dict from warnings import warn +import itertools import numpy as np import precice @@ -417,14 +418,6 @@ def _initialize(self) -> None: self._has_sim_crashed = [False] * self._local_number_of_sims self._neighbor_list = [None] * self._local_number_of_sims # DECLARATION self._interpolation = Interpolation(self._logger) - for count in range(self._local_number_of_sims): - self._neighbor_list[ - count - ] = self._interpolation.get_nearest_neighbor_indices_local( - self._mesh_vertex_coords, - self._mesh_vertex_coords[count], - self._number_of_nearest_neighbors, - ) micro_problem = getattr( importlib.import_module( @@ -627,7 +620,7 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: ) ) micro_sims_output[unset_sim] = self._interpolate_output_for_crashed_sim( - micro_sims_output, unset_sim + micro_sims_input, micro_sims_output, unset_sim ) return micro_sims_output @@ -731,7 +724,7 @@ def _solve_micro_simulations_with_adaptivity( ) micro_sims_output[unset_sim] = self._interpolate_output_for_crashed_sim( - micro_sims_output, unset_sim, active_sim_ids + micro_sims_input, micro_sims_output, unset_sim, active_sim_ids ) # For each inactive simulation, copy data from most similar active simulation @@ -763,13 +756,20 @@ def _solve_micro_simulations_with_adaptivity( return micro_sims_output def _interpolate_output_for_crashed_sim( - self, micro_sims_output: list, unset_sim: int, active_sim_ids: np.ndarray = None + self, + micro_sims_input: list, + micro_sims_output: list, + unset_sim: int, + active_sim_ids: np.ndarray = None, ) -> dict: """ Using the output of neighboring simulations, interpolate the output for a crashed simulation. Parameters ---------- + micro_sims_input : list + List of dicts in which keys are names of data and the values are the data which are required inputs to + solve a micro simulation. micro_sims_output : list Output of local micro simulations. unset_sim : int @@ -782,30 +782,64 @@ def _interpolate_output_for_crashed_sim( output_interpol : dict Result of the interpolation in which keys are names of data and the values are the data. """ + # Find neighbors of the crashed simulation in active and non-crashed simulations + # set iteration length to only iterate over active simulations + if self._is_adaptivity_on: + iter_length = active_sim_ids + else: + iter_length = range(len(micro_sims_input)) + micro_sims_active_input_lists = [] # DECLARATION + micro_sims_active_values = [] # DECLARATION + # Turn crashed sim macro parameters into list to use as coordinate for interpolation + crashed_position = [] # DECLARATION + for value in micro_sims_input[unset_sim].values(): + if isinstance(value, np.ndarray) or isinstance(value, list): + crashed_position.extend(value) + else: + crashed_position.append(value) + # Turn active sim macro parameters into list to use as coordinates for interpolation based on parameters + for i in iter_length: + if not self._has_sim_crashed[i]: + # collect macro data at one vertex + intermediate = [] # DECLARATION + for value in micro_sims_input[i].values(): + if isinstance(value, np.ndarray) or isinstance(value, list): + intermediate.extend(value) + else: + intermediate.append(value) + # Create lists of macro data for interpolation + micro_sims_active_input_lists.append(intermediate) + micro_sims_active_values.append(micro_sims_output[i].copy()) + # Find nearest neighbors + nearest_neighbors = self._interpolation.get_nearest_neighbor_indices_local( + micro_sims_active_input_lists, + crashed_position, + self._number_of_nearest_neighbors, + ) + # Interpolate output_interpol = dict() - coord = [] # DECLARATION + interpol_space = [] # DECLARATION interpol_values = [] # DECLARATION # Collect neighbor vertices for interpolation - for neighbor in self._neighbor_list[unset_sim]: - # Only include neighbors that have not crashed and remove data not required for interpolation - if not self._has_sim_crashed[neighbor]: - if self._is_adaptivity_on: - if neighbor in active_sim_ids: - coord.append(self._mesh_vertex_coords[neighbor].copy()) - interpol_values.append(micro_sims_output[neighbor].copy()) - interpol_values[-1].pop("micro_sim_time", None) - interpol_values[-1].pop("active_state", None) - interpol_values[-1].pop("active_steps", None) - else: - coord.append(self._mesh_vertex_coords[neighbor].copy()) - interpol_values.append(micro_sims_output[neighbor].copy()) - interpol_values[-1].pop("micro_sim_time", None) - if len(coord) == 0: + for neighbor in nearest_neighbors: + # Remove data not required for interpolation from values + if self._is_adaptivity_on: + interpol_space.append(micro_sims_active_input_lists[neighbor].copy()) + interpol_values.append(micro_sims_active_values[neighbor].copy()) + interpol_values[-1].pop("micro_sim_time", None) + interpol_values[-1].pop("active_state", None) + interpol_values[-1].pop("active_steps", None) + else: + interpol_space.append(micro_sims_active_input_lists[neighbor].copy()) + interpol_values.append(micro_sims_active_values[neighbor].copy()) + interpol_values[-1].pop("micro_sim_time", None) + if len(interpol_space) == 0: self._logger.error( "No neighbors available for interpolation at macro vertex {}.".format( self._mesh_vertex_coords[unset_sim] ) ) + return None else: # Interpolate for each parameter for key in interpol_values[0].keys(): @@ -814,7 +848,7 @@ def _interpolate_output_for_crashed_sim( for elems in range(len(interpol_values)): key_values.append(interpol_values[elems][key]) output_interpol[key] = self._interpolation.inv_dist_weighted_interp( - coord, self._mesh_vertex_coords[unset_sim], key_values + interpol_space, crashed_position, key_values ) # Reintroduce removed information if self._is_micro_solve_time_required: diff --git a/tests/unit/test_interpolation.py b/tests/unit/test_interpolation.py index cbaf5409..fd21ac60 100644 --- a/tests/unit/test_interpolation.py +++ b/tests/unit/test_interpolation.py @@ -30,27 +30,10 @@ def test_nearest_neighbor(self): Test if finding nearest neighbor works as expected if interpolation point itself is not part of neighbor coordinates. """ - neighbors = [[0, 2, 0], [0, 3, 0], [0, 0, 4], [-5, 0, 0]] + neighbors = [[0, 2, 0], [0, 3, 0], [0, 0, 4], [-5, 0, 0], [0, 0, 0]] inter_coord = [0, 0, 0] - expected_nearest_neighbor_index = [0, 1] - k = 2 - interpolation = Interpolation(MagicMock()) - nearest_neighbor_index = interpolation.get_nearest_neighbor_indices_local( - neighbors, inter_coord, k - ) - self.assertListEqual( - nearest_neighbor_index.tolist(), expected_nearest_neighbor_index - ) - - def test_nearest_neighbor_with_point_its_own_neighbor(self): - """ - Test if finding nearest neighbor works as expected when the - interpolation point is part of the coordinate list. - """ - neighbors = [[0, 0, 0], [0, 3, 0], [0, 0, 4], [-5, 0, 0]] - inter_coord = [0, 0, 0] - expected_nearest_neighbor_index = [1, 2] - k = 2 + expected_nearest_neighbor_index = [4, 0, 1] + k = 3 interpolation = Interpolation(MagicMock()) nearest_neighbor_index = interpolation.get_nearest_neighbor_indices_local( neighbors, inter_coord, k diff --git a/tests/unit/test_micro_simulation_crash_handling.py b/tests/unit/test_micro_simulation_crash_handling.py index 3e4b33ca..1f7de99f 100644 --- a/tests/unit/test_micro_simulation_crash_handling.py +++ b/tests/unit/test_micro_simulation_crash_handling.py @@ -39,7 +39,7 @@ def test_crash_handling(self): expected_crash_scalar_data = 55 / 49 manager = micro_manager.MicroManager("micro-manager-config_crash.json") - + manager._number_of_nearest_neighbors = 3 # reduce number of neighbors to 3 manager._local_number_of_sims = 4 manager._has_sim_crashed = [False] * 4 manager._mesh_vertex_coords = np.array( @@ -86,6 +86,7 @@ def test_crash_handling_with_adaptivity(self): manager = micro_manager.MicroManager("micro-manager-config_crash.json") + manager._number_of_nearest_neighbors = 3 # reduce number of neighbors to 3 manager._local_number_of_sims = 5 manager._micro_sims_active_steps = np.zeros(5, dtype=np.int32) manager._has_sim_crashed = [False] * 5 From 597590ea464ea1718f332ed21ced6b80c9eeb6ae Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Mon, 29 Apr 2024 10:47:00 +0200 Subject: [PATCH 19/23] Format simulation crash handling --- micro_manager/interpolation.py | 4 +-- micro_manager/micro_manager.py | 27 ++++++++++--------- tests/unit/test_interpolation.py | 18 ++++++++----- .../test_micro_simulation_crash_handling.py | 8 ++---- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/micro_manager/interpolation.py b/micro_manager/interpolation.py index 13b6b55a..f60a3824 100644 --- a/micro_manager/interpolation.py +++ b/micro_manager/interpolation.py @@ -46,9 +46,7 @@ def get_nearest_neighbor_indices_local( return neighbor_indices - def inv_dist_weighted_interp( - self, neighbors: np.ndarray, point: np.ndarray, values - ): + def interpolate(self, neighbors: np.ndarray, point: np.ndarray, values): """ Interpolate a value at a point using inverse distance weighting. .. math:: diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index dc38a3a3..cf711703 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -90,6 +90,7 @@ def __init__(self, config_file: str) -> None: self._is_micro_solve_time_required = self._config.write_micro_solve_time() + # Parameter for interpolation in case of simulation crash self._crash_threshold = 0.2 self._number_of_nearest_neighbors = 4 @@ -414,9 +415,8 @@ def _initialize(self) -> None: self._micro_sims = [None] * self._local_number_of_sims # DECLARATION - # setup for simulation crashes + # Setup for simulation crashes self._has_sim_crashed = [False] * self._local_number_of_sims - self._neighbor_list = [None] * self._local_number_of_sims # DECLARATION self._interpolation = Interpolation(self._logger) micro_problem = getattr( @@ -771,7 +771,7 @@ def _interpolate_output_for_crashed_sim( List of dicts in which keys are names of data and the values are the data which are required inputs to solve a micro simulation. micro_sims_output : list - Output of local micro simulations. + List dicts containing output of local micro simulations. unset_sim : int Index of the crashed simulation in the list of all local simulations currently interpolating. active_sim_ids : numpy.ndarray, optional @@ -783,32 +783,32 @@ def _interpolate_output_for_crashed_sim( Result of the interpolation in which keys are names of data and the values are the data. """ # Find neighbors of the crashed simulation in active and non-crashed simulations - # set iteration length to only iterate over active simulations + # Set iteration length to only iterate over active simulations if self._is_adaptivity_on: iter_length = active_sim_ids else: iter_length = range(len(micro_sims_input)) micro_sims_active_input_lists = [] # DECLARATION micro_sims_active_values = [] # DECLARATION - # Turn crashed sim macro parameters into list to use as coordinate for interpolation + # Turn crashed simulation macro parameters into list to use as coordinate for interpolation crashed_position = [] # DECLARATION for value in micro_sims_input[unset_sim].values(): if isinstance(value, np.ndarray) or isinstance(value, list): crashed_position.extend(value) else: crashed_position.append(value) - # Turn active sim macro parameters into list to use as coordinates for interpolation based on parameters + # Turn active simulation macro parameters into lists to use as coordinates for interpolation based on parameters for i in iter_length: if not self._has_sim_crashed[i]: - # collect macro data at one vertex - intermediate = [] # DECLARATION + # Collect macro data at one macro vertex + intermediate_list = [] # DECLARATION for value in micro_sims_input[i].values(): if isinstance(value, np.ndarray) or isinstance(value, list): - intermediate.extend(value) + intermediate_list.extend(value) else: - intermediate.append(value) + intermediate_list.append(value) # Create lists of macro data for interpolation - micro_sims_active_input_lists.append(intermediate) + micro_sims_active_input_lists.append(intermediate_list) micro_sims_active_values.append(micro_sims_output[i].copy()) # Find nearest neighbors nearest_neighbors = self._interpolation.get_nearest_neighbor_indices_local( @@ -817,7 +817,6 @@ def _interpolate_output_for_crashed_sim( self._number_of_nearest_neighbors, ) # Interpolate - output_interpol = dict() interpol_space = [] # DECLARATION interpol_values = [] # DECLARATION # Collect neighbor vertices for interpolation @@ -833,6 +832,8 @@ def _interpolate_output_for_crashed_sim( interpol_space.append(micro_sims_active_input_lists[neighbor].copy()) interpol_values.append(micro_sims_active_values[neighbor].copy()) interpol_values[-1].pop("micro_sim_time", None) + + output_interpol = dict() if len(interpol_space) == 0: self._logger.error( "No neighbors available for interpolation at macro vertex {}.".format( @@ -847,7 +848,7 @@ def _interpolate_output_for_crashed_sim( # Collect values of current parameter from neighboring simulations for elems in range(len(interpol_values)): key_values.append(interpol_values[elems][key]) - output_interpol[key] = self._interpolation.inv_dist_weighted_interp( + output_interpol[key] = self._interpolation.interpolate( interpol_space, crashed_position, key_values ) # Reintroduce removed information diff --git a/tests/unit/test_interpolation.py b/tests/unit/test_interpolation.py index fd21ac60..3e786d66 100644 --- a/tests/unit/test_interpolation.py +++ b/tests/unit/test_interpolation.py @@ -12,18 +12,23 @@ def test_local_interpolation(self): coords = [[-2, 0, 0], [-1, 0, 0], [2, 0, 0]] inter_point = [1, 0, 0] vector_data = [[-2, -2, -2], [-1, -1, -1], [2, 2, 2]] - expected_vector_data = [55 / 49, 55 / 49, 55 / 49] scalar_data = [[-2], [-1], [2]] - expected_scalar_data = 55 / 49 + expected_vector_interpolation_output = [55 / 49, 55 / 49, 55 / 49] + expected_scalar_interpolation_output = 55 / 49 + interpolation = Interpolation(MagicMock()) - interpolated_vector_data = interpolation.inv_dist_weighted_interp( + interpolated_vector_data = interpolation.interpolate( coords, inter_point, vector_data ) - interpolated_scalar_data = interpolation.inv_dist_weighted_interp( + interpolated_scalar_data = interpolation.interpolate( coords, inter_point, scalar_data ) - self.assertTrue(np.allclose(interpolated_vector_data, expected_vector_data)) - self.assertAlmostEqual(interpolated_scalar_data, expected_scalar_data) + self.assertTrue( + np.allclose(interpolated_vector_data, expected_vector_interpolation_output) + ) + self.assertAlmostEqual( + interpolated_scalar_data, expected_scalar_interpolation_output + ) def test_nearest_neighbor(self): """ @@ -34,6 +39,7 @@ def test_nearest_neighbor(self): inter_coord = [0, 0, 0] expected_nearest_neighbor_index = [4, 0, 1] k = 3 + interpolation = Interpolation(MagicMock()) nearest_neighbor_index = interpolation.get_nearest_neighbor_indices_local( neighbors, inter_coord, k diff --git a/tests/unit/test_micro_simulation_crash_handling.py b/tests/unit/test_micro_simulation_crash_handling.py index 1f7de99f..073950d1 100644 --- a/tests/unit/test_micro_simulation_crash_handling.py +++ b/tests/unit/test_micro_simulation_crash_handling.py @@ -45,7 +45,6 @@ def test_crash_handling(self): manager._mesh_vertex_coords = np.array( [[-2, 0, 0], [-1, 0, 0], [1, 0, 0], [2, 0, 0]] ) - manager._neighbor_list = np.array([[1, 2, 3], [0, 2, 3], [0, 1, 3], [0, 1, 2]]) manager._is_adaptivity_on = ( False # make sure adaptivity is off overriding config ) @@ -53,7 +52,7 @@ def test_crash_handling(self): micro_sims_output = manager._solve_micro_simulations(macro_data) - # Crashed data has interpolated value + # Crashed simulation has interpolated value data_crashed = micro_sims_output[2] self.assertEqual(data_crashed["micro-scalar-data"], expected_crash_scalar_data) self.assertListEqual( @@ -93,9 +92,6 @@ def test_crash_handling_with_adaptivity(self): manager._mesh_vertex_coords = np.array( [[-2, 0, 0], [-1, 0, 0], [1, 0, 0], [2, 0, 0], [1, 1, 0]] ) - manager._neighbor_list = np.array( - [[1, 2, 3, 4], [0, 2, 3, 4], [0, 1, 3, 4], [0, 1, 2, 4], [0, 1, 2, 3]] - ) manager._micro_sims = [MicroSimulation(i) for i in range(5)] is_sim_active = np.array([True, True, True, True, False]) @@ -104,7 +100,7 @@ def test_crash_handling_with_adaptivity(self): macro_data, is_sim_active, sim_is_associated_to ) - # Crashed data has interpolated value + # Crashed simulation has interpolated value data_crashed = micro_sims_output[2] self.assertEqual(data_crashed["micro-scalar-data"], expected_crash_scalar_data) self.assertListEqual( From bd48ce46790b149a543d61464a3444cb23c0c034 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Mon, 29 Apr 2024 11:14:55 +0200 Subject: [PATCH 20/23] Adapt error for no available neighbor --- micro_manager/interpolation.py | 2 +- micro_manager/micro_manager.py | 58 ++++++++++++++++------------------ 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/micro_manager/interpolation.py b/micro_manager/interpolation.py index f60a3824..707175f9 100644 --- a/micro_manager/interpolation.py +++ b/micro_manager/interpolation.py @@ -33,7 +33,7 @@ def get_nearest_neighbor_indices_local( assert len(all_local_coords) > 0, "No local coordinates provided." if len(all_local_coords) < k: self._logger.info( - "Number of neighbors {} is larger than the number of neighbors {}. Setting k to {}.".format( + "Number of desired neighbors k = {} is larger than the number of available neighbors {}. Setting k = {}.".format( k, len(all_local_coords), len(all_local_coords) ) ) diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index cf711703..90dd4c3e 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -811,11 +811,19 @@ def _interpolate_output_for_crashed_sim( micro_sims_active_input_lists.append(intermediate_list) micro_sims_active_values.append(micro_sims_output[i].copy()) # Find nearest neighbors - nearest_neighbors = self._interpolation.get_nearest_neighbor_indices_local( - micro_sims_active_input_lists, - crashed_position, - self._number_of_nearest_neighbors, - ) + if len(micro_sims_active_input_lists) == 0: + self._logger.error( + "No active neighbors available for interpolation at macro vertex {}. Value cannot be interpolated".format( + self._mesh_vertex_coords[unset_sim] + ) + ) + return None + else: + nearest_neighbors = self._interpolation.get_nearest_neighbor_indices_local( + micro_sims_active_input_lists, + crashed_position, + self._number_of_nearest_neighbors, + ) # Interpolate interpol_space = [] # DECLARATION interpol_values = [] # DECLARATION @@ -833,33 +841,23 @@ def _interpolate_output_for_crashed_sim( interpol_values.append(micro_sims_active_values[neighbor].copy()) interpol_values[-1].pop("micro_sim_time", None) + # Interpolate for each parameter output_interpol = dict() - if len(interpol_space) == 0: - self._logger.error( - "No neighbors available for interpolation at macro vertex {}.".format( - self._mesh_vertex_coords[unset_sim] - ) + for key in interpol_values[0].keys(): + key_values = [] # DECLARATION + # Collect values of current parameter from neighboring simulations + for elems in range(len(interpol_values)): + key_values.append(interpol_values[elems][key]) + output_interpol[key] = self._interpolation.interpolate( + interpol_space, crashed_position, key_values ) - return None - else: - # Interpolate for each parameter - for key in interpol_values[0].keys(): - key_values = [] # DECLARATION - # Collect values of current parameter from neighboring simulations - for elems in range(len(interpol_values)): - key_values.append(interpol_values[elems][key]) - output_interpol[key] = self._interpolation.interpolate( - interpol_space, crashed_position, key_values - ) - # Reintroduce removed information - if self._is_micro_solve_time_required: - output_interpol["micro_sim_time"] = 0 - if self._is_adaptivity_on: - output_interpol["active_state"] = 1 - output_interpol["active_steps"] = self._micro_sims_active_steps[ - unset_sim - ] - return output_interpol + # Reintroduce removed information + if self._is_micro_solve_time_required: + output_interpol["micro_sim_time"] = 0 + if self._is_adaptivity_on: + output_interpol["active_state"] = 1 + output_interpol["active_steps"] = self._micro_sims_active_steps[unset_sim] + return output_interpol def main(): From 57ec072de233fcc59f7094c263c564aacfdd13c6 Mon Sep 17 00:00:00 2001 From: Torben Schiz <49746900+tjwsch@users.noreply.github.com> Date: Fri, 3 May 2024 08:57:03 +0200 Subject: [PATCH 21/23] Apply text improvement suggestions from code review Co-authored-by: Ishaan Desai --- micro_manager/interpolation.py | 8 ++++---- micro_manager/micro_manager.py | 4 ++-- tests/unit/test_micro_simulation_crash_handling.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/micro_manager/interpolation.py b/micro_manager/interpolation.py index 707175f9..aa19cd17 100644 --- a/micro_manager/interpolation.py +++ b/micro_manager/interpolation.py @@ -14,7 +14,7 @@ def get_nearest_neighbor_indices_local( k: int, ) -> np.ndarray: """ - Get the indices of the k nearest neighbors of a point in a list of coordinates. + Get local indices of the k nearest neighbors of a point. Parameters ---------- @@ -28,12 +28,12 @@ def get_nearest_neighbor_indices_local( Returns ------ neighbor_indices : np.ndarray - Indices of the k nearest neighbors in all local points. + Local indices of the k nearest neighbors in all local points. """ assert len(all_local_coords) > 0, "No local coordinates provided." if len(all_local_coords) < k: self._logger.info( - "Number of desired neighbors k = {} is larger than the number of available neighbors {}. Setting k = {}.".format( + "Number of desired neighbors k = {} is larger than the number of available neighbors {}. Resetting k = {}.".format( k, len(all_local_coords), len(all_local_coords) ) ) @@ -48,7 +48,7 @@ def get_nearest_neighbor_indices_local( def interpolate(self, neighbors: np.ndarray, point: np.ndarray, values): """ - Interpolate a value at a point using inverse distance weighting. + Interpolate a value at a point using inverse distance weighting. (https://en.wikipedia.org/wiki/Inverse_distance_weighting) .. math:: f(x) = (\sum_{i=1}^{n} \frac{f_i}{\Vert x_i - x \Vert^2}) / (\sum_{j=1}^{n} \frac{1}{\Vert x_j - x \Vert^2}) diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 90dd4c3e..694ec8d8 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -788,8 +788,8 @@ def _interpolate_output_for_crashed_sim( iter_length = active_sim_ids else: iter_length = range(len(micro_sims_input)) - micro_sims_active_input_lists = [] # DECLARATION - micro_sims_active_values = [] # DECLARATION + micro_sims_active_input_lists = [] + micro_sims_active_values = [] # Turn crashed simulation macro parameters into list to use as coordinate for interpolation crashed_position = [] # DECLARATION for value in micro_sims_input[unset_sim].values(): diff --git a/tests/unit/test_micro_simulation_crash_handling.py b/tests/unit/test_micro_simulation_crash_handling.py index 073950d1..a52ad290 100644 --- a/tests/unit/test_micro_simulation_crash_handling.py +++ b/tests/unit/test_micro_simulation_crash_handling.py @@ -26,7 +26,7 @@ def solve(self, macro_data, dt): class TestSimulationCrashHandling(TestCase): def test_crash_handling(self): """ - Test if the micro manager catches a simulation crash and handles it adequately. + Test if the Micro Manager catches a simulation crash and handles it adequately. A crash if caught by interpolation within _solve_micro_simulations. """ From 832fb3e5e8b353160a9a0af59e1bf81572b023af Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Fri, 3 May 2024 10:18:48 +0200 Subject: [PATCH 22/23] Apply suggestions from crash handling review --- micro_manager/interpolation.py | 23 +++++------ micro_manager/micro_manager.py | 73 ++++++++++++++++------------------ 2 files changed, 46 insertions(+), 50 deletions(-) diff --git a/micro_manager/interpolation.py b/micro_manager/interpolation.py index aa19cd17..9fa08d62 100644 --- a/micro_manager/interpolation.py +++ b/micro_manager/interpolation.py @@ -7,9 +7,9 @@ def __init__(self, logger): self._logger = logger - def get_nearest_neighbor_indices_local( + def get_nearest_neighbor_indices( self, - all_local_coords: np.ndarray, + coords: np.ndarray, inter_point: np.ndarray, k: int, ) -> np.ndarray: @@ -18,7 +18,7 @@ def get_nearest_neighbor_indices_local( Parameters ---------- - all_local_coords : list + coords : list List of coordinates of all points. inter_point : list | np.ndarray Coordinates of the point for which the neighbors are to be found. @@ -30,15 +30,14 @@ def get_nearest_neighbor_indices_local( neighbor_indices : np.ndarray Local indices of the k nearest neighbors in all local points. """ - assert len(all_local_coords) > 0, "No local coordinates provided." - if len(all_local_coords) < k: + if len(coords) < k: self._logger.info( "Number of desired neighbors k = {} is larger than the number of available neighbors {}. Resetting k = {}.".format( - k, len(all_local_coords), len(all_local_coords) + k, len(coords), len(coords) ) ) - k = len(all_local_coords) - neighbors = NearestNeighbors(n_neighbors=k).fit(all_local_coords) + k = len(coords) + neighbors = NearestNeighbors(n_neighbors=k).fit(coords) neighbor_indices = neighbors.kneighbors( [inter_point], return_distance=False @@ -68,17 +67,17 @@ def interpolate(self, neighbors: np.ndarray, point: np.ndarray, values): """ interpol_val = 0 summed_weights = 0 - # iterate over all neighbors + # Iterate over all neighbors for inx in range(len(neighbors)): - # compute the squared norm of the difference between interpolation point and neighbor + # Compute the squared norm of the difference between interpolation point and neighbor norm = np.linalg.norm(np.array(neighbors[inx]) - np.array(point)) ** 2 # If interpolation point is already part of the data it is returned as the interpolation result # This avoids division by zero if norm < 1e-16: return values[inx] - # update interpolation value + # Update interpolation value interpol_val += values[inx] / norm - # extend normalization factor + # Extend normalization factor summed_weights += 1 / norm return interpol_val / summed_weights diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 694ec8d8..2d5dbc39 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -20,7 +20,6 @@ from copy import deepcopy from typing import Dict from warnings import warn -import itertools import numpy as np import precice @@ -90,7 +89,7 @@ def __init__(self, config_file: str) -> None: self._is_micro_solve_time_required = self._config.write_micro_solve_time() - # Parameter for interpolation in case of simulation crash + # Parameter for interpolation in case of a simulation crash self._crash_threshold = 0.2 self._number_of_nearest_neighbors = 4 @@ -417,7 +416,7 @@ def _initialize(self) -> None: # Setup for simulation crashes self._has_sim_crashed = [False] * self._local_number_of_sims - self._interpolation = Interpolation(self._logger) + self._interpolant = Interpolation(self._logger) micro_problem = getattr( importlib.import_module( @@ -607,21 +606,20 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: self._has_sim_crashed[count] = True # Interpolate result for crashed simulation - if self._has_sim_crashed.count(True) > 0: - unset_sims = [ - count for count, value in enumerate(micro_sims_output) if value is None - ] - - # Iterate over all crashed simulations to interpolate output - for unset_sim in unset_sims: - self._logger.info( - "Interpolating output for crashed simulation at macro vertex {}.".format( - self._mesh_vertex_coords[unset_sim] - ) - ) - micro_sims_output[unset_sim] = self._interpolate_output_for_crashed_sim( - micro_sims_input, micro_sims_output, unset_sim + unset_sims = [ + count for count, value in enumerate(micro_sims_output) if value is None + ] + + # Iterate over all crashed simulations to interpolate output + for unset_sim in unset_sims: + self._logger.info( + "Interpolating output for crashed simulation at macro vertex {}.".format( + self._mesh_vertex_coords[unset_sim] ) + ) + micro_sims_output[unset_sim] = self._interpolate_output_for_crashed_sim( + micro_sims_input, micro_sims_output, unset_sim + ) return micro_sims_output @@ -709,23 +707,22 @@ def _solve_micro_simulations_with_adaptivity( self._has_sim_crashed[active_id] = True # Interpolate result for crashed simulation - if self._has_sim_crashed.count(True) > 0: - unset_sims = [] - for active_id in active_sim_ids: - if micro_sims_output[active_id] is None: - unset_sims.append(active_id) - - # Iterate over all crashed simulations to interpolate output - for unset_sim in unset_sims: - self._logger.info( - "Interpolating output for crashed simulation at macro vertex {}.".format( - self._mesh_vertex_coords[unset_sim] - ) - ) + unset_sims = [] + for active_id in active_sim_ids: + if micro_sims_output[active_id] is None: + unset_sims.append(active_id) - micro_sims_output[unset_sim] = self._interpolate_output_for_crashed_sim( - micro_sims_input, micro_sims_output, unset_sim, active_sim_ids + # Iterate over all crashed simulations to interpolate output + for unset_sim in unset_sims: + self._logger.info( + "Interpolating output for crashed simulation at macro vertex {}.".format( + self._mesh_vertex_coords[unset_sim] ) + ) + + micro_sims_output[unset_sim] = self._interpolate_output_for_crashed_sim( + micro_sims_input, micro_sims_output, unset_sim, active_sim_ids + ) # For each inactive simulation, copy data from most similar active simulation if self._adaptivity_type == "global": @@ -791,7 +788,7 @@ def _interpolate_output_for_crashed_sim( micro_sims_active_input_lists = [] micro_sims_active_values = [] # Turn crashed simulation macro parameters into list to use as coordinate for interpolation - crashed_position = [] # DECLARATION + crashed_position = [] for value in micro_sims_input[unset_sim].values(): if isinstance(value, np.ndarray) or isinstance(value, list): crashed_position.extend(value) @@ -801,7 +798,7 @@ def _interpolate_output_for_crashed_sim( for i in iter_length: if not self._has_sim_crashed[i]: # Collect macro data at one macro vertex - intermediate_list = [] # DECLARATION + intermediate_list = [] for value in micro_sims_input[i].values(): if isinstance(value, np.ndarray) or isinstance(value, list): intermediate_list.extend(value) @@ -819,14 +816,14 @@ def _interpolate_output_for_crashed_sim( ) return None else: - nearest_neighbors = self._interpolation.get_nearest_neighbor_indices_local( + nearest_neighbors = self._interpolant.get_nearest_neighbor_indices( micro_sims_active_input_lists, crashed_position, self._number_of_nearest_neighbors, ) # Interpolate - interpol_space = [] # DECLARATION - interpol_values = [] # DECLARATION + interpol_space = [] + interpol_values = [] # Collect neighbor vertices for interpolation for neighbor in nearest_neighbors: # Remove data not required for interpolation from values @@ -848,7 +845,7 @@ def _interpolate_output_for_crashed_sim( # Collect values of current parameter from neighboring simulations for elems in range(len(interpol_values)): key_values.append(interpol_values[elems][key]) - output_interpol[key] = self._interpolation.interpolate( + output_interpol[key] = self._interpolant.interpolate( interpol_space, crashed_position, key_values ) # Reintroduce removed information From c58011771b04284cf05679fa68460e900231d9d0 Mon Sep 17 00:00:00 2001 From: Torben Schiz Date: Fri, 3 May 2024 10:22:58 +0200 Subject: [PATCH 23/23] Adapt interpolation test --- tests/unit/test_interpolation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_interpolation.py b/tests/unit/test_interpolation.py index 3e786d66..d1296411 100644 --- a/tests/unit/test_interpolation.py +++ b/tests/unit/test_interpolation.py @@ -41,7 +41,7 @@ def test_nearest_neighbor(self): k = 3 interpolation = Interpolation(MagicMock()) - nearest_neighbor_index = interpolation.get_nearest_neighbor_indices_local( + nearest_neighbor_index = interpolation.get_nearest_neighbor_indices( neighbors, inter_coord, k ) self.assertListEqual(