a shot at more graceful timeout recovery

intra2net · Jan 31, 2025 · bec980d · bec980d
1 parent 50c9621
commit bec980d
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 6 deletions.
diff --git a/avocado_i2n/cartgraph/graph.py b/avocado_i2n/cartgraph/graph.py
@@ -2123,10 +2123,9 @@ async def traverse_object_trees(
                 # despite ergodicity we ended at the same node (no other work)
                 if next in occupied_at:
                     if occupied_wait > test_duration:
-                        raise RuntimeError(
-                            f"Worker {worker.id} spent {occupied_wait:.2f} seconds waiting for "
-                            f"occupied nodes of maximum test duration {test_duration:.2f}: "
-                            + ", ".join(n.id for n in occupied_at)
+                        # allow reentrancy as best shot at recovering from an otherwise fatal error
+                        next.params["max_concurrent_tries"] = (
+                            self.params.get_numeric("max_concurrent_tries", 0) + 1
                         )
                     occupied_wait += occupied_timeout
                 else:

diff --git a/selftests/isolation/test_cartesian_graph.py b/selftests/isolation/test_cartesian_graph.py
@@ -2865,8 +2865,7 @@ def test_traverse_one_leaf_with_occupation_timeout(self):
             {"shortname": "^internal.automated.on_customize.vm1", "vms": "^vm1$", "nets": "^net1$"},
         ]
 
-        with self.assertRaisesRegex(RuntimeError, r"^Worker .+ spent [\d\.]+ seconds waiting for occupied nodes"):
-            self._run_traversal(graph, {"test_timeout": "1"})
+        self._run_traversal(graph, {"test_timeout": "1"})
 
     def test_traverse_two_objects_without_setup(self):
         """Test a two-object test traversal without a reusable setup."""