Cleaner report on completed stages

docs/release_notes.md

@@ -8,6 +8,31 @@ We are tracking two types of major changes:
 
 If you are creating the release notes for a new version, please see the [template](#template-version-majorminorpatch). Release notes should capture all of the significant changes since the last numbered package release.
 
+# Version 1.1.0
+
+This version focuses on improving the clarity of the telemetry reported.
+
+## Users
+
+### User Improvements
+
+- Report splits `stages_completed` into stage status and duration.
+
+## User Breaking Changes
+
+None.
+
+## Developers
+
+### Developer Improvements
+
+None
+
+### Developer Breaking Changes
+
+- `build.Status.COMPLETED_BUILD` is now called `build.Status.COMPLETED_BUILD`
+- `COMPLETED_BUILD_STAGES` column in the report was removed.
+
 # Version 1.0.0
 
 This version focuses on cleaning up technical debts and most of the changes are not visible to users. It removes cumbersome requirements for developers, removes unused features to streamline the codebase, and also clarifying some API naming schemes.

src/turnkeyml/analyze/script.py

@@ -272,7 +272,7 @@ def explore_invocation(
             )
 
             stats.save_model_eval_stat(
-                fs.Keys.BUILD_STATUS, fs.FunctionStatus.SUCCESSFUL
+                fs.Keys.BUILD_STATUS, fs.FunctionStatus.COMPLETED
             )
 
             model_to_benchmark = build_state.results[0]
@@ -317,7 +317,7 @@ def explore_invocation(
                 )
 
             stats.save_model_eval_stat(
-                fs.Keys.BENCHMARK_STATUS, fs.FunctionStatus.SUCCESSFUL
+                fs.Keys.BENCHMARK_STATUS, fs.FunctionStatus.COMPLETED
             )
 
             invocation_info.status_message = "Model successfully benchmarked!"

src/turnkeyml/build/ignition.py

@@ -324,7 +324,7 @@ def load_or_make_state(
 
             if (
                 model_type == build.ModelType.UNKNOWN
-                and state.build_status == build.Status.SUCCESSFUL_BUILD
+                and state.build_status == build.Status.COMPLETED_BUILD
             ):
                 msg = (
                     "Model caching is disabled for successful builds against custom Sequences. "

src/turnkeyml/build/stage.py

@@ -3,6 +3,7 @@
 import time
 import os
 import copy
+import enum
 from typing import List, Tuple
 from multiprocessing import Process
 import psutil
@@ -50,6 +51,11 @@ def _name_is_file_safe(name: str):
 
 
 class Stage(abc.ABC):
+    class Status(enum.Enum):
+        NOT_STARTED = "not_started"
+        COMPLETED = "completed"
+        INCOMPLETE = "incomplete"
+
     def status_line(self, successful, verbosity):
         """
         Print a line of status information for this Stage into the monitor.
@@ -83,6 +89,8 @@ def __init__(
         _name_is_file_safe(unique_name)
 
         self.unique_name = unique_name
+        self.status_key = f"stage_status:{unique_name}"
+        self.duration_key = f"stage_duration:{unique_name}"
         self.monitor_message = monitor_message
         self.progress = None
         self.logfile_path = None
@@ -137,12 +145,12 @@ def fire_helper(self, state: build.State) -> Tuple[build.State, int]:
         else:
             self.status_line(successful=True, verbosity=state.monitor)
 
-            # Stages should not set build.Status.SUCCESSFUL_BUILD, as that is
+            # Stages should not set build.Status.COMPLETED_BUILD, as that is
             # reserved for Sequence.launch()
-            if state.build_status == build.Status.SUCCESSFUL_BUILD:
+            if state.build_status == build.Status.COMPLETED_BUILD:
                 raise exp.StageError(
                     "TurnkeyML Stages are not allowed to set "
-                    "`state.build_status == build.Status.SUCCESSFUL_BUILD`, "
+                    "`state.build_status == build.Status.COMPLETED_BUILD`, "
                     "however that has happened. If you are a plugin developer, "
                     "do not do this. If you are a user, please file an issue at "
                     "https://github.com/onnx/turnkeyml/issues."
@@ -268,7 +276,7 @@ def launch(self, state: build.State) -> build.State:
 
         if state.build_status == build.Status.NOT_STARTED:
             state.build_status = build.Status.PARTIAL_BUILD
-        elif state.build_status == build.Status.SUCCESSFUL_BUILD:
+        elif state.build_status == build.Status.COMPLETED_BUILD:
             msg = """
             build_model() is running a build on a model that already built successfully, which
             should not happen because the build should have loaded from cache or rebuilt from scratch.
@@ -280,13 +288,23 @@ def launch(self, state: build.State) -> build.State:
         # Collect telemetry for the build
         stats = fs.Stats(state.cache_dir, state.config.build_name, state.evaluation_id)
         stats.save_model_eval_stat(
-            fs.Keys.ALL_BUILD_STAGES,
+            fs.Keys.SELECTED_SEQUENCE_OF_STAGES,
             self.get_names(),
         )
 
+        # At the beginning of a sequence no stage has started
+        for stage in self.stages:
+            stats.save_model_eval_stat(stage.status_key, Stage.Status.NOT_STARTED.value)
+            stats.save_model_eval_stat(stage.duration_key, "-")
+
         # Run the build
         try:
             for stage in self.stages:
+                # Set status as incomplete, since stage just started
+                stats.save_model_eval_stat(
+                    stage.status_key, Stage.Status.INCOMPLETE.value
+                )
+
                 # Collect telemetry about the stage
                 state.current_build_stage = stage.unique_name
                 start_time = time.time()
@@ -297,17 +315,17 @@ def launch(self, state: build.State) -> build.State:
                 # Collect telemetry about the stage
                 execution_time = time.time() - start_time
 
-                stats.save_model_eval_sub_stat(
-                    parent_key=fs.Keys.COMPLETED_BUILD_STAGES,
-                    key=stage.unique_name,
-                    value=execution_time,
+                # Set status as completed
+                stats.save_model_eval_stat(
+                    stage.status_key, Stage.Status.COMPLETED.value
                 )
+                stats.save_model_eval_stat(stage.duration_key, execution_time)
 
         except exp.StageError as e:
             # Advance the cursor below the monitor so
             # we can print an error message
             stage_depth_in_sequence = self.get_depth() - self.get_names().index(
-                stage.unique_name
+                stage.unique_name  # pylint: disable=undefined-loop-variable
             )
             stdout_lines_to_advance = stage_depth_in_sequence - 2
             cursor_down = "\n" * stdout_lines_to_advance
@@ -319,7 +337,7 @@ def launch(self, state: build.State) -> build.State:
 
         else:
             state.current_build_stage = None
-            state.build_status = build.Status.SUCCESSFUL_BUILD
+            state.build_status = build.Status.COMPLETED_BUILD
 
             # We use a deepcopy here because the Stage framework supports
             # intermediate_results of any type, including model objects in memory.

src/turnkeyml/build_api.py

@@ -105,7 +105,7 @@ def build_model(
 
     # Return a cached build if possible, otherwise prepare the model State for
     # a build
-    if state.build_status == build.Status.SUCCESSFUL_BUILD:
+    if state.build_status == build.Status.COMPLETED_BUILD:
         # Successful builds can be loaded from cache and returned with
         # no additional steps
         additional_msg = " (build_name auto-selected)" if config.auto_name else ""

src/turnkeyml/cli/report.py

@@ -78,12 +78,6 @@ def summary_spreadsheets(args) -> None:
 
                         # Copy the build-specific stats
                         for key, value in build.items():
-                            # Break each value in "completed build stages" into its own column
-                            # to make analysis easier
-                            if key == fs.Keys.COMPLETED_BUILD_STAGES:
-                                for subkey, subvalue in value.items():
-                                    evaluation_stats[subkey] = subvalue
-
                             # If a build or benchmark is still marked as "running" at
                             # reporting time, it
                             # must have been killed by a time out, out-of-memory (OOM), or some
@@ -93,7 +87,8 @@ def summary_spreadsheets(args) -> None:
                             ) and value == fs.FunctionStatus.RUNNING:
                                 value = fs.FunctionStatus.KILLED
 
-                            evaluation_stats[key] = value
+                            # Add stats ensuring that those are all in lower case
+                            evaluation_stats[key.lower()] = value
 
                         all_evaluation_stats.append(evaluation_stats)
                 except yaml.scanner.ScannerError:
@@ -108,6 +103,9 @@ def summary_spreadsheets(args) -> None:
                 if header not in column_headers:
                     column_headers.append(header)
 
+        # Sort all columns alphabetically
+        column_headers = sorted(column_headers)
+
         # Add each build to the report
         for evaluation_stats in all_evaluation_stats:
             # Start with a dictionary where all of the values are "-". If a build

src/turnkeyml/common/build.py

@@ -141,7 +141,7 @@ class Status(enum.Enum):
     NOT_STARTED = "not_started"
     PARTIAL_BUILD = "partial_build"
     BUILD_RUNNING = "build_running"
-    SUCCESSFUL_BUILD = "successful_build"
+    COMPLETED_BUILD = "completed_build"
     FAILED_BUILD = "failed_build"
 
 

src/turnkeyml/common/filesystem.py

@@ -322,11 +322,7 @@ class Keys:
     # ONNX model input tensor dimensions
     ONNX_INPUT_DIMENSIONS = "onnx_input_dimensions"
     # List of all build stages in the Sequence
-    ALL_BUILD_STAGES = "all_build_stages"
-    # Map of build stages that completed successfully to the
-    # execution time for that stage. We can figure out if any build
-    # stages failed if all_build_stages != completed_build_stages.keys().
-    COMPLETED_BUILD_STAGES = "completed_build_stages"
+    SELECTED_SEQUENCE_OF_STAGES = "selected_sequence_of_stages"
     # Location of the most up-to-date ONNX file for this build. If the
     # build completed successfully, this is the final ONNX file.
     ONNX_FILE = "onnx_file"
@@ -362,7 +358,7 @@ class Keys:
 
 class FunctionStatus:
     RUNNING = "running"
-    SUCCESSFUL = "successful"
+    COMPLETED = "completed"
     FAILED = "failed"
     KILLED = "killed"
 

src/turnkeyml/version.py

@@ -1 +1 @@
-__version__ = "1.0.0"
+__version__ = "1.1.0"

test/build_model.py

@@ -105,7 +105,7 @@ def full_compilation_pytorch_model():
         monitor=False,
         cache_dir=cache_location,
     )
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 def full_compilation_keras_subclass_model():
@@ -118,7 +118,7 @@ def full_compilation_keras_subclass_model():
         monitor=False,
         cache_dir=cache_location,
     )
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 def full_compilation_keras_sequential_model():
@@ -131,7 +131,7 @@ def full_compilation_keras_sequential_model():
         monitor=False,
         cache_dir=cache_location,
     )
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 def full_compilation_onnx_model():
@@ -152,7 +152,7 @@ def full_compilation_onnx_model():
         monitor=False,
         cache_dir=cache_location,
     )
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 def full_compilation_hummingbird_rf():
@@ -167,7 +167,7 @@ def full_compilation_hummingbird_rf():
         monitor=False,
         cache_dir=cache_location,
     )
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 def full_compilation_hummingbird_xgb():
@@ -182,7 +182,7 @@ def full_compilation_hummingbird_xgb():
         monitor=False,
         cache_dir=cache_location,
     )
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 def full_compilation_hummingbird_lgbm():
@@ -197,7 +197,7 @@ def full_compilation_hummingbird_lgbm():
         monitor=False,
         cache_dir=cache_location,
     )
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 def full_compilation_hummingbird_kn():
@@ -212,7 +212,7 @@ def full_compilation_hummingbird_kn():
         monitor=False,
         cache_dir=cache_location,
     )
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 def scriptmodule_functional_check():
@@ -229,7 +229,7 @@ def scriptmodule_functional_check():
         monitor=False,
         cache_dir=cache_location,
     )
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 def custom_stage():
@@ -280,7 +280,7 @@ def fire(self, state):
         cache_dir=cache_location,
     )
 
-    return state.build_status == build.Status.SUCCESSFUL_BUILD
+    return state.build_status == build.Status.COMPLETED_BUILD
 
 
 class FullyCustomStage(stage.Stage):
@@ -580,7 +580,7 @@ def test_013_set_onnx_opset(self):
             sequence=sequences.optimize_fp16,
         )
 
-        assert state.build_status == build.Status.SUCCESSFUL_BUILD
+        assert state.build_status == build.Status.COMPLETED_BUILD
 
         onnx_model = onnx.load(state.results[0])
         model_opset = getattr(onnx_model.opset_import[0], "version", None)
@@ -599,7 +599,7 @@ def test_014_export_only(self):
             sequence=sequences.onnx_fp32,
         )
 
-        assert state.build_status == build.Status.SUCCESSFUL_BUILD
+        assert state.build_status == build.Status.COMPLETED_BUILD
         assert os.path.exists(export.base_onnx_file(state))
         assert not os.path.exists(export.opt_onnx_file(state))
 
@@ -635,7 +635,7 @@ def test_015_receive_onnx(self):
         )
 
         # Make sure the build was successful
-        assert state.build_status == build.Status.SUCCESSFUL_BUILD
+        assert state.build_status == build.Status.COMPLETED_BUILD
 
         # Get ONNX file's opset
         onnx_model = onnx.load(onnx_file)