Cleaner report on completed stages

docs/release_notes.md

@@ -8,6 +8,30 @@ We are tracking two types of major changes:
 
 If you are creating the release notes for a new version, please see the [template](#template-version-majorminorpatch). Release notes should capture all of the significant changes since the last numbered package release.
 
+# Version 1.1.0
+
+This version focuses on improving the clarity of the telemetry reported.
+
+## Users
+
+### User Improvements
+
+- Report splits `stages_completed` into stage status and duration.
+
+## User Breaking Changes
+
+None.
+
+## Developers
+
+### Developer Improvements
+
+None
+
+### Developer Breaking Changes
+
+None
+
 # Version 1.0.0
 
 This version focuses on cleaning up technical debts and most of the changes are not visible to users. It removes cumbersome requirements for developers, removes unused features to streamline the codebase, and also clarifying some API naming schemes.

src/turnkeyml/build/stage.py

@@ -280,7 +280,7 @@ def launch(self, state: build.State) -> build.State:
         # Collect telemetry for the build
         stats = fs.Stats(state.cache_dir, state.config.build_name, state.evaluation_id)
         stats.save_model_eval_stat(
-            fs.Keys.ALL_BUILD_STAGES,
+            fs.Keys.SELECTED_SEQUENCE_OF_STAGES,
             self.get_names(),
         )
 

src/turnkeyml/cli/report.py

@@ -78,12 +78,41 @@ def summary_spreadsheets(args) -> None:
 
                         # Copy the build-specific stats
                         for key, value in build.items():
-                            # Break each value in "completed build stages" into its own column
-                            # to make analysis easier
+                            # Break each value in "completed build stages" into status and duration
+                            # to make the analysis of this data easier
                             if key == fs.Keys.COMPLETED_BUILD_STAGES:
+<<<<<<< HEAD
+                                previous_state_incomplete = False
+                                for stage in build[fs.Keys.SELECTED_SEQUENCE_OF_STAGES]:
+                                    duration_column_name = f"stage_duration: {stage}"
+                                    state_column_name = f"stage_status: {stage}"
+                                    if stage in build[fs.Keys.COMPLETED_BUILD_STAGES]:
+                                        evaluation_stats[
+                                            state_column_name
+                                        ] = "COMPLETED"
+                                        evaluation_stats[duration_column_name] = build[
+                                            fs.Keys.COMPLETED_BUILD_STAGES
+                                        ][stage]
+                                    elif not previous_state_incomplete:
+                                        previous_state_incomplete = True
+                                        evaluation_stats[
+                                            state_column_name
+                                        ] = "INCOMPLETE"
+                                        evaluation_stats[duration_column_name] = "-"
+                                    else:
+                                        evaluation_stats[
+                                            state_column_name
+                                        ] = "NOT STARTED"
+                                        evaluation_stats[duration_column_name] = "-"
+
+                                # Do not add the raw version of COMPLETED_BUILD_STAGES to the report
+                                continue
+
+=======
                                 for subkey, subvalue in value.items():
                                     evaluation_stats[subkey] = subvalue
 
+>>>>>>> main
                             # If a build or benchmark is still marked as "running" at
                             # reporting time, it
                             # must have been killed by a time out, out-of-memory (OOM), or some
@@ -93,7 +122,12 @@ def summary_spreadsheets(args) -> None:
                             ) and value == fs.FunctionStatus.RUNNING:
                                 value = fs.FunctionStatus.KILLED
 
+<<<<<<< HEAD
+                            # Add stats ensuring that those are all in lower case
+                            evaluation_stats[key.lower()] = value
+=======
                             evaluation_stats[key] = value
+>>>>>>> main
 
                         all_evaluation_stats.append(evaluation_stats)
                 except yaml.scanner.ScannerError:
@@ -108,6 +142,9 @@ def summary_spreadsheets(args) -> None:
                 if header not in column_headers:
                     column_headers.append(header)
 
+        # Sort all columns alphabetically
+        column_headers = sorted(column_headers)
+
         # Add each build to the report
         for evaluation_stats in all_evaluation_stats:
             # Start with a dictionary where all of the values are "-". If a build

src/turnkeyml/common/filesystem.py

@@ -322,10 +322,10 @@ class Keys:
     # ONNX model input tensor dimensions
     ONNX_INPUT_DIMENSIONS = "onnx_input_dimensions"
     # List of all build stages in the Sequence
-    ALL_BUILD_STAGES = "all_build_stages"
+    SELECTED_SEQUENCE_OF_STAGES = "selected_sequence_of_stages"
     # Map of build stages that completed successfully to the
     # execution time for that stage. We can figure out if any build
-    # stages failed if all_build_stages != completed_build_stages.keys().
+    # stages failed if selected_sequence_of_stages != completed_build_stages.keys().
     COMPLETED_BUILD_STAGES = "completed_build_stages"
     # Location of the most up-to-date ONNX file for this build. If the
     # build completed successfully, this is the final ONNX file.

src/turnkeyml/version.py

@@ -1 +1 @@
-__version__ = "1.0.0"
+__version__ = "1.1.0"

test/cli.py

@@ -957,7 +957,7 @@ def test_026_cli_report(self):
             "device",
             "mean_latency",
             "throughput",
-            "all_build_stages",
+            "selected_sequence_of_stages",
             "completed_build_stages",
         ]
         linear_summary = summary[1]
@@ -995,19 +995,18 @@ def test_026_cli_report(self):
 
         # Make sure the report.get_dict() API works
         result_dict = report.get_dict(
-            summary_csv_path, ["all_build_stages", "completed_build_stages"]
+            summary_csv_path,
+            [
+                "selected_sequence_of_stages",
+                "stage_duration: export_pytorch",
+                "stage_status: export_pytorch",
+            ],
         )
         for result in result_dict.values():
-            # All of the models should have exported to ONNX, so the "onnx_exported" value
-            # should be True for all of them
-            assert "export_pytorch" in yaml.safe_load(result["all_build_stages"])
-            assert (
-                "export_pytorch"
-                in yaml.safe_load(result["completed_build_stages"]).keys()
-            )
-            assert (
-                yaml.safe_load(result["completed_build_stages"])["export_pytorch"] > 0
-            )
+            # All of the models should have exported to ONNX
+            assert "export_pytorch" in result["selected_sequence_of_stages"]
+            assert result["stage_status: export_pytorch"] == "COMPLETED"
+            assert result["stage_duration: export_pytorch"] > 0
 
 
 if __name__ == "__main__":