Merge pull request #176 from specklesystems/2.17

Ensure stable report generation; update debug instructions
specklesystems · Nov 6, 2023 · da8097e · da8097e
2 parents 682a481 + 4b7483f
commit da8097e
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 39 deletions.
diff --git a/.gitignore b/.gitignore
@@ -124,3 +124,4 @@ zip-build*
 *.json 
 */PyQt-UI*
 specklepy_qt_ui*
+*requirements.txt
diff --git a/README.md b/README.md
@@ -157,14 +157,14 @@ Though it is not required, we recommend installing these plugins from the QGIS P
 
 #### Visual Studio Code
 
-First, you'll need to uncomment these 2 lines in the `__init__.py` file:
+First, you'll need to change the _debug value to True in `plugin_utils/installer.py` file and verify the VS Code Python extension path:
 
 ```python
-    # from speckle.utils import enable_remote_debugging
-    # enable_remote_debugging()
+    _debug = True
+    _vs_code_directory = os.path.expanduser("~\.vscode\extensions\ms-python.python-2023.20.0\pythonFiles\lib\python")
 ```
 
-This will automatically setup `ptvsd` if it's not already installed, and start listening to port `5678`.
+This will automatically setup `debugpy` if it's not already installed, and start listening to port `5678`.
 
 In VS Code, you can use the built in python debugger. You'll need to create a debug configuration by creating a `launch.json` file.
 
@@ -199,10 +199,5 @@ That's all there is to it! Now any breakpoints you create should be hit.
 
 ![successful debugging in vs code](https://user-images.githubusercontent.com/7717434/129324011-42ebd156-ba6b-4eca-8b67-22300eb462fc.png)
 
-> If you want to have the debugger wait for you to connect using VSCode, you can uncomment this line in the `speckle/utils.py` file:
->
-> ```python
->   #ptvsd.wait_for_attach()
-> ```
 
 Enjoy!
diff --git a/plugin_utils/installer.py b/plugin_utils/installer.py
@@ -12,6 +12,9 @@
 
 _user_data_env_var = "SPECKLE_USERDATA_PATH"
 _debug = False
+_vs_code_directory = os.path.expanduser(
+    "~\.vscode\extensions\ms-python.python-2023.20.0\pythonFiles\lib\python"
+)
 
 
 def _path() -> Optional[Path]:
@@ -147,7 +150,7 @@ def _dependencies_installed(requirements: str, path: str) -> bool:
         entry = f"{d.key}=={d.version}"
         if entry in requirements:
             requirements = requirements.replace(entry, "")
-
+    requirements = requirements.replace(" ", "").replace(";", "").replace(",", "")
     if len(requirements) > 0:
         return False
     print("Dependencies already installed")
@@ -160,6 +163,31 @@ def install_requirements(host_application: str) -> None:
     # dependencies
     requirements = get_requirements_path().read_text().replace("\n", "")
     path = str(connector_installation_path(host_application))
+
+    print(f"Installing debugpy to {path}")
+    from subprocess import run
+
+    if _debug is True:
+        try:
+            import debugpy
+        except:
+            completed_process = run(
+                [
+                    PYTHON_PATH,
+                    "-m",
+                    "pip",
+                    "install",
+                    "-t",
+                    str(path),
+                    "debugpy==1.8.0",
+                ],
+                capture_output=True,
+                text=True,
+            )
+            if completed_process.returncode != 0:
+                m = f"Failed to install debugpy through pip. Disable debug mode or install debugpy manually. Full log: {completed_process}"
+                raise Exception(completed_process)
+
     if _dependencies_installed(requirements, path):
         return
 
@@ -240,10 +268,7 @@ def startDegugger() -> None:
             import debugpy
             import shutil
 
-            directory = os.path.expanduser(
-                "~\.vscode\extensions\ms-python.python-2023.14.0\pythonFiles\lib\python"
-            )
-            sys.path.append(directory)
+            sys.path.append(_vs_code_directory)
             debugpy.configure(python=shutil.which("python"))
 
             try:

diff --git a/speckle/converter/layers/__init__.py b/speckle/converter/layers/__init__.py
@@ -632,12 +632,14 @@ def addExcelMainThread(obj: Tuple):
         #print("12")
         for item in report_features:
             dataStorage.latestActionReport.append(item)
+        dataStorage.latestConversionTime = datetime.now()
 
     except Exception as e:
         logToUser(e, level = 2, func = inspect.stack()[0][3], plugin = plugin.dockwidget)
         # report
         obj_type = "Vector Layer"
         dataStorage.latestActionReport.append({"speckle_id": f"{val_id} {finalName}", "obj_type": obj_type, "errors": f"{e}"})
+        dataStorage.latestConversionTime = datetime.now()
 
 
 
@@ -749,12 +751,14 @@ def addNonGeometryMainThread(obj: Tuple):
             dataStorage.latestActionReport.append({"speckle_id": f"{layer_id} {finalName}", "obj_type": obj_type, "errors": f"{all_feature_errors_count} features failed"})
         for item in report_features:
             dataStorage.latestActionReport.append(item)
+        dataStorage.latestConversionTime = datetime.now()
 
     except Exception as e:
         logToUser(e, level = 2, func = inspect.stack()[0][3], plugin = plugin.dockwidget)
         # report
         obj_type = geom_print + "Vector Layer"
         dataStorage.latestActionReport.append({"speckle_id": f"{layer_id} {finalName}", "obj_type": obj_type, "errors": f"{e}"})
+        dataStorage.latestConversionTime = datetime.now()
 
 def geometryLayerToNative(layerContentList: List[Base], layerName: str, val_id: str, streamBranch: str, plugin, matrix = None):
     #print("01_____GEOMETRY layer to native")
@@ -1026,12 +1030,14 @@ def addBimMainThread(obj: Tuple):
             dataStorage.latestActionReport.append({"speckle_id": f"{layer_id} {finalName}", "obj_type": obj_type, "errors": f"{all_feature_errors_count} features failed"})
         for item in report_features:
             dataStorage.latestActionReport.append(item)
+        dataStorage.latestConversionTime = datetime.now()
 
     except Exception as e:
         logToUser(e, level = 2, func = inspect.stack()[0][3], plugin = plugin.dockwidget)
         # report 
         obj_type = geom_print + "Vector Layer"
         dataStorage.latestActionReport.append({"speckle_id": f"{layer_id} {finalName}", "obj_type": obj_type, "errors": f"{e}"})
+        dataStorage.latestConversionTime = datetime.now()
 
 
 def cadVectorLayerToNative(geomList: List[Base], layerName: str, val_id: str, geomType: str, streamBranch: str, plugin, matrix = None) -> QgsVectorLayer: 
@@ -1225,12 +1231,14 @@ def addCadMainThread(obj: Tuple):
             dataStorage.latestActionReport.append({"speckle_id": f"{layer_id} {finalName}", "obj_type": obj_type, "errors": f"{all_feature_errors_count} features failed"})
         for item in report_features:
             dataStorage.latestActionReport.append(item)
+        dataStorage.latestConversionTime = datetime.now()
 
     except Exception as e:
         logToUser(e, level = 2, func = inspect.stack()[0][3], plugin = plugin.dockwidget)
         # report
         obj_type = geom_print + "Vector Layer"
         dataStorage.latestActionReport.append({"speckle_id": f"{layer_id} {finalName}", "obj_type": obj_type, "errors": f"{e}"})
+        dataStorage.latestConversionTime = datetime.now()
 
 def vectorLayerToNative(layer: Layer or VectorLayer, streamBranch: str, nameBase: str, plugin):
     try:
@@ -1478,12 +1486,14 @@ def addVectorMainThread(obj: Tuple):
 
         for item in report_features:
             dataStorage.latestActionReport.append(item)
+        dataStorage.latestConversionTime = datetime.now()
 
     except Exception as e:
         logToUser(e, level = 2, func = inspect.stack()[0][3], plugin = plugin.dockwidget)
         # report
         obj_type = "Vector Layer"
         dataStorage.latestActionReport.append({"speckle_id": f"{layer.id} {finalName}", "obj_type": obj_type, "errors": f"{e}"})
+        dataStorage.latestConversionTime = datetime.now()
 
 
 def rasterLayerToNative(layer: RasterLayer, streamBranch: str, nameBase: str, plugin):
@@ -1694,9 +1704,10 @@ def addRasterMainThread(obj: Tuple):
 
         #report on receive:
         dataStorage.latestActionReport.append({"speckle_id": f"{layer.id} {finalName}", "obj_type": "Raster Layer", "errors": ""})
+        dataStorage.latestConversionTime = datetime.now()
 
     except Exception as e:
         logToUser(e, level = 2, func = inspect.stack()[0][3], plugin = plugin.dockwidget)
         #report on receive:
         dataStorage.latestActionReport.append({"speckle_id": f"{layer.id} {finalName}", "obj_type": "Raster Layer", "errors": f"Receiving layer {layer.name} failed"})
-          
+        dataStorage.latestConversionTime = datetime.now()
diff --git a/speckle_qgis.py b/speckle_qgis.py
@@ -447,7 +447,6 @@ def onSend(self, message: str):
         try:
             if not self.dockwidget:
                 return
-            # self.dockwidget.showWait()
 
             projectCRS = self.project.crs()
 
@@ -521,9 +520,14 @@ def onSend(self, message: str):
                 name="QGIS commit",
                 elements=[],
             )
+
+            # conversions
+            time_start_conversion = datetime.now()
             base_obj = convertSelectedLayers(
                 base_obj, layers, tree_structure, projectCRS, self
             )
+            time_end_conversion = datetime.now()
+
             if (
                 base_obj is None
                 or base_obj.elements is None
@@ -562,12 +566,11 @@ def onSend(self, message: str):
             logToUser(e, level=2, func=inspect.stack()[0][3], plugin=self.dockwidget)
             return
 
+        # data transfer
+        time_start_transfer = datetime.now()
         try:
             # this serialises the block and sends it to the transport
             objId = operations.send(base=base_obj, transports=[transport])
-            self.dataStorage.latestActionTime = str(
-                datetime.now().strftime("%d/%m/%Y, %H:%M:%S")
-            )
         except Exception as e:
             logToUser(
                 "Error sending data: " + str(e),
@@ -576,6 +579,7 @@ def onSend(self, message: str):
                 plugin=self.dockwidget,
             )
             return
+        time_end_transfer = datetime.now()
 
         self.dockwidget.signal_cancel_operation.emit("cancel")
 
@@ -589,6 +593,16 @@ def onSend(self, message: str):
                 source_application="QGIS" + self.gis_version.split(".")[0],
             )
 
+            # add time stats to the report
+            self.dataStorage.latestActionTime = str(
+                datetime.now().strftime("%d/%m/%Y, %H:%M:%S")
+            )
+            self.dataStorage.latestTransferTime = str(
+                time_end_transfer - time_start_transfer
+            )
+            self.dataStorage.latestConversionTime = str(
+                time_end_conversion - time_start_conversion
+            )
             try:
                 metr_filter = "Selected" if bySelection is True else "Saved"
                 metr_main = True if branchName == "main" else False
@@ -624,6 +638,12 @@ def onSend(self, message: str):
                         "savedStreams": metr_saved_streams,
                         "projectedCRS": metr_projected,
                         "customCRS": metr_crs,
+                        "time_conversion": (
+                            time_end_conversion - time_start_conversion
+                        ).total_seconds(),
+                        "time_transfer": (
+                            time_end_transfer - time_start_transfer
+                        ).total_seconds(),
                     },
                 )
             except:
@@ -660,7 +680,8 @@ def onSend(self, message: str):
                     plugin=self.dockwidget,
                 )
 
-            # time.sleep(0.3)
+            self.dockwidget.msgLog.dataStorage = self.dataStorage
+
             logToUser(
                 "Data sent to '"
                 + str(streamName)
@@ -760,7 +781,10 @@ def onReceive(self):
             if transport == None:
                 return
 
+            # data transfer
+            time_start_transfer = datetime.now()
             commitObj = operations.receive(objId, transport, None)
+            time_end_transfer = datetime.now()
 
             projectCRS = self.project.crs()
             units = str(QgsUnitTypes.encodeUnit(projectCRS.mapUnits()))
@@ -781,22 +805,6 @@ def onReceive(self):
             metr_projected = True if not projectCRS.isGeographic() else False
             if self.project.crs().isValid() is False:
                 metr_projected = None
-            try:
-                metrics.track(
-                    metrics.RECEIVE,
-                    self.dataStorage.active_account,
-                    {
-                        "hostAppFullVersion": self.gis_version,
-                        "sourceHostAppVersion": app_full,
-                        "sourceHostApp": app,
-                        "isMultiplayer": commit.authorId != client_id,
-                        "connector_version": str(self.version),
-                        "projectedCRS": metr_projected,
-                        "customCRS": metr_crs,
-                    },
-                )
-            except:
-                metrics.track(metrics.RECEIVE, self.dataStorage.active_account)
 
             client.commit.received(
                 streamId,
@@ -816,7 +824,6 @@ def onReceive(self):
                         func=inspect.stack()[0][3],
                         plugin=self.dockwidget,
                     )
-            # logger.log(f"Succesfully received {objId}")
 
         except Exception as e:
             logToUser(
@@ -869,13 +876,21 @@ def onReceive(self):
 
             self.dataStorage.latestActionLayers = []
             self.dataStorage.latestActionReport = []
+
+            # conversions
+            time_start_conversion = self.dataStorage.latestConversionTime = datetime.now()
             traverseObject(self, commitObj, callback, check, str(newGroupName), "")
+            time_end_conversion = self.dataStorage.latestConversionTime
+
+            # add time stats to the report
             self.dataStorage.latestActionTime = str(
                 datetime.now().strftime("%d/%m/%Y, %H:%M:%S")
             )
+            self.dataStorage.latestTransferTime = str(
+                time_end_transfer - time_start_transfer
+            )
 
-            url: str = constructCommitURL(streamWrapper, branch.id, commit.id)
-
+            self.dockwidget.msgLog.dataStorage = self.dataStorage
             # if self.dockwidget.experimental.isChecked(): time.sleep(3)
             logToUser(
                 "Data received",
@@ -885,6 +900,26 @@ def onReceive(self):
                 report=True,
             )
 
+            try:
+                metrics.track(
+                    metrics.RECEIVE,
+                    self.dataStorage.active_account,
+                    {
+                        "hostAppFullVersion": self.gis_version,
+                        "sourceHostAppVersion": app_full,
+                        "sourceHostApp": app,
+                        "isMultiplayer": commit.authorId != client_id,
+                        "connector_version": str(self.version),
+                        "projectedCRS": metr_projected,
+                        "customCRS": metr_crs,
+                        "time_transfer": (
+                            time_end_transfer - time_start_transfer
+                        ).total_seconds(),
+                    },
+                )
+            except:
+                metrics.track(metrics.RECEIVE, self.dataStorage.active_account)
+
         except Exception as e:
             # if self.dockwidget.experimental.isChecked(): time.sleep(1)
             logToUser(