fix: a few fixes to website loader

dyana/loaders/base/dyana.py

@@ -33,6 +33,7 @@ def __init__(self, gpu: bool = False):
         self._network: dict[str, dict[str, dict[str, int]]] = {"start": get_network_stats()}
         self._imports_at_start = get_current_imports()
         self._additionals: dict[str, t.Any] = {}
+        self._extra: dict[str, t.Any] = {}
 
     def track_memory(self, event: str) -> None:
         self._ram[event] = get_peak_rss()
@@ -51,6 +52,9 @@ def track_error(self, event: str, error: str) -> None:
     def track(self, key: str, value: t.Any) -> None:
         self._additionals[key] = value
 
+    def track_extra(self, key: str, value: t.Any) -> None:
+        self._extra[key] = value
+
     def as_dict(self) -> dict[str, t.Any]:
         imports_at_end = get_current_imports()
         imported = {k: imports_at_end[k] for k in imports_at_end if k not in self._imports_at_start}
@@ -63,7 +67,7 @@ def as_dict(self) -> dict[str, t.Any]:
             "disk": self._disk,
             "network": self._network,
             "errors": self._errors,
-            "extra": {"imports": imported},
+            "extra": {"imports": imported, **self._extra},
         } | self._additionals
 
         if self._gpu:

dyana/loaders/website/main.py

@@ -144,7 +144,7 @@ def analyze_page_content(driver: webdriver.Chrome) -> dict[str, Any]:
             network_logs = driver.get_log("performance")
 
             # Add all metrics to profiler
-            profiler.extra = {
+            for key, value in {
                 "load_time": load_time,
                 "performance_metrics": metrics,
                 "content_analysis": content_analysis,
@@ -153,14 +153,15 @@ def analyze_page_content(driver: webdriver.Chrome) -> dict[str, Any]:
                 "title": driver.title,
                 "url": driver.current_url,
                 "status_code": driver.execute_script("return window.performance.getEntries()[0].responseStatus"),
-            }
+            }.items():
+                profiler.track_extra(key, value)
         except Exception as e:
             profiler.track_error("metrics", str(e))
 
         # Take screenshot
         try:
             screenshot = driver.get_screenshot_as_base64()
-            profiler.extra["screenshot"] = screenshot
+            profiler.track_extra("screenshot", screenshot)
         except Exception as e:
             profiler.track_error("screenshot", str(e))
 

dyana/loaders/website/settings.yml

@@ -11,5 +11,3 @@ args:
   - name: wait-for-timeout
     description: Timeout in seconds for page load and element wait.
     required: false
-    type: int
-    default: "60"

dyana/view.py

@@ -394,5 +394,5 @@ def view_extra(run: dict[str, t.Any]) -> None:
             fn_name = f"view_extra_{k}"
             if fn_name in globals():
                 globals()[fn_name](k, v)
-            else:
-                view_extra_unknown(k, v)
+            # else:
+            #    view_extra_unknown(k, v)