Fix benchmarking issues

base/testing/benchmarker.py

@@ -19,7 +19,7 @@
 
 
 class Benchmarker:
-    _thread: Thread | None
+    _thread: Thread | None = None
     _stop_flag: Event = Event()
 
     _sandbox_directory: Path
@@ -127,12 +127,11 @@ def start_benchmarking(self, contest: Contest, submissions: dict[Key, Repository
 
         logger.info(f"Started benchmarking for {len(submissions)} submissions")
 
-        thread = self._thread
-        if thread and thread.is_alive():
+        if self._thread and self._thread.is_alive():
             logger.info("Attempting to cancel previous benchmarking")
             self._stop_flag.set()
-            thread.join(timeout=60)
-            if thread.is_alive():
+            self._thread.join(timeout=60)
+            if self._thread.is_alive():
                 logger.warning("Benchmarking was not stopped gracefully.")
             else:
                 logger.info("Benchmarking was stopped gracefully.")

validator/base_validator/api_data.py

@@ -1,7 +1,7 @@
 from pydantic import BaseModel
 
 from base.checkpoint import Key, Uid, Benchmarks
-from base.contest import ContestId, RepositoryInfo, BenchmarkState
+from base.contest import ContestId, RepositoryInfo, BenchmarkState, Metrics
 
 
 class ApiMetadata(BaseModel):
@@ -13,6 +13,7 @@ class BenchmarkingResults(BaseModel):
     state: BenchmarkState
     benchmarks: Benchmarks
     invalid_submissions: set[Key]
+    baseline: Metrics | None
     average_benchmark_time: float | None
 
 

validator/submission_tester/api.py

@@ -119,6 +119,7 @@ def state(request: Request) -> BenchmarkingResults:
         state=benchmarker.state,
         benchmarks=benchmarker.benchmarks,
         invalid_submissions=benchmarker.invalid_submissions,
+        baseline=benchmarker.baseline.metrics if benchmarker.baseline else None,
         average_benchmark_time=average_benchmark_time,
     )
 

validator/weight_setting/benchmarking_api.py

@@ -78,8 +78,8 @@ def _authentication_headers(keypair: Keypair):
 def send_submissions_to_api(version: str, all_apis: list[BenchmarkingApi], submissions: Submissions):
     submissions_by_contest: dict[ContestId, dict[Key, RepositoryInfo]] = defaultdict(lambda: {})
 
-    for key, info in submissions.items():
-        submissions_by_contest[info.contest_id][key] = info.repository
+    for key, submission in submissions.items():
+        submissions_by_contest[submission.contest_id][key] = submission.repository_info
 
     contest_api_assignment: dict[ContestId, list[BenchmarkingApi]] = defaultdict(lambda: [])