diff --git a/base/testing/benchmarker.py b/base/testing/benchmarker.py index 5922ac3f..cabe7198 100644 --- a/base/testing/benchmarker.py +++ b/base/testing/benchmarker.py @@ -19,7 +19,7 @@ class Benchmarker: - _thread: Thread | None + _thread: Thread | None = None _stop_flag: Event = Event() _sandbox_directory: Path @@ -127,12 +127,11 @@ def start_benchmarking(self, contest: Contest, submissions: dict[Key, Repository logger.info(f"Started benchmarking for {len(submissions)} submissions") - thread = self._thread - if thread and thread.is_alive(): + if self._thread and self._thread.is_alive(): logger.info("Attempting to cancel previous benchmarking") self._stop_flag.set() - thread.join(timeout=60) - if thread.is_alive(): + self._thread.join(timeout=60) + if self._thread.is_alive(): logger.warning("Benchmarking was not stopped gracefully.") else: logger.info("Benchmarking was stopped gracefully.") diff --git a/validator/base_validator/api_data.py b/validator/base_validator/api_data.py index 4b879d75..455c7e2e 100644 --- a/validator/base_validator/api_data.py +++ b/validator/base_validator/api_data.py @@ -1,7 +1,7 @@ from pydantic import BaseModel from base.checkpoint import Key, Uid, Benchmarks -from base.contest import ContestId, RepositoryInfo, BenchmarkState +from base.contest import ContestId, RepositoryInfo, BenchmarkState, Metrics class ApiMetadata(BaseModel): @@ -13,6 +13,7 @@ class BenchmarkingResults(BaseModel): state: BenchmarkState benchmarks: Benchmarks invalid_submissions: set[Key] + baseline: Metrics | None average_benchmark_time: float | None diff --git a/validator/submission_tester/api.py b/validator/submission_tester/api.py index 55daf1b7..989f9362 100644 --- a/validator/submission_tester/api.py +++ b/validator/submission_tester/api.py @@ -119,6 +119,7 @@ def state(request: Request) -> BenchmarkingResults: state=benchmarker.state, benchmarks=benchmarker.benchmarks, invalid_submissions=benchmarker.invalid_submissions, + baseline=benchmarker.baseline.metrics if benchmarker.baseline else None, average_benchmark_time=average_benchmark_time, ) diff --git a/validator/weight_setting/benchmarking_api.py b/validator/weight_setting/benchmarking_api.py index b7d826b6..bae70f53 100644 --- a/validator/weight_setting/benchmarking_api.py +++ b/validator/weight_setting/benchmarking_api.py @@ -78,8 +78,8 @@ def _authentication_headers(keypair: Keypair): def send_submissions_to_api(version: str, all_apis: list[BenchmarkingApi], submissions: Submissions): submissions_by_contest: dict[ContestId, dict[Key, RepositoryInfo]] = defaultdict(lambda: {}) - for key, info in submissions.items(): - submissions_by_contest[info.contest_id][key] = info.repository + for key, submission in submissions.items(): + submissions_by_contest[submission.contest_id][key] = submission.repository_info contest_api_assignment: dict[ContestId, list[BenchmarkingApi]] = defaultdict(lambda: [])