Skip to content

Commit

Permalink
Reset baseline when starting benchmarking
Browse files Browse the repository at this point in the history
  • Loading branch information
MsRandom committed Nov 16, 2024

Verified

This commit was signed with the committer’s verified signature.
gastaldi George Gastaldi
1 parent f4fa4ea commit 71f4587
Showing 5 changed files with 9 additions and 4 deletions.
2 changes: 1 addition & 1 deletion neuron/neuron/checkpoint.py
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@
Uid: TypeAlias = int
Key: TypeAlias = str

BENCHMARKS_VERSION = 12
BENCHMARKS_VERSION = 13
SPEC_VERSION = 7
REVISION_LENGTH = 7

2 changes: 1 addition & 1 deletion validator/base_validator/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from .metrics import *
from .auto_updater import *

API_VERSION = "5.2.4"
API_VERSION = "5.2.5"
2 changes: 1 addition & 1 deletion validator/submission_tester/api.py
Original file line number Diff line number Diff line change
@@ -111,7 +111,7 @@ def start_benchmarking(
with benchmarker.lock:
timestamp = time.time_ns()

if timestamp - benchmarker.start_timestamp < 600_000_000_000:
if timestamp - benchmarker.start_timestamp < 120_000_000_000:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail="Started recently",
5 changes: 5 additions & 0 deletions validator/submission_tester/benchmarker.py
Original file line number Diff line number Diff line change
@@ -85,6 +85,7 @@ def _start_benchmarking(self, submissions: dict[Key, ModelRepositoryInfo]):
self.submission_times.clear()
self.inputs = random_inputs()
self.done = False
self.baseline = None

while not self.baseline and not self.cancelled_event.is_set():
try:
@@ -117,6 +118,10 @@ def _start_benchmarking(self, submissions: dict[Key, ModelRepositoryInfo]):
self.done = True

def start_benchmarking(self, submissions: dict[Key, ModelRepositoryInfo]):
if not submissions:
logger.warning("No submissions to benchmark")
return

logger.info(f"Starting benchmarking for {len(submissions)} submissions")

if self.thread and self.thread.is_alive():
2 changes: 1 addition & 1 deletion validator/weight_setting/validator.py
Original file line number Diff line number Diff line change
@@ -53,7 +53,7 @@
from .wandb_args import add_wandb_args
from .winner_selection import get_scores, get_contestant_scores, get_tiers, get_contestant_tier

VALIDATOR_VERSION: tuple[int, int, int] = (5, 2, 6)
VALIDATOR_VERSION: tuple[int, int, int] = (5, 2, 7)
VALIDATOR_VERSION_STRING = ".".join(map(str, VALIDATOR_VERSION))

WEIGHTS_VERSION = (

0 comments on commit 71f4587

Please sign in to comment.