Set min similarity to 0.7, fix validator merge

womboai · Nov 25, 2024 · a94bf71 · a94bf71
1 parent 3c4dd4c
commit a94bf71
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 7 deletions.
diff --git a/neuron/neuron/contest.py b/neuron/neuron/contest.py
@@ -9,7 +9,7 @@
 from .device import Device, CudaDevice, Gpu
 from .output_comparator import OutputComparator, ImageOutputComparator
 
-SIMILARITY_SCORE_THRESHOLD = 0.8
+SIMILARITY_SCORE_THRESHOLD = 0.7
 
 
 class MetricType(IntEnum):

diff --git a/neuron/neuron/submission_tester/clone.sh b/neuron/neuron/submission_tester/clone.sh
@@ -5,6 +5,6 @@ set -e
 REPOSITORY_URL=$1
 REVISION=$2
 
-find . -mindepth 1 -delete
+find . -mindepth 1 -delete || true
 git clone "$REPOSITORY_URL" .
 git checkout "$REVISION"
diff --git a/validator/weight_setting/validator.py b/validator/weight_setting/validator.py
@@ -39,7 +39,7 @@
     get_submissions,
     BENCHMARKS_VERSION,
     CheckpointBenchmark,
-    MetricData, ModelRepositoryInfo,
+    MetricData,
 )
 from neuron.device import ContestDeviceValidationError
 from .benchmarking_api import BenchmarkingApi, send_submissions_to_api
@@ -522,7 +522,7 @@ def set_weights(self):
                     logger.warning(f"Not setting weights for hotkey {hotkey} as their submission was not found")
                     self.reset_miner(uid)
 
-        contestants = get_contestant_scores(benchmarks, self.baseline_metrics)
+        contestants = get_contestant_scores(CURRENT_CONTEST, benchmarks, self.baseline_metrics)
         tiers = get_tiers(contestants)
         blocks = [info.block if info else None for info in self.contest_state.miner_info]
         weights = get_scores(tiers, blocks, len(self.metagraph.nodes))
@@ -572,7 +572,7 @@ def get_miner_submissions(self) -> list[MinerModelInfo | None]:
             block=self.block,
         )
 
-    def start_benchmarking(self, submissions: dict[Key, ModelRepositoryInfo]):
+    def start_benchmarking(self, submissions: dict[Key, MinerModelInfo]):
         return send_submissions_to_api(self.benchmarking_apis, submissions)
 
     @staticmethod
@@ -655,7 +655,7 @@ def do_step(self, block: int):
                     nodes = self.metagraph_nodes()
 
                     submissions = {
-                        nodes[uid].hotkey: self.contest_state.miner_info[uid].repository
+                        nodes[uid].hotkey: self.contest_state.miner_info[uid]
                         for uid in remaining
                     }
 
@@ -728,7 +728,7 @@ def do_step(self, block: int):
             nodes = self.metagraph_nodes()
 
             submissions = {
-                nodes[uid].hotkey: self.contest_state.miner_info[uid].repository
+                nodes[uid].hotkey: self.contest_state.miner_info[uid]
                 for uid in self.non_tested_miners()
             }