HIP 103 the second try (#745)

* HIP 103 the second try

* Use more of the terminology from the HIP

* Adjust assignments and urbanization code to be more testable

* Get everything working with tests

* Hopefully improve speed of assigning

* Fix for timeout

* Address comments

* Add test for oracle boosting

* Fmt

* Best I could come up with

* Fix

* Update mobile_verifier/tests/boosting_oracles.rs

Co-authored-by: andymck <[email protected]>

* PR comments

* Switch to master for helium-proto

---------

Co-authored-by: andymck <[email protected]>

Cargo.toml

@@ -63,7 +63,7 @@ sqlx = {version = "0", features = [
 helium-anchor-gen = {git = "https://github.com/helium/helium-anchor-gen.git"}
 helium-crypto = {version = "0.8.1", features=["sqlx-postgres", "multisig"]}
 helium-proto = {git = "https://github.com/helium/proto", branch = "master", features = ["services"]}
-hextree = "*"
+hextree = {git = "https://github.com/jaykickliter/HexTree", branch = "main", features = ["disktree"]}
 solana-client = "1.16"
 solana-sdk = "1.16"
 solana-program = "1.16"

file_store/src/file_info.rs

@@ -144,8 +144,8 @@ pub const MAPPER_MSG: &str = "mapper_msg";
 pub const COVERAGE_OBJECT: &str = "coverage_object";
 pub const COVERAGE_OBJECT_INGEST_REPORT: &str = "coverage_object_ingest_report";
 pub const SENIORITY_UPDATE: &str = "seniority_update";
-
 pub const BOOSTED_HEX_UPDATE: &str = "boosted_hex_update";
+pub const ORACLE_BOOSTING_REPORT: &str = "oracle_boosting_report";
 
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)]
 #[serde(rename_all = "snake_case")]
@@ -188,6 +188,7 @@ pub enum FileType {
     WifiHeartbeat,
     WifiHeartbeatIngestReport,
     BoostedHexUpdate,
+    OracleBoostingReport,
 }
 
 impl fmt::Display for FileType {
@@ -235,6 +236,7 @@ impl fmt::Display for FileType {
             Self::CoverageObjectIngestReport => COVERAGE_OBJECT_INGEST_REPORT,
             Self::SeniorityUpdate => SENIORITY_UPDATE,
             Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE,
+            Self::OracleBoostingReport => ORACLE_BOOSTING_REPORT,
         };
         f.write_str(s)
     }
@@ -285,6 +287,7 @@ impl FileType {
             Self::CoverageObjectIngestReport => COVERAGE_OBJECT_INGEST_REPORT,
             Self::SeniorityUpdate => SENIORITY_UPDATE,
             Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE,
+            Self::OracleBoostingReport => ORACLE_BOOSTING_REPORT,
         }
     }
 }
@@ -335,6 +338,7 @@ impl FromStr for FileType {
             COVERAGE_OBJECT_INGEST_REPORT => Self::CoverageObjectIngestReport,
             SENIORITY_UPDATE => Self::SeniorityUpdate,
             BOOSTED_HEX_UPDATE => Self::BoostedHexUpdate,
+            ORACLE_BOOSTING_REPORT => Self::OracleBoostingReport,
             _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))),
         };
         Ok(result)

file_store/src/file_sink.rs

@@ -216,6 +216,18 @@ impl FileSinkClient {
         }
     }
 
+    /// Writes all messages to the file sink, return the last oneshot
+    pub async fn write_all(
+        &self,
+        items: impl IntoIterator<Item = impl prost::Message>,
+    ) -> Result<Option<oneshot::Receiver<Result>>> {
+        let mut last_oneshot = None;
+        for item in items {
+            last_oneshot = Some(self.write(item, &[]).await?);
+        }
+        Ok(last_oneshot)
+    }
+
     pub async fn commit(&self) -> Result<oneshot::Receiver<Result<FileManifest>>> {
         let (on_commit_tx, on_commit_rx) = oneshot::channel();
         self.sender

mobile_config/src/boosted_hex_info.rs

@@ -102,12 +102,12 @@ pub struct BoostedHex {
 }
 
 impl BoostedHexes {
-    pub async fn new(hexes: Vec<BoostedHexInfo>) -> anyhow::Result<Self> {
-        let mut map = HashMap::new();
-        for info in hexes {
-            map.insert(info.location, info);
-        }
-        Ok(Self { hexes: map })
+    pub fn new(hexes: Vec<BoostedHexInfo>) -> Self {
+        let hexes = hexes
+            .into_iter()
+            .map(|info| (info.location, info))
+            .collect();
+        Self { hexes }
     }
 
     pub async fn get_all(

mobile_verifier/Cargo.toml

@@ -13,7 +13,7 @@ thiserror = {workspace = true}
 serde =  {workspace = true}
 serde_json = {workspace = true}
 h3o = {workspace = true, features = ["geo"]}
-hextree = "0"
+hextree = {workspace = true}
 http-serde = {workspace = true}
 clap = {workspace = true}
 sqlx = {workspace = true}

mobile_verifier/migrations/26_urbanized.sql

@@ -0,0 +1,3 @@
+CREATE TYPE oracle_assignment AS ENUM ('a', 'b', 'c');
+
+ALTER TABLE hexes ADD COLUMN urbanized oracle_assignment;

mobile_verifier/src/boosting_oracles/assignment.rs

@@ -0,0 +1,109 @@
+use helium_proto::services::poc_mobile::oracle_boosting_hex_assignment;
+use rust_decimal::Decimal;
+use rust_decimal_macros::dec;
+use std::fmt;
+
+#[derive(Copy, Clone, PartialEq, Eq, Debug, sqlx::Type)]
+#[sqlx(type_name = "oracle_assignment")]
+#[sqlx(rename_all = "lowercase")]
+pub enum Assignment {
+    A,
+    B,
+    C,
+}
+
+impl From<Assignment> for oracle_boosting_hex_assignment::Assignment {
+    fn from(assignment: Assignment) -> Self {
+        match assignment {
+            Assignment::A => Self::A,
+            Assignment::B => Self::B,
+            Assignment::C => Self::C,
+        }
+    }
+}
+
+impl From<Assignment> for i32 {
+    fn from(assignment: Assignment) -> i32 {
+        oracle_boosting_hex_assignment::Assignment::from(assignment) as i32
+    }
+}
+
+impl fmt::Display for Assignment {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use Assignment::*;
+
+        match self {
+            A => write!(f, "a"),
+            B => write!(f, "b"),
+            C => write!(f, "c"),
+        }
+    }
+}
+
+pub fn urbanization_multiplier(urbanization: Assignment) -> Decimal {
+    use Assignment::*;
+
+    match urbanization {
+        A => dec!(1.0),
+        B => dec!(0.25),
+        C => dec!(0.0),
+    }
+}
+
+#[allow(dead_code)]
+pub fn urbanization_and_footfall_multiplier(
+    footfall: Assignment,
+    urbanization: Assignment,
+) -> Decimal {
+    use Assignment::*;
+
+    match (footfall, urbanization) {
+        (A, A) => dec!(1.0),
+        (A, B) => dec!(1.0),
+        (B, A) => dec!(0.75),
+        (B, B) => dec!(0.50),
+        (C, A) => dec!(0.40),
+        (C, B) => dec!(0.10),
+        (A, C) => dec!(0.00),
+        (B, C) => dec!(0.00),
+        (C, C) => dec!(0.00),
+    }
+}
+
+#[allow(dead_code)]
+pub fn boosting_oracles_multiplier(
+    footfall: Assignment,
+    landtype: Assignment,
+    urbanization: Assignment,
+) -> Decimal {
+    use Assignment::*;
+
+    match (footfall, landtype, urbanization) {
+        // POI ≥ 1 Urbanized
+        (A, A, A) => dec!(1.00),
+        (A, B, A) => dec!(1.00),
+        (A, C, A) => dec!(1.00),
+        // POI ≥ 1 Not Urbanized
+        (A, A, B) => dec!(1.00),
+        (A, B, B) => dec!(1.00),
+        (A, C, B) => dec!(1.00),
+        // Point of Interest Urbanized
+        (B, A, A) => dec!(0.70),
+        (B, B, A) => dec!(0.70),
+        (B, C, A) => dec!(0.70),
+        // Point of Interest Not Urbanized
+        (B, A, B) => dec!(0.50),
+        (B, B, B) => dec!(0.50),
+        (B, C, B) => dec!(0.50),
+        // No POI Urbanized
+        (C, A, A) => dec!(0.40),
+        (C, B, A) => dec!(0.30),
+        (C, C, A) => dec!(0.05),
+        // No POI Not Urbanized
+        (C, A, B) => dec!(0.20),
+        (C, B, B) => dec!(0.15),
+        (C, C, B) => dec!(0.03),
+        // Outside of USA
+        (_, _, C) => dec!(0.00),
+    }
+}

mobile_verifier/src/boosting_oracles/mod.rs

@@ -0,0 +1,70 @@
+pub mod assignment;
+
+use std::collections::HashMap;
+
+use crate::geofence::GeofenceValidator;
+pub use assignment::Assignment;
+use hextree::disktree::DiskTreeMap;
+
+pub trait DiskTreeLike: Send + Sync + 'static {
+    fn get(&self, cell: hextree::Cell) -> hextree::Result<Option<(hextree::Cell, &[u8])>>;
+}
+
+impl DiskTreeLike for DiskTreeMap {
+    fn get(&self, cell: hextree::Cell) -> hextree::Result<Option<(hextree::Cell, &[u8])>> {
+        self.get(cell)
+    }
+}
+
+impl DiskTreeLike for HashMap<hextree::Cell, Vec<u8>> {
+    fn get(&self, cell: hextree::Cell) -> hextree::Result<Option<(hextree::Cell, &[u8])>> {
+        Ok(self.get(&cell).map(|x| (cell, x.as_slice())))
+    }
+}
+
+pub struct MockDiskTree;
+
+impl DiskTreeLike for MockDiskTree {
+    fn get(&self, cell: hextree::Cell) -> hextree::Result<Option<(hextree::Cell, &[u8])>> {
+        Ok(Some((cell, &[])))
+    }
+}
+
+pub struct Urbanization<DT, GF> {
+    urbanized: DT,
+    usa_geofence: GF,
+}
+
+impl<DT, GF> Urbanization<DT, GF> {
+    pub fn new(urbanized: DT, usa_geofence: GF) -> Self {
+        Self {
+            urbanized,
+            usa_geofence,
+        }
+    }
+}
+
+impl<DT, GF> Urbanization<DT, GF>
+where
+    DT: DiskTreeLike,
+    GF: GeofenceValidator<u64>,
+{
+    fn is_urbanized(&self, location: u64) -> anyhow::Result<bool> {
+        let cell = hextree::Cell::from_raw(location)?;
+        let result = self.urbanized.get(cell)?;
+        Ok(result.is_some())
+    }
+
+    pub fn hex_assignment(&self, hex: u64) -> anyhow::Result<Assignment> {
+        let assignment = if self.usa_geofence.in_valid_region(&hex) {
+            if self.is_urbanized(hex)? {
+                Assignment::A
+            } else {
+                Assignment::B
+            }
+        } else {
+            Assignment::C
+        };
+        Ok(assignment)
+    }
+}