From 787749a620298f1f8407c5a60e4fa1a99502f6be Mon Sep 17 00:00:00 2001 From: Andrew McKenzie Date: Fri, 5 Jan 2024 15:20:50 +0000 Subject: [PATCH] add witness lag validation --- Cargo.lock | 4 +- iot_verifier/src/poc.rs | 208 ++++++++++++++++++++++++----- iot_verifier/tests/runner_tests.rs | 28 ++++ 3 files changed, 201 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94967cd0f..0e5afaa47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,7 +1197,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#a7c24d010911a9de4ce0c16eaef6b6eb2d306975" +source = "git+https://github.com/helium/proto?branch=master#e3258b53a170315fb321d38dc9f451fd80538de8" dependencies = [ "base64 0.21.0", "byteorder", @@ -3047,7 +3047,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#a7c24d010911a9de4ce0c16eaef6b6eb2d306975" +source = "git+https://github.com/helium/proto?branch=master#e3258b53a170315fb321d38dc9f451fd80538de8" dependencies = [ "bytes", "prost", diff --git a/iot_verifier/src/poc.rs b/iot_verifier/src/poc.rs index 5976268e6..2bee5171f 100644 --- a/iot_verifier/src/poc.rs +++ b/iot_verifier/src/poc.rs @@ -52,6 +52,11 @@ lazy_static! { /// from density scaling calculations and not finding a value on subsequent lookups /// would disqualify the hotspot from validating further beacons static ref DEFAULT_TX_SCALE: Decimal = Decimal::new(2000, 4); + /// max permitted lag between the first witness and all subsequent witnesses + static ref MAX_WITNESS_LAG: Duration = Duration::milliseconds(1500); + /// max permitted lag between the beaconer and a witness + static ref MAX_BEACON_TO_WITNESS_LAG: Duration = Duration::milliseconds(4000); + } #[derive(Debug, PartialEq)] pub struct InvalidResponse { @@ -190,43 +195,47 @@ impl Poc { let mut failed_witnesses: Vec = Vec::new(); let mut existing_gateways: Vec = Vec::new(); let witnesses = self.witness_reports.clone(); - for witness_report in witnesses { - // have we already processed a witness report from this gateway ? - // if not, run verifications - // if so, skip verifications and declare the report a dup - if !existing_gateways.contains(&witness_report.report.pub_key) { - // not a dup, run the verifications - match self - .verify_witness( - deny_list, - &witness_report, - beacon_info, - gateway_cache, - hex_density_map, - ) - .await - { - Ok(verified_witness) => { - // track which gateways we have saw a witness report from - existing_gateways.push(verified_witness.report.pub_key.clone()); - verified_witnesses.push(verified_witness) + if !witnesses.is_empty() { + let witness_earliest_received_ts = witnesses[0].received_timestamp; + for witness_report in witnesses { + // have we already processed a witness report from this gateway ? + // if not, run verifications + // if so, skip verifications and declare the report a dup + if !existing_gateways.contains(&witness_report.report.pub_key) { + // not a dup, run the verifications + match self + .verify_witness( + deny_list, + &witness_report, + beacon_info, + gateway_cache, + hex_density_map, + witness_earliest_received_ts, + ) + .await + { + Ok(verified_witness) => { + // track which gateways we have saw a witness report from + existing_gateways.push(verified_witness.report.pub_key.clone()); + verified_witnesses.push(verified_witness) + } + Err(_) => failed_witnesses.push(witness_report), } - Err(_) => failed_witnesses.push(witness_report), + } else { + // the report is a dup + let dup_witness = IotVerifiedWitnessReport::invalid( + InvalidReason::Duplicate, + None, + &witness_report.report, + witness_report.received_timestamp, + None, + // if location is None, default gain and elevation to zero + 0, + 0, + InvalidParticipantSide::Witness, + ); + verified_witnesses.push(dup_witness) } - } else { - // the report is a dup - let dup_witness = IotVerifiedWitnessReport::invalid( - InvalidReason::Duplicate, - None, - &witness_report.report, - witness_report.received_timestamp, - None, - // if location is None, default gain and elevation to zero - 0, - 0, - InvalidParticipantSide::Witness, - ); - verified_witnesses.push(dup_witness) } } let resp = VerifyWitnessesResult { @@ -243,6 +252,7 @@ impl Poc { beaconer_info: &GatewayInfo, gateway_cache: &GatewayCache, hex_density_map: &HexDensityMap, + witness_first_ts: DateTime, ) -> Result> { let witness = &witness_report.report; let witness_pub_key = witness.pub_key.clone(); @@ -304,6 +314,7 @@ impl Poc { &witness_info, &self.beacon_report, beaconer_metadata, + witness_first_ts, ) { Ok(()) => { let tx_scale = hex_density_map @@ -378,6 +389,7 @@ pub fn do_beacon_verifications( Ok(()) } +#[allow(clippy::too_many_arguments)] pub fn do_witness_verifications( deny_list: &DenyList, entropy_start: DateTime, @@ -386,6 +398,7 @@ pub fn do_witness_verifications( witness_info: &GatewayInfo, beacon_report: &IotBeaconIngestReport, beaconer_metadata: &GatewayMetadata, + witness_first_ts: DateTime, ) -> GenericVerifyResult { tracing::debug!( "verifying witness from gateway: {:?}", @@ -416,6 +429,11 @@ pub fn do_witness_verifications( entropy_end, witness_report.received_timestamp, )?; + verify_witness_lag( + beacon_report.received_timestamp, + witness_first_ts, + witness_report.received_timestamp, + )?; verify_witness_data(&beacon_report.report.data, &witness_report.report.data)?; verify_gw_capability(witness_info.is_full_hotspot)?; verify_witness_freq( @@ -658,6 +676,38 @@ fn verify_gw_capability(is_full_hotspot: bool) -> GenericVerifyResult { Ok(()) } +/// verify witness lag +/// if the first received event is the beacon then, +/// all witnesses must be received within MAX_BEACON_TO_WITNESS_LAG of the beacon +/// if the first received event is a witness then, +/// all subsequent witnesses must be received within MAX_WITNESS_LAG of that first witness +fn verify_witness_lag( + beacon_received_ts: DateTime, + first_witness_ts: DateTime, + received_ts: DateTime, +) -> GenericVerifyResult { + let (first_event_ts, max_permitted_lag) = if beacon_received_ts <= first_witness_ts { + (beacon_received_ts, *MAX_BEACON_TO_WITNESS_LAG) + } else { + (first_witness_ts, *MAX_WITNESS_LAG) + }; + let this_witness_lag = received_ts - first_event_ts; + if this_witness_lag > max_permitted_lag { + tracing::debug!( + reason = ?InvalidReason::TooLate, + %received_ts, + %beacon_received_ts, + %first_witness_ts, + "witness verification failed" + ); + return Err(InvalidResponse { + reason: InvalidReason::TooLate, + details: None, + }); + } + Ok(()) +} + /// verify witness report is not in response to its own beacon fn verify_self_witness( beacon_pub_key: &PublicKeyBinary, @@ -1259,6 +1309,57 @@ mod tests { ); } + #[test] + fn test_verify_witness_lag() { + let now = Utc::now(); + // a beacon is received first and our test witness is within the acceptable lag from that beacon + assert!(verify_witness_lag( + now - Duration::seconds(60), + now - Duration::seconds(59), + now - Duration::seconds(58) + ) + .is_ok()); + // a witness is received first and our test witness is within the acceptable lag from that first witness + assert!(verify_witness_lag( + now - Duration::seconds(60), + now - Duration::seconds(64), + now - Duration::seconds(63) + ) + .is_ok()); + // a beacon is received first and our test witness is over the acceptable lag from that beacon + assert_eq!( + Err(InvalidResponse { + reason: InvalidReason::TooLate, + details: None + }), + verify_witness_lag( + now - Duration::seconds(60), + now - Duration::seconds(59), + now - Duration::seconds(55) + ) + ); + + // a witness is received first and our test witness is over the acceptable lag from that first witness + assert_eq!( + Err(InvalidResponse { + reason: InvalidReason::TooLate, + details: None + }), + verify_witness_lag( + now - Duration::seconds(55), + now - Duration::seconds(60), + now - Duration::seconds(58) + ) + ); + // a witness is received first and our test witness is that same first witness + assert!(verify_witness_lag( + now - Duration::seconds(55), + now - Duration::seconds(60), + now - Duration::seconds(60) + ) + .is_ok()); + } + #[test] fn test_verify_self_witness() { let key1 = PublicKeyBinary::from_str(PUBKEY1).unwrap(); @@ -1604,6 +1705,7 @@ mod tests { &witness_info, &beacon_report, &beaconer_metadata, + witness_report1.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1623,6 +1725,7 @@ mod tests { &witness_info, &beacon_report, &beaconer_metadata, + witness_report2.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1642,6 +1745,7 @@ mod tests { &witness_info, &beacon_report, &beaconer_metadata, + witness_report3.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1662,6 +1766,7 @@ mod tests { &witness_info4, &beacon_report, &beaconer_metadata, + witness_report4.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1681,6 +1786,7 @@ mod tests { &witness_info, &beacon_report, &beaconer_metadata, + witness_report5.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1701,6 +1807,7 @@ mod tests { &witness_info6, &beacon_report, &beaconer_metadata, + witness_report6.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1721,6 +1828,7 @@ mod tests { &witness_info7, &beacon_report, &beaconer_metadata, + witness_report7.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1741,6 +1849,7 @@ mod tests { &witness_info8, &beacon_report, &beaconer_metadata, + witness_report8.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1760,6 +1869,7 @@ mod tests { &witness_info, &beacon_report, &beaconer_metadata, + witness_report9.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1780,6 +1890,7 @@ mod tests { &witness_info10, &beacon_report, &beaconer_metadata, + witness_report10.received_timestamp, ); assert_eq!( Err(InvalidResponse { @@ -1789,9 +1900,10 @@ mod tests { resp10 ); - // for completeness, confirm our valid witness report is sane + // test witness lag from first received event let witness_report11 = valid_witness_report(PUBKEY2, entropy_start + Duration::minutes(2)); let witness_info11 = witness_gateway_info(Some(LOC4), ProtoRegion::Eu868, true); + let resp11 = do_witness_verifications( &deny_list, entropy_start, @@ -1800,8 +1912,30 @@ mod tests { &witness_info11, &beacon_report, &beaconer_metadata, + witness_report11.received_timestamp - Duration::milliseconds(6000), + ); + assert_eq!( + Err(InvalidResponse { + reason: InvalidReason::TooLate, + details: None + }), + resp11 + ); + + // for completeness, confirm our valid witness report is sane + let witness_report12 = valid_witness_report(PUBKEY2, entropy_start + Duration::minutes(2)); + let witness_info12 = witness_gateway_info(Some(LOC4), ProtoRegion::Eu868, true); + let resp12 = do_witness_verifications( + &deny_list, + entropy_start, + entropy_end, + &witness_report12, + &witness_info12, + &beacon_report, + &beaconer_metadata, + witness_report12.received_timestamp, ); - assert_eq!(Ok(()), resp11); + assert_eq!(Ok(()), resp12); } fn beaconer_gateway_info( diff --git a/iot_verifier/tests/runner_tests.rs b/iot_verifier/tests/runner_tests.rs index 56a945d13..1de1598b5 100644 --- a/iot_verifier/tests/runner_tests.rs +++ b/iot_verifier/tests/runner_tests.rs @@ -164,6 +164,34 @@ async fn valid_beacon_and_witness(pool: PgPool) -> anyhow::Result<()> { Ok(()) } +#[sqlx::test] +async fn valid_beacon_and_no_witness(pool: PgPool) -> anyhow::Result<()> { + let mut ctx = TestContext::setup(pool.clone()).await?; + + // test with a valid beacon and no witnesses + let beacon_to_inject = common::create_valid_beacon_report(common::BEACONER1, ctx.entropy_ts); + common::inject_beacon_report(pool.clone(), beacon_to_inject.clone()).await?; + ctx.runner.handle_db_tick().await?; + + let valid_poc = ctx.valid_pocs.receive_valid_poc().await; + assert_eq!(0, valid_poc.selected_witnesses.len()); + assert_eq!(0, valid_poc.unselected_witnesses.len()); + let valid_beacon = valid_poc.beacon_report.unwrap().report.clone().unwrap(); + // assert the pubkeys in the outputted reports + // match those which we injected + assert_eq!( + PublicKeyBinary::from(valid_beacon.pub_key.clone()), + PublicKeyBinary::from_str(common::BEACONER1).unwrap() + ); + // assert the beacon report outputted to filestore + // is unmodified from that submitted + assert_eq!( + valid_beacon, + LoraBeaconReportReqV1::from(beacon_to_inject.clone()) + ); + Ok(()) +} + #[sqlx::test] async fn invalid_beacon_gateway_not_found(pool: PgPool) -> anyhow::Result<()> { let mut ctx = TestContext::setup(pool.clone()).await?;