rpc: sync state api

Cargo.toml

@@ -9,5 +9,5 @@ members = [
 resolver = "2"
 
 [workspace.dependencies]
-miden-crypto = { git = "https://github.com/0xPolygonMiden/crypto", branch = "next", default-features = false, features = ["std"] }
+miden-crypto = { package = "miden-crypto", git = "https://github.com/0xPolygonMiden/crypto.git", branch = "next", features = ["std"] }
 miden_objects = { package = "miden-objects", git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "main", default-features = false }

proto/Cargo.toml

@@ -12,6 +12,7 @@ rust-version = "1.67"
 [dependencies]
 hex = { version = "0.4" }
 miden-crypto = { workspace = true }
+miden_objects = { workspace = true }
 prost = { version = "0.12" }
 tonic = { version = "0.10" }
 

proto/proto/account_id.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+package account_id;
+
+message AccountId {
+    // A miden account is defined with a little bit of proof-of-work, the id itself
+    // is defined as the first word of a hash digest. For this reason account ids
+    // can be considered as random values, because of that the encoding bellow uses
+    // fixed 64 bits, instead of zig-zag encoding.
+    fixed64 id = 1;
+}

proto/proto/block_header.proto

@@ -4,24 +4,24 @@ package block_header;
 import "digest.proto";
 
 message BlockHeader {
-    /// the hash of the previous blocks header.
+    // the hash of the previous blocks header.
     digest.Digest prev_hash = 1;
-    /// a unique sequential number of the current block.
+    // a unique sequential number of the current block.
     uint32 block_num = 2;
-    /// a commitment to an MMR of the entire chain where each block is a leaf.
+    // a commitment to an MMR of the entire chain where each block is a leaf.
     digest.Digest chain_root = 3;
-    /// a commitment to account database.
+    // a commitment to account database.
     digest.Digest account_root = 4;
-    /// a commitment to the nullifier database.
+    // a commitment to the nullifier database.
     digest.Digest nullifier_root = 5;
-    /// a commitment to all notes created in the current block.
+    // a commitment to all notes created in the current block.
     digest.Digest note_root = 6;
-    /// a commitment to a set of transaction batches executed as a part of this block.
+    // a commitment to a set of transaction batches executed as a part of this block.
     digest.Digest batch_root = 7;
-    /// a hash of a STARK proof attesting to the correct state transition.
+    // a hash of a STARK proof attesting to the correct state transition.
     digest.Digest proof_hash = 8;
-    /// specifies the version of the protocol.
+    // specifies the version of the protocol.
     uint32 version = 9;
-    /// the time when the block was created.
+    // the time when the block was created.
     uint64 timestamp = 10;
 }

proto/proto/merkle.proto

@@ -0,0 +1,8 @@
+syntax = "proto3";
+package merkle;
+
+import "digest.proto";
+
+message MerklePath {
+    repeated digest.Digest siblings = 1;
+}

proto/proto/mmr.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+package mmr;
+
+import "digest.proto";
+
+message MmrDelta {
+    uint64 forest = 1;
+    repeated digest.Digest data = 2;
+}

proto/proto/note.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+package note;
+
+import "digest.proto";
+import "merkle.proto";
+
+message Note {
+    uint32 block_num = 1;
+    uint32 note_index = 2;
+    digest.Digest note_hash = 3;
+    uint64 sender  = 4;
+    uint64 tag = 5;
+    uint32 num_assets = 6;
+    merkle.MerklePath merkle_path = 7;
+}

proto/proto/requests.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+package requests;
+
+import "account_id.proto";
+import "digest.proto";
+
+message CheckNullifiersRequest {
+    repeated digest.Digest nullifiers = 1;
+}
+
+message FetchBlockHeaderByNumberRequest {
+    // The block number of the target block.
+    //
+    // If not provided, means latest know block.
+    optional uint32 block_num = 1;
+}
+
+// State synchronization request.
+message SyncStateRequest {
+    // Send updates to the client starting at this block.
+    uint32 block_num = 1;
+
+    repeated account_id.AccountId account_ids = 2;
+
+    // Tags and nullifiers are filters, both filters correspond to the high
+    // 16bits of the real values shifted to the right `>> 48`.
+    repeated uint32 note_tags = 3;
+    repeated uint32 nullifiers = 4;
+}

proto/proto/responses.proto

@@ -0,0 +1,62 @@
+syntax = "proto3";
+package responses;
+
+import "account_id.proto";
+import "block_header.proto";
+import "digest.proto";
+import "merkle.proto";
+import "mmr.proto";
+import "tsmt.proto";
+
+message CheckNullifiersResponse {
+    // Each requested nullifier has its corresponding nullifier proof at the
+    // same position.
+    repeated tsmt.NullifierProof proofs = 1;
+}
+
+message FetchBlockHeaderByNumberResponse {
+    block_header.BlockHeader block_header = 1;
+}
+
+message AccountHashUpdate {
+    account_id.AccountId account_id = 1;
+    digest.Digest account_hash = 2;
+    uint32 block_num = 3;
+}
+
+message NullifierUpdate {
+    digest.Digest nullifier = 1;
+    uint32 block_num = 2;
+}
+
+message NoteSyncRecord {
+    uint32 note_index = 2;
+    digest.Digest note_hash = 3;
+    uint64 sender  = 4;
+    uint64 tag = 5;
+    uint32 num_assets = 6;
+    merkle.MerklePath merkle_path = 7;
+}
+
+message SyncStateResponse {
+    // number of the latest block in the chain
+    uint32 chain_tip = 1;
+
+    // block header of the block with the first note matching the specified criteria
+    block_header.BlockHeader block_header = 2;
+
+    // data needed to update the partial MMR from `block_ref` to `block_header.block_num`
+    mmr.MmrDelta mmr_delta = 3;
+
+    // Merkle path in the updated chain MMR to the block at `block_header.block_num`
+    merkle.MerklePath block_path = 4;
+
+    // a list of account hashes updated after `block_ref` but not after `block_header.block_num`
+    repeated AccountHashUpdate accounts = 5;
+
+    // a list of all notes together with the Merkle paths from `block_header.note_root`
+    repeated NoteSyncRecord notes = 6;
+
+    // a list of nullifiers created between `block_ref` and `block_header.block_num`
+    repeated NullifierUpdate nullifiers = 7;
+}

proto/proto/rpc.proto

@@ -2,32 +2,11 @@
 syntax = "proto3";
 package rpc;
 
-import "block_header.proto";
-import "digest.proto";
-import "tsmt.proto";
-
-message CheckNullifiersRequest {
-    repeated digest.Digest nullifiers = 1;
-}
-
-message CheckNullifiersResponse {
-    // Each requested nullifier has its corresponding nullifier proof at the
-    // same position.
-    repeated tsmt.NullifierProof proofs = 1;
-}
-
-message FetchBlockHeaderByNumberRequest {
-    // The block number of the target block.
-    //
-    // If not provided, means latest know block.
-    optional uint64 block_num = 1;
-}
-
-message FetchBlockHeaderByNumberResponse {
-    block_header.BlockHeader block_header = 1;
-}
+import "requests.proto";
+import "responses.proto";
 
 service Api {
-    rpc CheckNullifiers(CheckNullifiersRequest) returns (CheckNullifiersResponse) {}
-    rpc FetchBlockHeaderByNumber(FetchBlockHeaderByNumberRequest) returns (FetchBlockHeaderByNumberResponse) {}
+    rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {}
+    rpc FetchBlockHeaderByNumber(requests.FetchBlockHeaderByNumberRequest) returns (responses.FetchBlockHeaderByNumberResponse) {}
+    rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {}
 }

proto/proto/store.proto

@@ -4,32 +4,11 @@
 syntax = "proto3";
 package store;
 
-import "block_header.proto";
-import "digest.proto";
-import "tsmt.proto";
-
-message CheckNullifiersRequest {
-    repeated digest.Digest nullifiers = 1;
-}
-
-message CheckNullifiersResponse {
-    // Each requested nullifier has its corresponding nullifier proof at the
-    // same position.
-    repeated tsmt.NullifierProof proofs = 1;
-}
-
-message FetchBlockHeaderByNumberRequest {
-    // The block number of the target block.
-    //
-    // If not provided, means latest know block.
-    optional uint64 block_num = 1;
-}
-
-message FetchBlockHeaderByNumberResponse {
-    block_header.BlockHeader block_header = 1;
-}
+import "requests.proto";
+import "responses.proto";
 
 service Api {
-    rpc CheckNullifiers(CheckNullifiersRequest) returns (CheckNullifiersResponse) {}
-    rpc FetchBlockHeaderByNumber(FetchBlockHeaderByNumberRequest) returns (FetchBlockHeaderByNumberResponse) {}
+    rpc CheckNullifiers(requests.CheckNullifiersRequest) returns (responses.CheckNullifiersResponse) {}
+    rpc FetchBlockHeaderByNumber(requests.FetchBlockHeaderByNumberRequest) returns (responses.FetchBlockHeaderByNumberResponse) {}
+    rpc SyncState(requests.SyncStateRequest) returns (responses.SyncStateResponse) {}
 }

proto/src/conversion.rs

@@ -1,13 +1,10 @@
-use crate::digest;
-use crate::error;
-use crate::tsmt;
-use miden_crypto::hash::rpo::RpoDigest;
-use miden_crypto::merkle::MerklePath;
-use miden_crypto::merkle::TieredSmtProof;
-use miden_crypto::Felt;
-use miden_crypto::FieldElement;
-use miden_crypto::StarkField;
-use miden_crypto::Word;
+use crate::{account_id, block_header, digest, error, merkle, mmr, note, responses, tsmt};
+use miden_crypto::{
+    hash::rpo::RpoDigest,
+    merkle::{MerklePath, MmrDelta, TieredSmtProof},
+    Felt, FieldElement, StarkField, Word,
+};
+use miden_objects::BlockHeader;
 
 impl From<[u64; 4]> for digest::Digest {
     fn from(value: [u64; 4]) -> Self {
@@ -117,3 +114,87 @@ impl TryFrom<&digest::Digest> for RpoDigest {
         value.clone().try_into()
     }
 }
+
+impl TryFrom<block_header::BlockHeader> for BlockHeader {
+    type Error = error::ParseError;
+
+    fn try_from(value: block_header::BlockHeader) -> Result<Self, Self::Error> {
+        Ok(BlockHeader::new(
+            value.prev_hash.ok_or(error::ParseError::ProtobufMissingData)?.try_into()?,
+            value.block_num.into(),
+            value.chain_root.ok_or(error::ParseError::ProtobufMissingData)?.try_into()?,
+            value.account_root.ok_or(error::ParseError::ProtobufMissingData)?.try_into()?,
+            value.nullifier_root.ok_or(error::ParseError::ProtobufMissingData)?.try_into()?,
+            value.note_root.ok_or(error::ParseError::ProtobufMissingData)?.try_into()?,
+            value.batch_root.ok_or(error::ParseError::ProtobufMissingData)?.try_into()?,
+            value.proof_hash.ok_or(error::ParseError::ProtobufMissingData)?.try_into()?,
+            value.version.into(),
+            value.timestamp.into(),
+        ))
+    }
+}
+
+impl TryFrom<&block_header::BlockHeader> for BlockHeader {
+    type Error = error::ParseError;
+
+    fn try_from(value: &block_header::BlockHeader) -> Result<Self, Self::Error> {
+        value.clone().try_into()
+    }
+}
+
+impl TryFrom<mmr::MmrDelta> for MmrDelta {
+    type Error = error::ParseError;
+
+    fn try_from(value: mmr::MmrDelta) -> Result<Self, Self::Error> {
+        let data: Result<Vec<RpoDigest>, error::ParseError> =
+            value.data.into_iter().map(|v| v.try_into()).collect();
+
+        Ok(MmrDelta {
+            forest: value.forest as usize,
+            data: data?,
+        })
+    }
+}
+
+impl From<MmrDelta> for mmr::MmrDelta {
+    fn from(value: MmrDelta) -> Self {
+        let data: Vec<digest::Digest> = value.data.into_iter().map(|v| v.into()).collect();
+
+        mmr::MmrDelta {
+            forest: value.forest as u64,
+            data,
+        }
+    }
+}
+
+impl From<MerklePath> for merkle::MerklePath {
+    fn from(value: MerklePath) -> Self {
+        let siblings: Vec<digest::Digest> = value.nodes().iter().map(|v| (*v).into()).collect();
+        merkle::MerklePath { siblings }
+    }
+}
+
+impl From<note::Note> for responses::NoteSyncRecord {
+    fn from(value: note::Note) -> Self {
+        Self {
+            note_index: value.note_index,
+            note_hash: value.note_hash,
+            sender: value.sender,
+            tag: value.tag,
+            num_assets: value.num_assets,
+            merkle_path: value.merkle_path,
+        }
+    }
+}
+
+impl From<account_id::AccountId> for u64 {
+    fn from(value: account_id::AccountId) -> Self {
+        value.id
+    }
+}
+
+impl From<u64> for account_id::AccountId {
+    fn from(value: u64) -> Self {
+        account_id::AccountId { id: value }
+    }
+}

proto/src/error.rs

@@ -6,6 +6,7 @@ pub enum ParseError {
     MissingLeafKey,
     NotAValidFelt,
     InvalidProof,
+    ProtobufMissingData,
 }
 
 impl std::error::Error for ParseError {}
@@ -32,6 +33,7 @@ impl std::fmt::Display for ParseError {
             ParseError::InvalidProof => {
                 write!(f, "Received TSMT proof is invalid")
             },
+            ParseError::ProtobufMissingData => write!(f, "Protobuf message missing data"),
         }
     }
 }