refactor: forester (#897)

examples/token-escrow/Anchor.toml

@@ -7,12 +7,15 @@ skip-lint = false
 [programs.localnet]
 token_escrow = "GRLu2hKaAiMbxpkAM1HeXzks9YeGuz18SEgXEizVvPqX"
 
+[programs.testnet]
+token_escrow = "GRLu2hKaAiMbxpkAM1HeXzks9YeGuz18SEgXEizVvPqX"
+
 [registry]
 url = "https://api.apr.dev"
 
 [provider]
-cluster = "Localnet"
-wallet = "/home/ananas/.config/solana/id.json"
+cluster = "testnet"
+wallet = "$HOME/.config/solana/id.json"
 
 [scripts]
 test = "yarn run ts-mocha -p ./tsconfig.json -t 1000000 tests/**/*.ts"

forester/.env.example

@@ -0,0 +1 @@
+export FORESTER_PAYER=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64]

forester/.gitignore

@@ -1,2 +1,3 @@
 /target
 .idea
+.env

forester/README.md

@@ -2,24 +2,40 @@
 
 ## Description
 
-Forester is a service for nullifying the state merkle trees.
-It subscribes to the nullifier queue and nullifies the state merkle tree leaves.
+Forester is a service for nullifying the state and address merkle trees.
+It subscribes to the nullifier queue and nullifies merkle tree leaves.
 
 ## Configuration
 
 Forester requires a configuration file, `forester.toml`, specifying necessary keys:
-- `merkle_tree`: Address of the State Merkle tree.
-- `nullifier_queue`: Address of the Nullifier queue.
-- `payer`: The key pair for the payer.
+- `STATE_MERKLE_TREE_PUBKEY`: Address of the State Merkle tree.
+- `NULLIFIER_QUEUE_PUBKEY`: Address of the State Nullifier queue.
+- `ADDRESS_MERKLE_TREE_PUBKEY`: Address of the Address Merkle tree.
+- `ADDRESS_MERKLE_TREE_QUEUE_PUBKEY`: Address of the Address queue.
+- `REGISTRY_PUBKEY`: Address of the Registry program.
+
+
+To setup your environment properly, copy `.env.example` to `.env` 
+and update the `FORESTER_PAYER` field with your appropriate key. 
+
+Alternatively, if you prefer to use a terminal profile file, 
+add the key to your `~/.zshrc` (zsh) or `~/.bashrc` (bash) 
+by including this line: `export FORESTER_PAYER=your_value_here`.
+Substitute `your_value_here` with your actual key. 
+
+Remember to restart your terminal or source your terminal profile for the changes to take effect.
 
 ## Usage
 
 1. Run the service:
 To subscribe to nullify the state merkle tree, use the following command:
 `cargo run -- subscribe`
-
 2. To manually nullify state merkle tree leaves, use the following command:
-`cargo run -- nullify`
+`cargo run -- nullify-state`
+3. To manually nullify address merkle tree leaves, use the following command:
+`cargo run -- nullify-addresses`
+4. To manually nullify state *and* address merkle tree leaves, use the following command:
+   `cargo run -- nullify`
 
 
 ## TODO

forester/forester.toml

@@ -1,10 +1,15 @@
-PAYER = [17, 34, 231, 31, 83, 147, 93, 173, 61, 164, 25, 0, 204, 82, 234, 91, 202, 187, 228, 110, 146,
-    97, 112, 131, 180, 164, 96, 220, 57, 207, 65, 107, 2, 99, 226, 251, 88, 66, 92, 33, 25, 216,
-    211, 185, 112, 203, 212, 238, 105, 144, 72, 121, 176, 253, 106, 168, 115, 158, 154, 188, 62,
-    255, 166, 81]
+STATE_MERKLE_TREE_PUBKEY="5bdFnXU47QjzGpzHfXnxcEi5WXyxzEAZzd1vrE39bf1W"
+NULLIFIER_QUEUE_PUBKEY="44J4oDXpjPAbzHCSc24q7NEiPekss4sAbLd8ka4gd9CZ"
+REGISTRY_PUBKEY="7Z9Yuy3HkBCc2Wf3xzMGnz6qpV4n7ciwcoEMGKqhAnj1"
+ADDRESS_MERKLE_TREE_PUBKEY="C83cpRN6oaafjNgMQJvaYgAz592EP5wunKvbokeTKPLn"
+ADDRESS_MERKLE_TREE_QUEUE_PUBKEY="HNjtNrjt6irUPYEgxhx2Vcs42koK9fxzm3aFLHVaaRWz"
 
-STATE_MERKLE_TREE_PUBKEY = "5bdFnXU47QjzGpzHfXnxcEi5WXyxzEAZzd1vrE39bf1W"
-NULLIFIER_QUEUE_PUBKEY = "44J4oDXpjPAbzHCSc24q7NEiPekss4sAbLd8ka4gd9CZ"
-REGISTRY_PUBKEY = "7Z9Yuy3HkBCc2Wf3xzMGnz6qpV4n7ciwcoEMGKqhAnj1"
-ADDRESS_MERKLE_TREE_PUBKEY = "C83cpRN6oaafjNgMQJvaYgAz592EP5wunKvbokeTKPLn"
-ADDRESS_MERKLE_TREE_QUEUE_PUBKEY = "HNjtNrjt6irUPYEgxhx2Vcs42koK9fxzm3aFLHVaaRWz"
+RPC_URL="http://localhost:8899"
+WS_RPC_URL="ws://localhost:8900"
+INDEXER_URL="http://localhost:8784"
+PROVER_URL="http://localhost:3001"
+
+CONCURRENCY_LIMIT=10
+BATCH_SIZE=10
+MAX_RETRIES=5
+MAX_CONCURRENT_BATCHES=1

forester/package.json

@@ -9,22 +9,25 @@
     "start-validator": "../cli/test_bin/run test-validator --indexer-db-url=postgres://photon:photon@localhost:5432/postgres",
     "restart-validator": "pnpm restart-db && pnpm wait-for-db && pnpm migrate-db && pnpm start-validator",
     "build": "cargo build",
-    "test": "RUST_MIN_STACK=98388608 RUSTFLAGS=\"-D warnings\" cargo test --package forester -- --nocapture --test-threads=1",
+    "test": "RUST_MIN_STACK=98388608 RUSTFLAGS=\"-D warnings\" cargo test --package forester -- --test-threads=1 --nocapture",
+
+    "test-sync": "pnpm restart-validator && pnpm transfer-sync && pnpm nullify-state",
 
     "interop-nullify-test": "RUST_MIN_STACK=98388608 RUSTFLAGS=\"-D warnings\" cargo test test_photon_interop_nullify_account -- --nocapture",
     "interop-address-test": "RUST_MIN_STACK=98388608 RUSTFLAGS=\"-D warnings\" cargo test test_photon_interop_address -- --nocapture",
 
     "test-indexer": "RUST_BACKTRACE=1 RUST_MIN_STACK=98388608 cargo test test_indexer -- --nocapture",
     "address-tree-test": "RUST_BACKTRACE=1 RUST_MIN_STACK=98388608 cargo test empty_address_tree_test -- --nocapture",
     "rebuild-photon": "pkill photon && cd ../../photon && cargo build && cp ./target/debug/photon ./target/debug/photon-migration ../light-protocol/.local/cargo/bin",
-    "tree-info": "RUST_MIN_STACK=8388608 cargo test tree_info_test -- --nocapture",
+    "tree-info": "RUST_LOG=info RUST_MIN_STACK=8388608 cargo test tree_info_test -- --nocapture",
     "queue-info": "RUST_MIN_STACK=8388608 cargo test queue_info_test -- --nocapture",
-    "test-10-sync": "pnpm restart-validator && pnpm transfer-sync && pnpm nullify && pnpm transfer-sync && pnpm nullify && pnpm transfer-sync && pnpm nullify",
     "test-1000": "pnpm restart-validator && pnpm transfer && pnpm nullify && pnpm transfer && pnpm nullify",
     "transfer-100k": "ts-node scripts/generate_100k.ts",
-    "subscribe": "RUST_MIN_STACK=8388608 cargo run --release -- subscribe",
-    "nullify": "RUST_MIN_STACK=8388608 cargo run --release -- nullify",
-    "reindex": "RUST_MIN_STACK=8388608 cargo run --release -- index",
+    "subscribe": "RUST_MIN_STACK=8388608 cargo run -- subscribe",
+    "nullify-state": "RUST_MIN_STACK=8388608 cargo run -- nullify-state",
+    "nullify-addresses": "RUST_MIN_STACK=8388608 cargo run -- nullify-addresses",
+    "nullify": "RUST_MIN_STACK=8388608 cargo run -- nullify",
+    "reindex": "RUST_MIN_STACK=8388608 cargo run -- index",
     "dump-accounts": "./scripts/dump.sh",
     "transfer": "ts-node ./scripts/generate.ts",
     "transfer-10": "ts-node ./scripts/generate_10.ts",

forester/scripts/generate_sync.ts

@@ -30,15 +30,34 @@ async function transferAsync(i: number, rpc: Rpc, payer: Signer, bobPublicKey: P
     console.log(`transfer ${i} of ${TOTAL_NUMBER_OF_TRANSFERS}: ${transferSig}`);
 }
 
+function localRpc(): Rpc {
+    let validatorUrl = 'http://0.0.0.0:8899';
+    let photonUrl = 'http://0.0.0.0:8784';
+    let proverUrl = 'http://0.0.0.0:3001';
+
+    return createRpc(validatorUrl, photonUrl, proverUrl);
+}
+
+function zkTestnetRpc(): Rpc {
+    let validatorUrl = 'https://zk-testnet.helius.dev:8899';
+    let photonUrl = 'https://zk-testnet.helius.dev:8784';
+    let proverUrl = 'https://zk-testnet.helius.dev:3001';
+
+    return createRpc(validatorUrl, photonUrl, proverUrl);
+}
+
 async function prefillNullifierQueue() {
-    const rpc = createRpc();
+
+    const rpc = localRpc();
     const payer = Keypair.fromSecretKey(Uint8Array.from(payerKeypair));
+    const tx1 = await airdropSol({connection: rpc, lamports: LAMPORTS, recipientPublicKey: payer.publicKey});
+
     const bob = Keypair.fromSecretKey(Uint8Array.from(bobKeypair));
 
-    const tx1 = await airdropSol({connection: rpc, lamports: LAMPORTS, recipientPublicKey: payer.publicKey});
-    console.log('tx1', tx1);
+    console.log('Airdropping SOL to payer and bob...');
     const tx2 = await airdropSol({connection: rpc, lamports: LAMPORTS, recipientPublicKey: bob.publicKey});
     console.log('tx2', tx2);
+    console.log('Airdrop completed.');
 
     const payerBalance = await rpc.getBalance(payer.publicKey);
     console.log('payer balance', payerBalance);

forester/src/cli.rs

@@ -8,6 +8,8 @@ pub struct Cli {
 }
 #[derive(Subcommand)]
 pub enum Commands {
+    NullifyState,
+    NullifyAddresses,
     Nullify,
     Subscribe,
     Index,

forester/src/nullifier/config.rs → forester/src/config.rs

@@ -1,8 +1,10 @@
+use crate::external_services_config::ExternalServicesConfig;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::Keypair;
 
-pub struct Config {
-    pub server_url: String,
+#[derive(Debug)]
+pub struct ForesterConfig {
+    pub external_services: ExternalServicesConfig,
     pub nullifier_queue_pubkey: Pubkey,
     pub state_merkle_tree_pubkey: Pubkey,
     pub address_merkle_tree_pubkey: Pubkey,
@@ -12,12 +14,13 @@ pub struct Config {
     pub concurrency_limit: usize,
     pub batch_size: usize,
     pub max_retries: usize,
+    pub max_concurrent_batches: usize,
 }
 
-impl Clone for Config {
+impl Clone for ForesterConfig {
     fn clone(&self) -> Self {
         Self {
-            server_url: self.server_url.clone(),
+            external_services: self.external_services.clone(),
             nullifier_queue_pubkey: self.nullifier_queue_pubkey,
             state_merkle_tree_pubkey: self.state_merkle_tree_pubkey,
             address_merkle_tree_pubkey: self.address_merkle_tree_pubkey,
@@ -27,6 +30,7 @@ impl Clone for Config {
             concurrency_limit: self.concurrency_limit,
             batch_size: self.batch_size,
             max_retries: self.max_retries,
+            max_concurrent_batches: self.max_concurrent_batches,
         }
     }
 }

forester/src/errors.rs

@@ -22,11 +22,42 @@ pub enum ForesterError {
     BincodeError(#[from] Box<bincode::ErrorKind>),
     #[error("Indexer can't find any proofs")]
     NoProofsFound,
+    #[error("Max retries reached")]
+    MaxRetriesReached,
     #[error("error: {0:?}")]
     Custom(String),
     #[error("unknown error")]
     Unknown,
 }
+impl ForesterError {
+    pub fn to_owned(&self) -> Self {
+        match self {
+            ForesterError::RpcError(e) => ForesterError::Custom(format!("RPC Error: {:?}", e)),
+            ForesterError::DeserializeError(e) => {
+                ForesterError::Custom(format!("Deserialize Error: {:?}", e))
+            }
+            ForesterError::CopyMerkleTreeError(e) => {
+                ForesterError::Custom(format!("Copy Merkle Tree Error: {:?}", e))
+            }
+            ForesterError::AccountCompressionError(e) => {
+                ForesterError::Custom(format!("Account Compression Error: {:?}", e))
+            }
+            ForesterError::HashSetError(e) => {
+                ForesterError::Custom(format!("HashSet Error: {:?}", e))
+            }
+            ForesterError::PhotonApiError(e) => {
+                ForesterError::Custom(format!("Photon API Error: {:?}", e))
+            }
+            ForesterError::BincodeError(e) => {
+                ForesterError::Custom(format!("Bincode Error: {:?}", e))
+            }
+            ForesterError::NoProofsFound => ForesterError::NoProofsFound,
+            ForesterError::MaxRetriesReached => ForesterError::MaxRetriesReached,
+            ForesterError::Custom(s) => ForesterError::Custom(s.clone()),
+            ForesterError::Unknown => ForesterError::Unknown,
+        }
+    }
+}
 
 #[derive(Error, Debug)]
 pub enum PhotonApiErrorWrapper {

forester/src/external_services_config.rs

@@ -0,0 +1,8 @@
+#[derive(Debug, Clone)]
+pub struct ExternalServicesConfig {
+    pub rpc_url: String,
+    pub ws_rpc_url: String,
+    pub indexer_url: String,
+    pub prover_url: String,
+    pub derivation: String,
+}