From 2c5da772946757fa333f58fe6b69a9b2e9abc71a Mon Sep 17 00:00:00 2001 From: Philip Hayes Date: Fri, 7 Jun 2024 17:06:18 -0700 Subject: [PATCH] blockstream-electrs: init at 0.4.1-unstable-2024-09-30 --- .../bl/blockstream-electrs/bitcoind-v28.patch | 112 ++++++++++++++++++ .../bl/blockstream-electrs/package.nix | 97 +++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 pkgs/by-name/bl/blockstream-electrs/bitcoind-v28.patch create mode 100644 pkgs/by-name/bl/blockstream-electrs/package.nix diff --git a/pkgs/by-name/bl/blockstream-electrs/bitcoind-v28.patch b/pkgs/by-name/bl/blockstream-electrs/bitcoind-v28.patch new file mode 100644 index 00000000000000..3847cb672975ee --- /dev/null +++ b/pkgs/by-name/bl/blockstream-electrs/bitcoind-v28.patch @@ -0,0 +1,112 @@ +commit 1c29aff27b4cf9a14b3cdc0a0cf0e9685b857a49 +Author: Philip Kannegaard Hayes +Date: Thu Nov 14 16:52:04 2024 -0800 + + bitcoind: support v28.0 new default blk*.dat xor'ing + + See: + +diff --git a/src/daemon.rs b/src/daemon.rs +index 6190f15..6808907 100644 +--- a/src/daemon.rs ++++ b/src/daemon.rs +@@ -1,12 +1,13 @@ + use std::cell::OnceCell; + use std::collections::{HashMap, HashSet}; +-use std::env; ++use std::convert::TryFrom; + use std::io::{BufRead, BufReader, Lines, Write}; + use std::net::{SocketAddr, TcpStream}; + use std::path::PathBuf; + use std::str::FromStr; + use std::sync::{Arc, Mutex}; + use std::time::Duration; ++use std::{env, fs, io}; + + use base64::prelude::{Engine, BASE64_STANDARD}; + use error_chain::ChainedError; +@@ -393,6 +394,26 @@ impl Daemon { + Ok(paths) + } + ++ /// bitcoind v28.0+ defaults to xor-ing all blk*.dat files with this key, ++ /// stored in the blocks dir. ++ /// See: ++ pub fn read_blk_file_xor_key(&self) -> Result> { ++ // From: ++ let path = self.blocks_dir.join("xor.dat"); ++ let bytes = match fs::read(path) { ++ Ok(bytes) => bytes, ++ Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None), ++ Err(err) => return Err(err).chain_err(|| "failed to read daemon xor.dat file"), ++ }; ++ let xor_key: [u8; 8] = <[u8; 8]>::try_from(bytes.as_slice()).chain_err(|| { ++ format!( ++ "xor.dat unexpected length: actual: {}, expected: 8", ++ bytes.len() ++ ) ++ })?; ++ Ok(Some(xor_key)) ++ } ++ + pub fn magic(&self) -> u32 { + self.network.magic() + } +diff --git a/src/new_index/fetch.rs b/src/new_index/fetch.rs +index d7637ee..3778a88 100644 +--- a/src/new_index/fetch.rs ++++ b/src/new_index/fetch.rs +@@ -111,6 +111,7 @@ fn blkfiles_fetcher( + ) -> Result>> { + let magic = daemon.magic(); + let blk_files = daemon.list_blk_files()?; ++ let xor_key = daemon.read_blk_file_xor_key()?; + + let chan = SyncChannel::new(1); + let sender = chan.sender(); +@@ -118,7 +119,7 @@ fn blkfiles_fetcher( + let mut entry_map: HashMap = + new_headers.into_iter().map(|h| (*h.hash(), h)).collect(); + +- let parser = blkfiles_parser(blkfiles_reader(blk_files), magic); ++ let parser = blkfiles_parser(blkfiles_reader(blk_files, xor_key), magic); + Ok(Fetcher::from( + chan.into_receiver(), + spawn_thread("blkfiles_fetcher", move || { +@@ -151,7 +152,7 @@ fn blkfiles_fetcher( + )) + } + +-fn blkfiles_reader(blk_files: Vec) -> Fetcher> { ++fn blkfiles_reader(blk_files: Vec, xor_key: Option<[u8; 8]>) -> Fetcher> { + let chan = SyncChannel::new(1); + let sender = chan.sender(); + +@@ -160,8 +161,11 @@ fn blkfiles_reader(blk_files: Vec) -> Fetcher> { + spawn_thread("blkfiles_reader", move || { + for path in blk_files { + trace!("reading {:?}", path); +- let blob = fs::read(&path) ++ let mut blob = fs::read(&path) + .unwrap_or_else(|e| panic!("failed to read {:?}: {:?}", path, e)); ++ if let Some(xor_key) = xor_key { ++ blkfile_apply_xor_key(xor_key, &mut blob); ++ } + sender + .send(blob) + .unwrap_or_else(|_| panic!("failed to send {:?} contents", path)); +@@ -170,6 +174,14 @@ fn blkfiles_reader(blk_files: Vec) -> Fetcher> { + ) + } + ++/// By default, bitcoind v28.0+ applies an 8-byte "xor key" over each "blk*.dat" ++/// file. We have xor again to undo this transformation. ++fn blkfile_apply_xor_key(xor_key: [u8; 8], blob: &mut [u8]) { ++ for (i, blob_i) in blob.iter_mut().enumerate() { ++ *blob_i ^= xor_key[i & 0x7]; ++ } ++} ++ + fn blkfiles_parser(blobs: Fetcher>, magic: u32) -> Fetcher> { + let chan = SyncChannel::new(1); + let sender = chan.sender(); diff --git a/pkgs/by-name/bl/blockstream-electrs/package.nix b/pkgs/by-name/bl/blockstream-electrs/package.nix new file mode 100644 index 00000000000000..39ff192f01fc61 --- /dev/null +++ b/pkgs/by-name/bl/blockstream-electrs/package.nix @@ -0,0 +1,97 @@ +{ + bitcoind, + darwin, + electrum, + fetchFromGitHub, + lib, + rocksdb_8_3, + rustPlatform, + stdenv, +}: + +rustPlatform.buildRustPackage { + pname = "blockstream-electrs"; + version = "0.4.1-unstable-2024-09-30"; + + src = fetchFromGitHub { + owner = "Blockstream"; + repo = "electrs"; + rev = "dca8c5996a4b69e1544a481e1809d0deaf54c771"; + hash = "sha256-HAgdJZArFsoX28Ijn+VxjAUwLV9wUW/DLZYyjSiB4TQ="; + }; + + patches = [ + # Support bitcoind v28's new default blocksxor flag + # Upstream PR: + ./bitcoind-v28.patch + ]; + + useFetchCargoVendor = true; + cargoHash = "sha256-63aDrQ1RgyR6LP3yEzyuqM7PtwCDslA6bznTAlCOsI8="; + + nativeBuildInputs = [ + # Needed for librocksdb-sys + rustPlatform.bindgenHook + ]; + + buildInputs = lib.optionals stdenv.hostPlatform.isDarwin [ + darwin.apple_sdk.frameworks.Security + ]; + + env = { + # Dynamically link rocksdb + ROCKSDB_INCLUDE_DIR = "${rocksdb_8_3}/include"; + ROCKSDB_LIB_DIR = "${rocksdb_8_3}/lib"; + + # External binaries for integration tests are provided via nixpkgs. Skip + # trying to download them. + BITCOIND_SKIP_DOWNLOAD = true; + ELECTRUMD_SKIP_DOWNLOAD = true; + ELEMENTSD_SKIP_DOWNLOAD = true; + }; + + # Only build the service + cargoBuildFlags = [ + "--package=electrs" + "--bin=electrs" + ]; + + # Some upstream dev-dependencies (electrumd, elementsd) currently fail to + # build on non-x86_64-linux platforms, even if downloading is skipped. + # TODO(phlip9): submit a PR to fix this + doCheck = stdenv.hostPlatform.system == "x86_64-linux"; + + # Build tests in debug mode to reduce build time + checkType = "debug"; + + # Integration tests require us to pass in some external deps via env + preCheck = '' + export BITCOIND_EXE=${bitcoind}/bin/bitcoind + export ELECTRUMD_EXE=${electrum}/bin/electrum + ''; + + # Make sure the final binary actually runs + doInstallCheck = true; + installCheckPhase = '' + $out/bin/electrs --version + ''; + + meta = { + description = "Efficient re-implementation of Electrum Server in Rust"; + longDescription = '' + A blockchain index engine and HTTP API written in Rust based on + [romanz/electrs](https://github.com/romanz/electrs). + + Used as the backend for the [Esplora block explorer](https://github.com/Blockstream/esplora) + powering . + + API documentation [is available here](https://github.com/blockstream/esplora/blob/master/API.md). + + Documentation for the database schema and indexing process [is available here](https://github.com/Blockstream/electrs/blob/new-index/doc/schema.md). + ''; + homepage = "https://github.com/Blockstream/electrs"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ phlip9 ]; + mainProgram = "electrs"; + }; +}