Indexing Blocks & Trasactions (#23)

- Create tables on start up if not exists
- Write all the database types and insertion statements
- store blocks and transaction methods
- update the init logic that populates finalized blocks
- update the sync logic that populates beyond finalized block

Cargo.toml

@@ -8,8 +8,12 @@ license = "GPL-3.0-or-later"
 [dependencies]
 anyhow = "1"
 clap = { version = "4", features = ["derive", "env"] }
-ethrpc = { version = "0.0.8", features = ["http"] }
 rusqlite = { version = "0.30.0", features = ["extra_check"] }
+# Waiting on https://github.com/nlordell/ethrpc-rs/pull/9
+#ethrpc = { version = "0.0.8", features = ["http"] }
+ethrpc = { git = "https://github.com/bh2smith/ethrpc-rs", rev = "04ba07064fcb9ac8cf71e02d2c168819499b3616", features = [
+    "http",
+] }
 serde = { version = "1", features = ["derive"] }
 solabi = "0.2.0"
 tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] }
@@ -18,9 +22,10 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 url = { version = "2", features = ["serde"] }
 futures = "0.3"
-tokio-postgres = "0.7"
+tokio-postgres = { version = "0.7", features = ["with-time-0_3"] }
 pg_bigdecimal = "0.1.5"
 dotenv = "0.15.0"
+chrono = { version = "0.4.31", default-features = false, features = ["std"] }
 
 [dev-dependencies]
 hex-literal = "0.4"

kubernetes/config/configmap.yaml

@@ -7,8 +7,8 @@ data:
   arak.toml: |
 
     [indexer]
-    page-size = 1000
-    poll-interval = 10
+    page-size = 2000
+    poll-interval = 0.1
 
     ## ERC721: https://eips.ethereum.org/EIPS/eip-721#specification
     [[event]]

src/database/date_util.rs

@@ -0,0 +1,44 @@
+use std::time::SystemTime;
+
+use chrono::{DateTime, Utc};
+
+/// Converts unix timestamp seconds into DateTime Strings
+/// Accepts a given `format` string or uses standard default.
+pub fn systemtime_to_string(system_time: SystemTime, format: Option<&str>) -> String {
+    let date_format = format.unwrap_or("%Y-%m-%d %H:%M:%S");
+    let datetime_utc: DateTime<Utc> = system_time.into();
+    datetime_utc.format(date_format).to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::Duration;
+
+    #[test]
+    fn date_conversions() {
+        // Time Zero
+        assert_eq!(
+            systemtime_to_string(SystemTime::UNIX_EPOCH, None),
+            "1970-01-01 00:00:00"
+        );
+
+        // First Ethereum Block: https://etherscan.io/block/1
+        assert_eq!(
+            systemtime_to_string(
+                SystemTime::UNIX_EPOCH + Duration::from_secs(1438262788),
+                None
+            ),
+            "2015-07-30 13:26:28"
+        );
+
+        // Different Formats
+        assert_eq!(
+            systemtime_to_string(
+                SystemTime::UNIX_EPOCH,
+                Some("Date(%Y-%m-%d)-Time(%H:%M:%S)")
+            ),
+            "Date(1970-01-01)-Time(00:00:00)"
+        );
+    }
+}

src/database/mod.rs

@@ -1,3 +1,4 @@
+mod date_util;
 mod event_to_tables;
 mod event_visitor;
 mod keywords;
@@ -8,8 +9,11 @@ use {
     anyhow::Result,
     futures::future::BoxFuture,
     solabi::{abi::EventDescriptor, ethprim::Address, value::Value},
+    std::time::SystemTime,
 };
 
+use solabi::Digest;
+
 pub use self::{postgres::Postgres, sqlite::Sqlite};
 
 /// Block indexing information.
@@ -20,12 +24,22 @@ pub struct Block {
 }
 
 /// Block indexing information attached to an event.
+/// Also used for `blocks` and `transactions` (which are not events).
 #[derive(Debug)]
 pub struct EventBlock<'a> {
     pub event: &'a str,
     pub block: Block,
 }
 
+impl<'a> EventBlock<'a> {
+    pub fn is_event(&self) -> bool {
+        // TODO(bh2smith) - note that this implies blocks and transactions are "reserved" keywords
+        //  So we should not allow events to have these names. OR this can be done differently.
+        //  Maybe we should use _blocks and _transactions!
+        !["blocks", "transactions"].contains(&self.event)
+    }
+}
+
 /// An uncled block. All logs for this block or newer are considered invalid.
 #[derive(Debug)]
 pub struct Uncle<'a> {
@@ -44,6 +58,23 @@ pub struct Log<'a> {
     pub fields: Vec<Value>,
 }
 
+/// A basic Ethereum block.
+#[derive(Debug)]
+pub struct BlockTime {
+    pub number: u64,
+    pub timestamp: SystemTime,
+}
+
+/// A basic Ethereum transaction.
+#[derive(Debug, Default)]
+pub struct Transaction {
+    pub block_number: u64,
+    pub index: u64,
+    pub hash: Digest,
+    pub from: Address,
+    pub to: Option<Address>,
+}
+
 /// Abstraction over specific SQL like backends.
 ///
 /// All methods either succeed in full or error without having applied any
@@ -91,6 +122,8 @@ pub trait Database {
         &'a mut self,
         blocks: &'a [EventBlock],
         logs: &'a [Log],
+        block_times: &'a [BlockTime],
+        transactions: &'a [Transaction],
     ) -> BoxFuture<'a, Result<()>>;
 
     /// Removes logs from the specified event's uncled blocks.