From 3a9c70ef5d1834037add953c9d2f6d6feb369bca Mon Sep 17 00:00:00 2001 From: refcell Date: Wed, 26 Jun 2024 18:05:01 -0400 Subject: [PATCH] feat(examples): Send Logs to Loki (#321) * feat(examples): loki tracing * feat(examples): loki tracing * feat(examples): loki tracing * fixes --- Cargo.lock | 93 +++++++++++++++++++++++++- examples/trusted-sync/Cargo.toml | 3 +- examples/trusted-sync/run.sh | 2 + examples/trusted-sync/src/cli.rs | 18 +++++ examples/trusted-sync/src/main.rs | 3 +- examples/trusted-sync/src/telemetry.rs | 25 +++++-- justfile | 2 + 7 files changed, 137 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f24f314fa..e3211a5ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2194,6 +2194,16 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "loki-api" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "674883a98273598ac3aad4301724c56734bea90574c5033af067e8f9fb5eb399" +dependencies = [ + "prost", + "prost-types", +] + [[package]] name = "lru" version = "0.12.3" @@ -2749,6 +2759,38 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost", +] + [[package]] name = "protobuf" version = "2.28.0" @@ -3381,6 +3423,12 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + [[package]] name = "socket2" version = "0.5.7" @@ -3816,6 +3864,17 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -3827,6 +3886,37 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-loki" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea6023f9fe4b69267ccd3ed7d203d931c43c5f82dbaa0f07202bc17193a5f43" +dependencies = [ + "loki-api", + "reqwest", + "serde", + "serde_json", + "snap", + "tokio", + "tokio-stream", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-serde", + "tracing-subscriber", + "url", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.18" @@ -3838,7 +3928,7 @@ dependencies = [ "smallvec", "thread_local", "tracing-core", - "tracing-log", + "tracing-log 0.2.0", ] [[package]] @@ -3859,6 +3949,7 @@ dependencies = [ "serde", "tokio", "tracing", + "tracing-loki", "tracing-subscriber", ] diff --git a/examples/trusted-sync/Cargo.toml b/examples/trusted-sync/Cargo.toml index d8babdbf8..839755239 100644 --- a/examples/trusted-sync/Cargo.toml +++ b/examples/trusted-sync/Cargo.toml @@ -18,11 +18,12 @@ kona-derive = { path = "../../crates/derive", version = "0.0.2", features = ["se # Custom dependencies lazy_static = "1.5.0" +tracing-loki = "0.2.5" reqwest = "0.12" actix-web = "4.8.0" prometheus = { version = "0.13.4", features = ["process"] } tokio = { version = "1.37.0", features = ["full"] } -tracing-subscriber = "0.3.18" +tracing-subscriber = { version = "0.3.18", features = ["fmt"] } clap = { version = "4.5.4", features = ["derive", "env"] } serde = { version = "1.0.198", features = ["derive"] } alloy-provider = { version = "0.1", default-features = false } diff --git a/examples/trusted-sync/run.sh b/examples/trusted-sync/run.sh index e19268898..7ae8213f5 100644 --- a/examples/trusted-sync/run.sh +++ b/examples/trusted-sync/run.sh @@ -7,4 +7,6 @@ --start-l2-block $START_L2_BLOCK \ --metrics-server-addr $METRICS_SERVER_ADDR \ --metrics-server-port $METRICS_SERVER_PORT \ + --loki-server-addr $LOKI_SERVER_ADDR \ + --loki-server-port $LOKI_SERVER_PORT \ -vvv diff --git a/examples/trusted-sync/src/cli.rs b/examples/trusted-sync/src/cli.rs index e82475bc0..ed5e6c40e 100644 --- a/examples/trusted-sync/src/cli.rs +++ b/examples/trusted-sync/src/cli.rs @@ -9,6 +9,8 @@ const L2_RPC_URL: &str = "L2_RPC_URL"; const BEACON_URL: &str = "BEACON_URL"; const DEFAULT_METRICS_SERVER_ADDR: &str = "127.0.0.1"; const DEFAULT_METRICS_SERVER_PORT: u16 = 9000; +const DEFAULT_LOKI_SERVER_ADDR: &str = "127.0.0.1"; +const DEFAULT_LOKI_SERVER_PORT: u16 = 3133; /// The host binary CLI application arguments. #[derive(Parser, Clone, serde::Serialize, serde::Deserialize)] @@ -34,6 +36,12 @@ pub struct Cli { /// The metrics server port. #[clap(long, help = "Port of the metrics server")] pub metrics_server_port: Option, + /// The address of the loki server. + #[clap(long, help = "Address of the loki server")] + pub loki_server_addr: Option, + /// The loki server port. + #[clap(long, help = "Port of the loki server")] + pub loki_server_port: Option, } impl Cli { @@ -48,6 +56,16 @@ impl Cli { ) } + /// Returns the full loki server address. + pub fn loki_addr(&self) -> Url { + let str = format!( + "http://{}:{}", + self.loki_server_addr.clone().unwrap_or_else(|| DEFAULT_LOKI_SERVER_ADDR.to_string()), + self.loki_server_port.unwrap_or(DEFAULT_LOKI_SERVER_PORT) + ); + Url::parse(&str).expect("Failed to parse loki server address") + } + /// Returns the l1 rpc url from CLI or environment variable. pub fn l1_rpc_url(&self) -> Result { let url = if let Some(s) = self.l1_rpc_url.clone() { diff --git a/examples/trusted-sync/src/main.rs b/examples/trusted-sync/src/main.rs index 1c165c8b1..4edad3aac 100644 --- a/examples/trusted-sync/src/main.rs +++ b/examples/trusted-sync/src/main.rs @@ -14,7 +14,8 @@ const LOG_TARGET: &str = "trusted-sync"; #[actix_web::main] async fn main() -> Result<()> { let cfg = cli::Cli::parse(); - telemetry::init(cfg.v)?; + let loki_addr = cfg.loki_addr(); + telemetry::init(cfg.v, loki_addr).await?; let addr = cfg.metrics_server_addr(); let handle = tokio::spawn(async { sync(cfg).await }); tokio::select! { diff --git a/examples/trusted-sync/src/telemetry.rs b/examples/trusted-sync/src/telemetry.rs index 693ab38c6..8613f7f17 100644 --- a/examples/trusted-sync/src/telemetry.rs +++ b/examples/trusted-sync/src/telemetry.rs @@ -1,15 +1,28 @@ use anyhow::{anyhow, Result}; +use reqwest::Url; use tracing::Level; +use tracing_subscriber::{layer::SubscriberExt, prelude::*}; -pub fn init(v: u8) -> Result<()> { - let subscriber = tracing_subscriber::fmt() - .with_max_level(match v { +pub async fn init(v: u8, addr: Url) -> Result<()> { + let (loki_layer, task) = tracing_loki::builder() + .label("environment", "production") + .map_err(|e| anyhow!(e))? + .extra_field("pid", format!("{}", std::process::id())) + .map_err(|e| anyhow!(e))? + .build_url(addr) + .map_err(|e| anyhow!(e))?; + + let std_layer = tracing_subscriber::fmt::Layer::default().with_writer( + std::io::stdout.with_max_level(match v { 0 => Level::ERROR, 1 => Level::WARN, 2 => Level::INFO, 3 => Level::DEBUG, _ => Level::TRACE, - }) - .finish(); - tracing::subscriber::set_global_default(subscriber).map_err(|e| anyhow!(e)) + }), + ); + let subscriber = tracing_subscriber::registry().with(loki_layer).with(std_layer); + tracing::subscriber::set_global_default(subscriber).map_err(|e| anyhow!(e))?; + tokio::spawn(task); + Ok(()) } diff --git a/justfile b/justfile index f2519169b..54b11e786 100644 --- a/justfile +++ b/justfile @@ -102,4 +102,6 @@ docker-run-ts: -e START_L2_BLOCK=$START_L2_BLOCK \ -e METRICS_SERVER_ADDR=$METRICS_SERVER_ADDR \ -e METRICS_SERVER_PORT=$METRICS_SERVER_PORT \ + -e LOKI_SERVER_ADDR=$LOKI_SERVER_ADDR \ + -e LOKI_SERVER_PORT=$LOKI_SERVER_PORT \ trusted-sync