From 8796c42436e3374c43a353d19d5b90c2cde85a72 Mon Sep 17 00:00:00 2001 From: Gaius Date: Wed, 7 Aug 2024 11:06:59 +0800 Subject: [PATCH] feat: add zstd compression for grpc (#658) Signed-off-by: Gaius --- Cargo.lock | 36 ++++++++++++++----- Cargo.toml | 18 +++++----- .../src/grpc/dfdaemon_download.rs | 5 +++ dragonfly-client/src/grpc/dfdaemon_upload.rs | 5 +++ 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d18d8598..c9a54251 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1120,7 +1120,7 @@ dependencies = [ [[package]] name = "dragonfly-client" -version = "0.1.95" +version = "0.1.96" dependencies = [ "anyhow", "blake3", @@ -1190,7 +1190,7 @@ dependencies = [ [[package]] name = "dragonfly-client-backend" -version = "0.1.95" +version = "0.1.96" dependencies = [ "dragonfly-api", "dragonfly-client-core", @@ -1212,7 +1212,7 @@ dependencies = [ [[package]] name = "dragonfly-client-config" -version = "0.1.95" +version = "0.1.96" dependencies = [ "bytesize", "bytesize-serde", @@ -1233,7 +1233,7 @@ dependencies = [ [[package]] name = "dragonfly-client-core" -version = "0.1.95" +version = "0.1.96" dependencies = [ "hyper 1.4.0", "hyper-util", @@ -1248,7 +1248,7 @@ dependencies = [ [[package]] name = "dragonfly-client-init" -version = "0.1.95" +version = "0.1.96" dependencies = [ "anyhow", "clap", @@ -1264,7 +1264,7 @@ dependencies = [ [[package]] name = "dragonfly-client-storage" -version = "0.1.95" +version = "0.1.96" dependencies = [ "base16ct", "chrono", @@ -1288,7 +1288,7 @@ dependencies = [ [[package]] name = "dragonfly-client-util" -version = "0.1.95" +version = "0.1.96" dependencies = [ "base16ct", "blake3", @@ -1758,7 +1758,7 @@ dependencies = [ [[package]] name = "hdfs" -version = "0.1.95" +version = "0.1.96" dependencies = [ "dragonfly-client-backend", "dragonfly-client-core", @@ -4865,7 +4865,6 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "flate2", "h2 0.4.2", "http 1.1.0", "http-body 1.0.0", @@ -4883,6 +4882,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "zstd", ] [[package]] @@ -5691,6 +5691,24 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +[[package]] +name = "zstd" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.9+zstd.1.5.5" diff --git a/Cargo.toml b/Cargo.toml index 98853de2..68eee337 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.1.95" +version = "0.1.96" authors = ["The Dragonfly Developers"] homepage = "https://d7y.io/" repository = "https://github.com/dragonflyoss/client.git" @@ -22,13 +22,13 @@ readme = "README.md" edition = "2021" [workspace.dependencies] -dragonfly-client = { path = "dragonfly-client", version = "0.1.95" } -dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.95" } -dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.95" } -dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.95" } -dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.95" } -dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.95" } -dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.95" } +dragonfly-client = { path = "dragonfly-client", version = "0.1.96" } +dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.96" } +dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.96" } +dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.96" } +dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.96" } +dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.96" } +dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.96" } thiserror = "1.0" dragonfly-api = "2.0.143" reqwest = { version = "0.12.4", features = ["stream", "native-tls", "default-tls", "rustls-tls"] } @@ -59,7 +59,7 @@ serde = { version = "1.0", features = ["derive"] } serde_yaml = "0.9" serde_json = "1.0" http = "1" -tonic = { version = "0.12.1", features = ["gzip"] } +tonic = { version = "0.12.1", features = ["zstd"] } tokio = { version = "1.39.2", features = ["full"] } tokio-util = { version = "0.7.11", features = ["full"] } tokio-stream = "0.1.15" diff --git a/dragonfly-client/src/grpc/dfdaemon_download.rs b/dragonfly-client/src/grpc/dfdaemon_download.rs index 80fe842e..78fd4ab7 100644 --- a/dragonfly-client/src/grpc/dfdaemon_download.rs +++ b/dragonfly-client/src/grpc/dfdaemon_download.rs @@ -52,6 +52,7 @@ use tokio::fs; use tokio::net::{UnixListener, UnixStream}; use tokio::sync::mpsc; use tokio_stream::wrappers::{ReceiverStream, UnixListenerStream}; +use tonic::codec::CompressionEncoding; use tonic::{ transport::{Channel, Endpoint, Server, Uri}, Code, Request, Response, Status, @@ -90,6 +91,8 @@ impl DfdaemonDownloadServer { task, cache_task, }) + .send_compressed(CompressionEncoding::Zstd) + .accept_compressed(CompressionEncoding::Zstd) .max_decoding_message_size(usize::MAX) .max_encoding_message_size(usize::MAX); @@ -1001,6 +1004,8 @@ impl DfdaemonDownloadClient { }) .or_err(ErrorType::ConnectError)?; let client = DfdaemonDownloadGRPCClient::new(channel) + .send_compressed(CompressionEncoding::Zstd) + .accept_compressed(CompressionEncoding::Zstd) .max_decoding_message_size(usize::MAX) .max_encoding_message_size(usize::MAX); Ok(Self { client }) diff --git a/dragonfly-client/src/grpc/dfdaemon_upload.rs b/dragonfly-client/src/grpc/dfdaemon_upload.rs index 5cba838a..ed89ab06 100644 --- a/dragonfly-client/src/grpc/dfdaemon_upload.rs +++ b/dragonfly-client/src/grpc/dfdaemon_upload.rs @@ -47,6 +47,7 @@ use std::time::{Duration, Instant}; use tokio::io::AsyncReadExt; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; +use tonic::codec::CompressionEncoding; use tonic::{ transport::{Channel, Server}, Code, Request, Response, Status, @@ -85,6 +86,8 @@ impl DfdaemonUploadServer { task, cache_task, }) + .send_compressed(CompressionEncoding::Zstd) + .accept_compressed(CompressionEncoding::Zstd) .max_decoding_message_size(usize::MAX) .max_encoding_message_size(usize::MAX); @@ -1006,6 +1009,8 @@ impl DfdaemonUploadClient { }) .or_err(ErrorType::ConnectError)?; let client = DfdaemonUploadGRPCClient::new(channel) + .send_compressed(CompressionEncoding::Zstd) + .accept_compressed(CompressionEncoding::Zstd) .max_decoding_message_size(usize::MAX) .max_encoding_message_size(usize::MAX); Ok(Self { client })