From 25ed2960b70954047630fe5c32f6659fefe85e1a Mon Sep 17 00:00:00 2001 From: Gaius Date: Thu, 5 Dec 2024 17:56:37 +0800 Subject: [PATCH] feat: add gzip, deflate, brotli, zstd compressed for reqwest Signed-off-by: Gaius --- Cargo.lock | 53 +++++++++++++++++++ Cargo.toml | 2 +- dragonfly-client-backend/src/http.rs | 8 +++ .../src/object_storage.rs | 4 ++ dragonfly-client-config/src/dfdaemon.rs | 2 +- 5 files changed, 67 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e99f7853..fd8f6bd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,6 +59,21 @@ dependencies = [ "equator", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.20" @@ -195,6 +210,22 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd", + "zstd-safe", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -404,6 +435,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -3583,6 +3635,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ + "async-compression", "base64 0.22.1", "bytes", "encoding_rs", diff --git a/Cargo.toml b/Cargo.toml index be60cbb9..7bb2d805 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.123" } dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.123" } thiserror = "1.0" dragonfly-api = "=2.0.173" -reqwest = { version = "0.12.4", features = ["stream", "native-tls", "default-tls", "rustls-tls"] } +reqwest = { version = "0.12.4", features = ["stream", "native-tls", "default-tls", "rustls-tls", "gzip", "brotli", "zstd", "deflate"] } reqwest-middleware = "0.4" rcgen = { version = "0.12.1", features = ["x509-parser"] } hyper = { version = "1.5", features = ["full"] } diff --git a/dragonfly-client-backend/src/http.rs b/dragonfly-client-backend/src/http.rs index 2e0c2010..7b089a46 100644 --- a/dragonfly-client-backend/src/http.rs +++ b/dragonfly-client-backend/src/http.rs @@ -52,6 +52,10 @@ impl HTTP { .with_no_client_auth(); let client = reqwest::Client::builder() + .gzip(true) + .brotli(true) + .zstd(true) + .deflate(true) .use_preconfigured_tls(client_config_builder) .pool_max_idle_per_host(super::POOL_MAX_IDLE_PER_HOST) .tcp_keepalive(super::KEEP_ALIVE_INTERVAL) @@ -87,6 +91,10 @@ impl HTTP { .with_no_client_auth(); let client = reqwest::Client::builder() + .gzip(true) + .brotli(true) + .zstd(true) + .deflate(true) .use_preconfigured_tls(client_config_builder) .build()?; diff --git a/dragonfly-client-backend/src/object_storage.rs b/dragonfly-client-backend/src/object_storage.rs index b24a6308..2386d0c9 100644 --- a/dragonfly-client-backend/src/object_storage.rs +++ b/dragonfly-client-backend/src/object_storage.rs @@ -181,6 +181,10 @@ impl ObjectStorage { pub fn new(scheme: Scheme) -> ClientResult { // Initialize the reqwest client. let client = reqwest::Client::builder() + .gzip(true) + .brotli(true) + .zstd(true) + .deflate(true) .pool_max_idle_per_host(super::POOL_MAX_IDLE_PER_HOST) .tcp_keepalive(super::KEEP_ALIVE_INTERVAL) .http2_keep_alive_timeout(super::HTTP2_KEEP_ALIVE_TIMEOUT) diff --git a/dragonfly-client-config/src/dfdaemon.rs b/dragonfly-client-config/src/dfdaemon.rs index ed87e94a..08217630 100644 --- a/dragonfly-client-config/src/dfdaemon.rs +++ b/dragonfly-client-config/src/dfdaemon.rs @@ -119,7 +119,7 @@ fn default_download_rate_limit() -> ByteSize { /// default_download_piece_timeout is the default timeout for downloading a piece from source. #[inline] fn default_download_piece_timeout() -> Duration { - Duration::from_secs(10) + Duration::from_secs(15) } /// default_download_concurrent_piece_count is the default number of concurrent pieces to download.