From 2bebec7cce410db52774996d255540bae2023906 Mon Sep 17 00:00:00 2001 From: Jose Quintana <1700322+joseluisq@users.noreply.github.com> Date: Thu, 20 Apr 2023 00:56:05 +0200 Subject: [PATCH] feat: zstd auto-compression support (#197) * feat: zstd auto-compression support * feat: `zstd` pre-compressed files support * chore: update `headers-accept-encoding` which supports zstd `content-encoding` dependency --- Cargo.lock | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 4 ++-- src/compression.rs | 21 ++++++++++++++++++++- src/compression_static.rs | 2 ++ 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8fdfc5..ea11cd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,6 +59,8 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", + "zstd", + "zstd-safe", ] [[package]] @@ -171,6 +173,9 @@ name = "cc" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -442,9 +447,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "headers-accept-encoding" -version = "0.3.8" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "450f8482d4d4ec4db6d82d0e8ce6c3cd5421dd469bc0104e43c635c11a645314" +checksum = "f31dd148a6d59ca552b4b30a018188701f588c0d398da9e280a455775e450d69" dependencies = [ "base64 0.13.1", "bitflags", @@ -620,6 +625,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + +[[package]] name = "js-sys" version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -841,6 +855,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] name = "proc-macro-error" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1700,3 +1720,33 @@ name = "zeroize" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 138b25b..d8499e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,13 +43,13 @@ http2 = ["tls"] [dependencies] anyhow = "1.0" -async-compression = { version = "0.3", default-features = false, features = ["brotli", "deflate", "gzip", "tokio"] } +async-compression = { version = "0.3", default-features = false, features = ["brotli", "deflate", "gzip", "zstd", "tokio"] } bcrypt = "0.14" bytes = "1.4" form_urlencoded = "1.1" futures-util = { version = "0.3", default-features = false, features = ["sink"] } globset = { version = "0.4", features = ["serde1"] } -headers = { package = "headers-accept-encoding", version = "0.3" } +headers = { package = "headers-accept-encoding", version = "1.0" } http = "0.2" http-serde = "1.1" humansize = { version = "2.1", features = ["impl_style"] } diff --git a/src/compression.rs b/src/compression.rs index a39b1c1..110723b 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -3,7 +3,7 @@ // Part of the file is borrowed from * -use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder}; +use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder, ZstdEncoder}; use bytes::Bytes; use futures_util::Stream; use headers::{AcceptEncoding, ContentCoding, ContentType, HeaderMap, HeaderMapExt}; @@ -91,6 +91,10 @@ pub fn auto( let (head, body) = resp.into_parts(); return Ok(brotli(head, body.into())); } + if encoding == ContentCoding::ZSTD { + let (head, body) = resp.into_parts(); + return Ok(zstd(head, body.into())); + } } Ok(resp) @@ -149,6 +153,21 @@ pub fn brotli( Response::from_parts(head, body) } +/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response) +/// using zstd, adding `content-encoding: zstd` to the Response's [`HeaderMap`](hyper::HeaderMap) +pub fn zstd( + mut head: http::response::Parts, + body: CompressableBody, +) -> Response { + tracing::trace!("compressing response body on the fly using zstd"); + + let body = Body::wrap_stream(ReaderStream::new(ZstdEncoder::new(StreamReader::new(body)))); + let header = create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::ZSTD); + head.headers.remove(CONTENT_LENGTH); + head.headers.append(CONTENT_ENCODING, header); + Response::from_parts(head, body) +} + /// Given an optional existing encoding header, appends to the existing or creates a new one. pub fn create_encoding_header(existing: Option, coding: ContentCoding) -> HeaderValue { if let Some(val) = existing { diff --git a/src/compression_static.rs b/src/compression_static.rs index 85578b2..8948cf6 100644 --- a/src/compression_static.rs +++ b/src/compression_static.rs @@ -36,6 +36,8 @@ pub async fn precompressed_variant<'a>( Some(ContentCoding::GZIP | ContentCoding::DEFLATE) => "gz", // https://peazip.github.io/brotli-compressed-file-format.html Some(ContentCoding::BROTLI) => "br", + // https://datatracker.ietf.org/doc/html/rfc8878 + Some(ContentCoding::ZSTD) => "zst", _ => { tracing::trace!( "preferred encoding based on the file extension was not determined, skipping" -- libgit2 1.7.2