From a8a529cbdb8c7cffec8a84c2d5d753687b9c2e91 Mon Sep 17 00:00:00 2001 From: Jose Quintana Date: Mon, 3 May 2021 01:15:29 +0200 Subject: [PATCH] chore: compression via accept-encoding for text-based files only --- Cargo.lock | 124 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------- Cargo.toml | 5 ++++- src/compression.rs | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/handler.rs | 7 ++++--- src/lib.rs | 1 + 5 files changed, 288 insertions(+), 14 deletions(-) create mode 100644 src/compression.rs diff --git a/Cargo.lock b/Cargo.lock index 7ac142d..4ccedc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,27 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. [[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192ec435945d87bc2f70992b4d818154b5feede43c09fb7592146374eac90a6" + +[[package]] +name = "alloc-stdlib" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] name = "ansi_term" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -16,6 +37,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" [[package]] +name = "async-compression" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] name = "autocfg" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -43,6 +78,27 @@ dependencies = [ ] [[package]] +name = "brotli" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f29919120f08613aadcd4383764e00526fc9f18b6c0895814faeed0dd78613e" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1052e1c3b8d4d80eb84a8b94f0a1498797b5fb96314c001156a1c761940ef4ec" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -102,6 +158,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" [[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -111,6 +176,24 @@ dependencies = [ ] [[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "flate2" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -196,14 +279,14 @@ dependencies = [ [[package]] name = "headers" version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0b7591fb62902706ae8e7aaff416b1b0fa2c0fd0878b46dc13baa3712d8a855" +source = "git+https://github.com/joseluisq/hyper-headers.git?branch=headers_encoding#ca704fcb605adf33f327d0f5a41d5072606058a1" dependencies = [ "base64", "bitflags", "bytes", "headers-core", "http", + "itertools", "mime", "sha-1", "time", @@ -212,8 +295,7 @@ dependencies = [ [[package]] name = "headers-core" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +source = "git+https://github.com/joseluisq/hyper-headers.git?branch=headers_encoding#ca704fcb605adf33f327d0f5a41d5072606058a1" dependencies = [ "http", ] @@ -294,6 +376,15 @@ dependencies = [ ] [[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] name = "itoa" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -373,6 +464,16 @@ dependencies = [ ] [[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] name = "mio" version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -539,9 +640,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efb2352a0f4d4b128f734b5c44c79ff80117351138733f12f982fe3e2b13343" +checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b" dependencies = [ "regex-syntax", ] @@ -558,9 +659,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.24" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00efb87459ba4f6fb2169d20f68565555688e1250ee6825cdf6254f8b48fafb2" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "ryu" @@ -638,9 +739,11 @@ name = "static-web-server" version = "2.0.0-beta.3" dependencies = [ "anyhow", + "async-compression", "bytes", "futures", "headers", + "http", "hyper", "jemallocator", "mime_guess", @@ -648,6 +751,7 @@ dependencies = [ "num_cpus", "once_cell", "percent-encoding", + "pin-project", "signal", "structopt", "tokio", @@ -822,9 +926,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705096c6f83bf68ea5d357a6aa01829ddbdac531b357b45abeca842938085baa" +checksum = "aa5553bf0883ba7c9cbe493b085c29926bd41b66afc31ff72cf17ff4fb60dcd5" dependencies = [ "ansi_term", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 25a0772..e6a1cc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,9 @@ path = "src/bin/server.rs" hyper = { version = "0.14", features = ["stream", "http1", "tcp", "server"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "fs", "io-util"], default-features = false } futures = { version = "0.3", default-features = false } -headers = "0.3" +async-compression = { version = "0.3", features = ["brotli", "deflate", "gzip", "tokio"] } +headers = { git = "https://github.com/joseluisq/hyper-headers.git", branch = "headers_encoding" } +http = "0.2" tokio-util = { version = "0.6", features = ["io"] } anyhow = "1.0" tracing = "0.1" @@ -39,6 +41,7 @@ percent-encoding = "2.1" structopt = { version = "0.3", default-features = false } num_cpus = { version = "1.13" } once_cell = "1.7" +pin-project = "1.0" [target.'cfg(not(windows))'.dependencies.nix] version = "0.14" diff --git a/src/compression.rs b/src/compression.rs new file mode 100644 index 0000000..1d6c9e5 --- /dev/null +++ b/src/compression.rs @@ -0,0 +1,165 @@ +// Compression handler that compress the body of a response. +// -> Part of the file is borrowed from https://github.com/seanmonstar/warp/pull/513 + +use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder}; +use bytes::Bytes; +use futures::Stream; +use headers::{AcceptEncoding, ContentCoding, ContentType, HeaderMap, HeaderMapExt}; +use http::header::HeaderValue; +use hyper::{ + header::{CONTENT_ENCODING, CONTENT_LENGTH}, + Body, Response, +}; +use pin_project::pin_project; +use std::convert::TryFrom; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio_util::io::{ReaderStream, StreamReader}; + +use crate::error::Result; + +/// Contains a fixed list of common text-based MIME types in order to apply compression. +pub const TEXT_MIME_TYPES: [&str; 16] = [ + "text/html", + "text/css", + "text/javascript", + "text/xml", + "text/plain", + "text/x-component", + "application/javascript", + "application/x-javascript", + "application/json", + "application/xml", + "application/rss+xml", + "application/atom+xml", + "font/truetype", + "font/opentype", + "application/vnd.ms-fontobject", + "image/svg+xml", +]; + +/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response) +/// using `gzip`, `deflate` or `brotli` if is specified in the `Accept-Encoding` header, adding +/// `content-encoding: ` to the Response's [`HeaderMap`](hyper::HeaderMap) +/// It also provides the ability to apply compression for text-based MIME types only. +pub fn auto(headers: &HeaderMap, resp: Response) -> Result> { + // Skip compression for non-text-based MIME types + if let Some(content_type) = resp.headers().typed_get::() { + let content_type = content_type.to_string(); + if !TEXT_MIME_TYPES.iter().any(|h| *h == content_type) { + return Ok(resp); + } + } + + if let Some(ref accept_encoding) = headers.typed_get::() { + if let Some(encoding) = accept_encoding.prefered_encoding() { + if encoding == ContentCoding::GZIP { + let (head, body) = resp.into_parts(); + return Ok(gzip(head, body.into())); + } + if encoding == ContentCoding::DEFLATE { + let (head, body) = resp.into_parts(); + return Ok(deflate(head, body.into())); + } + if encoding == ContentCoding::BROTLI { + let (head, body) = resp.into_parts(); + return Ok(brotli(head, body.into())); + } + } + } + + Ok(resp) +} + +/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response) +/// using gzip, adding `content-encoding: gzip` to the Response's [`HeaderMap`](hyper::HeaderMap) +pub fn gzip( + mut head: http::response::Parts, + body: CompressableBody, +) -> Response { + let body = Body::wrap_stream(ReaderStream::new(GzipEncoder::new(StreamReader::new(body)))); + let header = create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::GZIP); + head.headers.remove(CONTENT_LENGTH); + head.headers.append(CONTENT_ENCODING, header); + Response::from_parts(head, body) +} + +/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response) +/// using deflate, adding `content-encoding: deflate` to the Response's [`HeaderMap`](hyper::HeaderMap) +pub fn deflate( + mut head: http::response::Parts, + body: CompressableBody, +) -> Response { + let body = Body::wrap_stream(ReaderStream::new(DeflateEncoder::new(StreamReader::new( + body, + )))); + let header = create_encoding_header( + head.headers.remove(CONTENT_ENCODING), + ContentCoding::DEFLATE, + ); + head.headers.remove(CONTENT_LENGTH); + head.headers.append(CONTENT_ENCODING, header); + Response::from_parts(head, body) +} + +/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response) +/// using brotli, adding `content-encoding: br` to the Response's [`HeaderMap`](hyper::HeaderMap) +pub fn brotli( + mut head: http::response::Parts, + body: CompressableBody, +) -> Response { + let body = Body::wrap_stream(ReaderStream::new(BrotliEncoder::new(StreamReader::new( + body, + )))); + let header = + create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::BROTLI); + head.headers.remove(CONTENT_LENGTH); + head.headers.append(CONTENT_ENCODING, header); + Response::from_parts(head, body) +} + +/// Given an optional existing encoding header, appends to the existing or creates a new one. +fn create_encoding_header(existing: Option, coding: ContentCoding) -> HeaderValue { + if let Some(val) = existing { + if let Ok(str_val) = val.to_str() { + return HeaderValue::try_from(&format!("{}, {}", str_val, coding.to_string())) + .unwrap_or_else(|_| coding.into()); + } + } + coding.into() +} +/// A wrapper around any type that implements [`Stream`](futures::Stream) to be +/// compatible with async_compression's Stream based encoders. +#[pin_project] +#[derive(Debug)] +pub struct CompressableBody +where + S: Stream>, + E: std::error::Error, +{ + #[pin] + pub body: S, +} + +impl Stream for CompressableBody +where + S: Stream>, + E: std::error::Error, +{ + type Item = std::io::Result; + + fn poll_next(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll> { + use std::io::{Error, ErrorKind}; + + let pin = self.project(); + // TODO: Use `.map_err()` (https://github.com/rust-lang/rust/issues/63514) once it is stabilized + S::poll_next(pin.body, ctx) + .map(|err| err.map(|res| res.map_err(|_| Error::from(ErrorKind::InvalidData)))) + } +} + +impl From for CompressableBody { + fn from(body: Body) -> Self { + CompressableBody { body } + } +} diff --git a/src/handler.rs b/src/handler.rs index 5dfe141..e7a7f0b 100644 --- a/src/handler.rs +++ b/src/handler.rs @@ -1,13 +1,14 @@ use hyper::{Body, Request, Response}; use std::path::Path; -use crate::static_files; +use crate::{compression, static_files}; use crate::{error::Result, error_page}; /// Main server request handler. pub async fn handle_request(base: &Path, req: Request) -> Result> { - match static_files::handle_request(base, req.headers(), req.uri().path()).await { - Ok(resp) => Ok(resp), + let headers = req.headers(); + match static_files::handle_request(base, headers, req.uri().path()).await { + Ok(resp) => compression::auto(headers, resp), Err(status) => error_page::get_error_response(req.method(), &status), } } diff --git a/src/lib.rs b/src/lib.rs index add858b..dd10823 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ #[macro_use] extern crate anyhow; +pub mod compression; pub mod config; pub mod error_page; pub mod handler; -- libgit2 1.7.2