From 137aefa2bedfdf5fa6e8c9adbc5c676e55a3da4e Mon Sep 17 00:00:00 2001 From: Jose Quintana Date: Tue, 27 Apr 2021 21:57:57 +0200 Subject: [PATCH] feat: auto compression filter based on accept-encoding header --- Cargo.lock | 24 +++++++++++++++++++----- src/compression.rs | 19 +++++++++++++++++++ src/lib.rs | 1 + src/server.rs | 249 ++++++++++++++++++++++++++++++++++++++++++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 4 files changed, 81 insertions(+), 212 deletions(-) create mode 100644 src/compression.rs diff --git a/Cargo.lock b/Cargo.lock index 233a2e7..a4d5eb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,6 +182,12 @@ dependencies = [ ] [[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] name = "flate2" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -314,14 +320,14 @@ checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" [[package]] name = "headers" version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0b7591fb62902706ae8e7aaff416b1b0fa2c0fd0878b46dc13baa3712d8a855" +source = "git+https://github.com/joseluisq/hyper-headers.git?branch=headers_encoding#ca704fcb605adf33f327d0f5a41d5072606058a1" dependencies = [ "base64", "bitflags", "bytes", "headers-core", "http", + "itertools", "mime", "sha-1", "time", @@ -330,8 +336,7 @@ dependencies = [ [[package]] name = "headers-core" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +source = "git+https://github.com/joseluisq/hyper-headers.git?branch=headers_encoding#ca704fcb605adf33f327d0f5a41d5072606058a1" dependencies = [ "http", ] @@ -423,6 +428,15 @@ dependencies = [ ] [[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] name = "itoa" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1139,7 +1153,7 @@ dependencies = [ [[package]] name = "warp" version = "0.3.1" -source = "git+https://github.com/joseluisq/warp.git?branch=0.3.x#ca25ca76e62d3c1438f8b87c522120a629ec945a" +source = "git+https://github.com/joseluisq/warp.git?branch=0.3.x#f638f8958addb953501a08d427aae64a4c4f5a21" dependencies = [ "async-compression", "bytes", diff --git a/src/compression.rs b/src/compression.rs new file mode 100644 index 0000000..7dd180d --- /dev/null +++ b/src/compression.rs @@ -0,0 +1,19 @@ +/// Contains a common fixed list of text-based MIME types in order to apply compression. +pub const TEXT_MIME_TYPES: [&str; 16] = [ + "text/html", + "text/css", + "text/javascript", + "text/xml", + "text/plain", + "text/x-component", + "application/javascript", + "application/x-javascript", + "application/json", + "application/xml", + "application/rss+xml", + "application/atom+xml", + "font/truetype", + "font/opentype", + "application/vnd.ms-fontobject", + "image/svg+xml", +]; diff --git a/src/lib.rs b/src/lib.rs index df11e98..91a5d4c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ extern crate anyhow; pub mod cache; +pub mod compression; pub mod config; pub mod cors; pub mod filters; diff --git a/src/server.rs b/src/server.rs index 90c69e2..bc887fe 100644 --- a/src/server.rs +++ b/src/server.rs @@ -3,8 +3,11 @@ use std::path::PathBuf; use structopt::StructOpt; use warp::Filter; -use crate::config::{Config, CONFIG}; -use crate::{cache, cors, filters, helpers, logger, rejection, Result}; +use crate::{cache, cors, helpers, logger, rejection, Result}; +use crate::{ + compression::TEXT_MIME_TYPES, + config::{Config, CONFIG}, +}; /// Define a multi-thread HTTP or HTTP/2 web server. pub struct Server { @@ -74,50 +77,16 @@ impl Server { let http2_tls_cert_path = &opts.http2_tls_cert; let http2_tls_key_path = &opts.http2_tls_key; - // Spawn a new Tokio asynchronous server task determined by the given compression type (gzip, brotli or none) - // TODO: this can be simplified by replicating something similar to `warp::compression::auto()` but skipping `deflate - // see Warp PR #513 https://github.com/seanmonstar/warp/pull/513 - tokio::task::spawn(async move { - if opts.compression == "brotli" { - run_server_with_brotli_compression( - addr, - root_dir, - http2, - http2_tls_cert_path, - http2_tls_key_path, - cors_filter_opt, - cors_allowed_origins, - ) - .await; - return; - } - - if opts.compression == "gzip" { - run_server_with_gzip_compression( - addr, - root_dir, - http2, - http2_tls_cert_path, - http2_tls_key_path, - cors_filter_opt, - cors_allowed_origins, - ) - .await; - return; - } - - // Fallback HTTP or HTTP/2 server with no compression - run_server_with_no_compression( - addr, - root_dir, - http2, - http2_tls_cert_path, - http2_tls_key_path, - cors_filter_opt, - cors_allowed_origins, - ) - .await - }); + // Spawn a new Tokio asynchronous server task determined by the given options + tokio::task::spawn(run_server_with_options( + addr, + root_dir, + http2, + http2_tls_cert_path, + http2_tls_key_path, + cors_filter_opt, + cors_allowed_origins, + )); handle_signals(); @@ -125,101 +94,14 @@ impl Server { } } -#[cfg(not(windows))] -/// Handle incoming signals for Unix-like OS's only -fn handle_signals() { - use crate::signals; - - signals::wait(|sig: signals::Signal| { - let code = signals::as_int(sig); - tracing::warn!("Signal {} caught. Server execution exited.", code); - std::process::exit(code) - }); -} - -#[cfg(windows)] -fn handle_signals() { - // TODO: Windows signals... -} - -/// It creates and starts a Warp HTTP or HTTP/2 server with Brotli compression. -pub async fn run_server_with_brotli_compression( - addr: SocketAddr, - root_dir: PathBuf, - http2: bool, - http2_tls_cert_path: &'static str, - http2_tls_key_path: &'static str, - cors_filter_opt: Option, - cors_allowed_origins: String, -) { - // Base fs directory filter - let base_fs_dir_filter = warp::fs::dir(root_dir.clone()) - .map(cache::control_headers) - .with(warp::trace::request()) - .recover(rejection::handle_rejection); - - // Public HEAD endpoint - let public_head = warp::head().and(base_fs_dir_filter.clone()); - - // Public GET endpoint (default) - let public_get_default = warp::get().and(base_fs_dir_filter); - - // Current fs directory filter - let fs_dir_filter = warp::fs::dir(root_dir) - .map(cache::control_headers) - .with(warp::compression::brotli(true)) - .with(warp::trace::request()) - .recover(rejection::handle_rejection); - - // Determine CORS filter - if let Some(cors_filter) = cors_filter_opt { - tracing::info!( - cors_enabled = ?true, - allowed_origins = ?cors_allowed_origins - ); - - let public_head = public_head.with(cors_filter.clone()); - let public_get_default = public_get_default.with(cors_filter.clone()); - - let public_get = warp::get() - .and(filters::has_accept_encoding("br")) - .and(fs_dir_filter) - .with(cors_filter.clone()); - - let server = warp::serve(public_head.or(public_get).or(public_get_default)); - - if http2 { - server - .tls() - .cert_path(http2_tls_cert_path) - .key_path(http2_tls_key_path) - .run(addr) - .await - } else { - server.run(addr).await - } - } else { - let public_get = warp::get() - .and(filters::has_accept_encoding("br")) - .and(fs_dir_filter); - - let server = warp::serve(public_head.or(public_get).or(public_get_default)); - - if http2 { - server - .tls() - .cert_path(http2_tls_cert_path) - .key_path(http2_tls_key_path) - .run(addr) - .await - } else { - server.run(addr).await - } +impl Default for Server { + fn default() -> Self { + Self::new() } } -/// It creates and starts a Warp HTTP or HTTP/2 server with GZIP compression. -pub async fn run_server_with_gzip_compression( +/// It creates and starts a Warp HTTP or HTTP/2 server with its options. +pub async fn run_server_with_options( addr: SocketAddr, root_dir: PathBuf, http2: bool, @@ -243,7 +125,14 @@ pub async fn run_server_with_gzip_compression( // Current fs directory filter let fs_dir_filter = warp::fs::dir(root_dir) .map(cache::control_headers) - .with(warp::compression::gzip(true)) + .with(warp::compression::auto(|headers| { + // Skip compression for non-text-based MIME types + if let Some(content_type) = headers.get("content-type") { + !TEXT_MIME_TYPES.iter().any(|h| h == content_type) + } else { + false + } + })) .with(warp::trace::request()) .recover(rejection::handle_rejection); @@ -256,11 +145,7 @@ pub async fn run_server_with_gzip_compression( let public_head = public_head.with(cors_filter.clone()); let public_get_default = public_get_default.with(cors_filter.clone()); - - let public_get = warp::get() - .and(filters::has_accept_encoding("gzip")) - .and(fs_dir_filter) - .with(cors_filter.clone()); + let public_get = warp::get().and(fs_dir_filter).with(cors_filter.clone()); let server = warp::serve(public_head.or(public_get).or(public_get_default)); @@ -275,9 +160,7 @@ pub async fn run_server_with_gzip_compression( server.run(addr).await } } else { - let public_get = warp::get() - .and(filters::has_accept_encoding("gzip")) - .and(fs_dir_filter); + let public_get = warp::get().and(fs_dir_filter); let server = warp::serve(public_head.or(public_get).or(public_get_default)); @@ -294,67 +177,19 @@ pub async fn run_server_with_gzip_compression( } } -/// It creates and starts a Warp HTTP or HTTP/2 server with no compression. -pub async fn run_server_with_no_compression( - addr: SocketAddr, - root_dir: PathBuf, - http2: bool, - http2_tls_cert_path: &'static str, - http2_tls_key_path: &'static str, - cors_filter_opt: Option, - cors_allowed_origins: String, -) { - // Base fs directory filter - let base_fs_dir_filter = warp::fs::dir(root_dir.clone()) - .map(cache::control_headers) - .with(warp::trace::request()) - .recover(rejection::handle_rejection); - - // Public HEAD endpoint - let public_head = warp::head().and(base_fs_dir_filter.clone()); - - // Public GET endpoint (default) - let public_get_default = warp::get().and(base_fs_dir_filter); - - // Determine CORS filter - if let Some(cors_filter) = cors_filter_opt { - tracing::info!( - cors_enabled = ?true, - allowed_origins = ?cors_allowed_origins - ); - - let public_get = public_get_default.with(cors_filter.clone()); - - let server = warp::serve(public_head.or(public_get)); - - if http2 { - server - .tls() - .cert_path(http2_tls_cert_path) - .key_path(http2_tls_key_path) - .run(addr) - .await - } else { - server.run(addr).await - } - } else { - let server = warp::serve(public_head.or(public_get_default)); +#[cfg(not(windows))] +/// Handle incoming signals for Unix-like OS's only +fn handle_signals() { + use crate::signals; - if http2 { - server - .tls() - .cert_path(http2_tls_cert_path) - .key_path(http2_tls_key_path) - .run(addr) - .await - } else { - server.run(addr).await - } - } + signals::wait(|sig: signals::Signal| { + let code = signals::as_int(sig); + tracing::warn!("Signal {} caught. Server execution exited.", code); + std::process::exit(code) + }); } -impl Default for Server { - fn default() -> Self { - Self::new() - } +#[cfg(windows)] +fn handle_signals() { + // TODO: Windows signals... } -- libgit2 1.7.2