index : static-web-server.git

ascending towards madness

author Jose Quintana <joseluisquintana20@gmail.com> 2021-05-02 23:15:29.0 +00:00:00
committer Jose Quintana <joseluisquintana20@gmail.com> 2021-05-02 23:15:29.0 +00:00:00
commit 
a8a529cbdb8c7cffec8a84c2d5d753687b9c2e91 [patch]
tree 
ed6be729603973dcefec2c41661ef4f3ac9a2620
parent 
d453567468cb15e2c8b4213c875497ce0d7e53e8
download 
a8a529cbdb8c7cffec8a84c2d5d753687b9c2e91.tar.gz

chore: compression via accept-encoding for text-based files only




Diff


 Cargo.lock         | 124 +++++++++++++++++++++++++++++++++++++----
 Cargo.toml         |   5 +-
 src/compression.rs | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 src/handler.rs     |   7 +-
 src/lib.rs         |   1 +-
 5 files changed, 288 insertions(+), 14 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7ac142d..4ccedc2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,6 +1,27 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"

[[package]]
name = "alloc-no-stdlib"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5192ec435945d87bc2f70992b4d818154b5feede43c09fb7592146374eac90a6"

[[package]]
name = "alloc-stdlib"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2"
dependencies = [
 "alloc-no-stdlib",
]

[[package]]
name = "ansi_term"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -16,6 +37,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b"

[[package]]
name = "async-compression"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6"
dependencies = [
 "brotli",
 "flate2",
 "futures-core",
 "memchr",
 "pin-project-lite",
 "tokio",
]

[[package]]
name = "autocfg"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -43,6 +78,27 @@ dependencies = [
]

[[package]]
name = "brotli"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f29919120f08613aadcd4383764e00526fc9f18b6c0895814faeed0dd78613e"
dependencies = [
 "alloc-no-stdlib",
 "alloc-stdlib",
 "brotli-decompressor",
]

[[package]]
name = "brotli-decompressor"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1052e1c3b8d4d80eb84a8b94f0a1498797b5fb96314c001156a1c761940ef4ec"
dependencies = [
 "alloc-no-stdlib",
 "alloc-stdlib",
]

[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -102,6 +158,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"

[[package]]
name = "crc32fast"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
dependencies = [
 "cfg-if 1.0.0",
]

[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -111,6 +176,24 @@ dependencies = [
]

[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"

[[package]]
name = "flate2"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0"
dependencies = [
 "cfg-if 1.0.0",
 "crc32fast",
 "libc",
 "miniz_oxide",
]

[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -196,14 +279,14 @@ dependencies = [
[[package]]
name = "headers"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0b7591fb62902706ae8e7aaff416b1b0fa2c0fd0878b46dc13baa3712d8a855"
source = "git+https://github.com/joseluisq/hyper-headers.git?branch=headers_encoding#ca704fcb605adf33f327d0f5a41d5072606058a1"
dependencies = [
 "base64",
 "bitflags",
 "bytes",
 "headers-core",
 "http",
 "itertools",
 "mime",
 "sha-1",
 "time",
@@ -212,8 +295,7 @@ dependencies = [
[[package]]
name = "headers-core"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429"
source = "git+https://github.com/joseluisq/hyper-headers.git?branch=headers_encoding#ca704fcb605adf33f327d0f5a41d5072606058a1"
dependencies = [
 "http",
]
@@ -294,6 +376,15 @@ dependencies = [
]

[[package]]
name = "itertools"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
dependencies = [
 "either",
]

[[package]]
name = "itoa"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -373,6 +464,16 @@ dependencies = [
]

[[package]]
name = "miniz_oxide"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
dependencies = [
 "adler",
 "autocfg",
]

[[package]]
name = "mio"
version = "0.7.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -539,9 +640,9 @@ dependencies = [

[[package]]
name = "regex"
version = "1.5.2"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efb2352a0f4d4b128f734b5c44c79ff80117351138733f12f982fe3e2b13343"
checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b"
dependencies = [
 "regex-syntax",
]
@@ -558,9 +659,9 @@ dependencies = [

[[package]]
name = "regex-syntax"
version = "0.6.24"
version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00efb87459ba4f6fb2169d20f68565555688e1250ee6825cdf6254f8b48fafb2"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"

[[package]]
name = "ryu"
@@ -638,9 +739,11 @@ name = "static-web-server"
version = "2.0.0-beta.3"
dependencies = [
 "anyhow",
 "async-compression",
 "bytes",
 "futures",
 "headers",
 "http",
 "hyper",
 "jemallocator",
 "mime_guess",
@@ -648,6 +751,7 @@ dependencies = [
 "num_cpus",
 "once_cell",
 "percent-encoding",
 "pin-project",
 "signal",
 "structopt",
 "tokio",
@@ -822,9 +926,9 @@ dependencies = [

[[package]]
name = "tracing-subscriber"
version = "0.2.17"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "705096c6f83bf68ea5d357a6aa01829ddbdac531b357b45abeca842938085baa"
checksum = "aa5553bf0883ba7c9cbe493b085c29926bd41b66afc31ff72cf17ff4fb60dcd5"
dependencies = [
 "ansi_term",
 "chrono",
diff --git a/Cargo.toml b/Cargo.toml
index 25a0772..e6a1cc1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -28,7 +28,9 @@ path = "src/bin/server.rs"
hyper = { version = "0.14", features = ["stream", "http1", "tcp", "server"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "fs", "io-util"], default-features = false }
futures = { version = "0.3", default-features = false }
headers = "0.3"
async-compression = { version = "0.3", features = ["brotli", "deflate", "gzip", "tokio"] }
headers = { git = "https://github.com/joseluisq/hyper-headers.git", branch = "headers_encoding" }
http = "0.2"
tokio-util = { version = "0.6", features = ["io"] }
anyhow = "1.0"
tracing = "0.1"
@@ -39,6 +41,7 @@ percent-encoding = "2.1"
structopt = { version = "0.3", default-features = false }
num_cpus = { version = "1.13" }
once_cell = "1.7"
pin-project = "1.0"

[target.'cfg(not(windows))'.dependencies.nix]
version = "0.14"
diff --git a/src/compression.rs b/src/compression.rs
new file mode 100644
index 0000000..1d6c9e5
--- /dev/null
+++ b/src/compression.rs
@@ -0,0 +1,165 @@
// Compression handler that compress the body of a response.
// -> Part of the file is borrowed from https://github.com/seanmonstar/warp/pull/513

use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder};
use bytes::Bytes;
use futures::Stream;
use headers::{AcceptEncoding, ContentCoding, ContentType, HeaderMap, HeaderMapExt};
use http::header::HeaderValue;
use hyper::{
    header::{CONTENT_ENCODING, CONTENT_LENGTH},
    Body, Response,
};
use pin_project::pin_project;
use std::convert::TryFrom;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio_util::io::{ReaderStream, StreamReader};

use crate::error::Result;

/// Contains a fixed list of common text-based MIME types in order to apply compression.
pub const TEXT_MIME_TYPES: [&str; 16] = [
    "text/html",
    "text/css",
    "text/javascript",
    "text/xml",
    "text/plain",
    "text/x-component",
    "application/javascript",
    "application/x-javascript",
    "application/json",
    "application/xml",
    "application/rss+xml",
    "application/atom+xml",
    "font/truetype",
    "font/opentype",
    "application/vnd.ms-fontobject",
    "image/svg+xml",
];

/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response)
/// using `gzip`, `deflate` or `brotli` if is specified in the `Accept-Encoding` header, adding
/// `content-encoding: <coding>` to the Response's [`HeaderMap`](hyper::HeaderMap)
/// It also provides the ability to apply compression for text-based MIME types only.
pub fn auto(headers: &HeaderMap<HeaderValue>, resp: Response<Body>) -> Result<Response<Body>> {
    // Skip compression for non-text-based MIME types
    if let Some(content_type) = resp.headers().typed_get::<ContentType>() {
        let content_type = content_type.to_string();
        if !TEXT_MIME_TYPES.iter().any(|h| *h == content_type) {
            return Ok(resp);
        }
    }

    if let Some(ref accept_encoding) = headers.typed_get::<AcceptEncoding>() {
        if let Some(encoding) = accept_encoding.prefered_encoding() {
            if encoding == ContentCoding::GZIP {
                let (head, body) = resp.into_parts();
                return Ok(gzip(head, body.into()));
            }
            if encoding == ContentCoding::DEFLATE {
                let (head, body) = resp.into_parts();
                return Ok(deflate(head, body.into()));
            }
            if encoding == ContentCoding::BROTLI {
                let (head, body) = resp.into_parts();
                return Ok(brotli(head, body.into()));
            }
        }
    }

    Ok(resp)
}

/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response)
/// using gzip, adding `content-encoding: gzip` to the Response's [`HeaderMap`](hyper::HeaderMap)
pub fn gzip(
    mut head: http::response::Parts,
    body: CompressableBody<Body, hyper::Error>,
) -> Response<Body> {
    let body = Body::wrap_stream(ReaderStream::new(GzipEncoder::new(StreamReader::new(body))));
    let header = create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::GZIP);
    head.headers.remove(CONTENT_LENGTH);
    head.headers.append(CONTENT_ENCODING, header);
    Response::from_parts(head, body)
}

/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response)
/// using deflate, adding `content-encoding: deflate` to the Response's [`HeaderMap`](hyper::HeaderMap)
pub fn deflate(
    mut head: http::response::Parts,
    body: CompressableBody<Body, hyper::Error>,
) -> Response<Body> {
    let body = Body::wrap_stream(ReaderStream::new(DeflateEncoder::new(StreamReader::new(
        body,
    ))));
    let header = create_encoding_header(
        head.headers.remove(CONTENT_ENCODING),
        ContentCoding::DEFLATE,
    );
    head.headers.remove(CONTENT_LENGTH);
    head.headers.append(CONTENT_ENCODING, header);
    Response::from_parts(head, body)
}

/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response)
/// using brotli, adding `content-encoding: br` to the Response's [`HeaderMap`](hyper::HeaderMap)
pub fn brotli(
    mut head: http::response::Parts,
    body: CompressableBody<Body, hyper::Error>,
) -> Response<Body> {
    let body = Body::wrap_stream(ReaderStream::new(BrotliEncoder::new(StreamReader::new(
        body,
    ))));
    let header =
        create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::BROTLI);
    head.headers.remove(CONTENT_LENGTH);
    head.headers.append(CONTENT_ENCODING, header);
    Response::from_parts(head, body)
}

/// Given an optional existing encoding header, appends to the existing or creates a new one.
fn create_encoding_header(existing: Option<HeaderValue>, coding: ContentCoding) -> HeaderValue {
    if let Some(val) = existing {
        if let Ok(str_val) = val.to_str() {
            return HeaderValue::try_from(&format!("{}, {}", str_val, coding.to_string()))
                .unwrap_or_else(|_| coding.into());
        }
    }
    coding.into()
}
/// A wrapper around any type that implements [`Stream`](futures::Stream) to be
/// compatible with async_compression's Stream based encoders.
#[pin_project]
#[derive(Debug)]
pub struct CompressableBody<S, E>
where
    S: Stream<Item = Result<Bytes, E>>,
    E: std::error::Error,
{
    #[pin]
    pub body: S,
}

impl<S, E> Stream for CompressableBody<S, E>
where
    S: Stream<Item = Result<Bytes, E>>,
    E: std::error::Error,
{
    type Item = std::io::Result<Bytes>;

    fn poll_next(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
        use std::io::{Error, ErrorKind};

        let pin = self.project();
        // TODO: Use `.map_err()` (https://github.com/rust-lang/rust/issues/63514) once it is stabilized
        S::poll_next(pin.body, ctx)
            .map(|err| err.map(|res| res.map_err(|_| Error::from(ErrorKind::InvalidData))))
    }
}

impl From<Body> for CompressableBody<Body, hyper::Error> {
    fn from(body: Body) -> Self {
        CompressableBody { body }
    }
}
diff --git a/src/handler.rs b/src/handler.rs
index 5dfe141..e7a7f0b 100644
--- a/src/handler.rs
+++ b/src/handler.rs
@@ -1,13 +1,14 @@
use hyper::{Body, Request, Response};
use std::path::Path;

use crate::static_files;
use crate::{compression, static_files};
use crate::{error::Result, error_page};

/// Main server request handler.
pub async fn handle_request(base: &Path, req: Request<Body>) -> Result<Response<Body>> {
    match static_files::handle_request(base, req.headers(), req.uri().path()).await {
        Ok(resp) => Ok(resp),
    let headers = req.headers();
    match static_files::handle_request(base, headers, req.uri().path()).await {
        Ok(resp) => compression::auto(headers, resp),
        Err(status) => error_page::get_error_response(req.method(), &status),
    }
}
diff --git a/src/lib.rs b/src/lib.rs
index add858b..dd10823 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -3,6 +3,7 @@
#[macro_use]
extern crate anyhow;

pub mod compression;
pub mod config;
pub mod error_page;
pub mod handler;