From 48f9458fed9a9d4f4b0f7a20bd6cdf5df51a5258 Mon Sep 17 00:00:00 2001 From: Jose Quintana <1700322+joseluisq@users.noreply.github.com> Date: Fri, 23 Sep 2022 23:15:44 +0200 Subject: [PATCH] Support for serving pre-compressed files (#139) * feat: support for serving pre-compressed files via new boolean `compression-static` option * refactor: continue workflow when the pre-compressed file is not found * chore: preliminary precompressed response * refactor: pre-compressed file variant handling * refactor: bump up pinned rust version to 1.59.0 on ci * refactor: compression_static module * refactor: optimize file metadata search for pre-compressed variant * tests: compression static * refactor: some tests and static file variables --- .github/workflows/devel.yml | 2 +- .gitignore | 1 + src/compression.rs | 54 +++++++++++++++++++++++++++++++++--------------------- src/compression_static.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/directory_listing.rs | 4 ++-- src/handler.rs | 8 ++++++-- src/lib.rs | 1 + src/server.rs | 5 +++++ src/settings/cli.rs | 10 ++++++++++ src/settings/file.rs | 3 +++ src/settings/mod.rs | 5 +++++ src/static_files.rs | 186 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------------------------- tests/compression_static.rs | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/dir_listing.rs | 12 ++++++++---- tests/fixtures/public/index.html.gz | Bin 0 -> 332 bytes tests/static_files.rs | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------- tests/toml/config.toml | 3 +++ 17 files changed, 439 insertions(+), 111 deletions(-) create mode 100644 src/compression_static.rs create mode 100644 tests/compression_static.rs create mode 100644 tests/fixtures/public/index.html.gz diff --git a/.github/workflows/devel.yml b/.github/workflows/devel.yml index a80696f..d4b6f8d 100644 --- a/.github/workflows/devel.yml +++ b/.github/workflows/devel.yml @@ -48,7 +48,7 @@ jobs: # Specific Rust channels - build: pinned os: ubuntu-20.04 - rust: 1.56.1 + rust: 1.59.0 - build: stable os: ubuntu-20.04 rust: stable diff --git a/.gitignore b/.gitignore index 3831ff4..c8150a9 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ docs/*/**.html !sample.env !/docs +!/tests/fixtures/**/* diff --git a/src/compression.rs b/src/compression.rs index 2357d80..54e0212 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -44,6 +44,14 @@ pub const TEXT_MIME_TYPES: [&str; 24] = [ "application/wasm", ]; +/// Try to get the prefered `content-encoding` via the `accept-encoding` header. +pub fn get_prefered_encoding(headers: &HeaderMap) -> Option { + if let Some(ref accept_encoding) = headers.typed_get::() { + return accept_encoding.prefered_encoding(); + } + None +} + /// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response) /// using `gzip`, `deflate` or `brotli` if is specified in the `Accept-Encoding` header, adding /// `content-encoding: ` to the Response's [`HeaderMap`](hyper::HeaderMap) @@ -59,28 +67,26 @@ pub fn auto( } // Compress response based on Accept-Encoding header - if let Some(ref accept_encoding) = headers.typed_get::() { - if let Some(encoding) = accept_encoding.prefered_encoding() { - // Skip compression for non-text-based MIME types - if let Some(content_type) = resp.headers().typed_get::() { - let content_type = &content_type.to_string(); - if !TEXT_MIME_TYPES.iter().any(|h| *h == content_type) { - return Ok(resp); - } + if let Some(encoding) = get_prefered_encoding(headers) { + // Skip compression for non-text-based MIME types + if let Some(content_type) = resp.headers().typed_get::() { + let content_type = &content_type.to_string(); + if !TEXT_MIME_TYPES.iter().any(|h| *h == content_type) { + return Ok(resp); } + } - if encoding == ContentCoding::GZIP { - let (head, body) = resp.into_parts(); - return Ok(gzip(head, body.into())); - } - if encoding == ContentCoding::DEFLATE { - let (head, body) = resp.into_parts(); - return Ok(deflate(head, body.into())); - } - if encoding == ContentCoding::BROTLI { - let (head, body) = resp.into_parts(); - return Ok(brotli(head, body.into())); - } + if encoding == ContentCoding::GZIP { + let (head, body) = resp.into_parts(); + return Ok(gzip(head, body.into())); + } + if encoding == ContentCoding::DEFLATE { + let (head, body) = resp.into_parts(); + return Ok(deflate(head, body.into())); + } + if encoding == ContentCoding::BROTLI { + let (head, body) = resp.into_parts(); + return Ok(brotli(head, body.into())); } } @@ -93,6 +99,8 @@ pub fn gzip( mut head: http::response::Parts, body: CompressableBody, ) -> Response { + tracing::trace!("compressing response body on the fly using gzip"); + let body = Body::wrap_stream(ReaderStream::new(GzipEncoder::new(StreamReader::new(body)))); let header = create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::GZIP); head.headers.remove(CONTENT_LENGTH); @@ -106,6 +114,8 @@ pub fn deflate( mut head: http::response::Parts, body: CompressableBody, ) -> Response { + tracing::trace!("compressing response body on the fly using deflate"); + let body = Body::wrap_stream(ReaderStream::new(DeflateEncoder::new(StreamReader::new( body, )))); @@ -124,6 +134,8 @@ pub fn brotli( mut head: http::response::Parts, body: CompressableBody, ) -> Response { + tracing::trace!("compressing response body on the fly using brotli"); + let body = Body::wrap_stream(ReaderStream::new(BrotliEncoder::new(StreamReader::new( body, )))); @@ -135,7 +147,7 @@ pub fn brotli( } /// Given an optional existing encoding header, appends to the existing or creates a new one. -fn create_encoding_header(existing: Option, coding: ContentCoding) -> HeaderValue { +pub fn create_encoding_header(existing: Option, coding: ContentCoding) -> HeaderValue { if let Some(val) = existing { if let Ok(str_val) = val.to_str() { return HeaderValue::from_str(&[str_val, ", ", coding.to_static()].concat()) diff --git a/src/compression_static.rs b/src/compression_static.rs new file mode 100644 index 0000000..37782f0 --- /dev/null +++ b/src/compression_static.rs @@ -0,0 +1,57 @@ +use headers::{ContentCoding, HeaderMap, HeaderValue}; +use std::{fs::Metadata, path::PathBuf, sync::Arc}; + +use crate::{ + compression, + static_files::{file_metadata, ArcPath}, +}; + +/// Search for the pre-compressed variant of the given file path. +pub async fn precompressed_variant( + file_path: PathBuf, + headers: &HeaderMap, +) -> Option<(ArcPath, Metadata, &str)> { + let mut precompressed = None; + + tracing::trace!( + "preparing pre-compressed file path variant of {}", + file_path.display() + ); + + // Determine prefered-encoding extension if available + let precomp_ext = match compression::get_prefered_encoding(headers) { + // https://zlib.net/zlib_faq.html#faq39 + Some(ContentCoding::GZIP | ContentCoding::DEFLATE) => Some("gz"), + // https://peazip.github.io/brotli-compressed-file-format.html + Some(ContentCoding::BROTLI) => Some("br"), + _ => None, + }; + + if precomp_ext.is_none() { + tracing::trace!("preferred encoding based on the file extension was not determined"); + } + + // Try to find the pre-compressed metadata variant for the given file path + if let Some(ext) = precomp_ext { + let mut filepath_precomp = file_path; + let filename = filepath_precomp.file_name().unwrap().to_str().unwrap(); + let precomp_file_name = [filename, ".", ext].concat(); + filepath_precomp.set_file_name(precomp_file_name); + + tracing::trace!( + "getting metadata for pre-compressed file variant {}", + filepath_precomp.display() + ); + + if let Ok((meta, _)) = file_metadata(&filepath_precomp).await { + tracing::trace!("pre-compressed file variant found, serving it directly"); + + let encoding = if ext == "gz" { "gzip" } else { ext }; + precompressed = Some((ArcPath(Arc::new(filepath_precomp)), meta, encoding)); + } + + // Note: In error case like "no such file or dir" the workflow just continues + } + + precompressed +} diff --git a/src/directory_listing.rs b/src/directory_listing.rs index 27cc7ac..5f48f79 100644 --- a/src/directory_listing.rs +++ b/src/directory_listing.rs @@ -14,7 +14,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use crate::Result; /// Provides directory listing support for the current request. -/// Note that this function highly depends on `static_files::path_from_tail()` function +/// Note that this function highly depends on `static_files::get_composed_metadata()` function /// which must be called first. See `static_files::handle()` for more details. pub fn auto_index<'a>( method: &'a Method, @@ -28,7 +28,7 @@ pub fn auto_index<'a>( // Note: it's safe to call `parent()` here since `filepath` // value always refer to a path with file ending and under // a root directory boundary. - // See `path_from_tail()` function which sanitizes the requested + // See `get_composed_metadata()` function which sanitizes the requested // path before to be delegated here. let parent = filepath.parent().unwrap_or(filepath); diff --git a/src/handler.rs b/src/handler.rs index 07f7f13..ec1ee08 100644 --- a/src/handler.rs +++ b/src/handler.rs @@ -15,6 +15,7 @@ pub struct RequestHandlerOpts { // General options pub root_dir: PathBuf, pub compression: bool, + pub compression_static: bool, pub dir_listing: bool, pub dir_listing_order: u8, pub cors: Option, @@ -54,6 +55,7 @@ impl RequestHandler { let dir_listing_order = self.opts.dir_listing_order; let log_remote_addr = self.opts.log_remote_address; let redirect_trailing_slash = self.opts.redirect_trailing_slash; + let compression_static = self.opts.compression_static; let mut cors_headers: Option = None; @@ -144,6 +146,7 @@ impl RequestHandler { } } + // Advanced options if let Some(advanced) = &self.opts.advanced_opts { // Redirects if let Some(parts) = redirects::get_redirection(uri_path, &advanced.redirects) { @@ -188,10 +191,11 @@ impl RequestHandler { dir_listing, dir_listing_order, redirect_trailing_slash, + compression_static, }) .await { - Ok(mut resp) => { + Ok((mut resp, is_precompressed)) => { // Append CORS headers if they are present if let Some(cors_headers) = cors_headers { if !cors_headers.is_empty() { @@ -203,7 +207,7 @@ impl RequestHandler { } // Auto compression based on the `Accept-Encoding` header - if self.opts.compression { + if self.opts.compression && !is_precompressed { resp = match compression::auto(method, headers, resp) { Ok(res) => res, Err(err) => { diff --git a/src/lib.rs b/src/lib.rs index 4a3a7cb..43ef7fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ extern crate serde; pub mod basic_auth; pub mod compression; +pub mod compression_static; pub mod control_headers; pub mod cors; pub mod custom_headers; diff --git a/src/server.rs b/src/server.rs index 40a6b76..2e95e38 100644 --- a/src/server.rs +++ b/src/server.rs @@ -138,6 +138,10 @@ impl Server { let compression = general.compression; tracing::info!("auto compression: enabled={}", compression); + // Check pre-compressed files based on the `Accept-Encoding` header + let compression_static = general.compression_static; + tracing::info!("compression static: enabled={}", compression_static); + // Directory listing option let dir_listing = general.directory_listing; tracing::info!("directory listing: enabled={}", dir_listing); @@ -183,6 +187,7 @@ impl Server { opts: Arc::from(RequestHandlerOpts { root_dir, compression, + compression_static, dir_listing, dir_listing_order, cors, diff --git a/src/settings/cli.rs b/src/settings/cli.rs index 912c194..543ff47 100644 --- a/src/settings/cli.rs +++ b/src/settings/cli.rs @@ -118,6 +118,16 @@ pub struct General { #[structopt( long, + parse(try_from_str), + default_value = "false", + env = "SERVER_COMPRESSION_STATIC" + )] + /// Check for a pre-compressed file on disk and serve it if available. + /// The order of precedence is determined by the `Accept-Encoding` header. + pub compression_static: bool, + + #[structopt( + long, short = "z", parse(try_from_str), default_value = "false", diff --git a/src/settings/file.rs b/src/settings/file.rs index 4c7eaa3..00bda76 100644 --- a/src/settings/file.rs +++ b/src/settings/file.rs @@ -93,6 +93,9 @@ pub struct General { // Compression pub compression: Option, + // Check for a pre-compressed file on disk + pub compression_static: Option, + // Error pages pub page404: Option, pub page50x: Option, diff --git a/src/settings/mod.rs b/src/settings/mod.rs index 9904a37..2e6821e 100644 --- a/src/settings/mod.rs +++ b/src/settings/mod.rs @@ -67,6 +67,7 @@ impl Settings { let mut config_file = opts.config_file.clone(); let mut cache_control_headers = opts.cache_control_headers; let mut compression = opts.compression; + let mut compression_static = opts.compression_static; let mut page404 = opts.page404; let mut page50x = opts.page50x; let mut http2 = opts.http2; @@ -127,6 +128,9 @@ impl Settings { if let Some(v) = general.compression { compression = v } + if let Some(v) = general.compression_static { + compression_static = v + } if let Some(v) = general.page404 { page404 = v } @@ -288,6 +292,7 @@ impl Settings { config_file, cache_control_headers, compression, + compression_static, page404, page50x, http2, diff --git a/src/static_files.rs b/src/static_files.rs index 850dac9..6ef4bd4 100644 --- a/src/static_files.rs +++ b/src/static_files.rs @@ -3,12 +3,13 @@ use bytes::{Bytes, BytesMut}; use futures_util::future::Either; -use futures_util::{future, ready, stream, FutureExt, Stream, StreamExt, TryFutureExt}; +use futures_util::{future, ready, stream, FutureExt, Stream, StreamExt}; use headers::{ AcceptRanges, ContentLength, ContentRange, ContentType, HeaderMap, HeaderMapExt, HeaderValue, IfModifiedSince, IfRange, IfUnmodifiedSince, LastModified, Range, }; -use hyper::{Body, Method, Response, StatusCode}; +use http::header::CONTENT_LENGTH; +use hyper::{header::CONTENT_ENCODING, Body, Method, Response, StatusCode}; use percent_encoding::percent_decode_str; use std::fs::Metadata; use std::future::Future; @@ -23,7 +24,7 @@ use tokio::fs::File as TkFile; use tokio::io::AsyncSeekExt; use tokio_util::io::poll_read_buf; -use crate::{directory_listing, Result}; +use crate::{compression_static, directory_listing, Result}; /// Arc `PathBuf` reference wrapper since Arc doesn't implement AsRef. #[derive(Clone, Debug)] @@ -45,11 +46,12 @@ pub struct HandleOpts<'a> { pub dir_listing: bool, pub dir_listing_order: u8, pub redirect_trailing_slash: bool, + pub compression_static: bool, } /// Entry point to handle incoming requests which map to specific files /// on file system and return a file response. -pub async fn handle<'a>(opts: &HandleOpts<'a>) -> Result, StatusCode> { +pub async fn handle<'a>(opts: &HandleOpts<'a>) -> Result<(Response, bool), StatusCode> { let method = opts.method; let uri_path = opts.uri_path; @@ -58,14 +60,23 @@ pub async fn handle<'a>(opts: &HandleOpts<'a>) -> Result, StatusC return Err(StatusCode::METHOD_NOT_ALLOWED); } - let base = Arc::new(opts.base_path.into()); - let (filepath, meta, auto_index) = path_from_tail(base, uri_path).await?; + let headers_opt = opts.headers; + let compression_static_opt = opts.compression_static; - // NOTE: `auto_index` appends an `index.html` to an `uri_path` of kind directory only. + let base = Arc::::new(opts.base_path.into()); + let file_path = sanitize_path(base.as_ref(), uri_path)?; + + let (file_path, meta, is_dir, precompressed_variant) = + composed_file_metadata(file_path, headers_opt, compression_static_opt).await?; + + // `is_precompressed` relates to `opts.compression_static` value + let is_precompressed = precompressed_variant.is_some(); + + // NOTE: `is_dir` means an "auto index" for the current directory request // Check for a trailing slash on the current directory path // and redirect if that path doesn't end with the slash char - if opts.redirect_trailing_slash && auto_index && !uri_path.ends_with('/') { + if opts.redirect_trailing_slash && is_dir && !uri_path.ends_with('/') { let uri = [uri_path, "/"].concat(); let loc = match HeaderValue::from_str(uri.as_str()) { Ok(val) => val, @@ -78,11 +89,12 @@ pub async fn handle<'a>(opts: &HandleOpts<'a>) -> Result, StatusC let mut resp = Response::new(Body::empty()); resp.headers_mut().insert(hyper::header::LOCATION, loc); *resp.status_mut() = StatusCode::PERMANENT_REDIRECT; + tracing::trace!("uri doesn't end with a slash so redirecting permanently"); - return Ok(resp); + return Ok((resp, is_precompressed)); } - // Respond the permitted communication options + // Respond with the permitted communication options if method == Method::OPTIONS { let mut resp = Response::new(Body::empty()); *resp.status_mut() = StatusCode::NO_CONTENT; @@ -93,66 +105,138 @@ pub async fn handle<'a>(opts: &HandleOpts<'a>) -> Result, StatusC Method::GET, ])); resp.headers_mut().typed_insert(AcceptRanges::bytes()); - return Ok(resp); + + return Ok((resp, is_precompressed)); } // Directory listing - // 1. Check if "directory listing" feature is enabled + // Check if "directory listing" feature is enabled, // if current path is a valid directory and // if it does not contain an `index.html` file (if a proper auto index is generated) - if opts.dir_listing && auto_index && !filepath.as_ref().exists() { - return directory_listing::auto_index( + if opts.dir_listing && is_dir && !file_path.as_ref().exists() { + let resp = directory_listing::auto_index( method, uri_path, opts.uri_query, - filepath.as_ref(), + file_path.as_ref(), opts.dir_listing_order, ) - .await; + .await?; + + return Ok((resp, is_precompressed)); } - file_reply(opts.headers, (filepath, &meta, auto_index)).await + // Check for a pre-compressed file variant if present under the `opts.compression_static` context + if let Some(meta_precompressed) = precompressed_variant { + let (path_precomp, precomp_ext) = meta_precompressed; + + let mut resp = file_reply(headers_opt, file_path, &meta, Some(path_precomp)).await?; + + // Prepare corresponding headers to let know how to decode the payload + resp.headers_mut().remove(CONTENT_LENGTH); + resp.headers_mut() + .insert(CONTENT_ENCODING, precomp_ext.parse().unwrap()); + + return Ok((resp, is_precompressed)); + } + + let resp = file_reply(headers_opt, file_path, &meta, None).await?; + + Ok((resp, is_precompressed)) } -/// Convert an incoming uri into a valid and sanitized path then returns a tuple -// with the path as well as its file metadata and an auto index check if it's a directory. -fn path_from_tail( - base: Arc, - tail: &str, -) -> impl Future> + Send { - future::ready(sanitize_path(base.as_ref(), tail)).and_then(|mut buf| async { - match tokio::fs::metadata(&buf).await { - Ok(meta) => { - let mut auto_index = false; - if meta.is_dir() { - tracing::debug!("dir: appending index.html to directory path"); - buf.push("index.html"); - auto_index = true; +/// Returns the final composed metadata information (tuple) containing +/// the Arc `PathBuf` reference wrapper for the current `file_path` with its file metadata +/// as well as its optional pre-compressed variant. +async fn composed_file_metadata( + mut file_path: PathBuf, + headers: &HeaderMap, + compression_static: bool, +) -> Result<(ArcPath, Metadata, bool, Option<(ArcPath, &str)>), StatusCode> { + // First pre-compressed variant check for the given file path + let mut tried_precompressed = false; + if compression_static { + tried_precompressed = true; + if let Some((path, meta, ext)) = + compression_static::precompressed_variant(file_path.clone(), headers).await + { + return Ok((ArcPath(Arc::new(file_path)), meta, false, Some((path, ext)))); + } + } + + tracing::trace!("getting metadata for file {}", file_path.display()); + + match file_metadata(file_path.as_ref()).await { + Ok((mut meta, is_dir)) => { + if is_dir { + // Append a HTML index page by default if it's a directory path (`autoindex`) + tracing::debug!("dir: appending an index.html to the directory path"); + file_path.push("index.html"); + + // If file exists then overwrite the `meta` + // Also noting that it's still a directory request + if let Ok(meta_res) = file_metadata(file_path.as_ref()).await { + (meta, _) = meta_res } - tracing::trace!("dir: {:?}", buf); - Ok((ArcPath(Arc::new(buf)), meta, auto_index)) } - Err(err) => { - tracing::debug!("file not found: {} {:?}", buf.display(), err); - Err(StatusCode::NOT_FOUND) + + Ok((ArcPath(Arc::new(file_path)), meta, is_dir, None)) + } + Err(err) => { + // Second pre-compressed variant check for the given file path + if compression_static && !tried_precompressed { + if let Some((path, meta, ext)) = + compression_static::precompressed_variant(file_path.clone(), headers).await + { + return Ok((ArcPath(Arc::new(file_path)), meta, false, Some((path, ext)))); + } } + + Err(err) } - }) + } } -/// Reply with a file content. +/// Try to find the file system metadata for the given file path. +pub async fn file_metadata(file_path: &Path) -> Result<(Metadata, bool), StatusCode> { + match tokio::fs::metadata(file_path).await { + Ok(meta) => { + let is_dir = meta.is_dir(); + tracing::trace!("file found: {:?}", file_path); + Ok((meta, is_dir)) + } + Err(err) => { + tracing::debug!("file not found: {:?} {:?}", file_path, err); + Err(StatusCode::NOT_FOUND) + } + } +} + +/// Reply with the corresponding file content taking into account +/// its precompressed variant if any. +/// The `path` param should contains always the original requested file path and +/// the `meta` param value should corresponds to it. +/// However, if `path_precompressed` contains some value then +/// the `meta` param value will belong to the `path_precompressed` (precompressed file variant). fn file_reply<'a>( headers: &'a HeaderMap, - res: (ArcPath, &'a Metadata, bool), + path: ArcPath, + meta: &'a Metadata, + path_precompressed: Option, ) -> impl Future, StatusCode>> + Send + 'a { - let (path, meta, auto_index) = res; let conditionals = get_conditional_headers(headers); - TkFile::open(path.clone()).then(move |res| match res { - Ok(file) => Either::Left(file_conditional(file, path, meta, auto_index, conditionals)), + + let file_path = path_precompressed.unwrap_or_else(|| path.clone()); + + TkFile::open(file_path).then(move |res| match res { + Ok(file) => Either::Left(file_conditional(file, path, meta, conditionals)), Err(err) => { let status = match err.kind() { io::ErrorKind::NotFound => { - tracing::debug!("file not found: {:?}", path.as_ref().display()); + tracing::debug!( + "file can't be opened or not found: {:?}", + path.as_ref().display() + ); StatusCode::NOT_FOUND } io::ErrorKind::PermissionDenied => { @@ -187,6 +271,7 @@ fn get_conditional_headers(header_list: &HeaderMap) -> Conditionals } } +/// Sanitizes a base/tail paths and then it returns an unified one. fn sanitize_path(base: impl AsRef, tail: &str) -> Result { let path_decoded = match percent_decode_str(tail.trim_start_matches('/')).decode_utf8() { Ok(p) => p, @@ -198,7 +283,7 @@ fn sanitize_path(base: impl AsRef, tail: &str) -> Result Result, StatusCode> { - if auto_index { - match file.metadata().await { - Ok(meta) => Ok(response_body(file, &meta, path, conditionals)), - Err(err) => { - tracing::debug!("file metadata error: {}", err); - Err(StatusCode::INTERNAL_SERVER_ERROR) - } - } - } else { - Ok(response_body(file, meta, path, conditionals)) - } + Ok(response_body(file, meta, path, conditionals)) } fn response_body( diff --git a/tests/compression_static.rs b/tests/compression_static.rs new file mode 100644 index 0000000..cebc38c --- /dev/null +++ b/tests/compression_static.rs @@ -0,0 +1,122 @@ +#![forbid(unsafe_code)] +#![deny(warnings)] +#![deny(rust_2018_idioms)] +#![deny(dead_code)] + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use headers::HeaderMap; + use http::Method; + use std::path::PathBuf; + + use static_web_server::static_files::{self, HandleOpts}; + + fn public_dir() -> PathBuf { + PathBuf::from("docker/public/") + } + + #[tokio::test] + async fn compression_static_file_exists() { + let mut headers = HeaderMap::new(); + headers.insert( + http::header::ACCEPT_ENCODING, + "gzip, deflate, br".parse().unwrap(), + ); + + let index_gz_path = PathBuf::from("tests/fixtures/public/index.html.gz"); + let index_gz_path_public = public_dir().join("index.html.gz"); + std::fs::copy(&index_gz_path, &index_gz_path_public) + .expect("unexpected error copying fixture file"); + + let (mut resp, _) = static_files::handle(&HandleOpts { + method: &Method::GET, + headers: &headers, + base_path: &public_dir(), + uri_path: "index.html", + uri_query: None, + dir_listing: false, + dir_listing_order: 6, + redirect_trailing_slash: true, + compression_static: true, + }) + .await + .expect("unexpected error response on `handle` function"); + + let index_gz_buf = + std::fs::read(&index_gz_path).expect("unexpected error when reading index.html.gz"); + let index_gz_buf = Bytes::from(index_gz_buf); + + std::fs::remove_file(index_gz_path_public).unwrap(); + + let headers = resp.headers(); + + assert_eq!(resp.status(), 200); + assert!(!headers.contains_key("content-length")); + assert_eq!(headers["content-encoding"], "gzip"); + assert_eq!(headers["accept-ranges"], "bytes"); + assert!(!headers["last-modified"].is_empty()); + assert_eq!( + &headers["content-type"], "text/html", + "content-type is not html" + ); + + let body = hyper::body::to_bytes(resp.body_mut()) + .await + .expect("unexpected bytes error during `body` conversion"); + + assert_eq!( + body, index_gz_buf, + "body and index_gz_buf are not equal in length" + ); + } + + #[tokio::test] + async fn compression_static_file_does_not_exist() { + let mut headers = HeaderMap::new(); + headers.insert( + http::header::ACCEPT_ENCODING, + "gzip, deflate, br".parse().unwrap(), + ); + + let index_path_public = public_dir().join("assets/index.html"); + + let (mut resp, _) = static_files::handle(&HandleOpts { + method: &Method::GET, + headers: &headers, + base_path: &public_dir().join("assets/"), + uri_path: "index.html", + uri_query: None, + dir_listing: false, + dir_listing_order: 6, + redirect_trailing_slash: true, + compression_static: true, + }) + .await + .expect("unexpected error response on `handle` function"); + + let index_buf = + std::fs::read(&index_path_public).expect("unexpected error when reading index.html"); + let index_buf = Bytes::from(index_buf); + + let headers = resp.headers(); + + assert_eq!(resp.status(), 200); + assert!(headers.contains_key("content-length")); + assert_eq!(headers["accept-ranges"], "bytes"); + assert!(!headers["last-modified"].is_empty()); + assert_eq!( + &headers["content-type"], "text/html", + "content-type is not html" + ); + + let body = hyper::body::to_bytes(resp.body_mut()) + .await + .expect("unexpected bytes error during `body` conversion"); + + assert_eq!( + body, index_buf, + "body and index_gz_buf are not equal in length" + ); + } +} diff --git a/tests/dir_listing.rs b/tests/dir_listing.rs index 39db511..c6694b1 100644 --- a/tests/dir_listing.rs +++ b/tests/dir_listing.rs @@ -41,10 +41,11 @@ mod tests { dir_listing: true, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { assert_eq!(res.status(), 308); assert_eq!(res.headers()["location"], "/assets/"); } @@ -68,10 +69,11 @@ mod tests { dir_listing: true, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 200); assert_eq!(res.headers()["content-type"], "text/html; charset=utf-8"); @@ -105,10 +107,11 @@ mod tests { dir_listing: true, dir_listing_order: 6, redirect_trailing_slash: false, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 200); assert_eq!(res.headers()["content-type"], "text/html; charset=utf-8"); @@ -142,10 +145,11 @@ mod tests { dir_listing: true, dir_listing_order: 6, redirect_trailing_slash: false, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { assert_eq!(res.status(), 200); assert_eq!(res.headers()["content-type"], "text/markdown"); } diff --git a/tests/fixtures/public/index.html.gz b/tests/fixtures/public/index.html.gz new file mode 100644 index 0000000..869a105 Binary files /dev/null and b/tests/fixtures/public/index.html.gz differ diff --git a/tests/static_files.rs b/tests/static_files.rs index 40a9505..9908119 100644 --- a/tests/static_files.rs +++ b/tests/static_files.rs @@ -22,7 +22,7 @@ mod tests { #[tokio::test] async fn handle_file() { - let mut res = static_files::handle(&HandleOpts { + let (mut res, _) = static_files::handle(&HandleOpts { method: &Method::GET, headers: &HeaderMap::new(), base_path: &root_dir(), @@ -31,6 +31,7 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await .expect("unexpected error response on `handle` function"); @@ -61,7 +62,7 @@ mod tests { #[tokio::test] async fn handle_file_head() { - let mut res = static_files::handle(&HandleOpts { + let (mut res, _) = static_files::handle(&HandleOpts { method: &Method::HEAD, headers: &HeaderMap::new(), base_path: &root_dir(), @@ -70,6 +71,7 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await .expect("unexpected error response on `handle` function"); @@ -110,6 +112,7 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { @@ -125,7 +128,7 @@ mod tests { #[tokio::test] async fn handle_trailing_slash_redirection() { - let mut res = static_files::handle(&HandleOpts { + let (mut res, _) = static_files::handle(&HandleOpts { method: &Method::GET, headers: &HeaderMap::new(), base_path: &root_dir(), @@ -134,6 +137,7 @@ mod tests { dir_listing: false, dir_listing_order: 0, redirect_trailing_slash: true, + compression_static: false, }) .await .expect("unexpected error response on `handle` function"); @@ -159,10 +163,11 @@ mod tests { dir_listing: false, dir_listing_order: 0, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { assert_eq!(res.status(), 308); assert_eq!(res.headers()["location"], "assets/"); } @@ -183,10 +188,11 @@ mod tests { dir_listing: false, dir_listing_order: 0, redirect_trailing_slash: false, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { assert_eq!(res.status(), 200); } Err(status) => { @@ -212,10 +218,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { if uri.is_empty() { // it should redirect permanently assert_eq!(res.status(), 308); @@ -256,10 +263,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { assert_eq!(res.status(), 200); assert_eq!(res.headers()["content-length"], buf.len().to_string()); } @@ -282,6 +290,7 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { @@ -311,10 +320,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { assert_eq!(res.status(), 200); assert_eq!(res.headers()["content-length"], buf.len().to_string()); res @@ -340,10 +350,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 304); assert_eq!(res.headers().get("content-length"), None); let body = hyper::body::to_bytes(res.body_mut()) @@ -372,10 +383,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 200); let body = hyper::body::to_bytes(res.body_mut()) .await @@ -402,10 +414,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { assert_eq!(res.status(), 200); res } @@ -430,10 +443,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { assert_eq!(res.status(), 200); } Err(_) => { @@ -457,10 +471,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 412); let body = hyper::body::to_bytes(res.body_mut()) @@ -498,10 +513,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => match method { + Ok((mut res, _)) => match method { // The handle only accepts HEAD or GET request methods Method::GET | Method::HEAD => { let buf = fs::read(root_dir().join("index.html")) @@ -556,10 +572,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { let res = compression::auto(method, &headers, res) .expect("unexpected bytes error during body compression"); @@ -617,10 +634,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 206); assert_eq!( res.headers()["content-range"], @@ -658,10 +676,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 416); assert_eq!( res.headers()["content-range"], @@ -700,10 +719,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(res) => { + Ok((res, _)) => { assert_eq!(res.status(), 200); assert_eq!(res.headers()["content-length"], buf.len().to_string()); assert_eq!(res.headers().get("content-range"), None); @@ -734,10 +754,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 206); assert_eq!( res.headers()["content-range"], @@ -778,10 +799,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 206); assert_eq!( res.headers()["content-range"], @@ -819,10 +841,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 416); assert_eq!( res.headers()["content-range"], @@ -863,10 +886,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 416); assert_eq!( res.headers()["content-range"], @@ -905,10 +929,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 200); let body = hyper::body::to_bytes(res.body_mut()) .await @@ -942,10 +967,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 206); assert_eq!( res.headers()["content-range"], @@ -990,10 +1016,11 @@ mod tests { dir_listing: false, dir_listing_order: 6, redirect_trailing_slash: true, + compression_static: false, }) .await { - Ok(mut res) => { + Ok((mut res, _)) => { assert_eq!(res.status(), 206); assert_eq!( res.headers()["content-range"], diff --git a/tests/toml/config.toml b/tests/toml/config.toml index 1967c1d..d096246 100644 --- a/tests/toml/config.toml +++ b/tests/toml/config.toml @@ -52,6 +52,9 @@ log-remote-address = false #### Redirect to trailing slash in the requested directory uri redirect-trailing-slash = true +#### Check for existing pre-compressed files +compression-static = true + ### Windows Only #### Run the web server as a Windows Service -- libgit2 1.7.2