From d4427eb3e100313d8e407af55a8e288d5a9516cc Mon Sep 17 00:00:00 2001 From: Jose Quintana <1700322+joseluisq@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:44:38 +0100 Subject: [PATCH] feat: experimental Tokio Runtime metrics for Prometheus (#307) * feat: Prometheus metrics endpoint at /metrics Signed-off-by: Tom Plant * fix: add `experimental` prefix to metrics arg, env var, and logs Signed-off-by: Tom Plant * fix: disable tokio-metrics-collector on Windows Signed-off-by: Tom Plant * chore: address feedback * refactor: rename feature to `experimental-metrics` and add test * fix: freebsd ci tests * refactor: move dependencies to the unix target section --------- Signed-off-by: Tom Plant Co-authored-by: Jose Quintana --- .cargo/config.toml | 5 +++++ .cirrus.yml | 6 ++++-- Cargo.lock | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/handler.rs | 31 +++++++++++++++++++++++++++++++ src/server.rs | 20 ++++++++++++++++++++ src/settings/cli.rs | 13 +++++++++++++ src/settings/file.rs | 4 ++++ src/settings/mod.rs | 8 ++++++++ src/testing.rs | 2 ++ tests/experimental_metrics.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++++ tests/fixtures/toml/experimental_metrics.toml | 5 +++++ 12 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 tests/experimental_metrics.rs create mode 100644 tests/fixtures/toml/experimental_metrics.toml diff --git a/.cargo/config.toml b/.cargo/config.toml index 5d51715..a89bcf2 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -9,3 +9,8 @@ rustflags = ["-C", "target-feature=+crt-static"] rustflags = ["-C", "target-feature=+crt-static"] [target.i686-pc-windows-msvc] rustflags = ["-C", "link-arg=libvcruntime.lib"] + +# Required for RuntimeMetrics as of writing +[build] +rustflags = ["--cfg", "tokio_unstable"] +rustdocflags = ["--cfg", "tokio_unstable"] diff --git a/.cirrus.yml b/.cirrus.yml index 3af9b17..85adc05 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -8,7 +8,7 @@ freebsd_instance: task: only_if: $CIRRUS_TAG == '' env: - RUSTFLAGS: -Dwarnings + RUSTFLAGS: -Dwarnings --cfg tokio_unstable matrix: - name: freebsd-amd64-test env: @@ -21,6 +21,7 @@ task: - curl https://sh.rustup.rs -sSf --output rustup.sh - sh rustup.sh -y --profile minimal --default-toolchain stable - . $HOME/.cargo/env + - cp -r .cargo/config.toml $HOME/.cargo/ - rustc --version toolchain_script: - . $HOME/.cargo/env @@ -41,7 +42,7 @@ task: task: only_if: $CIRRUS_TAG != '' env: - RUSTFLAGS: -Dwarnings + RUSTFLAGS: -Dwarnings --cfg tokio_unstable GITHUB_TOKEN: ENCRYPTED[d1766ef328d83729917d2ffb875d64c35d1c0177edf8f32e66ec464daf5c1b7b145d65fc6c044a73fffe2235d3b38349] matrix: - name: freebsd-amd64-release @@ -55,6 +56,7 @@ task: - curl https://sh.rustup.rs -sSf --output rustup.sh - sh rustup.sh -y --profile minimal --default-toolchain stable - . $HOME/.cargo/env + - cp -r .cargo/config.toml $HOME/.cargo/ - rustc --version toolchain_script: - . $HOME/.cargo/env diff --git a/Cargo.lock b/Cargo.lock index 947d14b..59fb41b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -421,6 +421,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "futures-sink" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -439,9 +450,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -913,6 +926,27 @@ dependencies = [ ] [[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + +[[package]] name = "quote" version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1199,6 +1233,7 @@ dependencies = [ "num_cpus", "percent-encoding", "pin-project", + "prometheus", "regex", "rustls-pemfile", "serde", @@ -1209,6 +1244,7 @@ dependencies = [ "signal-hook-tokio", "tikv-jemallocator", "tokio", + "tokio-metrics-collector", "tokio-rustls", "tokio-util", "toml", @@ -1241,6 +1277,26 @@ dependencies = [ ] [[package]] +name = "thiserror" +version = "1.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a48fd946b02c0a526b2e9481c8e2a17755e47039164a86c4070446e3a4614d" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "thread_local" version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1300,6 +1356,31 @@ dependencies = [ ] [[package]] +name = "tokio-metrics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eace09241d62c98b7eeb1107d4c5c64ca3bd7da92e8c218c153ab3a78f9be112" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + +[[package]] +name = "tokio-metrics-collector" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767da47381602cc481653456823b3ebb600e83d5dd4e0293da9b5566c6c00f0" +dependencies = [ + "lazy_static", + "parking_lot", + "prometheus", + "tokio", + "tokio-metrics", +] + +[[package]] name = "tokio-rustls" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1311,6 +1392,17 @@ dependencies = [ ] [[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] name = "tokio-util" version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/Cargo.toml b/Cargo.toml index d36e749..178d50a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,8 @@ version = "0.5" [target.'cfg(unix)'.dependencies] signal-hook = { version = "0.3", features = ["extended-siginfo"] } signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"], default-features = false } +tokio-metrics-collector = "0.2" +prometheus = "0.13" [target.'cfg(windows)'.dependencies] windows-service = "0.6" diff --git a/src/handler.rs b/src/handler.rs index a2f5177..48cd289 100644 --- a/src/handler.rs +++ b/src/handler.rs @@ -80,6 +80,9 @@ pub struct RequestHandlerOpts { pub ignore_hidden_files: bool, /// Health endpoint feature. pub health: bool, + /// Metrics endpoint feature (experimental). + #[cfg(unix)] + pub experimental_metrics: bool, /// Maintenance mode feature. pub maintenance_mode: bool, /// Custom HTTP status for when entering into maintenance mode. @@ -122,6 +125,8 @@ impl RequestHandler { let compression_static = self.opts.compression_static; let ignore_hidden_files = self.opts.ignore_hidden_files; let health = self.opts.health; + #[cfg(unix)] + let experimental_metrics = self.opts.experimental_metrics; let index_files: Vec<&str> = self.opts.index_files.iter().map(|s| s.as_str()).collect(); let mut cors_headers: Option = None; @@ -129,6 +134,10 @@ impl RequestHandler { let health_request = health && uri_path == "/health" && (method.is_get() || method.is_head()); + #[cfg(unix)] + let metrics_request = + experimental_metrics && uri_path == "/metrics" && (method.is_get() || method.is_head()); + // Log request information with its remote address if available let mut remote_addr_str = String::new(); if log_remote_addr { @@ -192,6 +201,28 @@ impl RequestHandler { ); } + // Metrics endpoint check + #[cfg(unix)] + if metrics_request { + use prometheus::Encoder; + + let body = if method.is_get() { + let encoder = prometheus::TextEncoder::new(); + let mut buffer = Vec::new(); + encoder + .encode(&prometheus::default_registry().gather(), &mut buffer) + .unwrap(); + let data = String::from_utf8(buffer).unwrap(); + Body::from(data) + } else { + Body::empty() + }; + let mut resp = Response::new(body); + resp.headers_mut() + .typed_insert(ContentType::from(mime_guess::mime::TEXT_PLAIN_UTF_8)); + return Ok(resp); + } + // CORS if let Some(cors) = &self.opts.cors { match cors.check_request(method, headers) { diff --git a/src/server.rs b/src/server.rs index e13e8d6..3e3613c 100644 --- a/src/server.rs +++ b/src/server.rs @@ -292,6 +292,24 @@ impl Server { let health = general.health; server_info!("health endpoint: enabled={}", health); + // Metrics endpoint option (experimental) + #[cfg(unix)] + let experimental_metrics = general.experimental_metrics; + #[cfg(unix)] + server_info!( + "metrics endpoint (experimental): enabled={}", + experimental_metrics + ); + + #[cfg(unix)] + if experimental_metrics { + prometheus::default_registry() + .register(Box::new( + tokio_metrics_collector::default_runtime_collector(), + )) + .unwrap(); + } + // Maintenance mode option let maintenance_mode = general.maintenance_mode; let maintenance_mode_status = general.maintenance_mode_status; @@ -332,6 +350,8 @@ impl Server { ignore_hidden_files, index_files, health, + #[cfg(unix)] + experimental_metrics, maintenance_mode, maintenance_mode_status, maintenance_mode_file, diff --git a/src/settings/cli.rs b/src/settings/cli.rs index 71eaa2a..3f24843 100644 --- a/src/settings/cli.rs +++ b/src/settings/cli.rs @@ -400,6 +400,19 @@ pub struct General { /// This is especially useful with Kubernetes liveness and readiness probes. pub health: bool, + #[cfg(unix)] + #[arg( + long = "experimental-metrics", + default_value = "false", + default_missing_value("true"), + num_args(0..=1), + require_equals(true), + action = clap::ArgAction::Set, + env = "SERVER_EXPERIMENTAL_METRICS", + )] + /// Add a /metrics endpoint that returns a Prometheus metrics response. + pub experimental_metrics: bool, + #[arg( long, default_value = "false", diff --git a/src/settings/file.rs b/src/settings/file.rs index 6dd57db..05ae5a7 100644 --- a/src/settings/file.rs +++ b/src/settings/file.rs @@ -239,6 +239,10 @@ pub struct General { /// Health endpoint feature. pub health: Option, + #[cfg(unix)] + /// Metrics endpoint feature (experimental). + pub experimental_metrics: Option, + /// Maintenance mode feature. pub maintenance_mode: Option, diff --git a/src/settings/mod.rs b/src/settings/mod.rs index 60e4a64..b641676 100644 --- a/src/settings/mod.rs +++ b/src/settings/mod.rs @@ -149,6 +149,8 @@ impl Settings { let mut ignore_hidden_files = opts.ignore_hidden_files; let mut index_files = opts.index_files; let mut health = opts.health; + #[cfg(unix)] + let mut experimental_metrics = opts.experimental_metrics; let mut maintenance_mode = opts.maintenance_mode; let mut maintenance_mode_status = opts.maintenance_mode_status; @@ -294,6 +296,10 @@ impl Settings { if let Some(v) = general.health { health = v } + #[cfg(unix)] + if let Some(v) = general.experimental_metrics { + experimental_metrics = v + } if let Some(v) = general.index_files { index_files = v } @@ -547,6 +553,8 @@ impl Settings { ignore_hidden_files, index_files, health, + #[cfg(unix)] + experimental_metrics, maintenance_mode, maintenance_mode_status, maintenance_mode_file, diff --git a/src/testing.rs b/src/testing.rs index 9621210..75a637e 100644 --- a/src/testing.rs +++ b/src/testing.rs @@ -52,6 +52,8 @@ pub mod fixtures { ignore_hidden_files: opts.general.ignore_hidden_files, index_files: vec![opts.general.index_files], health: opts.general.health, + #[cfg(unix)] + experimental_metrics: opts.general.experimental_metrics, maintenance_mode: opts.general.maintenance_mode, maintenance_mode_status: opts.general.maintenance_mode_status, maintenance_mode_file: opts.general.maintenance_mode_file, diff --git a/tests/experimental_metrics.rs b/tests/experimental_metrics.rs new file mode 100644 index 0000000..597155d --- /dev/null +++ b/tests/experimental_metrics.rs @@ -0,0 +1,47 @@ +#![forbid(unsafe_code)] +#![deny(warnings)] +#![deny(rust_2018_idioms)] +#![deny(dead_code)] + +#[cfg(unix)] +pub mod tests { + use hyper::Request; + use std::net::SocketAddr; + + use static_web_server::testing::fixtures::{fixture_req_handler, REMOTE_ADDR}; + + #[tokio::test] + async fn experimental_metrics_enabled() { + let req_handler = fixture_req_handler("toml/experimental_metrics.toml"); + let remote_addr = Some(REMOTE_ADDR.parse::().unwrap()); + + let mut req = Request::default(); + *req.method_mut() = hyper::Method::GET; + *req.uri_mut() = "http://localhost/metrics".parse().unwrap(); + + prometheus::default_registry() + .register(Box::new( + tokio_metrics_collector::default_runtime_collector(), + )) + .unwrap(); + + match req_handler.handle(&mut req, remote_addr).await { + Ok(res) => { + assert_eq!(res.status(), 200); + assert_eq!(res.headers()["content-type"], "text/plain; charset=utf-8"); + + let body = hyper::body::to_bytes(res.into_body()) + .await + .expect("unexpected bytes error during `body` conversion"); + let body_str = std::str::from_utf8(&body).unwrap(); + + assert!(body_str.contains("tokio_budget_forced_yield_count 0")); + assert!(body_str.contains("tokio_total_local_schedule_count 0")); + assert!(body_str.contains("tokio_workers_count 1")); + } + Err(err) => { + panic!("unexpected error: {err}") + } + }; + } +} diff --git a/tests/fixtures/toml/experimental_metrics.toml b/tests/fixtures/toml/experimental_metrics.toml new file mode 100644 index 0000000..45c0e34 --- /dev/null +++ b/tests/fixtures/toml/experimental_metrics.toml @@ -0,0 +1,5 @@ +[general] + +root = "docker/public" + +experimental-metrics = true -- libgit2 1.7.2