From 8f8f3ca0194bde5a4c3cb0515a663821b7289a30 Mon Sep 17 00:00:00 2001 From: Jeremy Andrews Date: Wed, 14 Jul 2021 15:24:13 +0200 Subject: [PATCH 1/3] enable gzip and deflate in http client --- CHANGELOG.md | 1 + Cargo.toml | 2 ++ src/goose.rs | 13 +++++++++---- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff2aee4..322a1cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - documentation cleanup; properly rename `GooseDefault::RequestFormat` and fix links - always configure `GooseConfiguration.manager` and `GooseConfiguration.worker`; confirm Manager is enabled when setting `--expect-workers` - moved `GooseConfiguration`, `GooseDefault`, and `GooseDefaultType` into new `src/config.rs` file; standardized configuration precedence through internal `GooseConfigure` trait defining `get_value()` for all supported types; general improvements to configuration documentation + - enable [`gzip`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.gzip) and [`deflate`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.deflate) support in the client ## 0.12.0 July 8, 2021 - remove internal-only functions and structures from documentation, exposing only what's useful to consumers of the Goose library (API change) diff --git a/Cargo.toml b/Cargo.toml index d1a1ceda..e8965691 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,8 @@ rand = "0.8" regex = "1" reqwest = { version = "0.11", default-features = false, features = [ "cookies", + "deflate", + "gzip", "json", ] } serde = { version = "1.0", features = [ diff --git a/src/goose.rs b/src/goose.rs index 8c328d18..c6f42fe8 100644 --- a/src/goose.rs +++ b/src/goose.rs @@ -871,6 +871,8 @@ impl GooseUser { let client = Client::builder() .user_agent(APP_USER_AGENT) .cookie_store(true) + .deflate(true) + .gzip(true) .build()?; Ok(GooseUser { @@ -1929,8 +1931,8 @@ impl GooseUser { /// will only affect requests made during test teardown; /// - A manually built client is specific to a single Goose thread -- if you are /// generating a large load test with many users, each will need to manually build their - /// own client (typically you'd do this in a Task that is registered with `set_on_start()` - /// in each Task Set requiring a custom client; + /// own client (typically you'd do this in a Task that is registered with + /// [`GooseTask::set_on_start()`] in each Task Set requiring a custom client; /// - Manually building a client will completely replace the automatically built client /// with a brand new one, so any configuration, cookies or headers set in the previously /// built client will be gone; @@ -1940,7 +1942,8 @@ impl GooseUser { /// [`.cookie_store(true)`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.cookie_store). /// /// In the following example, the Goose client is configured with a different user agent, - /// sets a default header on every request, and stores cookies. + /// sets a default header on every request, stores cookies, and supports gzip and deflate + /// compression. /// /// # Example /// ```rust @@ -1958,7 +1961,9 @@ impl GooseUser { /// let builder = Client::builder() /// .default_headers(headers) /// .user_agent("custom user agent") - /// .cookie_store(true); + /// .cookie_store(true) + /// .deflate(true) + /// .gzip(true); /// /// user.set_client_builder(builder).await?; /// From 2bad5d14b9a117758798f82499bc580813e39dd9 Mon Sep 17 00:00:00 2001 From: Jeremy Andrews Date: Thu, 15 Jul 2021 10:27:35 +0200 Subject: [PATCH 2/3] enable gzip by default, introduce `--no-gzip` flag --- CHANGELOG.md | 4 +++- Cargo.toml | 1 - src/config.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ src/goose.rs | 8 +++----- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 322a1cc5..d4aec06f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog +## 0.12.2-dev + - enable [`gzip`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.gzip) support and set Accept-Encoding header by default in the client; disable with `--no-gzip` or `GooseDefault::NoGzip` + ## 0.12.1 July 15, 2021 - rename `rustls` feature to `rustls-tls` so `tests/controller.rs` can build with the `rustls` library; update `tungstenite` to `0.14` and `tokio-tungstenite` = `0.15` to allow building with `rustls` - documentation cleanup; properly rename `GooseDefault::RequestFormat` and fix links - always configure `GooseConfiguration.manager` and `GooseConfiguration.worker`; confirm Manager is enabled when setting `--expect-workers` - moved `GooseConfiguration`, `GooseDefault`, and `GooseDefaultType` into new `src/config.rs` file; standardized configuration precedence through internal `GooseConfigure` trait defining `get_value()` for all supported types; general improvements to configuration documentation - - enable [`gzip`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.gzip) and [`deflate`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.deflate) support in the client ## 0.12.0 July 8, 2021 - remove internal-only functions and structures from documentation, exposing only what's useful to consumers of the Goose library (API change) diff --git a/Cargo.toml b/Cargo.toml index e8965691..52a3d272 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,6 @@ rand = "0.8" regex = "1" reqwest = { version = "0.11", default-features = false, features = [ "cookies", - "deflate", "gzip", "json", ] } diff --git a/src/config.rs b/src/config.rs index c56ff5e6..815f4d2c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -76,6 +76,7 @@ const DEFAULT_PORT: &str = "5115"; /// --websocket-host HOST Sets WebSocket Controller host (default: 0.0.0.0) /// --websocket-port PORT Sets WebSocket Controller TCP port (default: 5117) /// --no-autostart Doesn't automatically start load test +/// --no-gzip Doesn't set the gzip Accept-Encoding header /// --co-mitigation STRATEGY Sets coordinated omission mitigation strategy /// --throttle-requests VALUE Sets maximum requests per second /// --sticky-follow Follows base_url redirect with subsequent requests @@ -201,6 +202,9 @@ pub struct GooseConfiguration { /// Doesn't automatically start load test #[options(no_short)] pub no_autostart: bool, + /// Doesn't set the gzip Accept-Encoding header + #[options(no_short)] + pub no_gzip: bool, /// Sets coordinated omission mitigation strategy #[options(no_short, meta = "STRATEGY")] pub co_mitigation: Option, @@ -294,6 +298,8 @@ pub(crate) struct GooseDefaults { pub no_websocket: Option, /// An optional default for not auto-starting the load test. pub no_autostart: Option, + /// An optional default for not setting the gzip Accept-Encoding header. + pub no_gzip: Option, /// An optional default for coordinated omission mitigation. pub co_mitigation: Option, /// An optional default to track additional status code metrics. @@ -386,6 +392,8 @@ pub enum GooseDefault { CoordinatedOmissionMitigation, /// An optional default for not automatically starting load test. NoAutoStart, + /// An optional default for not setting the gzip Accept-Encoding header. + NoGzip, /// An optional default to track additional status code metrics. StatusCodes, /// An optional default maximum requests per second. @@ -484,6 +492,7 @@ pub enum GooseDefault { /// - [`GooseDefault::NoTelnet`] /// - [`GooseDefault::NoWebSocket`] /// - [`GooseDefault::NoAutoStart`] +/// - [`GooseDefault::NoGzip`] /// - [`GooseDefault::StatusCodes`] /// - [`GooseDefault::StickyFollow`] /// - [`GooseDefault::Manager`] @@ -569,6 +578,7 @@ impl GooseDefaultType<&str> for GooseAttack { | GooseDefault::NoTelnet | GooseDefault::NoWebSocket | GooseDefault::NoAutoStart + | GooseDefault::NoGzip | GooseDefault::StatusCodes | GooseDefault::StickyFollow | GooseDefault::Manager @@ -655,6 +665,7 @@ impl GooseDefaultType for GooseAttack { | GooseDefault::NoTelnet | GooseDefault::NoWebSocket | GooseDefault::NoAutoStart + | GooseDefault::NoGzip | GooseDefault::StatusCodes | GooseDefault::StickyFollow | GooseDefault::Manager @@ -708,6 +719,7 @@ impl GooseDefaultType for GooseAttack { GooseDefault::NoTelnet => self.defaults.no_telnet = Some(value), GooseDefault::NoWebSocket => self.defaults.no_websocket = Some(value), GooseDefault::NoAutoStart => self.defaults.no_autostart = Some(value), + GooseDefault::NoGzip => self.defaults.no_gzip = Some(value), GooseDefault::StatusCodes => self.defaults.status_codes = Some(value), GooseDefault::StickyFollow => self.defaults.sticky_follow = Some(value), GooseDefault::Manager => self.defaults.manager = Some(value), @@ -800,6 +812,7 @@ impl GooseDefaultType for GooseAttack { | GooseDefault::NoTelnet | GooseDefault::NoWebSocket | GooseDefault::NoAutoStart + | GooseDefault::NoGzip | GooseDefault::StatusCodes | GooseDefault::StickyFollow | GooseDefault::Manager @@ -893,6 +906,7 @@ impl GooseDefaultType for GooseAttack { | GooseDefault::NoTelnet | GooseDefault::NoWebSocket | GooseDefault::NoAutoStart + | GooseDefault::NoGzip | GooseDefault::StatusCodes | GooseDefault::StickyFollow | GooseDefault::Manager @@ -1502,6 +1516,24 @@ impl GooseConfiguration { ]) .unwrap_or(false); + // Configure `no_gzip`. + self.no_gzip = self + .get_value(vec![ + // Use --no-gzip if set. + GooseValue { + value: Some(self.no_gzip), + filter: !self.no_gzip, + message: "no_gzip", + }, + // Use GooseDefault if not already set and not Worker. + GooseValue { + value: defaults.no_gzip, + filter: defaults.no_gzip.is_none() || self.worker, + message: "no_gzip", + }, + ]) + .unwrap_or(false); + self.co_mitigation = self.get_value(vec![ // Use --co-mitigation if set. GooseValue { @@ -1880,6 +1912,13 @@ impl GooseConfiguration { detail: "`configuration.no_autostart` can not be set in Worker mode." .to_string(), }); + // Can't set `no_gzip` on Worker. + } else if self.no_gzip { + return Err(GooseError::InvalidOption { + option: "`configuration.no_gzip`".to_string(), + value: true.to_string(), + detail: "`configuration.no_gzip` can not be set in Worker mode.".to_string(), + }); } else if self .co_mitigation .as_ref() @@ -2187,6 +2226,8 @@ mod test { .unwrap() .set_default(GooseDefault::NoAutoStart, true) .unwrap() + .set_default(GooseDefault::NoGzip, true) + .unwrap() .set_default(GooseDefault::ReportFile, report_file.as_str()) .unwrap() .set_default(GooseDefault::RequestLog, request_log.as_str()) @@ -2251,6 +2292,7 @@ mod test { assert!(goose_attack.defaults.no_telnet == Some(true)); assert!(goose_attack.defaults.no_websocket == Some(true)); assert!(goose_attack.defaults.no_autostart == Some(true)); + assert!(goose_attack.defaults.no_gzip == Some(true)); assert!(goose_attack.defaults.report_file == Some(report_file)); assert!(goose_attack.defaults.request_log == Some(request_log)); assert!(goose_attack.defaults.request_format == Some(GooseLogFormat::Raw)); diff --git a/src/goose.rs b/src/goose.rs index c6f42fe8..9b948d45 100644 --- a/src/goose.rs +++ b/src/goose.rs @@ -871,8 +871,8 @@ impl GooseUser { let client = Client::builder() .user_agent(APP_USER_AGENT) .cookie_store(true) - .deflate(true) - .gzip(true) + // Enable gzip unless `--no-gzip` flag is enabled. + .gzip(!configuration.no_gzip) .build()?; Ok(GooseUser { @@ -1942,8 +1942,7 @@ impl GooseUser { /// [`.cookie_store(true)`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.cookie_store). /// /// In the following example, the Goose client is configured with a different user agent, - /// sets a default header on every request, stores cookies, and supports gzip and deflate - /// compression. + /// sets a default header on every request, stores cookies, and supports gzip compression. /// /// # Example /// ```rust @@ -1962,7 +1961,6 @@ impl GooseUser { /// .default_headers(headers) /// .user_agent("custom user agent") /// .cookie_store(true) - /// .deflate(true) /// .gzip(true); /// /// user.set_client_builder(builder).await?; From 15dfa412722ba1b3b1bcebbeb5dd06a2f10b3d20 Mon Sep 17 00:00:00 2001 From: Jeremy Andrews Date: Thu, 15 Jul 2021 11:53:13 +0200 Subject: [PATCH 3/3] document how to enable brotli and/or deflate --- Cargo.toml | 2 +- src/goose.rs | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 52a3d272..bb42c502 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "goose" -version = "0.12.1" +version = "0.12.2-dev" authors = ["Jeremy Andrews "] edition = "2018" description = "A load testing framework inspired by Locust." diff --git a/src/goose.rs b/src/goose.rs index 9b948d45..7473632a 100644 --- a/src/goose.rs +++ b/src/goose.rs @@ -1913,7 +1913,8 @@ impl GooseUser { /// /// let builder = Client::builder() /// .user_agent(APP_USER_AGENT) - /// .cookie_store(true); + /// .cookie_store(true) + /// .gzip(true); /// ``` /// /// Alternatively, you can use this function to manually build a @@ -1968,6 +1969,24 @@ impl GooseUser { /// Ok(()) /// } /// ``` + /// + /// Reqwest also supports + /// [`brotli`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.brotli) and + /// [`deflate`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.deflate) compression. + /// + /// To enable either, you must enable the features in your load test's `Cargo.toml`, for example: + /// ```text + /// reqwest = { version = "^0.11.4", default-features = false, features = [ + /// "brotli", + /// "cookies", + /// "deflate", + /// "gzip", + /// "json", + /// ] } + /// ``` + /// + /// Once enabled, you can add `.brotli(true)` and/or `.deflate(true)` to your custom Client::builder(), + /// similar to how is documented above. pub async fn set_client_builder(&self, builder: ClientBuilder) -> Result<(), GooseTaskError> { *self.client.lock().await = builder.build()?;