Merge pull request #328 from jeremyandrews/compress

enable gzip and deflate in http client

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.12.2-dev
+  - enable [`gzip`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.gzip) support and set Accept-Encoding header by default in the client; disable with `--no-gzip` or `GooseDefault::NoGzip`
+
 ## 0.12.1 July 15, 2021
  - rename `rustls` feature to `rustls-tls` so `tests/controller.rs` can build with the `rustls` library; update `tungstenite` to `0.14` and `tokio-tungstenite` = `0.15` to allow building with `rustls`
   - documentation cleanup; properly rename `GooseDefault::RequestFormat` and fix links

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "goose"
-version = "0.12.1"
+version = "0.12.2-dev"
 authors = ["Jeremy Andrews <[email protected]>"]
 edition = "2018"
 description = "A load testing framework inspired by Locust."
@@ -28,6 +28,7 @@ rand = "0.8"
 regex = "1"
 reqwest = { version = "0.11",  default-features = false, features = [
     "cookies",
+    "gzip",
     "json",
 ] }
 serde = { version = "1.0", features = [

src/config.rs

@@ -76,6 +76,7 @@ const DEFAULT_PORT: &str = "5115";
 /// --websocket-host HOST      Sets WebSocket Controller host (default: 0.0.0.0)
 /// --websocket-port PORT      Sets WebSocket Controller TCP port (default: 5117)
 /// --no-autostart             Doesn't automatically start load test
+/// --no-gzip                  Doesn't set the gzip Accept-Encoding header
 /// --co-mitigation STRATEGY   Sets coordinated omission mitigation strategy
 /// --throttle-requests VALUE  Sets maximum requests per second
 /// --sticky-follow            Follows base_url redirect with subsequent requests
@@ -201,6 +202,9 @@ pub struct GooseConfiguration {
     /// Doesn't automatically start load test
     #[options(no_short)]
     pub no_autostart: bool,
+    /// Doesn't set the gzip Accept-Encoding header
+    #[options(no_short)]
+    pub no_gzip: bool,
     /// Sets coordinated omission mitigation strategy
     #[options(no_short, meta = "STRATEGY")]
     pub co_mitigation: Option<GooseCoordinatedOmissionMitigation>,
@@ -294,6 +298,8 @@ pub(crate) struct GooseDefaults {
     pub no_websocket: Option<bool>,
     /// An optional default for not auto-starting the load test.
     pub no_autostart: Option<bool>,
+    /// An optional default for not setting the gzip Accept-Encoding header.
+    pub no_gzip: Option<bool>,
     /// An optional default for coordinated omission mitigation.
     pub co_mitigation: Option<GooseCoordinatedOmissionMitigation>,
     /// An optional default to track additional status code metrics.
@@ -386,6 +392,8 @@ pub enum GooseDefault {
     CoordinatedOmissionMitigation,
     /// An optional default for not automatically starting load test.
     NoAutoStart,
+    /// An optional default for not setting the gzip Accept-Encoding header.
+    NoGzip,
     /// An optional default to track additional status code metrics.
     StatusCodes,
     /// An optional default maximum requests per second.
@@ -484,6 +492,7 @@ pub enum GooseDefault {
 ///  - [`GooseDefault::NoTelnet`]
 ///  - [`GooseDefault::NoWebSocket`]
 ///  - [`GooseDefault::NoAutoStart`]
+///  - [`GooseDefault::NoGzip`]
 ///  - [`GooseDefault::StatusCodes`]
 ///  - [`GooseDefault::StickyFollow`]
 ///  - [`GooseDefault::Manager`]
@@ -569,6 +578,7 @@ impl GooseDefaultType<&str> for GooseAttack {
             | GooseDefault::NoTelnet
             | GooseDefault::NoWebSocket
             | GooseDefault::NoAutoStart
+            | GooseDefault::NoGzip
             | GooseDefault::StatusCodes
             | GooseDefault::StickyFollow
             | GooseDefault::Manager
@@ -655,6 +665,7 @@ impl GooseDefaultType<usize> for GooseAttack {
             | GooseDefault::NoTelnet
             | GooseDefault::NoWebSocket
             | GooseDefault::NoAutoStart
+            | GooseDefault::NoGzip
             | GooseDefault::StatusCodes
             | GooseDefault::StickyFollow
             | GooseDefault::Manager
@@ -708,6 +719,7 @@ impl GooseDefaultType<bool> for GooseAttack {
             GooseDefault::NoTelnet => self.defaults.no_telnet = Some(value),
             GooseDefault::NoWebSocket => self.defaults.no_websocket = Some(value),
             GooseDefault::NoAutoStart => self.defaults.no_autostart = Some(value),
+            GooseDefault::NoGzip => self.defaults.no_gzip = Some(value),
             GooseDefault::StatusCodes => self.defaults.status_codes = Some(value),
             GooseDefault::StickyFollow => self.defaults.sticky_follow = Some(value),
             GooseDefault::Manager => self.defaults.manager = Some(value),
@@ -800,6 +812,7 @@ impl GooseDefaultType<GooseCoordinatedOmissionMitigation> for GooseAttack {
             | GooseDefault::NoTelnet
             | GooseDefault::NoWebSocket
             | GooseDefault::NoAutoStart
+            | GooseDefault::NoGzip
             | GooseDefault::StatusCodes
             | GooseDefault::StickyFollow
             | GooseDefault::Manager
@@ -893,6 +906,7 @@ impl GooseDefaultType<GooseLogFormat> for GooseAttack {
             | GooseDefault::NoTelnet
             | GooseDefault::NoWebSocket
             | GooseDefault::NoAutoStart
+            | GooseDefault::NoGzip
             | GooseDefault::StatusCodes
             | GooseDefault::StickyFollow
             | GooseDefault::Manager
@@ -1502,6 +1516,24 @@ impl GooseConfiguration {
             ])
             .unwrap_or(false);
 
+        // Configure `no_gzip`.
+        self.no_gzip = self
+            .get_value(vec![
+                // Use --no-gzip if set.
+                GooseValue {
+                    value: Some(self.no_gzip),
+                    filter: !self.no_gzip,
+                    message: "no_gzip",
+                },
+                // Use GooseDefault if not already set and not Worker.
+                GooseValue {
+                    value: defaults.no_gzip,
+                    filter: defaults.no_gzip.is_none() || self.worker,
+                    message: "no_gzip",
+                },
+            ])
+            .unwrap_or(false);
+
         self.co_mitigation = self.get_value(vec![
             // Use --co-mitigation if set.
             GooseValue {
@@ -1880,6 +1912,13 @@ impl GooseConfiguration {
                     detail: "`configuration.no_autostart` can not be set in Worker mode."
                         .to_string(),
                 });
+            // Can't set `no_gzip` on Worker.
+            } else if self.no_gzip {
+                return Err(GooseError::InvalidOption {
+                    option: "`configuration.no_gzip`".to_string(),
+                    value: true.to_string(),
+                    detail: "`configuration.no_gzip` can not be set in Worker mode.".to_string(),
+                });
             } else if self
                 .co_mitigation
                 .as_ref()
@@ -2187,6 +2226,8 @@ mod test {
             .unwrap()
             .set_default(GooseDefault::NoAutoStart, true)
             .unwrap()
+            .set_default(GooseDefault::NoGzip, true)
+            .unwrap()
             .set_default(GooseDefault::ReportFile, report_file.as_str())
             .unwrap()
             .set_default(GooseDefault::RequestLog, request_log.as_str())
@@ -2251,6 +2292,7 @@ mod test {
         assert!(goose_attack.defaults.no_telnet == Some(true));
         assert!(goose_attack.defaults.no_websocket == Some(true));
         assert!(goose_attack.defaults.no_autostart == Some(true));
+        assert!(goose_attack.defaults.no_gzip == Some(true));
         assert!(goose_attack.defaults.report_file == Some(report_file));
         assert!(goose_attack.defaults.request_log == Some(request_log));
         assert!(goose_attack.defaults.request_format == Some(GooseLogFormat::Raw));

src/goose.rs

@@ -871,6 +871,8 @@ impl GooseUser {
         let client = Client::builder()
             .user_agent(APP_USER_AGENT)
             .cookie_store(true)
+            // Enable gzip unless `--no-gzip` flag is enabled.
+            .gzip(!configuration.no_gzip)
             .build()?;
 
         Ok(GooseUser {
@@ -1911,7 +1913,8 @@ impl GooseUser {
     ///
     /// let builder = Client::builder()
     ///   .user_agent(APP_USER_AGENT)
-    ///   .cookie_store(true);
+    ///   .cookie_store(true)
+    ///   .gzip(true);
     /// ```
     ///
     /// Alternatively, you can use this function to manually build a
@@ -1929,8 +1932,8 @@ impl GooseUser {
     ///    will only affect requests made during test teardown;
     ///  - A manually built client is specific to a single Goose thread -- if you are
     ///    generating a large load test with many users, each will need to manually build their
-    ///    own client (typically you'd do this in a Task that is registered with `set_on_start()`
-    ///    in each Task Set requiring a custom client;
+    ///    own client (typically you'd do this in a Task that is registered with
+    ///   [`GooseTask::set_on_start()`] in each Task Set requiring a custom client;
     ///  - Manually building a client will completely replace the automatically built client
     ///    with a brand new one, so any configuration, cookies or headers set in the previously
     ///    built client will be gone;
@@ -1940,7 +1943,7 @@ impl GooseUser {
     ///    [`.cookie_store(true)`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.cookie_store).
     ///
     /// In the following example, the Goose client is configured with a different user agent,
-    /// sets a default header on every request, and stores cookies.
+    /// sets a default header on every request, stores cookies, and supports gzip compression.
     ///
     /// # Example
     /// ```rust
@@ -1958,13 +1961,32 @@ impl GooseUser {
     ///     let builder = Client::builder()
     ///         .default_headers(headers)
     ///         .user_agent("custom user agent")
-    ///         .cookie_store(true);
+    ///         .cookie_store(true)
+    ///         .gzip(true);
     ///
     ///     user.set_client_builder(builder).await?;
     ///
     ///     Ok(())
     /// }
     /// ```
+    ///
+    /// Reqwest also supports
+    /// [`brotli`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.brotli) and
+    /// [`deflate`](https://docs.rs/reqwest/*/reqwest/struct.ClientBuilder.html#method.deflate) compression.
+    ///
+    /// To enable either, you must enable the features in your load test's `Cargo.toml`, for example:
+    /// ```text
+    /// reqwest = { version = "^0.11.4",  default-features = false, features = [
+    ///     "brotli",
+    ///     "cookies",
+    ///     "deflate",
+    ///     "gzip",
+    ///     "json",
+    /// ] }
+    /// ```
+    ///
+    /// Once enabled, you can add `.brotli(true)` and/or `.deflate(true)` to your custom Client::builder(),
+    /// similar to how is documented above.
     pub async fn set_client_builder(&self, builder: ClientBuilder) -> Result<(), GooseTaskError> {
         *self.client.lock().await = builder.build()?;