feat(config): Add an option for upstream content encoding (#771)

Adds the `http.encoding` configuration option to disable or change compression of upstream store requests. It is planned that this is eventually applied to all upstream requests, hence the generic naming. By default, compression should be enabled by all Relays. Sentry applies more aggressive limits on the content-length of raw requests than after decompression. However, in the current state, compression can become a bottleneck in high throughput scenarios, in which case it may be more beneficial to disable compression.
getsentry · Sep 11, 2020 · edfc6bd · edfc6bd
1 parent 77d2f90
commit edfc6bd
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@
 - The module attribute on stack frames (`$frame.module`) and the (usually server side generated) attribute `culprit` can now be scrubbed with advanced data scrubbing. ([#744](https://github.com/getsentry/relay/pull/744))
 - Compress outgoing store requests for events and envelopes including attachements using `gzip` content encoding. ([#745](https://github.com/getsentry/relay/pull/745))
 - Retry sending events on network errors instead of dropping them. Also, Relay now buffers all requests until it has authenticated with the upstream. ([#747](//github.com/getsentry/relay/pull/747))
+- Add a configuration option to change content encoding of upstream store requests. The default is `gzip`, and other options are `identity`, `deflate`, or `br`. ([#771](https://github.com/getsentry/relay/pull/771))
 
 **Bug Fixes**:
 

diff --git a/relay-config/src/config.rs b/relay-config/src/config.rs
@@ -505,6 +505,20 @@ impl Default for Limits {
     }
 }
 
+/// Http content encoding for upstream store requests.
+#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HttpEncoding {
+    /// Identity function, no compression.
+    Identity,
+    /// Compression using a zlib header with deflate encoding.
+    Deflate,
+    /// Compression using gzip.
+    Gzip,
+    /// Compression using the brotli algorithm.
+    Br,
+}
+
 /// Controls authentication with upstream.
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(default)]
@@ -539,6 +553,21 @@ struct Http {
     /// attempts persist beyond the grace period, Relay suspends event submission and reverts into
     /// authentication mode.
     auth_grace_period: u64,
+    /// Content encoding to apply to upstream store requests.
+    ///
+    /// By default, Relay applies `gzip` content encoding to compress upstream requests. Compression
+    /// can be disabled to reduce CPU consumption, but at the expense of increased network traffic.
+    ///
+    /// This setting applies to all store requests of SDK data, including events, transactions,
+    /// envelopes and sessions. At the moment, this does not apply to Relay's internal queries.
+    ///
+    /// Available options are:
+    ///
+    ///  - `identity`: Disables compression.
+    ///  - `deflate`: Compression using a zlib header with deflate encoding.
+    ///  - `gzip` (default): Compression using gzip.
+    ///  - `br`: Compression using the brotli algorithm.
+    encoding: HttpEncoding,
 }
 
 impl Default for Http {
@@ -550,6 +579,7 @@ impl Default for Http {
             host_header: None,
             auth_interval: Some(600), // 10 minutes
             auth_grace_period: 10,
+            encoding: HttpEncoding::Gzip,
         }
     }
 }
@@ -1138,6 +1168,11 @@ impl Config {
         Duration::from_secs(self.values.http.auth_grace_period)
     }
 
+    /// Content encoding of upstream requests.
+    pub fn http_encoding(&self) -> HttpEncoding {
+        self.values.http.encoding
+    }
+
     /// Returns whether this Relay should emit outcomes.
     ///
     /// This is `true` either if `outcomes.emit_outcomes` is explicitly enabled, or if this Relay is

diff --git a/relay-server/src/actors/events.rs b/relay-server/src/actors/events.rs
@@ -13,7 +13,7 @@ use parking_lot::RwLock;
 use serde_json::Value as SerdeValue;
 
 use relay_common::{clone, metric, LogError};
-use relay_config::{Config, RelayMode};
+use relay_config::{Config, HttpEncoding, RelayMode};
 use relay_general::pii::{PiiAttachmentsProcessor, PiiProcessor};
 use relay_general::processor::{process_value, ProcessingState};
 use relay_general::protocol::{
@@ -1331,6 +1331,7 @@ impl Handler<HandleEnvelope> for EventManager {
         let outcome_producer = self.outcome_producer.clone();
         let captured_events = self.captured_events.clone();
         let capture = self.config.relay_mode() == RelayMode::Capture;
+        let http_encoding = self.config.http_encoding();
 
         #[cfg(feature = "processing")]
         let store_forwarder = self.store_forwarder.clone();
@@ -1452,11 +1453,18 @@ impl Handler<HandleEnvelope> for EventManager {
                             builder.header("User-Agent", user_agent);
                         }
 
+                        let content_encoding = match http_encoding {
+                            HttpEncoding::Identity => ContentEncoding::Identity,
+                            HttpEncoding::Deflate => ContentEncoding::Deflate,
+                            HttpEncoding::Gzip => ContentEncoding::Gzip,
+                            HttpEncoding::Br => ContentEncoding::Br,
+                        };
+
                         builder
+                            .content_encoding(content_encoding)
                             .header("X-Sentry-Auth", meta.auth_header())
                             .header("X-Forwarded-For", meta.forwarded_for())
                             .header("Content-Type", envelope::CONTENT_TYPE)
-                            .content_encoding(ContentEncoding::Gzip)
                             .body(envelope.to_vec().map_err(failure::Error::from)?)
                     },
                 );