inbound: Restrict the HTTP/1 idle timeout (linkerd#1931)

The inbound proxy may hold idle inbound connections to the application open for (by default) 20s. This can be problematic when a server sets a lesser idle timeout and that idle timeout matches some poll interval (like a Kubernetes probe or metrics scrape). This change reduces the HTTP/1 connection pool idle timeout to 3s. This should be short enough to avoid this problem in many cases but it is also large enough that connection pooling can help busy services. In the future, this configuration should be made dynamic via the inbound policy API. Signed-off-by: Oliver Gould <ver@buoyant.io>
personio · Sep 13, 2022 · f3b9c49 · f3b9c49
1 parent 88518a8
commit f3b9c49
Showing 1 changed file with 17 additions and 1 deletion.
diff --git a/linkerd/app/src/env.rs b/linkerd/app/src/env.rs
@@ -218,6 +218,9 @@ const ENV_INITIAL_STREAM_WINDOW_SIZE: &str = "LINKERD2_PROXY_HTTP2_INITIAL_STREA
 const ENV_INITIAL_CONNECTION_WINDOW_SIZE: &str =
     "LINKERD2_PROXY_HTTP2_INITIAL_CONNECTION_WINDOW_SIZE";
 
+const ENV_INBOUND_HTTP1_CONNECTION_POOL_IDLE_TIMEOUT: &str =
+    "LINKERD2_PROXY_INBOUND_HTTP1_CONNECTION_POOL_IDLE_TIMEOUT";
+
 // Default values for various configuration fields
 const DEFAULT_OUTBOUND_LISTEN_ADDR: &str = "127.0.0.1:4140";
 pub const DEFAULT_INBOUND_LISTEN_ADDR: &str = "0.0.0.0:4143";
@@ -253,6 +256,13 @@ const DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE: u32 = 1048576; // 1MB ~ 16 streams
 const DEFAULT_INBOUND_ROUTER_MAX_IDLE_AGE: Duration = Duration::from_secs(20);
 const DEFAULT_OUTBOUND_ROUTER_MAX_IDLE_AGE: Duration = Duration::from_secs(5);
 
+// XXX This default inbound connection idle timeout should be less than or equal
+// to the server's idle timeout so that we don't try to reuse a connection as it
+// is being timed out of the server.
+//
+// In the future this should be made configurable per-server from the proxy API.
+const DEFAULT_INBOUND_HTTP1_CONNECTION_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(3);
+
 // By default, we don't limit the number of connections a connection pol may
 // use, as doing so can severely impact CPU utilization for applications with
 // many concurrent requests. It's generally preferable to use the MAX_IDLE_AGE
@@ -495,6 +505,12 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
             inbound_cache_max_idle_age?.unwrap_or(DEFAULT_INBOUND_ROUTER_MAX_IDLE_AGE);
         let max_idle =
             inbound_max_idle_per_endpoint?.unwrap_or(DEFAULT_INBOUND_MAX_IDLE_CONNS_PER_ENDPOINT);
+        let connection_pool_timeout = parse(
+            strings,
+            ENV_INBOUND_HTTP1_CONNECTION_POOL_IDLE_TIMEOUT,
+            parse_duration,
+        )?
+        .unwrap_or(DEFAULT_INBOUND_HTTP1_CONNECTION_POOL_IDLE_TIMEOUT);
         let keepalive = Keepalive(inbound_connect_keepalive?);
         let connect = ConnectConfig {
             keepalive,
@@ -507,7 +523,7 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
             h2_settings,
             h1_settings: h1::PoolSettings {
                 max_idle,
-                idle_timeout: cache_max_idle_age,
+                idle_timeout: connection_pool_timeout,
             },
         };