Merge pull request #28876 from thockin/kube-proxy-scale-conntrack-by-…

…cores Automatic merge from submit-queue Scale kube-proxy conntrack limits by cores (new default behavior) For large machines we want more conntrack entries than smaller machines. Commence bike-shedding on names and values and semantics. I'd love to get this in as a patch to 1.3.x since we have had some trouble for users on large machines. Fixes #28867
kubernetes · Jul 16, 2016 · 6f98b69 · 6f98b69
2 parents 5b027a5 + 1f37281
commit 6f98b69
Show file tree

Hide file tree

Showing 11 changed files with 319 additions and 176 deletions.
diff --git a/cmd/kube-proxy/app/options/options.go b/cmd/kube-proxy/app/options/options.go
@@ -83,6 +83,9 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
 	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
 	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
 	fs.DurationVar(&s.UDPIdleTimeout.Duration, "udp-timeout", s.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxy-mode=userspace")
-	fs.Int32Var(&s.ConntrackMax, "conntrack-max", s.ConntrackMax, "Maximum number of NAT connections to track (0 to leave as-is)")
+	fs.Int32Var(&s.ConntrackMax, "conntrack-max", s.ConntrackMax,
+		"Maximum number of NAT connections to track (0 to leave as-is).")
+	fs.Int32Var(&s.ConntrackMaxPerCore, "conntrack-max-per-core", s.ConntrackMaxPerCore,
+		"Maximum number of NAT connections to track per CPU core (0 to leave as-is). This is only considered if conntrack-max is 0.")
 	fs.DurationVar(&s.ConntrackTCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", s.ConntrackTCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)")
 }
diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go
@@ -24,6 +24,7 @@ import (
 	"net"
 	"net/http"
 	_ "net/http/pprof"
+	"runtime"
 	"strconv"
 	"time"
 
@@ -298,8 +299,12 @@ func (s *ProxyServer) Run() error {
 
 	// Tune conntrack, if requested
 	if s.Conntracker != nil {
-		if s.Config.ConntrackMax > 0 {
-			err := s.Conntracker.SetMax(int(s.Config.ConntrackMax))
+		max, err := getConntrackMax(s.Config)
+		if err != nil {
+			return err
+		}
+		if max > 0 {
+			err := s.Conntracker.SetMax(max)
 			if err != nil {
 				if err != readOnlySysFSError {
 					return err
@@ -329,6 +334,18 @@ func (s *ProxyServer) Run() error {
 	return nil
 }
 
+func getConntrackMax(config *options.ProxyServerConfig) (int, error) {
+	if config.ConntrackMax > 0 && config.ConntrackMaxPerCore > 0 {
+		return -1, fmt.Errorf("invalid config: ConntrackMax and ConntrackMaxPerCore are mutually exclusive")
+	}
+	if config.ConntrackMax > 0 {
+		return int(config.ConntrackMax), nil
+	} else if config.ConntrackMaxPerCore > 0 {
+		return (int(config.ConntrackMaxPerCore) * runtime.NumCPU()), nil
+	}
+	return 0, nil
+}
+
 type nodeGetter interface {
 	Get(hostname string) (*api.Node, error)
 }

diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go
@@ -19,12 +19,14 @@ package app
 import (
 	"fmt"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
 	"k8s.io/kubernetes/cmd/kube-proxy/app/options"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/util/iptables"
 )
 
@@ -268,7 +270,6 @@ func Test_getProxyMode(t *testing.T) {
 // Config and iptinterface are not nil when CleanupAndExit is true.
 // To avoid proxy crash: https://github.com/kubernetes/kubernetes/pull/14736
 func TestProxyServerWithCleanupAndExit(t *testing.T) {
-
 	// creates default config
 	config := options.NewProxyConfig()
 
@@ -284,3 +285,51 @@ func TestProxyServerWithCleanupAndExit(t *testing.T) {
 	assert.NotNil(t, proxyserver.Config)
 	assert.NotNil(t, proxyserver.IptInterface)
 }
+
+func TestGetConntrackMax(t *testing.T) {
+	ncores := runtime.NumCPU()
+	testCases := []struct {
+		config   componentconfig.KubeProxyConfiguration
+		expected int
+		err      string
+	}{
+		{
+			config:   componentconfig.KubeProxyConfiguration{},
+			expected: 0,
+		},
+		{
+			config: componentconfig.KubeProxyConfiguration{
+				ConntrackMax: 12345,
+			},
+			expected: 12345,
+		},
+		{
+			config: componentconfig.KubeProxyConfiguration{
+				ConntrackMax:        12345,
+				ConntrackMaxPerCore: 67890,
+			},
+			expected: -1,
+			err:      "mutually exclusive",
+		},
+		{
+			config: componentconfig.KubeProxyConfiguration{
+				ConntrackMaxPerCore: 67890, // use this if other is 0
+			},
+			expected: 67890 * ncores,
+		},
+	}
+
+	for i, tc := range testCases {
+		cfg := options.ProxyServerConfig{KubeProxyConfiguration: tc.config}
+		x, e := getConntrackMax(&cfg)
+		if e != nil {
+			if tc.err == "" {
+				t.Errorf("[%d] unexpected error: %v", i, e)
+			} else if !strings.Contains(e.Error(), tc.err) {
+				t.Errorf("[%d] expected an error containing %q: %v", i, tc.err, e)
+			}
+		} else if x != tc.expected {
+			t.Errorf("[%d] expected %d, got %d", i, tc.expected, x)
+		}
+	}
+}
diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt
@@ -74,6 +74,7 @@ config-sync-period
 configure-cbr0
 configure-cloud-routes
 conntrack-max
+conntrack-max-per-core
 conntrack-tcp-timeout-established
 consumer-port
 consumer-service-name