Skip to content

Commit

Permalink
Enable iptables kube-proxy by default in master
Browse files Browse the repository at this point in the history
  • Loading branch information
@thockin
thockin committed Nov 14, 2015
1 parent 12ad0b2 commit 970c045
Show file tree
Hide file tree
Showing 16 changed files with 597 additions and 405 deletions.
65 changes: 36 additions & 29 deletions cmd/kube-proxy/app/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
fs.Var(&s.PortRange, "proxy-port-range", "Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.")
fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
fs.StringVar(&s.ProxyMode, "proxy-mode", "", "Which proxy mode to use: 'userspace' (older, stable) or 'iptables' (experimental). If blank, look at the Node object on the Kubernetes API and respect the '"+experimentalProxyModeAnnotation+"' annotation if provided. Otherwise use the best-available proxy (currently userspace, but may change in future versions). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.")
fs.StringVar(&s.ProxyMode, "proxy-mode", "", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If blank, look at the Node object on the Kubernetes API and respect the '"+experimentalProxyModeAnnotation+"' annotation if provided. Otherwise use the best-available proxy (currently iptables). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.")
fs.DurationVar(&s.IptablesSyncPeriod, "iptables-sync-period", s.IptablesSyncPeriod, "How often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.")
fs.DurationVar(&s.ConfigSyncPeriod, "config-sync-period", s.ConfigSyncPeriod, "How often configuration from the apiserver is refreshed. Must be greater than 0.")
fs.BoolVar(&s.MasqueradeAll, "masquerade-all", false, "If using the pure iptables proxy, SNAT everything")
Expand Down Expand Up @@ -238,17 +238,8 @@ func NewProxyServerDefault(config *ProxyServerConfig) (*ProxyServer, error) {
var proxier proxy.ProxyProvider
var endpointsHandler proxyconfig.EndpointsConfigHandler

useIptablesProxy := false
if mayTryIptablesProxy(config.ProxyMode, client.Nodes(), hostname) {
var err error
// guaranteed false on error, error only necessary for debugging
useIptablesProxy, err = iptables.ShouldUseIptablesProxier()
if err != nil {
glog.Errorf("Can't determine whether to use iptables proxy, using userspace proxier: %v", err)
}
}

if useIptablesProxy {
proxyMode := getProxyMode(config.ProxyMode, client.Nodes(), hostname, iptInterface)
if proxyMode == proxyModeIptables {
glog.V(2).Info("Using iptables Proxier.")
proxierIptables, err := iptables.NewProxier(iptInterface, execer, config.IptablesSyncPeriod, config.MasqueradeAll)
if err != nil {
Expand Down Expand Up @@ -340,27 +331,28 @@ type nodeGetter interface {
Get(hostname string) (*api.Node, error)
}

func mayTryIptablesProxy(proxyMode string, client nodeGetter, hostname string) bool {
if proxyMode == proxyModeIptables {
glog.V(1).Infof("Flag proxy-mode allows iptables proxy")
return true
func getProxyMode(proxyMode string, client nodeGetter, hostname string, iptver iptables.IptablesVersioner) string {
if proxyMode == proxyModeUserspace {
return proxyModeUserspace
} else if proxyMode == proxyModeIptables {
return tryIptablesProxy(iptver)
} else if proxyMode != "" {
glog.V(1).Infof("Flag proxy-mode=%q forbids iptables proxy", proxyMode)
return false
glog.V(1).Infof("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode)
return tryIptablesProxy(iptver)
}
// proxyMode == "" - choose the best option.
if client == nil {
glog.Errorf("Not trying iptables proxy: nodeGetter is nil")
return false
glog.Errorf("nodeGetter is nil: assuming iptables proxy")
return tryIptablesProxy(iptver)
}
node, err := client.Get(hostname)
if err != nil {
glog.Errorf("Not trying iptables proxy: can't get Node %q: %v", hostname, err)
return false
glog.Errorf("Can't get Node %q, assuming iptables proxy: %v", hostname, err)
return tryIptablesProxy(iptver)
}
if node == nil {
glog.Errorf("Not trying iptables proxy: got nil Node %q", hostname)
return false
glog.Errorf("Got nil Node %q, assuming iptables proxy: %v", hostname)
return tryIptablesProxy(iptver)
}
proxyMode, found := node.Annotations[betaProxyModeAnnotation]
if found {
Expand All @@ -372,12 +364,27 @@ func mayTryIptablesProxy(proxyMode string, client nodeGetter, hostname string) b
glog.V(1).Infof("Found experimental annotation %q = %q", experimentalProxyModeAnnotation, proxyMode)
}
}
if proxyMode == proxyModeIptables {
glog.V(1).Infof("Annotation allows iptables proxy")
return true
if proxyMode == proxyModeUserspace {
glog.V(1).Infof("Annotation demands userspace proxy")
return proxyModeUserspace
}
glog.V(1).Infof("Not trying iptables proxy: %+v", node)
return false
return tryIptablesProxy(iptver)
}

func tryIptablesProxy(iptver iptables.IptablesVersioner) string {
var err error
// guaranteed false on error, error only necessary for debugging
useIptablesProxy, err := iptables.CanUseIptablesProxier(iptver)
if err != nil {
glog.Errorf("Can't determine whether to use iptables proxy, using userspace proxier: %v", err)
return proxyModeUserspace
}
if useIptablesProxy {
return proxyModeIptables
}
// Fallback.
glog.V(1).Infof("Can't use iptables proxy, using userspace proxier: %v", err)
return proxyModeUserspace
}

func (s *ProxyServer) birthCry() {
Expand Down
194 changes: 166 additions & 28 deletions cmd/kube-proxy/app/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@ limitations under the License.
package app

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/util/iptables"
)

type fakeNodeInterface struct {
Expand All @@ -31,40 +33,176 @@ func (fake *fakeNodeInterface) Get(hostname string) (*api.Node, error) {
return &fake.node, nil
}

func Test_mayTryIptablesProxy(t *testing.T) {
type fakeIptablesVersioner struct {
version string // what to return
err error // what to return
}

func (fake *fakeIptablesVersioner) GetVersion() (string, error) {
return fake.version, fake.err
}

func Test_getProxyMode(t *testing.T) {
var cases = []struct {
flag string
annKey string
annVal string
expected bool
flag string
annotationKey string
annotationVal string
iptablesVersion string
iptablesError error
expected string
}{
{"userspace", "", "", false},
{"iptables", "", "", true},
{"", "", "", false},
{"", "net.experimental.kubernetes.io/proxy-mode", "userspace", false},
{"", "net.experimental.kubernetes.io/proxy-mode", "iptables", true},
{"", "net.experimental.kubernetes.io/proxy-mode", "other", false},
{"", "net.experimental.kubernetes.io/proxy-mode", "", false},
{"", "net.beta.kubernetes.io/proxy-mode", "userspace", false},
{"", "net.beta.kubernetes.io/proxy-mode", "iptables", true},
{"", "net.beta.kubernetes.io/proxy-mode", "other", false},
{"", "net.beta.kubernetes.io/proxy-mode", "", false},
{"", "proxy-mode", "iptables", false},
{"userspace", "net.experimental.kubernetes.io/proxy-mode", "userspace", false},
{"userspace", "net.experimental.kubernetes.io/proxy-mode", "iptables", false},
{"iptables", "net.experimental.kubernetes.io/proxy-mode", "userspace", true},
{"iptables", "net.experimental.kubernetes.io/proxy-mode", "iptables", true},
{"userspace", "net.beta.kubernetes.io/proxy-mode", "userspace", false},
{"userspace", "net.beta.kubernetes.io/proxy-mode", "iptables", false},
{"iptables", "net.beta.kubernetes.io/proxy-mode", "userspace", true},
{"iptables", "net.beta.kubernetes.io/proxy-mode", "iptables", true},
{ // flag says userspace
flag: "userspace",
expected: proxyModeUserspace,
},
{ // flag says iptables, error detecting version
flag: "iptables",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // flag says iptables, version too low
flag: "iptables",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // flag says iptables, version ok
flag: "iptables",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
{ // detect, error
flag: "",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // detect, version too low
flag: "",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // detect, version ok
flag: "",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
{ // annotation says userspace
flag: "",
annotationKey: "net.experimental.kubernetes.io/proxy-mode",
annotationVal: "userspace",
expected: proxyModeUserspace,
},
{ // annotation says iptables, error detecting
flag: "",
annotationKey: "net.experimental.kubernetes.io/proxy-mode",
annotationVal: "iptables",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // annotation says iptables, version too low
flag: "",
annotationKey: "net.experimental.kubernetes.io/proxy-mode",
annotationVal: "iptables",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // annotation says iptables, version ok
flag: "",
annotationKey: "net.experimental.kubernetes.io/proxy-mode",
annotationVal: "iptables",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
{ // annotation says something else, version ok
flag: "",
annotationKey: "net.experimental.kubernetes.io/proxy-mode",
annotationVal: "other",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
{ // annotation says nothing, version ok
flag: "",
annotationKey: "net.experimental.kubernetes.io/proxy-mode",
annotationVal: "",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
{ // annotation says userspace
flag: "",
annotationKey: "net.beta.kubernetes.io/proxy-mode",
annotationVal: "userspace",
expected: proxyModeUserspace,
},
{ // annotation says iptables, error detecting
flag: "",
annotationKey: "net.beta.kubernetes.io/proxy-mode",
annotationVal: "iptables",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // annotation says iptables, version too low
flag: "",
annotationKey: "net.beta.kubernetes.io/proxy-mode",
annotationVal: "iptables",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // annotation says iptables, version ok
flag: "",
annotationKey: "net.beta.kubernetes.io/proxy-mode",
annotationVal: "iptables",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
{ // annotation says something else, version ok
flag: "",
annotationKey: "net.beta.kubernetes.io/proxy-mode",
annotationVal: "other",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
{ // annotation says nothing, version ok
flag: "",
annotationKey: "net.beta.kubernetes.io/proxy-mode",
annotationVal: "",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
{ // flag says userspace, annotation disagrees
flag: "userspace",
annotationKey: "net.experimental.kubernetes.io/proxy-mode",
annotationVal: "iptables",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeUserspace,
},
{ // flag says iptables, annotation disagrees
flag: "iptables",
annotationKey: "net.experimental.kubernetes.io/proxy-mode",
annotationVal: "userspace",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
{ // flag says userspace, annotation disagrees
flag: "userspace",
annotationKey: "net.beta.kubernetes.io/proxy-mode",
annotationVal: "iptables",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeUserspace,
},
{ // flag says iptables, annotation disagrees
flag: "iptables",
annotationKey: "net.beta.kubernetes.io/proxy-mode",
annotationVal: "userspace",
iptablesVersion: iptables.MinCheckVersion,
expected: proxyModeIptables,
},
}
for i, c := range cases {
getter := &fakeNodeInterface{}
getter.node.Annotations = map[string]string{c.annKey: c.annVal}
r := mayTryIptablesProxy(c.flag, getter, "host")
getter.node.Annotations = map[string]string{c.annotationKey: c.annotationVal}
versioner := &fakeIptablesVersioner{c.iptablesVersion, c.iptablesError}
r := getProxyMode(c.flag, getter, "host", versioner)
if r != c.expected {
t.Errorf("Case[%d] Expected %t, got %t", i, c.expected, r)
t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions docs/admin/kube-proxy.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,13 @@ kube-proxy
--masquerade-all[=false]: If using the pure iptables proxy, SNAT everything
--master="": The address of the Kubernetes API server (overrides any value in kubeconfig)
--oom-score-adj=-999: The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]
--proxy-mode="": Which proxy mode to use: 'userspace' (older, stable) or 'iptables' (experimental). If blank, look at the Node object on the Kubernetes API and respect the 'net.experimental.kubernetes.io/proxy-mode' annotation if provided. Otherwise use the best-available proxy (currently userspace, but may change in future versions). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.
--proxy-mode="": Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If blank, look at the Node object on the Kubernetes API and respect the 'net.experimental.kubernetes.io/proxy-mode' annotation if provided. Otherwise use the best-available proxy (currently iptables). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.
--proxy-port-range=: Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.
--resource-container="/kube-proxy": Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).
--udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace
```

###### Auto generated by spf13/cobra on 23-Oct-2015
###### Auto generated by spf13/cobra on 9-Nov-2015


<!-- BEGIN MUNGE: GENERATED_ANALYTICS -->
Expand Down
Loading

0 comments on commit 970c045

Please sign in to comment.