Skip to content

Commit

Permalink
Remove Linux and Windows Kube-proxy Userspace mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@knabben
knabben committed Oct 5, 2022
1 parent 44a0b4e commit 7df6c02
Show file tree
Hide file tree
Showing 41 changed files with 87 additions and 7,961 deletions.
22 changes: 7 additions & 15 deletions cmd/kube-proxy/app/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
fs.Var(&utilflag.IPPortVar{Val: &o.config.MetricsBindAddress}, "metrics-bind-address", "The IP address with port for the metrics server to serve on (set to '0.0.0.0:10249' for all IPv4 interfaces and '[::]:10249' for all IPv6 interfaces). Set empty to disable. This parameter is ignored if a config file is specified by --config.")
fs.BoolVar(&o.config.BindAddressHardFail, "bind-address-hard-fail", o.config.BindAddressHardFail, "If true kube-proxy will treat failure to bind to a port as fatal and exit")
fs.Var(utilflag.PortRangeVar{Val: &o.config.PortRange}, "proxy-port-range", "Range of host ports (beginPort-endPort, single port or beginPort+offset, inclusive) that may be consumed in order to proxy service traffic. If (unspecified, 0, or 0-0) then ports will be randomly chosen.")
fs.Var(&o.config.Mode, "proxy-mode", "Which proxy mode to use: 'iptables' (Linux-only), 'ipvs' (Linux-only), 'kernelspace' (Windows-only), or 'userspace' (Linux/Windows, deprecated). The default value is 'iptables' on Linux and 'userspace' on Windows(will be 'kernelspace' in a future release). "+
fs.Var(&o.config.Mode, "proxy-mode", "Which proxy mode to use: on Linux this can be 'iptables' (default) or 'ipvs'. On Windows the only supported value is 'kernelspace'."+
"This parameter is ignored if a config file is specified by --config.")
fs.Var(cliflag.NewMapStringBool(&o.config.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
"Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")+"\n"+
Expand Down Expand Up @@ -191,7 +191,6 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
o.config.Conntrack.TCPCloseWaitTimeout.Duration,
"NAT timeout for TCP connections in the CLOSE_WAIT state")
fs.DurationVar(&o.config.ConfigSyncPeriod.Duration, "config-sync-period", o.config.ConfigSyncPeriod.Duration, "How often configuration from the apiserver is refreshed. Must be greater than 0.")
fs.DurationVar(&o.config.UDPIdleTimeout.Duration, "udp-timeout", o.config.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace")

fs.BoolVar(&o.config.IPVS.StrictARP, "ipvs-strict-arp", o.config.IPVS.StrictARP, "Enable strict ARP by setting arp_ignore to 1 and arp_announce to 2")
fs.BoolVar(&o.config.IPTables.MasqueradeAll, "masquerade-all", o.config.IPTables.MasqueradeAll, "If using the pure iptables proxy, SNAT all traffic sent via Service cluster IPs (this not commonly needed)")
Expand Down Expand Up @@ -539,7 +538,6 @@ type ProxyServer struct {
MetricsBindAddress string
BindAddressHardFail bool
EnableProfiling bool
UseEndpointSlices bool
OOMScoreAdj *int32
ConfigSyncPeriod time.Duration
HealthzServer healthcheck.ProxierHealthUpdater
Expand Down Expand Up @@ -738,26 +736,20 @@ func (s *ProxyServer) Run() error {
options.LabelSelector = labelSelector.String()
}))

// Create configs (i.e. Watches for Services and Endpoints or EndpointSlices)
// Create configs (i.e. Watches for Services and EndpointSlices)
// Note: RegisterHandler() calls need to happen before creation of Sources because sources
// only notify on changes, and the initial update (on process start) may be lost if no handlers
// are registered yet.
serviceConfig := config.NewServiceConfig(informerFactory.Core().V1().Services(), s.ConfigSyncPeriod)
serviceConfig.RegisterEventHandler(s.Proxier)
go serviceConfig.Run(wait.NeverStop)

if endpointsHandler, ok := s.Proxier.(config.EndpointsHandler); ok && !s.UseEndpointSlices {
endpointsConfig := config.NewEndpointsConfig(informerFactory.Core().V1().Endpoints(), s.ConfigSyncPeriod)
endpointsConfig.RegisterEventHandler(endpointsHandler)
go endpointsConfig.Run(wait.NeverStop)
} else {
endpointSliceConfig := config.NewEndpointSliceConfig(informerFactory.Discovery().V1().EndpointSlices(), s.ConfigSyncPeriod)
endpointSliceConfig.RegisterEventHandler(s.Proxier)
go endpointSliceConfig.Run(wait.NeverStop)
}
endpointSliceConfig := config.NewEndpointSliceConfig(informerFactory.Discovery().V1().EndpointSlices(), s.ConfigSyncPeriod)
endpointSliceConfig.RegisterEventHandler(s.Proxier)
go endpointSliceConfig.Run(wait.NeverStop)

// This has to start after the calls to NewServiceConfig and NewEndpointsConfig because those
// functions must configure their shared informer event handlers first.
// This has to start after the calls to NewServiceConfig because that
// function must configure its shared informer event handlers first.
informerFactory.Start(wait.NeverStop)

if utilfeature.DefaultFeatureGate.Enabled(features.TopologyAwareHints) {
Expand Down
57 changes: 13 additions & 44 deletions cmd/kube-proxy/app/server_others.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilnet "k8s.io/apimachinery/pkg/util/net"
clientset "k8s.io/client-go/kubernetes"
toolswatch "k8s.io/client-go/tools/watch"
"k8s.io/component-base/configz"
Expand All @@ -55,7 +54,6 @@ import (
"k8s.io/kubernetes/pkg/proxy/iptables"
"k8s.io/kubernetes/pkg/proxy/ipvs"
proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics"
"k8s.io/kubernetes/pkg/proxy/userspace"
proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
utilipset "k8s.io/kubernetes/pkg/util/ipset"
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
Expand Down Expand Up @@ -157,22 +155,20 @@ func newProxyServer(
var ipt [2]utiliptables.Interface
dualStack := true // While we assume that node supports, we do further checks below

if proxyMode != proxyconfigapi.ProxyModeUserspace {
// Create iptables handlers for both families, one is already created
// Always ordered as IPv4, IPv6
if primaryProtocol == utiliptables.ProtocolIPv4 {
ipt[0] = iptInterface
ipt[1] = utiliptables.New(execer, utiliptables.ProtocolIPv6)
} else {
ipt[0] = utiliptables.New(execer, utiliptables.ProtocolIPv4)
ipt[1] = iptInterface
}
// Create iptables handlers for both families, one is already created
// Always ordered as IPv4, IPv6
if primaryProtocol == utiliptables.ProtocolIPv4 {
ipt[0] = iptInterface
ipt[1] = utiliptables.New(execer, utiliptables.ProtocolIPv6)
} else {
ipt[0] = utiliptables.New(execer, utiliptables.ProtocolIPv4)
ipt[1] = iptInterface
}

for _, perFamilyIpt := range ipt {
if !perFamilyIpt.Present() {
klog.InfoS("kube-proxy running in single-stack mode, this ipFamily is not supported", "ipFamily", perFamilyIpt.Protocol())
dualStack = false
}
for _, perFamilyIpt := range ipt {
if !perFamilyIpt.Present() {
klog.V(0).InfoS("kube-proxy running in single-stack mode, this ipFamily is not supported", "ipFamily", perFamilyIpt.Protocol())
dualStack = false
}
}

Expand Down Expand Up @@ -320,31 +316,6 @@ func newProxyServer(
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
proxymetrics.RegisterMetrics()
} else {
klog.InfoS("Using userspace Proxier")
klog.InfoS("The userspace proxier is now deprecated and will be removed in a future release, please use 'iptables' or 'ipvs' instead")

// TODO this has side effects that should only happen when Run() is invoked.
proxier, err = userspace.NewProxier(
userspace.NewLoadBalancerRR(),
netutils.ParseIPSloppy(config.BindAddress),
iptInterface,
execer,
*utilnet.ParsePortRangeOrDie(config.PortRange),
config.IPTables.SyncPeriod.Duration,
config.IPTables.MinSyncPeriod.Duration,
config.UDPIdleTimeout.Duration,
config.NodePortAddresses,
)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
}

useEndpointSlices := true
if proxyMode == proxyconfigapi.ProxyModeUserspace {
// userspace mode doesn't support endpointslice.
useEndpointSlices = false
}

return &ProxyServer{
Expand All @@ -367,7 +338,6 @@ func newProxyServer(
OOMScoreAdj: config.OOMScoreAdj,
ConfigSyncPeriod: config.ConfigSyncPeriod.Duration,
HealthzServer: healthzServer,
UseEndpointSlices: useEndpointSlices,
}, nil
}

Expand Down Expand Up @@ -571,7 +541,6 @@ func cleanupAndExit() error {

var encounteredError bool
for _, ipt := range ipts {
encounteredError = userspace.CleanupLeftovers(ipt) || encounteredError
encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError
encounteredError = ipvs.CleanupLeftovers(ipvsInterface, ipt, ipsetInterface) || encounteredError
}
Expand Down
5 changes: 1 addition & 4 deletions cmd/kube-proxy/app/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,6 @@ metricsBindAddress: "%s"
mode: "%s"
oomScoreAdj: 17
portRange: "2-7"
udpIdleTimeout: 123ms
detectLocalMode: "ClusterCIDR"
detectLocal:
bridgeInterface: "cbr0"
Expand Down Expand Up @@ -263,7 +262,6 @@ nodePortAddresses:
Mode: kubeproxyconfig.ProxyMode(tc.mode),
OOMScoreAdj: pointer.Int32(17),
PortRange: "2-7",
UDPIdleTimeout: metav1.Duration{Duration: 123 * time.Millisecond},
NodePortAddresses: []string{"10.20.30.40/16", "fd00:1::0/64"},
DetectLocalMode: kubeproxyconfig.LocalModeClusterCIDR,
DetectLocal: kubeproxyconfig.DetectLocalConfiguration{
Expand Down Expand Up @@ -457,8 +455,7 @@ mode: ""
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
detectLocalMode: "BridgeInterface"
udpIdleTimeout: 250ms`)
detectLocalMode: "BridgeInterface"`)
if err != nil {
return nil, "", fmt.Errorf("unexpected error when writing content to temp kube-proxy config file: %v", err)
}
Expand Down
131 changes: 38 additions & 93 deletions cmd/kube-proxy/app/server_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import (

v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/client-go/tools/events"
"k8s.io/component-base/configz"
"k8s.io/component-base/metrics"
Expand All @@ -43,11 +42,7 @@ import (
proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme"
"k8s.io/kubernetes/pkg/proxy/healthcheck"
"k8s.io/kubernetes/pkg/proxy/winkernel"
"k8s.io/kubernetes/pkg/proxy/winuserspace"
utilnetsh "k8s.io/kubernetes/pkg/util/netsh"
utilnode "k8s.io/kubernetes/pkg/util/node"
"k8s.io/utils/exec"
netutils "k8s.io/utils/net"
)

// NewProxyServer returns a new ProxyServer.
Expand Down Expand Up @@ -101,75 +96,51 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, master string
healthzPort, _ = strconv.Atoi(port)
}

// Check if Kernel Space can be used.
canUseWinKernelProxy, err := winkernel.CanUseWinKernelProxier(winkernel.WindowsKernelCompatTester{})
if !canUseWinKernelProxy && err != nil {
return nil, err
}

var proxier proxy.Provider
proxyMode := getProxyMode(config.Mode, winkernel.WindowsKernelCompatTester{})
proxyMode := proxyconfigapi.ProxyModeKernelspace
dualStackMode := getDualStackMode(config.Winkernel.NetworkName, winkernel.DualStackCompatTester{})
if proxyMode == proxyconfigapi.ProxyModeKernelspace {
klog.InfoS("Using Kernelspace Proxier.")
if dualStackMode {
klog.InfoS("Creating dualStackProxier for Windows kernel.")

proxier, err = winkernel.NewDualStackProxier(
config.IPTables.SyncPeriod.Duration,
config.IPTables.MinSyncPeriod.Duration,
config.IPTables.MasqueradeAll,
int(*config.IPTables.MasqueradeBit),
config.ClusterCIDR,
hostname,
nodeIPTuple(config.BindAddress),
recorder,
healthzServer,
config.Winkernel,
healthzPort,
)
} else {

proxier, err = winkernel.NewProxier(
config.IPTables.SyncPeriod.Duration,
config.IPTables.MinSyncPeriod.Duration,
config.IPTables.MasqueradeAll,
int(*config.IPTables.MasqueradeBit),
config.ClusterCIDR,
hostname,
nodeIP,
recorder,
healthzServer,
config.Winkernel,
healthzPort,
)

}

if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}

winkernel.RegisterMetrics()
if dualStackMode {
klog.V(0).InfoS("Creating dualStackProxier for Windows kernel.")

proxier, err = winkernel.NewDualStackProxier(
config.IPTables.SyncPeriod.Duration,
config.IPTables.MinSyncPeriod.Duration,
config.IPTables.MasqueradeAll,
int(*config.IPTables.MasqueradeBit),
config.ClusterCIDR,
hostname,
nodeIPTuple(config.BindAddress),
recorder,
healthzServer,
config.Winkernel,
healthzPort,
)
} else {
klog.InfoS("Using userspace Proxier.")
klog.InfoS("The userspace proxier is now deprecated and will be removed in a future release, please use 'kernelspace' instead")
execer := exec.New()
var netshInterface utilnetsh.Interface
netshInterface = utilnetsh.New(execer)

proxier, err = winuserspace.NewProxier(
winuserspace.NewLoadBalancerRR(),
netutils.ParseIPSloppy(config.BindAddress),
netshInterface,
*utilnet.ParsePortRangeOrDie(config.PortRange),
// TODO @pires replace below with default values, if applicable
proxier, err = winkernel.NewProxier(
config.IPTables.SyncPeriod.Duration,
config.UDPIdleTimeout.Duration,
config.IPTables.MinSyncPeriod.Duration,
config.IPTables.MasqueradeAll,
int(*config.IPTables.MasqueradeBit),
config.ClusterCIDR,
hostname,
nodeIP,
recorder,
healthzServer,
config.Winkernel,
healthzPort,
)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
}
useEndpointSlices := true
if proxyMode == proxyconfigapi.ProxyModeUserspace {
// userspace mode doesn't support endpointslice.
useEndpointSlices = false
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
winkernel.RegisterMetrics()

return &ProxyServer{
Client: client,
EventClient: eventClient,
Expand All @@ -184,43 +155,17 @@ func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, master string
OOMScoreAdj: config.OOMScoreAdj,
ConfigSyncPeriod: config.ConfigSyncPeriod.Duration,
HealthzServer: healthzServer,
UseEndpointSlices: useEndpointSlices,
}, nil
}

func getDualStackMode(networkname string, compatTester winkernel.StackCompatTester) bool {
return compatTester.DualStackCompatible(networkname)
}

func getProxyMode(proxyMode proxyconfigapi.ProxyMode, kcompat winkernel.KernelCompatTester) proxyconfigapi.ProxyMode {
if proxyMode == proxyconfigapi.ProxyModeKernelspace {
return tryWinKernelSpaceProxy(kcompat)
}
return proxyconfigapi.ProxyModeUserspace
}

func detectNumCPU() int {
return goruntime.NumCPU()
}

func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) proxyconfigapi.ProxyMode {
// Check for Windows Kernel Version if we can support Kernel Space proxy
// Check for Windows Version

// guaranteed false on error, error only necessary for debugging
useWinKernelProxy, err := winkernel.CanUseWinKernelProxier(kcompat)
if err != nil {
klog.ErrorS(err, "Can't determine whether to use windows kernel proxy, using userspace proxier")
return proxyconfigapi.ProxyModeUserspace
}
if useWinKernelProxy {
return proxyconfigapi.ProxyModeKernelspace
}
// Fallback.
klog.V(1).InfoS("Can't use winkernel proxy, using userspace proxier")
return proxyconfigapi.ProxyModeUserspace
}

// cleanupAndExit cleans up after a previous proxy run
func cleanupAndExit() error {
return errors.New("--cleanup-and-exit is not implemented on Windows")
Expand Down
9 changes: 1 addition & 8 deletions pkg/generated/openapi/zz_generated.openapi.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 7df6c02

Please sign in to comment.