Skip to content

Commit

Permalink
Add extra addresses for listeners configuration to support dual stack…
Browse files Browse the repository at this point in the history 
… in Istio (istio#40634)

* add extra addresses for listeners configuration to support dual stack in Istio

* change code based on comments

* fixing bug for inboundCustomListener

* fixing bug for lds and inbound, and add unit test

* refactor some code and fix the unit test error

* fix lint-go

* code change based on comment

* change code based on comment

* add dual stack reachability integration test

* change the echo server config for IPFamilies

* add dual stack echo config for security integration test

* not add dual stack integraion test flag in this PR, but in another PR: istio#40983

* add dual stack proxy comment for dual stack unit test and retrigger all integration job

* Add WildcardAndLocalHost struct for holding the wildcard and localhost addresses to support dual stack.

* fix bug for bench_test when creating a node proxy

* changed code according to John's comments

* change oWildcardAndLocalHost into hostAddresses

* refactor code based on the comments

* move the HostAddresses into proxy node according to comments

* added comments and fixed nit

* change the code based on maintainer's comments

* maintain 2 maps to return wildCards and localHosts according to IP mode of proxy

* change util function name from BuildExtraAddresses into BuildAdditionalAddresses
  • Loading branch information
@zhlsunshine
zhlsunshine authored Sep 30, 2022
1 parent 698bf92 commit 0812983
Show file tree
Hide file tree
Showing 10 changed files with 606 additions and 363 deletions.
5 changes: 5 additions & 0 deletions pilot/pkg/model/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -909,6 +909,11 @@ func (node *Proxy) IsIPv6() bool {
return node.ipMode == IPv6
}

// GetIPMode returns proxy's ipMode
func (node *Proxy) GetIPMode() IPMode {
return node.ipMode
}

// ParseMetadata parses the opaque Metadata from an Envoy Node into string key-value pairs.
// Any non-string values are ignored.
func ParseMetadata(metadata *structpb.Struct) (*NodeMetadata, error) {
Expand Down
14 changes: 7 additions & 7 deletions pilot/pkg/networking/core/v1alpha3/httproute_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -536,12 +536,12 @@ func TestSidecarOutboundHTTPRouteConfigWithDuplicateHosts(t *testing.T) {

func TestSidecarOutboundHTTPRouteConfig(t *testing.T) {
services := []*model.Service{
buildHTTPService("bookinfo.com", visibility.Public, wildcardIP, "default", 9999, 70),
buildHTTPService("private.com", visibility.Private, wildcardIP, "default", 9999, 80),
buildHTTPService("bookinfo.com", visibility.Public, wildcardIPv4, "default", 9999, 70),
buildHTTPService("private.com", visibility.Private, wildcardIPv4, "default", 9999, 80),
buildHTTPService("test.com", visibility.Public, "8.8.8.8", "not-default", 8080),
buildHTTPService("test-private.com", visibility.Private, "9.9.9.9", "not-default", 80, 70),
buildHTTPService("test-private-2.com", visibility.Private, "9.9.9.10", "not-default", 60),
buildHTTPService("test-headless.com", visibility.Public, wildcardIP, "not-default", 8888),
buildHTTPService("test-headless.com", visibility.Public, wildcardIPv4, "not-default", 8888),
buildHTTPService("service-A.default.svc.cluster.local", visibility.Public, "", "default", 7777),
}

Expand Down Expand Up @@ -1393,12 +1393,12 @@ func TestSidecarOutboundHTTPRouteConfig(t *testing.T) {

func TestSelectVirtualService(t *testing.T) {
services := []*model.Service{
buildHTTPService("bookinfo.com", visibility.Public, wildcardIP, "default", 9999, 70),
buildHTTPService("private.com", visibility.Private, wildcardIP, "default", 9999, 80),
buildHTTPService("bookinfo.com", visibility.Public, wildcardIPv4, "default", 9999, 70),
buildHTTPService("private.com", visibility.Private, wildcardIPv4, "default", 9999, 80),
buildHTTPService("test.com", visibility.Public, "8.8.8.8", "not-default", 8080),
buildHTTPService("test-private.com", visibility.Private, "9.9.9.9", "not-default", 80, 70),
buildHTTPService("test-private-2.com", visibility.Private, "9.9.9.10", "not-default", 60),
buildHTTPService("test-headless.com", visibility.Public, wildcardIP, "not-default", 8888),
buildHTTPService("test-headless.com", visibility.Public, wildcardIPv4, "not-default", 8888),
}

servicesByName := make(map[host.Name]*model.Service, len(services))
Expand Down Expand Up @@ -1648,7 +1648,7 @@ func buildHTTPService(hostname string, v visibility.Instance, ip, namespace stri
ExportTo: map[visibility.Instance]bool{v: true},
},
}
if ip == wildcardIP {
if ip == wildcardIPv4 {
service.Resolution = model.Passthrough
}

Expand Down
64 changes: 50 additions & 14 deletions pilot/pkg/networking/core/v1alpha3/listener.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ func (lb *ListenerBuilder) buildSidecarOutboundListeners(node *model.Proxy,
) []*listener.Listener {
noneMode := node.GetInterceptionMode() == model.InterceptionNone

actualWildcard, actualLocalHostAddress := getActualWildcardAndLocalHost(node)
actualWildcards, actualLocalHosts := getWildcardsAndLocalHostForDualStack(node.GetIPMode())

var tcpListeners, httpListeners []*listener.Listener
// For conflict resolution
Expand Down Expand Up @@ -373,12 +373,23 @@ func (lb *ListenerBuilder) buildSidecarOutboundListeners(node *model.Proxy,
// If captureMode is not NONE, i.e., bindToPort is false, then
// we will bind to user specified IP (if any) or to the VIPs of services in
// this egress listener.
var extraBind []string
bind := egressListener.IstioListener.Bind
if bind == "" {
if bindToPort {
bind = actualLocalHostAddress
// the first local host would be the binding address and
// the rest would be the additional addresses
bind = actualLocalHosts[0]
if len(actualLocalHosts) > 1 {
extraBind = actualLocalHosts[1:]
}
} else {
bind = actualWildcard
// the first wildcard address would be the binding address and
// the rest would be the additional addresses
bind = actualWildcards[0]
if len(actualWildcards) > 1 {
extraBind = actualWildcards[1:]
}
}
}

Expand All @@ -389,12 +400,13 @@ func (lb *ListenerBuilder) buildSidecarOutboundListeners(node *model.Proxy,
bind: bind,
port: listenPort,
bindToPort: bindToPort,
extraBind: extraBind,
}

for _, service := range services {
listenerOpts.service = service
// Set service specific attributes here.
lb.buildSidecarOutboundListenerForPortOrUDS(listenerOpts, listenerMap, virtualServices, actualWildcard)
lb.buildSidecarOutboundListenerForPortOrUDS(listenerOpts, listenerMap, virtualServices, actualWildcards[0])
}
} else {
// This is a catch all egress listener with no port. This
Expand All @@ -421,12 +433,12 @@ func (lb *ListenerBuilder) buildSidecarOutboundListeners(node *model.Proxy,
e.locked = true
}

bind := ""
var bind string
if egressListener.IstioListener != nil && egressListener.IstioListener.Bind != "" {
bind = egressListener.IstioListener.Bind
}
if bindToPort && bind == "" {
bind = actualLocalHostAddress
bind = actualLocalHosts[0]
}

// Build ListenerOpts and PluginParams once and reuse across all Services to avoid unnecessary allocations.
Expand All @@ -438,6 +450,7 @@ func (lb *ListenerBuilder) buildSidecarOutboundListeners(node *model.Proxy,

for _, service := range services {
saddress := service.GetAddressForProxy(node)
sExtrAddresses := service.GetExtraAddressesForProxy(node)
for _, servicePort := range service.Ports {
// Skip ports we cannot bind to
if !node.CanBindToPort(bindToPort, uint32(servicePort.Port)) {
Expand All @@ -450,6 +463,15 @@ func (lb *ListenerBuilder) buildSidecarOutboundListeners(node *model.Proxy,

// bind might have been modified by below code, so reset it for every Service.
listenerOpts.bind = bind
// listenerOpts.extraBind should be specified at the same time for dual stack env
if len(actualLocalHosts) > 1 {
if bind == actualLocalHosts[0] {
listenerOpts.extraBind = actualLocalHosts[1:]
} else if len(sExtrAddresses) > 0 {
listenerOpts.extraBind = sExtrAddresses
}
}

// port depends on servicePort.
listenerOpts.port = servicePort
listenerOpts.service = service
Expand All @@ -470,7 +492,7 @@ func (lb *ListenerBuilder) buildSidecarOutboundListeners(node *model.Proxy,
// selected or scaled down, so we skip these as well. This leaves us with
// only a plain ServiceEntry with resolution NONE. In this case, we will
// fallback to a wildcard listener.
lb.buildSidecarOutboundListenerForPortOrUDS(listenerOpts, listenerMap, virtualServices, actualWildcard)
lb.buildSidecarOutboundListenerForPortOrUDS(listenerOpts, listenerMap, virtualServices, actualWildcards[0])
continue
}
for _, instance := range instances {
Expand All @@ -487,11 +509,11 @@ func (lb *ListenerBuilder) buildSidecarOutboundListeners(node *model.Proxy,
continue
}
listenerOpts.bind = instance.Endpoint.Address
lb.buildSidecarOutboundListenerForPortOrUDS(listenerOpts, listenerMap, virtualServices, actualWildcard)
lb.buildSidecarOutboundListenerForPortOrUDS(listenerOpts, listenerMap, virtualServices, actualWildcards[0])
}
} else {
// Standard logic for headless and non headless services
lb.buildSidecarOutboundListenerForPortOrUDS(listenerOpts, listenerMap, virtualServices, actualWildcard)
lb.buildSidecarOutboundListenerForPortOrUDS(listenerOpts, listenerMap, virtualServices, actualWildcards[0])
}
}
}
Expand Down Expand Up @@ -532,7 +554,7 @@ func (lb *ListenerBuilder) buildHTTPProxy(node *model.Proxy,
}

// enable HTTP PROXY port if necessary; this will add an RDS route for this port
_, listenAddress := getActualWildcardAndLocalHost(node)
_, actualLocalHosts := getWildcardsAndLocalHostForDualStack(node.GetIPMode())

httpOpts := &core.Http1ProtocolOptions{
AllowAbsoluteUrl: proto.BoolTrue,
Expand All @@ -544,7 +566,7 @@ func (lb *ListenerBuilder) buildHTTPProxy(node *model.Proxy,
opts := buildListenerOpts{
push: push,
proxy: node,
bind: listenAddress,
bind: actualLocalHosts[0],
port: &model.Port{Port: int(httpProxyPort)},
filterChainOpts: []*filterChainOpts{{
httpOpts: &httpListenerOpts{
Expand All @@ -560,6 +582,10 @@ func (lb *ListenerBuilder) buildHTTPProxy(node *model.Proxy,
bindToPort: true,
skipUserFilters: true,
}

if len(actualLocalHosts) > 1 {
opts.extraBind = actualLocalHosts[1:]
}
l := buildListener(opts, core.TrafficDirection_OUTBOUND)

// TODO: plugins for HTTP_PROXY mode, envoyfilter needs another listener match for SIDECAR_HTTP_PROXY
Expand All @@ -583,8 +609,12 @@ func buildSidecarOutboundHTTPListenerOptsForPortOrUDS(listenerMapKey *string,
) (bool, []*filterChainOpts) {
// first identify the bind if its not set. Then construct the key
// used to lookup the listener in the conflict map.
if len(listenerOpts.bind) == 0 { // no user specified bind. Use 0.0.0.0:Port
listenerOpts.bind = actualWildcard
if len(listenerOpts.bind) == 0 { // no user specified bind. Use 0.0.0.0:Port or [::]:Port
actualWildcards, _ := getWildcardsAndLocalHostForDualStack(listenerOpts.proxy.GetIPMode())
listenerOpts.bind = actualWildcards[0]
if len(actualWildcards) > 0 {
listenerOpts.extraBind = actualWildcards[1:]
}
}
*listenerMapKey = listenerOpts.bind + ":" + strconv.Itoa(listenerOpts.port.Port)

Expand Down Expand Up @@ -692,6 +722,7 @@ func buildSidecarOutboundTCPListenerOptsForPortOrUDS(listenerMapKey *string,
var destinationCIDR string
if len(listenerOpts.bind) == 0 {
svcListenAddress := listenerOpts.service.GetAddressForProxy(listenerOpts.proxy)
svcExtraListenAddress := listenerOpts.service.GetExtraAddressesForProxy(listenerOpts.proxy)
// We should never get an empty address.
// This is a safety guard, in case some platform adapter isn't doing things
// properly
Expand All @@ -704,6 +735,7 @@ func buildSidecarOutboundTCPListenerOptsForPortOrUDS(listenerMapKey *string,
destinationCIDR = svcListenAddress
listenerOpts.bind = actualWildcard
}
listenerOpts.extraBind = svcExtraListenAddress
}
}

Expand Down Expand Up @@ -1088,6 +1120,7 @@ type buildListenerOpts struct {
push *model.PushContext
proxy *model.Proxy
bind string
extraBind []string
port *model.Port
filterChainOpts []*filterChainOpts
bindToPort bool
Expand Down Expand Up @@ -1208,7 +1241,6 @@ func buildListener(opts buildListenerOpts, trafficDirection core.TrafficDirectio
if !opts.bindToPort {
bindToPort = proto.BoolFalse
}

// only use to exact_balance for tcp outbound listeners; virtualOutbound listener should
// not have this set per Envoy docs for redirected listeners
if opts.proxy.Metadata.OutboundListenerExactBalance && trafficDirection == core.TrafficDirection_OUTBOUND {
Expand All @@ -1229,6 +1261,8 @@ func buildListener(opts buildListenerOpts, trafficDirection core.TrafficDirectio
BindToPort: bindToPort,
ConnectionBalanceConfig: connectionBalance,
}
// add extra addresses for the listener
res.AdditionalAddresses = util.BuildAdditionalAddresses(opts.extraBind, uint32(opts.port.Port), opts.proxy)

if opts.proxy.Type != model.Router {
res.ListenerFiltersTimeout = opts.push.Mesh.ProtocolDetectionTimeout
Expand Down Expand Up @@ -1256,6 +1290,8 @@ func buildListener(opts buildListenerOpts, trafficDirection core.TrafficDirectio
},
EnableReusePort: proto.BoolTrue,
}
// add extra addresses for the listener
res.AdditionalAddresses = util.BuildAdditionalAddresses(opts.extraBind, uint32(opts.port.Port), opts.proxy)
}

accessLogBuilder.setListenerAccessLog(opts.push, opts.proxy, res, opts.class)
Expand Down
34 changes: 29 additions & 5 deletions pilot/pkg/networking/core/v1alpha3/listener_address.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ const (
InboundPassthroughBindIpv6 = "::6"
)

var (
wildCards map[model.IPMode][]string
localHosts map[model.IPMode][]string
)

// TODO: getActualWildcardAndLocalHost would be removed once the dual stack support in Istio
// getActualWildcardAndLocalHost will return corresponding Wildcard and LocalHost
// depending on value of proxy's IPAddresses.
func getActualWildcardAndLocalHost(node *model.Proxy) (string, string) {
Expand All @@ -52,14 +58,32 @@ func getPassthroughBindIP(node *model.Proxy) string {
return InboundPassthroughBindIpv6
}

// getSidecarInboundBindIP returns the IP that the proxy can bind to along with the sidecar specified port.
// getSidecarInboundBindIPs returns the IP that the proxy can bind to along with the sidecar specified port.
// It looks for an unicast address, if none found, then the default wildcard address is used.
// This will make the inbound listener bind to instance_ip:port instead of 0.0.0.0:port where applicable.
func getSidecarInboundBindIP(node *model.Proxy) string {
func getSidecarInboundBindIPs(node *model.Proxy) []string {
// Return the IP if its a global unicast address.
if len(node.GlobalUnicastIP) > 0 {
return node.GlobalUnicastIP
return []string{node.GlobalUnicastIP}
}
defaultInboundIP, _ := getActualWildcardAndLocalHost(node)
return defaultInboundIP
defaultInboundIPs, _ := getWildcardsAndLocalHostForDualStack(node.GetIPMode())
return defaultInboundIPs
}

func getWildcardsAndLocalHostForDualStack(ipMode model.IPMode) ([]string, []string) {
return wildCards[ipMode], localHosts[ipMode]
}

func init() {
// maintain 2 maps to return wildCards and localHosts according to IP mode of proxy
wildCards = make(map[model.IPMode][]string)
localHosts = make(map[model.IPMode][]string)

wildCards[model.IPv4] = []string{WildcardAddress}
wildCards[model.IPv6] = []string{WildcardIPv6Address}
wildCards[model.Dual] = []string{WildcardAddress, WildcardIPv6Address}

localHosts[model.IPv4] = []string{LocalhostAddress}
localHosts[model.IPv6] = []string{LocalhostIPv6Address}
localHosts[model.Dual] = []string{LocalhostAddress, LocalhostIPv6Address}
}
10 changes: 7 additions & 3 deletions pilot/pkg/networking/core/v1alpha3/listener_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,21 @@ func (lb *ListenerBuilder) buildVirtualOutboundListener() *ListenerBuilder {

filterChains := buildOutboundCatchAllNetworkFilterChains(lb.node, lb.push)

actualWildcard, _ := getActualWildcardAndLocalHost(lb.node)

actualWildcards, _ := getWildcardsAndLocalHostForDualStack(lb.node.GetIPMode())
// add an extra listener that binds to the port that is the recipient of the iptables redirect
ipTablesListener := &listener.Listener{
Name: model.VirtualOutboundListenerName,
Address: util.BuildAddress(actualWildcard, uint32(lb.push.Mesh.ProxyListenPort)),
Address: util.BuildAddress(actualWildcards[0], uint32(lb.push.Mesh.ProxyListenPort)),
Transparent: isTransparentProxy,
UseOriginalDst: proto.BoolTrue,
FilterChains: filterChains,
TrafficDirection: core.TrafficDirection_OUTBOUND,
}
// add extra addresses for the listener
if len(actualWildcards) > 1 {
ipTablesListener.AdditionalAddresses = util.BuildAdditionalAddresses(actualWildcards[1:], uint32(lb.push.Mesh.ProxyListenPort), lb.node)
}

class := model.OutboundListenerClass(lb.node.Type)
accessLogBuilder.setListenerAccessLog(lb.push, lb.node, ipTablesListener, class)
lb.virtualOutboundListener = ipTablesListener
Expand Down
11 changes: 8 additions & 3 deletions pilot/pkg/networking/core/v1alpha3/listener_builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,15 @@ func TestVirtualListenerBuilder(t *testing.T) {
}

var (
testServices = []*model.Service{buildService("test.com", wildcardIP, protocol.HTTP, tnow)}
testServices = []*model.Service{
buildService("test.com", wildcardIPv4, protocol.HTTP, tnow),
buildService("test.com", wildcardIPv6, protocol.HTTP, tnow),
}
testServicesWithQUIC = []*model.Service{
buildService("test.com", wildcardIP, protocol.HTTP, tnow),
buildService("quick.com", wildcardIP, protocol.UDP, tnow),
buildService("test.com", wildcardIPv4, protocol.HTTP, tnow),
buildService("test.com", wildcardIPv6, protocol.HTTP, tnow),
buildService("quick.com", wildcardIPv4, protocol.UDP, tnow),
buildService("quick.com", wildcardIPv6, protocol.UDP, tnow),
}
)

Expand Down
Loading

0 comments on commit 0812983

Please sign in to comment.