WIP - Add ELB proxy protocol support

kubernetes · Apr 20, 2016 · 0335ee9 · 0335ee9 · k8s-teamcity-mesosphere · Apr 21, 2016
1 parent f3f6ffa
commit 0335ee9
Show file tree

Hide file tree

Showing 2 changed files with 136 additions and 2 deletions.
diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go
@@ -69,6 +69,9 @@ const TagNameSubnetPublicELB = "kubernetes.io/role/elb"
 // This lets us define more advanced semantics in future.
 const ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/aws-load-balancer-internal"
 
+// TODO Add description here
+const ServiceAnnotationLoadBalancerProxyProtocol = "service.beta.kubernetes.io/aws-load-balancer-proxy-protocol"
+
 // We sometimes read to see if something exists; then try to create it if we didn't find it
 // This can fail once in a consistent system if done in parallel
 // In an eventually consistent system, it could fail unboundedly
@@ -140,6 +143,8 @@ type ELB interface {
 	DescribeLoadBalancers(*elb.DescribeLoadBalancersInput) (*elb.DescribeLoadBalancersOutput, error)
 	RegisterInstancesWithLoadBalancer(*elb.RegisterInstancesWithLoadBalancerInput) (*elb.RegisterInstancesWithLoadBalancerOutput, error)
 	DeregisterInstancesFromLoadBalancer(*elb.DeregisterInstancesFromLoadBalancerInput) (*elb.DeregisterInstancesFromLoadBalancerOutput, error)
+	CreateLoadBalancerPolicy(*elb.CreateLoadBalancerPolicyInput) (*elb.CreateLoadBalancerPolicyOutput, error)
+	SetLoadBalancerPoliciesForBackendServer(*elb.SetLoadBalancerPoliciesForBackendServerInput) (*elb.SetLoadBalancerPoliciesForBackendServerOutput, error)
 
 	DetachLoadBalancerFromSubnets(*elb.DetachLoadBalancerFromSubnetsInput) (*elb.DetachLoadBalancerFromSubnetsOutput, error)
 	AttachLoadBalancerToSubnets(*elb.AttachLoadBalancerToSubnetsInput) (*elb.AttachLoadBalancerToSubnetsOutput, error)
@@ -2146,6 +2151,14 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a
 		internalELB = true
 	}
 
+	// Determine if we need to set the Proxy protocol policy
+	// XXX see http://docs.aws.amazon.com/cli/latest/reference/elb/create-load-balancer-policy.html
+	proxyProtocol := false
+	// TODO Review how we want this annonation to look. This is likely temporary
+	if annotations[ServiceAnnotationLoadBalancerProxyProtocol] != "" {
+		proxyProtocol = true
+	}
+
 	// Find the subnets that the ELB will live in
 	subnetIDs, err := s.findELBSubnets(internalELB)
 	if err != nil {
@@ -2218,7 +2231,15 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a
 	}
 
 	// Build the load balancer itself
-	loadBalancer, err := s.ensureLoadBalancer(serviceName, loadBalancerName, listeners, subnetIDs, securityGroupIDs, internalELB)
+	loadBalancer, err := s.ensureLoadBalancer(
+		serviceName,
+		loadBalancerName,
+		listeners,
+		subnetIDs,
+		securityGroupIDs,
+		internalELB,
+		proxyProtocol,
+	)
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/cloudprovider/providers/aws/aws_loadbalancer.go b/pkg/cloudprovider/providers/aws/aws_loadbalancer.go
@@ -28,7 +28,9 @@ import (
 	"k8s.io/kubernetes/pkg/util/sets"
 )
 
-func (s *AWSCloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBalancerName string, listeners []*elb.Listener, subnetIDs []string, securityGroupIDs []string, internalELB bool) (*elb.LoadBalancerDescription, error) {
+const ProxyProtocolPolicyName = "k8s-proxyprotocol"
+
+func (s *AWSCloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBalancerName string, listeners []*elb.Listener, subnetIDs []string, securityGroupIDs []string, internalELB, proxyProtocol bool) (*elb.LoadBalancerDescription, error) {
 	loadBalancer, err := s.describeLoadBalancer(loadBalancerName)
 	if err != nil {
 		return nil, err
@@ -62,6 +64,32 @@ func (s *AWSCloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadB
 		if err != nil {
 			return nil, err
 		}
+
+		// XXX Do we always want to create this policy or should we do this lazily
+		// and create when requested
+		err = s.createProxyProtocolPolicy(loadBalancerName)
+		if err != nil {
+			return nil, err
+		}
+
+		if proxyProtocol {
+			for _, proxyListener := range listeners {
+				// TODO Refactor this out into a seperate method
+				request := &elb.SetLoadBalancerPoliciesForBackendServerInput{
+					InstancePort:     proxyListener.InstancePort,
+					LoadBalancerName: aws.String(loadBalancerName),
+					PolicyNames: []*string{
+						aws.String(ProxyProtocolPolicyName),
+					},
+				}
+				glog.V(2).Info("Enabling proxy protocol policy on node port %d", *proxyListener.InstancePort)
+				_, err := s.elb.SetLoadBalancerPoliciesForBackendServer(request)
+				if err != nil {
+					return nil, fmt.Errorf("error enabling AWS loadbalancer proxy protocol: %v", err)
+				}
+			}
+		}
+
 		dirty = true
 	} else {
 		// TODO: Sync internal vs non-internal
@@ -118,6 +146,8 @@ func (s *AWSCloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadB
 			}
 		}
 
+		// TODO(williamsandrew) Ensure proxy protocol policy is on the existing load balancer
+
 		{
 			// Sync listeners
 			listenerDescriptions := loadBalancer.ListenerDescriptions
@@ -186,8 +216,58 @@ func (s *AWSCloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadB
 				if err != nil {
 					return nil, fmt.Errorf("error creating AWS loadbalancer listeners: %v", err)
 				}
+
+				if proxyProtocol {
+					for _, newListener := range additions {
+						// TODO(williamsandrew) DRY this up
+						request := &elb.SetLoadBalancerPoliciesForBackendServerInput{
+							InstancePort:     newListener.InstancePort,
+							LoadBalancerName: aws.String(loadBalancerName),
+							PolicyNames: []*string{
+								aws.String(ProxyProtocolPolicyName),
+							},
+						}
+						glog.V(2).Info("Enabling proxy protocol policy on node port %d", *newListener.InstancePort)
+						_, err := s.elb.SetLoadBalancerPoliciesForBackendServer(request)
+						if err != nil {
+							return nil, fmt.Errorf("error enabling AWS loadbalancer proxy protocol: %v", err)
+						}
+					}
+				}
 				dirty = true
 			}
+
+			// Sync Proxy Protocol for existing listeners
+			for _, backendListener := range loadBalancer.BackendServerDescriptions {
+				var policies []*string = nil
+
+				if proxyProtocolEnabled(backendListener) {
+					if !proxyProtocol {
+						policies = []*string{}
+						glog.V(2).Info("Disabling proxy protocol policy on node port %d", *backendListener.InstancePort)
+					}
+				} else {
+					if proxyProtocol {
+						policies = []*string{
+							aws.String(ProxyProtocolPolicyName),
+						}
+						glog.V(2).Info("Enabling proxy protocol policy on node port %d", *backendListener.InstancePort)
+					}
+				}
+
+				if policies != nil {
+					// TODO(williamsandrew) DRY this up
+					request := &elb.SetLoadBalancerPoliciesForBackendServerInput{
+						InstancePort:     backendListener.InstancePort,
+						LoadBalancerName: aws.String(loadBalancerName),
+						PolicyNames:      policies,
+					}
+					_, err := s.elb.SetLoadBalancerPoliciesForBackendServer(request)
+					if err != nil {
+						return nil, fmt.Errorf("error enabling AWS loadbalancer proxy protocol: %v", err)
+					}
+				}
+			}
 		}
 	}
 
@@ -308,3 +388,36 @@ func (s *AWSCloud) ensureLoadBalancerInstances(loadBalancerName string, lbInstan
 
 	return nil
 }
+
+func (s *AWSCloud) createProxyProtocolPolicy(loadBalancerName string) error {
+	glog.V(2).Info("XXX creating proxy protocol policy")
+
+	policyRequest := &elb.CreateLoadBalancerPolicyInput{
+		LoadBalancerName: aws.String(loadBalancerName),
+		PolicyName:       aws.String(ProxyProtocolPolicyName),
+		PolicyTypeName:   aws.String("ProxyProtocolPolicyType"),
+		PolicyAttributes: []*elb.PolicyAttribute{
+			{
+				AttributeName:  aws.String("ProxyProtocol"),
+				AttributeValue: aws.String("true"),
+			},
+		},
+	}
+	glog.V(2).Info("Creating proxy protocol policy on load balancer")
+	_, err := s.elb.CreateLoadBalancerPolicy(policyRequest)
+	if err != nil {
+		return fmt.Errorf("error creating proxy protocol policy on load balancer: %v", err)
+	}
+
+	return nil
+}
+
+func proxyProtocolEnabled(backend *elb.BackendServerDescription) bool {
+	for _, policy := range backend.PolicyNames {
+		if orEmpty(policy) == ProxyProtocolPolicyName {
+			return true
+		}
+	}
+
+	return false
+}