Add support for HTTPS->HTTP ELB listeners through annotations

Moved listener creation to a separate function, which had the nice side effect of allowing tests (added eight cases).
kubernetes · Apr 14, 2016 · 20946ee · 20946ee · k8s-teamcity-mesosphere · Apr 14, 2016
1 parent d19bc6b
commit 20946ee
Show file tree

Hide file tree

Showing 2 changed files with 154 additions and 20 deletions.
diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go
@@ -69,6 +69,21 @@ const TagNameSubnetPublicELB = "kubernetes.io/role/elb"
 // This lets us define more advanced semantics in future.
 const ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/aws-load-balancer-internal"
 
+// Service annotation requesting a secure listener. Value is [InstanceProtocol=]CertARN
+// If InstanceProtocol is `http` (default) or `https`, an HTTPS listener that terminates the connection and parses headers is created.
+// If it is set to `ssl` or `tcp`, a "raw" SSL listener is used.
+// For more, see http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/elb-listener-config.html
+// CertARN is an IAM or CM certificate ARN, e.g. arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012
+const ServiceAnnotationLoadBalancerCertificate = "service.beta.kubernetes.io/aws-load-balancer-certarn"
+
+// Maps from instance protocol to ELB protocol
+var protocolMapping = map[string]string{
+	"https": "https",
+	"http":  "https",
+	"ssl":   "ssl",
+	"tcp":   "ssl",
+}
+
 // We sometimes read to see if something exists; then try to create it if we didn't find it
 // This can fail once in a consistent system if done in parallel
 // In an eventually consistent system, it could fail unboundedly
@@ -2098,6 +2113,39 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) {
 	return false, nil
 }
 
+func getListener(port api.ServicePort, annotations map[string]string) (*elb.Listener, error) {
+	loadBalancerPort := int64(port.Port)
+	instancePort := int64(port.NodePort)
+	protocol := strings.ToLower(string(port.Protocol))
+	instanceProtocol := protocol
+
+	listener := &elb.Listener{}
+	listener.InstancePort = &instancePort
+	listener.LoadBalancerPort = &loadBalancerPort
+	certID := annotations[ServiceAnnotationLoadBalancerCertificate]
+	if certID != "" {
+		parts := strings.Split(certID, "=")
+		if len(parts) == 1 {
+			protocol = "https"
+			instanceProtocol = "http"
+		} else if len(parts) == 2 {
+			instanceProtocol = strings.ToLower(parts[0])
+			protocol = protocolMapping[instanceProtocol]
+			if protocol == "" {
+				return nil, fmt.Errorf("Invalid protocol %s in %s", instanceProtocol, certID)
+			}
+			certID = parts[1]
+		} else {
+			return nil, fmt.Errorf("Invalid certificate annotation %s", certID)
+		}
+		listener.SSLCertificateId = &certID
+	}
+	listener.Protocol = &protocol
+	listener.InstanceProtocol = &instanceProtocol
+
+	return listener, nil
+}
+
 // EnsureLoadBalancer implements LoadBalancer.EnsureLoadBalancer
 func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, annotations map[string]string) (*api.LoadBalancerStatus, error) {
 	glog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)",
@@ -2112,10 +2160,21 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a
 		return nil, fmt.Errorf("requested load balancer with no ports")
 	}
 
+	// Figure out what mappings we want on the load balancer
+	listeners := []*elb.Listener{}
 	for _, port := range apiService.Spec.Ports {
 		if port.Protocol != api.ProtocolTCP {
 			return nil, fmt.Errorf("Only TCP LoadBalancer is supported for AWS ELB")
 		}
+		if port.NodePort == 0 {
+			glog.Errorf("Ignoring port without NodePort defined: %v", port)
+			continue
+		}
+		listener, err := getListener(port, annotations)
+		if err != nil {
+			return nil, err
+		}
+		listeners = append(listeners, listener)
 	}
 
 	if apiService.Spec.LoadBalancerIP != "" {
@@ -2197,26 +2256,6 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a
 	}
 	securityGroupIDs := []string{securityGroupID}
 
-	// Figure out what mappings we want on the load balancer
-	listeners := []*elb.Listener{}
-	for _, port := range apiService.Spec.Ports {
-		if port.NodePort == 0 {
-			glog.Errorf("Ignoring port without NodePort defined: %v", port)
-			continue
-		}
-		instancePort := int64(port.NodePort)
-		loadBalancerPort := int64(port.Port)
-		protocol := strings.ToLower(string(port.Protocol))
-
-		listener := &elb.Listener{}
-		listener.InstancePort = &instancePort
-		listener.LoadBalancerPort = &loadBalancerPort
-		listener.Protocol = &protocol
-		listener.InstanceProtocol = &protocol
-
-		listeners = append(listeners, listener)
-	}
-
 	// Build the load balancer itself
 	loadBalancer, err := s.ensureLoadBalancer(serviceName, loadBalancerName, listeners, subnetIDs, securityGroupIDs, internalELB)
 	if err != nil {

diff --git a/pkg/cloudprovider/providers/aws/aws_test.go b/pkg/cloudprovider/providers/aws/aws_test.go
@@ -1199,3 +1199,98 @@ func TestDescribeLoadBalancerOnEnsure(t *testing.T) {
 
 	c.EnsureLoadBalancer(&api.Service{ObjectMeta: api.ObjectMeta{Name: "myservice", UID: "id"}}, []string{}, map[string]string{})
 }
+
+func TestGetListener(t *testing.T) {
+	tests := []struct {
+		name string
+
+		lbPort       int64
+		instancePort int64
+		annotation   string
+
+		expectError      bool
+		lbProtocol       string
+		instanceProtocol string
+		certID           string
+		//listener    *elb.Listener
+	}{
+		{
+			"No annotation, passthrough",
+			80, 8000, "",
+			false, "tcp", "tcp", "",
+		},
+		{
+			"Invalid cert annotation, no protocol before equal sign",
+			443, 8000, "=foo",
+			true, "tcp", "tcp", "cert",
+		},
+		{
+			"Invalid cert annotation, bogus protocol before equal sign",
+			443, 8000, "bacon=foo",
+			true, "tcp", "tcp", "cert",
+		},
+		{
+			"Invalid cert annotation, too many equal signs",
+			443, 8000, "==",
+			true, "tcp", "tcp", "cert",
+		},
+		{
+			"HTTPS->HTTPS",
+			443, 8000, "https=cert",
+			false, "https", "https", "cert",
+		},
+		{
+			"HTTPS->HTTP",
+			443, 8000, "http=cert",
+			false, "https", "http", "cert",
+		},
+		{
+			"SSL->SSL",
+			443, 8000, "ssl=cert",
+			false, "ssl", "ssl", "cert",
+		},
+		{
+			"SSL->TCP",
+			443, 8000, "tcp=cert",
+			false, "ssl", "tcp", "cert",
+		},
+	}
+
+	for _, test := range tests {
+		t.Logf("Running test case %s", test.name)
+		annotations := make(map[string]string)
+		if test.annotation != "" {
+			annotations[ServiceAnnotationLoadBalancerCertificate] = test.annotation
+		}
+		l, err := getListener(api.ServicePort{
+			NodePort: int(test.instancePort),
+			Port:     int(test.lbPort),
+			Protocol: api.Protocol("tcp"),
+		}, annotations)
+		if test.expectError {
+			if err == nil {
+				t.Errorf("Should error for case %s", test.name)
+			}
+		} else {
+			if err != nil {
+				t.Errorf("Should succeed for case: %s, got %v", test.name, err)
+			} else {
+				var cert *string
+				if test.certID != "" {
+					cert = &test.certID
+				}
+				expected := &elb.Listener{
+					InstancePort:     &test.instancePort,
+					InstanceProtocol: &test.instanceProtocol,
+					LoadBalancerPort: &test.lbPort,
+					Protocol:         &test.lbProtocol,
+					SSLCertificateId: cert,
+				}
+				if !reflect.DeepEqual(l, expected) {
+					t.Errorf("Incorrect listener (%v vs %v) for case: %s",
+						l, expected, test.name)
+				}
+			}
+		}
+	}
+}