Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

AWS: SSL support for ELB listeners through annotations #23495

Merged
merged 5 commits into from
May 12, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 61 additions & 20 deletions pkg/cloudprovider/providers/aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,25 @@ const TagNameSubnetPublicELB = "kubernetes.io/role/elb"
// This lets us define more advanced semantics in future.
const ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/aws-load-balancer-internal"

// Service annotation requesting a secure listener. Value is a valid certificate ARN.
// For more, see http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/elb-listener-config.html
// CertARN is an IAM or CM certificate ARN, e.g. arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012
const ServiceAnnotationLoadBalancerCertificate = "service.beta.kubernetes.io/aws-load-balancer-ssl-cert"

// Service annotation specifying the protocol spoken by the backend (pod) behind a secure listener.
// Only inspected when `aws-load-balancer-ssl-cert` is used.
// If `http` (default) or `https`, an HTTPS listener that terminates the connection and parses headers is created.
// If set to `ssl` or `tcp`, a "raw" SSL listener is used.
const ServiceAnnotationLoadBalancerBEProtocol = "service.beta.kubernetes.io/aws-load-balancer-backend-protocol"

// Maps from backend protocol to ELB protocol
var backendProtocolMapping = map[string]string{
"https": "https",
"http": "https",
"ssl": "ssl",
"tcp": "ssl",
}

// We sometimes read to see if something exists; then try to create it if we didn't find it
// This can fail once in a consistent system if done in parallel
// In an eventually consistent system, it could fail unboundedly
Expand Down Expand Up @@ -2099,6 +2118,37 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) {
return false, nil
}

// buildListener creates a new listener from the given port, adding an SSL certificate
// if indicated by the appropriate annotations.
func buildListener(port api.ServicePort, annotations map[string]string) (*elb.Listener, error) {
loadBalancerPort := int64(port.Port)
instancePort := int64(port.NodePort)
protocol := strings.ToLower(string(port.Protocol))
instanceProtocol := protocol

listener := &elb.Listener{}
listener.InstancePort = &instancePort
listener.LoadBalancerPort = &loadBalancerPort
certID := annotations[ServiceAnnotationLoadBalancerCertificate]
if certID != "" {
instanceProtocol = annotations[ServiceAnnotationLoadBalancerBEProtocol]
if instanceProtocol == "" {
protocol = "ssl"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wondering if we should warn/error in the case when the user specifies an invalid annotation (i.e. not found in map, vs not found). But that can be a separate PR - I want to get this one merged!

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Edit: never mind... I need more coffee

instanceProtocol = "tcp"
} else {
protocol = backendProtocolMapping[instanceProtocol]
if protocol == "" {
return nil, fmt.Errorf("Invalid backend protocol %s for %s in %s", instanceProtocol, certID, ServiceAnnotationLoadBalancerBEProtocol)
}
}
listener.SSLCertificateId = &certID
}
listener.Protocol = &protocol
listener.InstanceProtocol = &instanceProtocol

return listener, nil
}

// EnsureLoadBalancer implements LoadBalancer.EnsureLoadBalancer
func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, annotations map[string]string) (*api.LoadBalancerStatus, error) {
glog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)",
Expand All @@ -2113,10 +2163,21 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a
return nil, fmt.Errorf("requested load balancer with no ports")
}

// Figure out what mappings we want on the load balancer
listeners := []*elb.Listener{}
for _, port := range apiService.Spec.Ports {
if port.Protocol != api.ProtocolTCP {
return nil, fmt.Errorf("Only TCP LoadBalancer is supported for AWS ELB")
}
if port.NodePort == 0 {
glog.Errorf("Ignoring port without NodePort defined: %v", port)
continue
}
listener, err := buildListener(port, annotations)
if err != nil {
return nil, err
}
listeners = append(listeners, listener)
}

if apiService.Spec.LoadBalancerIP != "" {
Expand Down Expand Up @@ -2198,26 +2259,6 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a
}
securityGroupIDs := []string{securityGroupID}

// Figure out what mappings we want on the load balancer
listeners := []*elb.Listener{}
for _, port := range apiService.Spec.Ports {
if port.NodePort == 0 {
glog.Errorf("Ignoring port without NodePort defined: %v", port)
continue
}
instancePort := int64(port.NodePort)
loadBalancerPort := int64(port.Port)
protocol := strings.ToLower(string(port.Protocol))

listener := &elb.Listener{}
listener.InstancePort = &instancePort
listener.LoadBalancerPort = &loadBalancerPort
listener.Protocol = &protocol
listener.InstanceProtocol = &protocol

listeners = append(listeners, listener)
}

// Build the load balancer itself
loadBalancer, err := s.ensureLoadBalancer(serviceName, loadBalancerName, listeners, subnetIDs, securityGroupIDs, internalELB)
if err != nil {
Expand Down
103 changes: 103 additions & 0 deletions pkg/cloudprovider/providers/aws/aws_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1199,3 +1199,106 @@ func TestDescribeLoadBalancerOnEnsure(t *testing.T) {

c.EnsureLoadBalancer(&api.Service{ObjectMeta: api.ObjectMeta{Name: "myservice", UID: "id"}}, []string{}, map[string]string{})
}

func TestBuildListener(t *testing.T) {
tests := []struct {
name string

lbPort int64
instancePort int64
backendProtocolAnnotation string
certAnnotation string

expectError bool
lbProtocol string
instanceProtocol string
certID string
}{
{
"No cert or BE protocol annotation, passthrough",
80, 7999, "", "",
false, "tcp", "tcp", "",
},
{
"Cert annotation without BE protocol specified, SSL->TCP",
80, 8000, "", "cert",
false, "ssl", "tcp", "cert",
},
{
"BE protocol without cert annotation, passthrough",
443, 8001, "https", "",
false, "tcp", "tcp", "",
},
{
"Invalid cert annotation, bogus backend protocol",
443, 8002, "bacon", "foo",
true, "tcp", "tcp", "cert",
},
{
"Invalid cert annotation, protocol followed by equal sign",
443, 8003, "http=", "=",
true, "tcp", "tcp", "cert",
},
{
"HTTPS->HTTPS",
443, 8004, "https", "cert",
false, "https", "https", "cert",
},
{
"HTTPS->HTTP",
443, 8005, "http", "cert",
false, "https", "http", "cert",
},
{
"SSL->SSL",
443, 8006, "ssl", "cert",
false, "ssl", "ssl", "cert",
},
{
"SSL->TCP",
443, 8007, "tcp", "cert",
false, "ssl", "tcp", "cert",
},
}

for _, test := range tests {
t.Logf("Running test case %s", test.name)
annotations := make(map[string]string)
if test.backendProtocolAnnotation != "" {
annotations[ServiceAnnotationLoadBalancerBEProtocol] = test.backendProtocolAnnotation
}
if test.certAnnotation != "" {
annotations[ServiceAnnotationLoadBalancerCertificate] = test.certAnnotation
}
l, err := buildListener(api.ServicePort{
NodePort: int32(test.instancePort),
Port: int32(test.lbPort),
Protocol: api.Protocol("tcp"),
}, annotations)
if test.expectError {
if err == nil {
t.Errorf("Should error for case %s", test.name)
}
} else {
if err != nil {
t.Errorf("Should succeed for case: %s, got %v", test.name, err)
} else {
var cert *string
if test.certID != "" {
cert = &test.certID
}
expected := &elb.Listener{
InstancePort: &test.instancePort,
InstanceProtocol: &test.instanceProtocol,
LoadBalancerPort: &test.lbPort,
Protocol: &test.lbProtocol,
SSLCertificateId: cert,
}
if !reflect.DeepEqual(l, expected) {
t.Errorf("Incorrect listener (%v vs expected %v) for case: %s",
l, expected, test.name)
}
}
}
}
}