Make master service IP static (no longer randomly assigned)

justinsb · Jan 31, 2015 · e83fd7b · e83fd7b
1 parent b40d079
commit e83fd7b
Show file tree

Hide file tree

Showing 9 changed files with 144 additions and 37 deletions.
diff --git a/cmd/integration/integration.go b/cmd/integration/integration.go
@@ -162,7 +162,7 @@ func startComponents(manifestURL string) (apiServerURL string) {
 		AdmissionControl:  admit.NewAlwaysAdmit(),
 		ReadWritePort:     portNumber,
 		ReadOnlyPort:      portNumber,
-		PublicAddress:     host,
+		PublicAddress:     net.ParseIP(host),
 	})
 	handler.delegate = m.Handler
 

diff --git a/cmd/kubernetes/kubernetes.go b/cmd/kubernetes/kubernetes.go
@@ -22,6 +22,7 @@ package main
 
 import (
 	"fmt"
+	"net"
 
 	kubeletapp "github.com/GoogleCloudPlatform/kubernetes/cmd/kubelet/app"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@@ -46,7 +47,7 @@ var (
 	masterServiceNamespace = flag.String("master_service_namespace", api.NamespaceDefault, "The namespace from which the kubernetes master services should be injected into pods")
 )
 
-func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr string, port int) {
+func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr net.IP, port int) {
 	machineList := []string{"localhost"}
 
 	standalone.RunApiServer(cl, etcdClient, addr, port, *masterServiceNamespace)
@@ -57,7 +58,7 @@ func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr string
 	standalone.SimpleRunKubelet(cl, nil, dockerClient, machineList[0], "/tmp/kubernetes", "", "127.0.0.1", 10250, *masterServiceNamespace, kubeletapp.ProbeVolumePlugins())
 }
 
-func newApiClient(addr string, port int) *client.Client {
+func newApiClient(addr net.IP, port int) *client.Client {
 	apiServerURL := fmt.Sprintf("http://%s:%d", addr, port)
 	cl := client.NewOrDie(&client.Config{Host: apiServerURL, Version: testapi.Version()})
 	return cl
@@ -73,7 +74,8 @@ func main() {
 	if err != nil {
 		glog.Fatalf("Failed to connect to etcd: %v", err)
 	}
-	startComponents(etcdClient, newApiClient(*addr, *port), *addr, *port)
+	address := net.ParseIP(*addr)
+	startComponents(etcdClient, newApiClient(address, *port), address, *port)
 	glog.Infof("Kubernetes API Server is up and running on http://%s:%d", *addr, *port)
 
 	select {}

diff --git a/pkg/master/master.go b/pkg/master/master.go
@@ -97,8 +97,8 @@ type Config struct {
 	// Defaults to 443 if not set.
 	ReadWritePort int
 
-	// If empty, the first result from net.InterfaceAddrs will be used.
-	PublicAddress string
+	// If nil, the first result from net.InterfaceAddrs will be used.
+	PublicAddress net.IP
 }
 
 // Master contains state for a Kubernetes cluster master/api server.
@@ -133,9 +133,14 @@ type Master struct {
 	v1beta3               bool
 	nodeIPCache           IPGetter
 
-	readOnlyServer  string
-	readWriteServer string
-	masterServices  *util.Runner
+	publicIP             net.IP
+	publicReadOnlyPort   int
+	publicReadWritePort  int
+	serviceReadOnlyIP    net.IP
+	serviceReadOnlyPort  int
+	serviceReadWriteIP   net.IP
+	serviceReadWritePort int
+	masterServices       *util.Runner
 
 	// "Outputs"
 	Handler         http.Handler
@@ -176,7 +181,7 @@ func setDefaults(c *Config) {
 	if c.ReadWritePort == 0 {
 		c.ReadWritePort = 443
 	}
-	for c.PublicAddress == "" {
+	for c.PublicAddress == nil {
 		// Find and use the first non-loopback address.
 		// TODO: potentially it'd be useful to skip the docker interface if it
 		// somehow is first in the list.
@@ -196,7 +201,7 @@ func setDefaults(c *Config) {
 				continue
 			}
 			found = true
-			c.PublicAddress = ip.String()
+			c.PublicAddress = ip
 			glog.Infof("Will report %v as public IP address.", ip)
 			break
 		}
@@ -244,6 +249,17 @@ func New(c *Config) *Master {
 		glog.Fatalf("master.New() called with config.KubeletClient == nil")
 	}
 
+	// Select the first two valid IPs from portalNet to use as the master service portalIPs
+	serviceReadOnlyIP, err := service.GetIndexedIP(c.PortalNet, 1)
+	if err != nil {
+		glog.Fatalf("Failed to generate service read-only IP for master service: %v", err)
+	}
+	serviceReadWriteIP, err := service.GetIndexedIP(c.PortalNet, 2)
+	if err != nil {
+		glog.Fatalf("Failed to generate service read-write IP for master service: %v", err)
+	}
+	glog.Infof("Setting master service IPs based on PortalNet subnet to %q (read-only) and %q (read-write).", serviceReadOnlyIP, serviceReadWriteIP)
+
 	m := &Master{
 		podRegistry:           etcd.NewRegistry(c.EtcdHelper, boundPodFactory),
 		controllerRegistry:    etcd.NewRegistry(c.EtcdHelper, nil),
@@ -268,9 +284,16 @@ func New(c *Config) *Master {
 		v1beta3:               c.EnableV1Beta3,
 		nodeIPCache:           NewIPCache(c.Cloud, util.RealClock{}, 30*time.Second),
 
-		masterCount:     c.MasterCount,
-		readOnlyServer:  net.JoinHostPort(c.PublicAddress, strconv.Itoa(int(c.ReadOnlyPort))),
-		readWriteServer: net.JoinHostPort(c.PublicAddress, strconv.Itoa(int(c.ReadWritePort))),
+		masterCount:         c.MasterCount,
+		publicIP:            c.PublicAddress,
+		publicReadOnlyPort:  c.ReadOnlyPort,
+		publicReadWritePort: c.ReadWritePort,
+		serviceReadOnlyIP:   serviceReadOnlyIP,
+		// TODO: serviceReadOnlyPort should be passed in as an argument, it may not always be 80
+		serviceReadOnlyPort: 80,
+		serviceReadWriteIP:  serviceReadWriteIP,
+		// TODO: serviceReadWritePort should be passed in as an argument, it may not always be 443
+		serviceReadWritePort: 443,
 	}
 
 	if c.RestfulContainer != nil {
@@ -443,7 +466,7 @@ func (m *Master) init(c *Config) {
 func (m *Master) InstallSwaggerAPI() {
 	// Enable swagger UI and discovery API
 	swaggerConfig := swagger.Config{
-		WebServicesUrl: m.readWriteServer,
+		WebServicesUrl: net.JoinHostPort(m.publicIP.String(), strconv.Itoa(int(m.publicReadWritePort))),
 		WebServices:    m.handlerContainer.RegisteredWebServices(),
 		// TODO: Parameterize the path?
 		ApiPath:         "/swaggerapi/",

diff --git a/pkg/master/publish.go b/pkg/master/publish.go
@@ -18,6 +18,8 @@ package master
 
 import (
 	"fmt"
+	"net"
+	"strconv"
 	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@@ -32,12 +34,11 @@ func (m *Master) serviceWriterLoop(stop chan struct{}) {
 		// TODO: when it becomes possible to change this stuff,
 		// stop polling and start watching.
 		// TODO: add endpoints of all replicas, not just the elected master.
-		if m.readWriteServer != "" {
-			// TODO: the public port should be part of the argument here, port will not always be 443
-			if err := m.createMasterServiceIfNeeded("kubernetes", 443); err != nil {
+		if m.serviceReadWriteIP != nil {
+			if err := m.createMasterServiceIfNeeded("kubernetes", m.serviceReadWriteIP, m.serviceReadWritePort); err != nil {
 				glog.Errorf("Can't create rw service: %v", err)
 			}
-			if err := m.ensureEndpointsContain("kubernetes", m.readWriteServer); err != nil {
+			if err := m.ensureEndpointsContain("kubernetes", net.JoinHostPort(m.publicIP.String(), strconv.Itoa(int(m.publicReadWritePort)))); err != nil {
 				glog.Errorf("Can't create rw endpoints: %v", err)
 			}
 		}
@@ -55,12 +56,11 @@ func (m *Master) roServiceWriterLoop(stop chan struct{}) {
 		// Update service & endpoint records.
 		// TODO: when it becomes possible to change this stuff,
 		// stop polling and start watching.
-		if m.readOnlyServer != "" {
-			// TODO: the public port should be part of the argument here, port will not always be 80
-			if err := m.createMasterServiceIfNeeded("kubernetes-ro", 80); err != nil {
+		if m.serviceReadOnlyIP != nil {
+			if err := m.createMasterServiceIfNeeded("kubernetes-ro", m.serviceReadOnlyIP, m.serviceReadOnlyPort); err != nil {
 				glog.Errorf("Can't create ro service: %v", err)
 			}
-			if err := m.ensureEndpointsContain("kubernetes-ro", m.readOnlyServer); err != nil {
+			if err := m.ensureEndpointsContain("kubernetes-ro", net.JoinHostPort(m.publicIP.String(), strconv.Itoa(int(m.publicReadOnlyPort)))); err != nil {
 				glog.Errorf("Can't create ro endpoints: %v", err)
 			}
 		}
@@ -75,7 +75,7 @@ func (m *Master) roServiceWriterLoop(stop chan struct{}) {
 
 // createMasterServiceIfNeeded will create the specified service if it
 // doesn't already exist.
-func (m *Master) createMasterServiceIfNeeded(serviceName string, port int) error {
+func (m *Master) createMasterServiceIfNeeded(serviceName string, serviceIP net.IP, servicePort int) error {
 	ctx := api.NewDefaultContext()
 	if _, err := m.serviceRegistry.GetService(ctx, serviceName); err == nil {
 		// The service already exists.
@@ -88,9 +88,10 @@ func (m *Master) createMasterServiceIfNeeded(serviceName string, port int) error
 			Labels:    map[string]string{"provider": "kubernetes", "component": "apiserver"},
 		},
 		Spec: api.ServiceSpec{
-			Port: port,
+			Port: servicePort,
 			// maintained by this code, not by the pod selector
 			Selector: nil,
+			PortalIP: serviceIP.String(),
 		},
 	}
 	// Kids, don't do this at home: this is a hack. There's no good way to call the business

diff --git a/pkg/master/server/server.go b/pkg/master/server/server.go
@@ -47,7 +47,7 @@ import (
 type APIServer struct {
 	Port                       int
 	Address                    util.IP
-	PublicAddressOverride      string
+	PublicAddressOverride      util.IP
 	ReadOnlyPort               int
 	APIRate                    float32
 	APIBurst                   int
@@ -80,6 +80,7 @@ func NewAPIServer() *APIServer {
 	s := APIServer{
 		Port:                   8080,
 		Address:                util.IP(net.ParseIP("127.0.0.1")),
+		PublicAddressOverride:  util.IP(net.ParseIP("")),
 		ReadOnlyPort:           7080,
 		APIRate:                10.0,
 		APIBurst:               200,
@@ -127,11 +128,10 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
 		"further assumed that port 443 on the cluster's public address is proxied to this "+
 		"port. This is performed by nginx in the default setup.")
 	fs.Var(&s.Address, "address", "The IP address on to serve on (set to 0.0.0.0 for all interfaces)")
-	fs.StringVar(&s.PublicAddressOverride, "public_address_override", s.PublicAddressOverride, ""+
-		"Public serving address. Read only port will be opened on this address, "+
-		"and it is assumed that port 443 at this address will be proxied/redirected "+
-		"to '-address':'-port'. If blank, the address in the first listed interface "+
-		"will be used.")
+	fs.Var(&s.PublicAddressOverride, "public_address_override", "Public serving address."+
+		"Read only port will be opened on this address, and it is assumed that port "+
+		"443 at this address will be proxied/redirected to '-address':'-port'. If "+
+		"blank, the address in the first listed interface will be used.")
 	fs.IntVar(&s.ReadOnlyPort, "read_only_port", s.ReadOnlyPort, ""+
 		"The port from which to serve read-only resources. If 0, don't serve on a "+
 		"read-only address. It is assumed that firewall rules are set up such that "+
@@ -251,7 +251,7 @@ func (s *APIServer) Run(_ []string) error {
 		CorsAllowedOriginList:  s.CorsAllowedOriginList,
 		ReadOnlyPort:           s.ReadOnlyPort,
 		ReadWritePort:          s.Port,
-		PublicAddress:          s.PublicAddressOverride,
+		PublicAddress:          net.IP(s.PublicAddressOverride),
 		Authenticator:          authenticator,
 		Authorizer:             authorizer,
 		AdmissionControl:       admissionController,
@@ -263,11 +263,11 @@ func (s *APIServer) Run(_ []string) error {
 	// We serve on 3 ports.  See docs/reaching_the_api.md
 	roLocation := ""
 	if s.ReadOnlyPort != 0 {
-		roLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(config.ReadOnlyPort))
+		roLocation = net.JoinHostPort(config.PublicAddress.String(), strconv.Itoa(config.ReadOnlyPort))
 	}
 	secureLocation := ""
 	if s.SecurePort != 0 {
-		secureLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(s.SecurePort))
+		secureLocation = net.JoinHostPort(config.PublicAddress.String(), strconv.Itoa(s.SecurePort))
 	}
 	rwLocation := net.JoinHostPort(s.Address.String(), strconv.Itoa(int(s.Port)))
 
@@ -317,7 +317,7 @@ func (s *APIServer) Run(_ []string) error {
 				if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
 					s.TLSCertFile = "/var/run/kubernetes/apiserver.crt"
 					s.TLSPrivateKeyFile = "/var/run/kubernetes/apiserver.key"
-					if err := util.GenerateSelfSignedCert(config.PublicAddress, s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
+					if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
 						glog.Errorf("Unable to generate self signed cert: %v", err)
 					} else {
 						glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)

diff --git a/pkg/registry/service/ip_allocator.go b/pkg/registry/service/ip_allocator.go
@@ -174,6 +174,18 @@ func (ipa *ipAllocator) AllocateNext() (net.IP, error) {
 	return nil, fmt.Errorf("can't find a free IP in %s", ipa.subnet)
 }
 
+// Returns the index-th IP from the specified subnet range.
+// For example, subnet "10.0.0.0/24" with index "2" will return the IP "10.0.0.2".
+// TODO(saad-ali): Move this (and any other functions that are independent of ipAllocator) to some
+// place more generic.
+func GetIndexedIP(subnet *net.IPNet, index int) (net.IP, error) {
+	ip := ipAdd(subnet.IP, index /* offset */)
+	if !subnet.Contains(ip) {
+		return nil, fmt.Errorf("can't generate IP with index %d from subnet. subnet too small. subnet: %q", index, subnet)
+	}
+	return ip, nil
+}
+
 func (ipa *ipAllocator) createRandomIp() net.IP {
 	ip := ipa.subnet.IP
 	mask := ipa.subnet.Mask

diff --git a/pkg/registry/service/ip_allocator_test.go b/pkg/registry/service/ip_allocator_test.go
@@ -214,6 +214,75 @@ func TestIPAdd(t *testing.T) {
 	}
 }
 
+func TestGenerateFirstTwoIPsFromSubnet(t *testing.T) {
+	// Arrange
+	testCases := []struct {
+		subnet        string
+		expected1stIP string
+		expected2ndIP string
+	}{
+		{"10.0.0.0/24", "10.0.0.1", "10.0.0.2"},
+		{"10.10.10.10/24", "10.10.10.1", "10.10.10.2"},
+		{"10.10.10.10/16", "10.10.0.1", "10.10.0.2"},
+		{"10.10.10.10/8", "10.0.0.1", "10.0.0.2"},
+		{"10.10.10.10/0", "0.0.0.1", "0.0.0.2"},
+		{"192.168.100.1/16", "192.168.0.1", "192.168.0.2"},
+		{"153.15.250.5/23", "153.15.250.1", "153.15.250.2"},
+		{"2001:db8::/48", "2001:db8::1", "2001:db8::2"},
+		{"2001:db8:123:255::/48", "2001:db8:123::1", "2001:db8:123::2"},
+		{"12.12.0.0/30", "12.12.0.1", "12.12.0.2"},
+	}
+
+	// Act & Assert
+	for _, testCase := range testCases {
+		_, subnet, _ := net.ParseCIDR(testCase.subnet)
+		firstIP, err := GetIndexedIP(subnet, 1)
+		if err != nil {
+			t.Errorf("Unexpected error: %v", err)
+		}
+		secondIP, err := GetIndexedIP(subnet, 2)
+		if err != nil {
+			t.Errorf("Unexpected error: %v", err)
+		}
+		if firstIP.String() != testCase.expected1stIP {
+			t.Errorf("Unexpected first IP: Expected <%q> Actual <%q>", testCase.expected1stIP, firstIP.String())
+		}
+		if secondIP.String() != testCase.expected2ndIP {
+			t.Errorf("Unexpected second IP: Expected <%q> Actual <%q>", testCase.expected2ndIP, secondIP.String())
+		}
+	}
+}
+
+func TestGetIndexedIPSubnetTooSmall(t *testing.T) {
+	// Arrange
+	testCases := []struct {
+		subnet string
+	}{
+		{"12.12.0.0/32"},
+		{"12.12.0.0/31"},
+	}
+
+	// Act & Assert
+	for _, testCase := range testCases {
+		_, subnet, _ := net.ParseCIDR(testCase.subnet)
+		secondIP, err := GetIndexedIP(subnet, 2)
+		if err == nil {
+			t.Errorf("Expected error but no error occured for subnet: ", testCase.subnet)
+		}
+		thirdIP, err := GetIndexedIP(subnet, 3)
+		if err == nil {
+			t.Errorf("Expected error but no error occured for subnet: ", testCase.subnet)
+		}
+		if secondIP != nil {
+			t.Errorf("Unexpected second IP: Expected nil Actual <%q>", thirdIP.String())
+		}
+		if thirdIP != nil {
+			t.Errorf("Unexpected third IP: Expected nil Actual <%q>", secondIP.String())
+		}
+
+	}
+}
+
 func TestCopyIP(t *testing.T) {
 	ip1 := net.ParseIP("1.2.3.4")
 	ip2 := copyIP(ip1)

diff --git a/pkg/standalone/standalone.go b/pkg/standalone/standalone.go
@@ -85,7 +85,7 @@ func GetAPIServerClient(authPath string, apiServerList util.StringList) (*client
 }
 
 // RunApiServer starts an API server in a go routine.
-func RunApiServer(cl *client.Client, etcdClient tools.EtcdClient, addr string, port int, masterServiceNamespace string) {
+func RunApiServer(cl *client.Client, etcdClient tools.EtcdClient, addr net.IP, port int, masterServiceNamespace string) {
 	handler := delegateHandler{}
 
 	helper, err := master.NewEtcdHelper(etcdClient, "")

diff --git a/test/integration/auth_test.go b/test/integration/auth_test.go
@@ -106,7 +106,7 @@ var aService string = `
   "apiVersion": "v1beta1",
   "id": "a",
   "port": 8000,
-  "portalIP": "10.0.0.1",
+  "portalIP": "10.0.0.100",
   "labels": { "name": "a" },
   "selector": { "name": "a" }
 }