Skip to content

Commit

Permalink
Merge pull request #28871 from vishh/gce-cp
Browse files Browse the repository at this point in the history 
Automatic merge from submit-queue

Do not query the metadata server to find out if running on GCE.  Retry metadata server query for gcr if running on gce.

Retry the logic for determining is gcr is enabled to workaround metadata unavailability.

Note: This patch does not retry fetching registry credentials.
  • Loading branch information
k8s-merge-robot authored Jul 18, 2016
2 parents 92f6b0c + ea1a459 commit 8eb0cf5
Show file tree
Hide file tree
Showing 3 changed files with 191 additions and 14 deletions.
86 changes: 80 additions & 6 deletions pkg/credentialprovider/gcp/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package gcp_credentials

import (
"encoding/json"
"io/ioutil"
"net/http"
"strings"
"time"
Expand All @@ -31,13 +32,20 @@ const (
metadataAttributes = metadataUrl + "instance/attributes/"
dockerConfigKey = metadataAttributes + "google-dockercfg"
dockerConfigUrlKey = metadataAttributes + "google-dockercfg-url"
serviceAccounts = metadataUrl + "instance/service-accounts/"
metadataScopes = metadataUrl + "instance/service-accounts/default/scopes"
metadataToken = metadataUrl + "instance/service-accounts/default/token"
metadataEmail = metadataUrl + "instance/service-accounts/default/email"
storageScopePrefix = "https://www.googleapis.com/auth/devstorage"
cloudPlatformScopePrefix = "https://www.googleapis.com/auth/cloud-platform"
googleProductName = "Google"
defaultServiceAccount = "default/"
)

// Product file path that contains the cloud service name.
// This is a variable instead of a const to enable testing.
var gceProductNameFile = "/sys/class/dmi/id/product_name"

// For these urls, the parts of the host name can be glob, for example '*.gcr.io" will match
// "foo.gcr.io" and "bar.gcr.io".
var containerRegistryUrls = []string{"container.cloud.google.com", "gcr.io", "*.gcr.io"}
Expand Down Expand Up @@ -98,10 +106,20 @@ func init() {
})
}

// Returns true if it finds a local GCE VM.
// Looks at a product file that is an undocumented API.
func onGCEVM() bool {
data, err := ioutil.ReadFile(gceProductNameFile)
if err != nil {
glog.V(2).Infof("Error while reading product_name: %v", err)
return false
}
return strings.Contains(string(data), googleProductName)
}

// Enabled implements DockerConfigProvider for all of the Google implementations.
func (g *metadataProvider) Enabled() bool {
_, err := credentialprovider.ReadUrl(metadataUrl, g.Client, metadataHeader)
return err == nil
return onGCEVM()
}

// LazyProvide implements DockerConfigProvider. Should never be called.
Expand Down Expand Up @@ -148,18 +166,74 @@ func (g *dockerConfigUrlKeyProvider) Provide() credentialprovider.DockerConfig {
return credentialprovider.DockerConfig{}
}

// runcWithBackoff runs input function `f` with an exponential backoff.
// Note that this method can block indefinitely.
func runWithBackoff(f func() ([]byte, error)) []byte {
var backoff = 100 * time.Millisecond
const maxBackoff = time.Minute
for {
value, err := f()
if err == nil {
return value
}
time.Sleep(backoff)
backoff = backoff * 2
if backoff > maxBackoff {
backoff = maxBackoff
}
}
}

// Enabled implements a special metadata-based check, which verifies the
// storage scope is available on the GCE VM.
// If running on a GCE VM, check if 'default' service account exists.
// If it does not exist, assume that registry is not enabled.
// If default service account exists, check if relevant scopes exist in the default service account.
// The metadata service can become temporarily inaccesible. Hence all requests to the metadata
// service will be retried until the metadata server returns a `200`.
// It is expected that "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/" will return a `200`
// and "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/scopes" will also return `200`.
// More information on metadata service can be found here - https://cloud.google.com/compute/docs/storing-retrieving-metadata
func (g *containerRegistryProvider) Enabled() bool {
value, err := credentialprovider.ReadUrl(metadataScopes+"?alt=json", g.Client, metadataHeader)
if err != nil {
if !onGCEVM() {
return false
}
// Given that we are on GCE, we should keep retrying until the metadata server responds.
value := runWithBackoff(func() ([]byte, error) {
value, err := credentialprovider.ReadUrl(serviceAccounts, g.Client, metadataHeader)
if err != nil {
glog.V(2).Infof("Failed to Get service accounts from gce metadata server: %v", err)
}
return value, err
})
// We expect the service account to return a list of account directories separated by newlines, e.g.,
// sv-account-name1/
// sv-account-name2/
// ref: https://cloud.google.com/compute/docs/storing-retrieving-metadata
defaultServiceAccountExists := false
for _, sa := range strings.Split(string(value), "\n") {
if strings.TrimSpace(sa) == defaultServiceAccount {
defaultServiceAccountExists = true
break
}
}
if !defaultServiceAccountExists {
glog.V(2).Infof("'default' service account does not exist. Found following service accounts: %q", string(value))
return false
}
url := metadataScopes + "?alt=json"
value = runWithBackoff(func() ([]byte, error) {
value, err := credentialprovider.ReadUrl(url, g.Client, metadataHeader)
if err != nil {
glog.V(2).Infof("Failed to Get scopes in default service account from gce metadata server: %v", err)
}
return value, err
})
var scopes []string
if err := json.Unmarshal([]byte(value), &scopes); err != nil {
if err := json.Unmarshal(value, &scopes); err != nil {
glog.Errorf("Failed to unmarshal scopes: %v", err)
return false
}

for _, v := range scopes {
// cloudPlatformScope implies storage scope.
if strings.HasPrefix(v, storageScopePrefix) || strings.HasPrefix(v, cloudPlatformScopePrefix) {
Expand Down
115 changes: 107 additions & 8 deletions pkg/credentialprovider/gcp/metadata_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"os"
"reflect"
"strings"
"testing"
Expand All @@ -31,6 +33,14 @@ import (
utilnet "k8s.io/kubernetes/pkg/util/net"
)

func createProductNameFile() (string, error) {
file, err := ioutil.TempFile("", "")
if err != nil {
return "", fmt.Errorf("failed to create temporary test file: %v", err)
}
return file.Name(), ioutil.WriteFile(file.Name(), []byte("Google"), 0600)
}

func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
registryUrl := "hello.kubernetes.io"
email := "foo@bar.baz"
Expand All @@ -44,6 +54,12 @@ func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
}
}`, registryUrl, email, auth)

var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
const probeEndpoint = "/computeMetadata/v1/"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the one metadata key.
Expand Down Expand Up @@ -111,6 +127,12 @@ func TestDockerKeyringFromGoogleDockerConfigMetadataUrl(t *testing.T) {
}
}`, registryUrl, email, auth)

var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
const probeEndpoint = "/computeMetadata/v1/"
const valueEndpoint = "/my/value"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -176,11 +198,19 @@ func TestContainerRegistryBasics(t *testing.T) {
token := &tokenBlob{AccessToken: "ya26.lots-of-indiscernible-garbage"}

const (
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
scopeEndpoint = defaultEndpoint + "scopes"
emailEndpoint = defaultEndpoint + "email"
tokenEndpoint = defaultEndpoint + "token"
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
scopeEndpoint = defaultEndpoint + "scopes"
emailEndpoint = defaultEndpoint + "email"
tokenEndpoint = defaultEndpoint + "token"
)
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)

server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the URL key and the value endpoint
if scopeEndpoint == r.URL.Path {
Expand All @@ -198,6 +228,9 @@ func TestContainerRegistryBasics(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
fmt.Fprintln(w, string(bytes))
} else if serviceAccountsEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, "default/\ncustom")
} else {
w.WriteHeader(http.StatusNotFound)
}
Expand Down Expand Up @@ -243,23 +276,77 @@ func TestContainerRegistryBasics(t *testing.T) {
}
}

func TestContainerRegistryNoServiceAccount(t *testing.T) {
const (
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the URL key and the value endpoint
if serviceAccountsEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
bytes, err := json.Marshal([]string{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
fmt.Fprintln(w, string(bytes))
} else {
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()

var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)

// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(server.URL + req.URL.Path)
},
})

provider := &containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
}

if provider.Enabled() {
t.Errorf("Provider is unexpectedly enabled")
}
}

func TestContainerRegistryNoStorageScope(t *testing.T) {
const (
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
scopeEndpoint = defaultEndpoint + "scopes"
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
scopeEndpoint = defaultEndpoint + "scopes"
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the URL key and the value endpoint
if scopeEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `["https://www.googleapis.com/auth/compute.read_write"]`)
} else if serviceAccountsEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, "default/\ncustom")
} else {
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()

var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)

// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
Expand All @@ -278,21 +365,33 @@ func TestContainerRegistryNoStorageScope(t *testing.T) {

func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) {
const (
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
scopeEndpoint = defaultEndpoint + "scopes"
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
scopeEndpoint = defaultEndpoint + "scopes"
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the URL key and the value endpoint
if scopeEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `["https://www.googleapis.com/auth/compute.read_write","https://www.googleapis.com/auth/cloud-platform.read-only"]`)
} else if serviceAccountsEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprintln(w, "default/\ncustom")
} else {
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()

var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)

// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
Expand Down
4 changes: 4 additions & 0 deletions pkg/credentialprovider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,11 @@ import (
// DockerConfigProvider is the interface that registered extensions implement
// to materialize 'dockercfg' credentials.
type DockerConfigProvider interface {
// Enabled returns true if the config provider is enabled.
// Implementations can be blocking - e.g. metadata server unavailable.
Enabled() bool
// Provide returns docker configuration.
// Implementations can be blocking - e.g. metadata server unavailable.
Provide() DockerConfig
// LazyProvide() gets called after URL matches have been performed, so the
// location used as the key in DockerConfig would be redundant.
Expand Down

0 comments on commit 8eb0cf5

Please sign in to comment.