From 2a4b252e4a781da337477d918c18b37ae12ea44a Mon Sep 17 00:00:00 2001 From: Amit Watve Date: Wed, 29 Jan 2020 17:39:35 -0800 Subject: [PATCH 1/7] remove redundant prefix --- pkg/cluster/internal/providers/docker/{docker_util.go => util.go} | 0 pkg/cluster/internal/providers/podman/{podman_util.go => util.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pkg/cluster/internal/providers/docker/{docker_util.go => util.go} (100%) rename pkg/cluster/internal/providers/podman/{podman_util.go => util.go} (100%) diff --git a/pkg/cluster/internal/providers/docker/docker_util.go b/pkg/cluster/internal/providers/docker/util.go similarity index 100% rename from pkg/cluster/internal/providers/docker/docker_util.go rename to pkg/cluster/internal/providers/docker/util.go diff --git a/pkg/cluster/internal/providers/podman/podman_util.go b/pkg/cluster/internal/providers/podman/util.go similarity index 100% rename from pkg/cluster/internal/providers/podman/podman_util.go rename to pkg/cluster/internal/providers/podman/util.go From 7fcea6fd8713b97fd43d5b3c43fa3bcbfffd510b Mon Sep 17 00:00:00 2001 From: Amit Watve Date: Wed, 29 Jan 2020 17:41:33 -0800 Subject: [PATCH 2/7] remove usernsremap --- .../internal/providers/podman/provision.go | 4 ---- pkg/cluster/internal/providers/podman/util.go | 15 --------------- 2 files changed, 19 deletions(-) diff --git a/pkg/cluster/internal/providers/podman/provision.go b/pkg/cluster/internal/providers/podman/provision.go index 587fc4a157..4b212111a2 100644 --- a/pkg/cluster/internal/providers/podman/provision.go +++ b/pkg/cluster/internal/providers/podman/provision.go @@ -141,10 +141,6 @@ func commonArgs(cluster string, cfg *config.Cluster) ([]string, error) { args = append(args, "-e", fmt.Sprintf("%s=%s", key, val)) } - // handle hosts that have user namespace remapping enabled - if usernsRemap() { - args = append(args, "--userns=host") - } return args, nil } diff --git a/pkg/cluster/internal/providers/podman/util.go b/pkg/cluster/internal/providers/podman/util.go index 29b21b71e2..17546fdabc 100644 --- a/pkg/cluster/internal/providers/podman/util.go +++ b/pkg/cluster/internal/providers/podman/util.go @@ -25,21 +25,6 @@ import ( "sigs.k8s.io/kind/pkg/exec" ) -// usernsRemap checks if userns-remap is enabled in podmand -func usernsRemap() bool { - cmd := exec.Command("podman", "info", "--format", "'{{json .SecurityOptions}}'") - lines, err := exec.CombinedOutputLines(cmd) - if err != nil { - return false - } - if len(lines) > 0 { - if strings.Contains(lines[0], "name=userns") { - return true - } - } - return false -} - func getPodmanVersion() (*version.Version, error) { cmd := exec.Command("podman", "--version") lines, err := exec.CombinedOutputLines(cmd) From ef69dbe31702e273b413f371cf9db8a9edf73b1f Mon Sep 17 00:00:00 2001 From: Amit Watve Date: Wed, 29 Jan 2020 17:44:19 -0800 Subject: [PATCH 3/7] fix ip format --- pkg/cluster/internal/providers/podman/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/internal/providers/podman/node.go b/pkg/cluster/internal/providers/podman/node.go index 07aa3bd8f7..6e5083b221 100644 --- a/pkg/cluster/internal/providers/podman/node.go +++ b/pkg/cluster/internal/providers/podman/node.go @@ -52,7 +52,7 @@ func (n *node) Role() (string, error) { func (n *node) IP() (ipv4 string, ipv6 string, err error) { // retrieve the IP address of the node using podman inspect cmd := exec.Command("podman", "inspect", - "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}", + "-f", "{{.NetworkSettings.IPAddress}},{{.NetworkSettings.GlobalIPv6Address}}", n.name, // ... against the "node" container ) lines, err := exec.CombinedOutputLines(cmd) From 20dfa0709b203bf327b1f6a7a4f69c5a153a62a6 Mon Sep 17 00:00:00 2001 From: Amit Watve Date: Wed, 29 Jan 2020 17:57:10 -0800 Subject: [PATCH 4/7] handle random port --- pkg/cluster/internal/providers/podman/provision.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/cluster/internal/providers/podman/provision.go b/pkg/cluster/internal/providers/podman/provision.go index 4b212111a2..0860f7138c 100644 --- a/pkg/cluster/internal/providers/podman/provision.go +++ b/pkg/cluster/internal/providers/podman/provision.go @@ -303,6 +303,11 @@ func generatePortMappings(clusterIPFamily config.ClusterIPFamily, portMappings . // generate the actual mapping arg protocol := string(pm.Protocol) hostPortBinding := net.JoinHostPort(pm.ListenAddress, fmt.Sprintf("%d", pm.HostPort)) + // Podman expects empty string instead of 0 to assign a random port + // https://github.com/containers/libpod/blob/master/pkg/spec/ports.go#L68-L69 + if strings.HasSuffix(hostPortBinding, ":0") { + hostPortBinding = strings.TrimSuffix(hostPortBinding, "0") + } args = append(args, fmt.Sprintf("--publish=%s:%d/%s", hostPortBinding, pm.ContainerPort, protocol)) } return args From f9366430af85bb9ae4c0af2cf7078df8dd2519e6 Mon Sep 17 00:00:00 2001 From: Amit Watve Date: Thu, 30 Jan 2020 13:12:45 -0800 Subject: [PATCH 5/7] fix apiserver info --- .../internal/providers/podman/provider.go | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/pkg/cluster/internal/providers/podman/provider.go b/pkg/cluster/internal/providers/podman/provider.go index ecc6b9cb52..d91f77b520 100644 --- a/pkg/cluster/internal/providers/podman/provider.go +++ b/pkg/cluster/internal/providers/podman/provider.go @@ -17,11 +17,14 @@ limitations under the License. package podman import ( + "encoding/json" "fmt" "net" + "strconv" "strings" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/kind/pkg/cluster/internal/providers/provider/common" "sigs.k8s.io/kind/pkg/cluster/nodes" "sigs.k8s.io/kind/pkg/errors" @@ -29,7 +32,6 @@ import ( "sigs.k8s.io/kind/pkg/log" "sigs.k8s.io/kind/pkg/cluster/internal/providers/provider" - "sigs.k8s.io/kind/pkg/cluster/internal/providers/provider/common" "sigs.k8s.io/kind/pkg/cluster/nodeutils" "sigs.k8s.io/kind/pkg/internal/apis/config" "sigs.k8s.io/kind/pkg/internal/cli" @@ -152,9 +154,8 @@ func (p *Provider) GetAPIServerEndpoint(cluster string) (string, error) { // retrieve the specific port mapping using podman inspect cmd := exec.Command( "podman", "inspect", - "--format", fmt.Sprintf( - "{{ with (index (index .NetworkSettings.Ports \"%d/tcp\") 0) }}{{ printf \"%%s\t%%s\" .HostIp .HostPort }}{{ end }}", common.APIServerInternalPort, - ), + "--format", + "{{ json .NetworkSettings.Ports }}", n.String(), ) lines, err := exec.OutputLines(cmd) @@ -164,13 +165,27 @@ func (p *Provider) GetAPIServerEndpoint(cluster string) (string, error) { if len(lines) != 1 { return "", errors.Errorf("network details should only be one line, got %d lines", len(lines)) } - parts := strings.Split(lines[0], "\t") - if len(parts) != 2 { - return "", errors.Errorf("network details should only be two parts, got %d", len(parts)) + + // portMapping maps to the standard CNI portmapping capability + // see: https://github.com/containernetworking/cni/blob/spec-v0.4.0/CONVENTIONS.md + type portMapping struct { + HostPort int32 `json:"hostPort"` + ContainerPort int32 `json:"containerPort"` + Protocol string `json:"protocol"` + HostIP string `json:"hostIP"` + } + + var portMappings []portMapping + if err := json.Unmarshal([]byte(lines[0]), &portMappings); err != nil { + return "", errors.Errorf("invalid network details: %v", err) + } + for _, pm := range portMappings { + if pm.ContainerPort == common.APIServerInternalPort && pm.Protocol == "tcp" { + return net.JoinHostPort(pm.HostIP, strconv.FormatInt(int64(pm.HostPort), 10)), nil + } } - // join host and port - return net.JoinHostPort(parts[0], parts[1]), nil + return "", errors.Errorf("unable to find apiserver endpoint information") } // node returns a new node handle for this provider From 14f388ccea4c1f9f3d7604e60e2ca090128e3095 Mon Sep 17 00:00:00 2001 From: Amit Watve Date: Thu, 30 Jan 2020 15:14:12 -0800 Subject: [PATCH 6/7] handle non support for tag and sha --- pkg/cluster/internal/providers/podman/images.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/internal/providers/podman/images.go b/pkg/cluster/internal/providers/podman/images.go index ac3035d997..fbd1103c53 100644 --- a/pkg/cluster/internal/providers/podman/images.go +++ b/pkg/cluster/internal/providers/podman/images.go @@ -38,7 +38,10 @@ func ensureNodeImages(logger log.Logger, status *cli.Status, cfg *config.Cluster // prints user friendly message friendlyImageName := image if strings.Contains(image, "@sha256:") { - friendlyImageName = strings.Split(image, "@sha256:")[0] + splits := strings.Split(image, "@sha256:") + friendlyImageName = splits[0] + // possibly trim tag and use sha + image = strings.Split(friendlyImageName, ":")[0] + "@sha256:" + splits[1] } status.Start(fmt.Sprintf("Ensuring node image (%s) 🖼", friendlyImageName)) if _, err := pullIfNotPresent(logger, image, 4); err != nil { From fc098e2976d594988b5895f007ce5d40e4f89848 Mon Sep 17 00:00:00 2001 From: Amit Watve Date: Thu, 30 Jan 2020 15:24:21 -0800 Subject: [PATCH 7/7] warn rootless --- pkg/cluster/internal/providers/podman/provider.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/internal/providers/podman/provider.go b/pkg/cluster/internal/providers/podman/provider.go index d91f77b520..b6d31bd8ad 100644 --- a/pkg/cluster/internal/providers/podman/provider.go +++ b/pkg/cluster/internal/providers/podman/provider.go @@ -20,19 +20,20 @@ import ( "encoding/json" "fmt" "net" + "os" "strconv" "strings" "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/kind/pkg/cluster/internal/providers/provider/common" "sigs.k8s.io/kind/pkg/cluster/nodes" + "sigs.k8s.io/kind/pkg/cluster/nodeutils" "sigs.k8s.io/kind/pkg/errors" "sigs.k8s.io/kind/pkg/exec" "sigs.k8s.io/kind/pkg/log" "sigs.k8s.io/kind/pkg/cluster/internal/providers/provider" - "sigs.k8s.io/kind/pkg/cluster/nodeutils" + "sigs.k8s.io/kind/pkg/cluster/internal/providers/provider/common" "sigs.k8s.io/kind/pkg/internal/apis/config" "sigs.k8s.io/kind/pkg/internal/cli" ) @@ -56,6 +57,11 @@ func (p *Provider) Provision(status *cli.Status, cluster string, cfg *config.Clu return err } + // kind doesn't currently work with podman rootless, surface a warning + if os.Geteuid() != 0 { + p.logger.Warn("podman provider may not work properly in rootless mode") + } + // TODO: validate cfg // ensure node images are pulled before actually provisioning if err := ensureNodeImages(p.logger, status, cfg); err != nil { @@ -181,7 +187,7 @@ func (p *Provider) GetAPIServerEndpoint(cluster string) (string, error) { } for _, pm := range portMappings { if pm.ContainerPort == common.APIServerInternalPort && pm.Protocol == "tcp" { - return net.JoinHostPort(pm.HostIP, strconv.FormatInt(int64(pm.HostPort), 10)), nil + return net.JoinHostPort(pm.HostIP, strconv.Itoa(int(pm.HostPort))), nil } }