From cabd2bb619261e58859feff165944da9e5287ecd Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Tue, 30 Jan 2018 09:33:48 +0800 Subject: [PATCH 1/2] Add experimental hyperv containers support on Windows --- pkg/kubelet/dockershim/docker_sandbox.go | 2 + pkg/kubelet/dockershim/helpers_linux.go | 4 ++ pkg/kubelet/dockershim/helpers_unsupported.go | 4 ++ pkg/kubelet/dockershim/helpers_windows.go | 40 +++++++++++++++++-- 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/dockershim/docker_sandbox.go b/pkg/kubelet/dockershim/docker_sandbox.go index 24d7879eeea35..331ecd2f22d60 100644 --- a/pkg/kubelet/dockershim/docker_sandbox.go +++ b/pkg/kubelet/dockershim/docker_sandbox.go @@ -587,6 +587,8 @@ func (ds *dockerService) makeSandboxDockerConfig(c *runtimeapi.PodSandboxConfig, return nil, fmt.Errorf("failed to generate sandbox security options for sandbox %q: %v", c.Metadata.Name, err) } hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...) + + applyExperimentalCreateConfig(createConfig, c.Annotations) return createConfig, nil } diff --git a/pkg/kubelet/dockershim/helpers_linux.go b/pkg/kubelet/dockershim/helpers_linux.go index f8a9a1fbbe066..a9e04b1875c5c 100644 --- a/pkg/kubelet/dockershim/helpers_linux.go +++ b/pkg/kubelet/dockershim/helpers_linux.go @@ -145,3 +145,7 @@ func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) { } return fmt.Sprintf(dockerNetNSFmt, c.State.Pid), nil } + +// applyExperimentalCreateConfig applys experimental configures from sandbox annotations. +func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) { +} diff --git a/pkg/kubelet/dockershim/helpers_unsupported.go b/pkg/kubelet/dockershim/helpers_unsupported.go index dfe3a097a4956..fe904426c11a7 100644 --- a/pkg/kubelet/dockershim/helpers_unsupported.go +++ b/pkg/kubelet/dockershim/helpers_unsupported.go @@ -53,3 +53,7 @@ func (ds *dockerService) determinePodIPBySandboxID(uid string) string { func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) { return "", fmt.Errorf("unsupported platform") } + +// applyExperimentalCreateConfig applys experimental configures from sandbox annotations. +func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) { +} diff --git a/pkg/kubelet/dockershim/helpers_windows.go b/pkg/kubelet/dockershim/helpers_windows.go index c93515daab276..499dc620c749c 100644 --- a/pkg/kubelet/dockershim/helpers_windows.go +++ b/pkg/kubelet/dockershim/helpers_windows.go @@ -29,6 +29,13 @@ import ( runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" ) +const ( + hypervIsolationAnnotationKey = "experimental.windows.kubernetes.io/isolation-type" + + // Refer https://aka.ms/hyperv-container. + hypervIsolation = "hyperv" +) + func DefaultMemorySwap() int64 { return 0 } @@ -40,6 +47,22 @@ func (ds *dockerService) getSecurityOpts(seccompProfile string, separator rune) return nil, nil } +func shouldIsolatedByHyperV(annotations map[string]string) bool { + v, ok := annotations[hypervIsolationAnnotationKey] + return ok && v == hypervIsolation +} + +// applyExperimentalCreateConfig applys experimental configures from sandbox annotations. +func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) { + if shouldIsolatedByHyperV(annotations) { + createConfig.HostConfig.Isolation = hypervIsolation + + if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" { + createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode("none") + } + } +} + func (ds *dockerService) updateCreateConfig( createConfig *dockertypes.ContainerCreateConfig, config *runtimeapi.ContainerConfig, @@ -47,11 +70,13 @@ func (ds *dockerService) updateCreateConfig( podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error { if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode != "" { createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode(networkMode) - } else { + } else if !shouldIsolatedByHyperV(sandboxConfig.Annotations) { // Todo: Refactor this call in future for calling methods directly in security_context.go modifyHostNetworkOptionForContainer(false, podSandboxID, createConfig.HostConfig) } + applyExperimentalCreateConfig(createConfig, sandboxConfig.Annotations) + return nil } @@ -87,8 +112,17 @@ func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string { // Todo: Add a kernel version check for more validation if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" { - // Do not return any IP, so that we would continue and get the IP of the Sandbox - ds.getIP(sandboxID, r) + if r.HostConfig.Isolation == hypervIsolation { + // Hyper-V only supports one container per Pod yet and the container will have a different + // IP address from sandbox. Return the first non-sandbox container IP as POD IP. + // TODO(feiskyer): remove this workaround after Hyper-V supports multiple containers per Pod. + if containerIP := ds.getIP(c.ID, r); containerIP != "" { + return containerIP + } + } else { + // Do not return any IP, so that we would continue and get the IP of the Sandbox + ds.getIP(sandboxID, r) + } } else { // On Windows, every container that is created in a Sandbox, needs to invoke CNI plugin again for adding the Network, // with the shared container name as NetNS info, From a6d0cd0f01ff9cb84efe355e9c893e9deca16cc2 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Tue, 30 Jan 2018 12:56:31 +0800 Subject: [PATCH 2/2] Add HyperVContainer feature gates --- pkg/features/kube_features.go | 7 +++++++ pkg/kubelet/dockershim/BUILD | 8 +++++++- pkg/kubelet/dockershim/helpers_windows.go | 7 +++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index cfdcf614953ed..86e991b4692e0 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -225,6 +225,12 @@ const ( // // Implement support for limiting pids in pods SupportPodPidsLimit utilfeature.Feature = "SupportPodPidsLimit" + + // owner: @feiskyer + // alpha: v1.10 + // + // Enable Hyper-V containers on Windows + HyperVContainer utilfeature.Feature = "HyperVContainer" ) func init() { @@ -265,6 +271,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS ResourceLimitsPriorityFunction: {Default: false, PreRelease: utilfeature.Alpha}, SupportIPVSProxyMode: {Default: false, PreRelease: utilfeature.Beta}, SupportPodPidsLimit: {Default: false, PreRelease: utilfeature.Alpha}, + HyperVContainer: {Default: false, PreRelease: utilfeature.Alpha}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: diff --git a/pkg/kubelet/dockershim/BUILD b/pkg/kubelet/dockershim/BUILD index 2a136a19e3375..ba420b45a5cba 100644 --- a/pkg/kubelet/dockershim/BUILD +++ b/pkg/kubelet/dockershim/BUILD @@ -124,7 +124,13 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/client-go/tools/remotecommand:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:windows": [ + "//pkg/features:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + ], + "//conditions:default": [], + }), ) go_test( diff --git a/pkg/kubelet/dockershim/helpers_windows.go b/pkg/kubelet/dockershim/helpers_windows.go index 499dc620c749c..4cd722d54e2db 100644 --- a/pkg/kubelet/dockershim/helpers_windows.go +++ b/pkg/kubelet/dockershim/helpers_windows.go @@ -26,6 +26,9 @@ import ( dockercontainer "github.com/docker/docker/api/types/container" dockerfilters "github.com/docker/docker/api/types/filters" "github.com/golang/glog" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" ) @@ -48,6 +51,10 @@ func (ds *dockerService) getSecurityOpts(seccompProfile string, separator rune) } func shouldIsolatedByHyperV(annotations map[string]string) bool { + if !utilfeature.DefaultFeatureGate.Enabled(features.HyperVContainer) { + return false + } + v, ok := annotations[hypervIsolationAnnotationKey] return ok && v == hypervIsolation }