Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support of hyperv isolation for windows containers #58751

Merged
merged 2 commits into from
Feb 1, 2018
Merged

Add support of hyperv isolation for windows containers #58751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cabd2bb
Add experimental hyperv containers support on Windows
feiskyer Jan 30, 2018
a6d0cd0
Add HyperVContainer feature gates
feiskyer Jan 30, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pkg/features/kube_features.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,12 @@ const (
//
// Implement support for limiting pids in pods
SupportPodPidsLimit utilfeature.Feature = "SupportPodPidsLimit"

// owner: @feiskyer
// alpha: v1.10
//
// Enable Hyper-V containers on Windows
HyperVContainer utilfeature.Feature = "HyperVContainer"
)

func init() {
Expand Down Expand Up @@ -265,6 +271,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
ResourceLimitsPriorityFunction: {Default: false, PreRelease: utilfeature.Alpha},
SupportIPVSProxyMode: {Default: false, PreRelease: utilfeature.Beta},
SupportPodPidsLimit: {Default: false, PreRelease: utilfeature.Alpha},
HyperVContainer: {Default: false, PreRelease: utilfeature.Alpha},

// inherited features from generic apiserver, relisted here to get a conflict if it is changed
// unintentionally on either side:
Expand Down
8 changes: 7 additions & 1 deletion pkg/kubelet/dockershim/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,13 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/client-go/tools/remotecommand:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
] + select({
"@io_bazel_rules_go//go/platform:windows": [
"//pkg/features:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
],
"//conditions:default": [],
}),
)

go_test(
Expand Down
2 changes: 2 additions & 0 deletions pkg/kubelet/dockershim/docker_sandbox.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,8 @@ func (ds *dockerService) makeSandboxDockerConfig(c *runtimeapi.PodSandboxConfig,
return nil, fmt.Errorf("failed to generate sandbox security options for sandbox %q: %v", c.Metadata.Name, err)
}
hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...)

applyExperimentalCreateConfig(createConfig, c.Annotations)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need apply the isolation specific config to sandbox? For Windows Hyper-V container, the POD infra container is not used and needs to be minimal, and therefore process container is sufficient.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is not used now, but not future. I think we'd better keep sandbox same with containers, WDYT?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's fine with me. We can reserve it as place holder.

return createConfig, nil
}

Expand Down
4 changes: 4 additions & 0 deletions pkg/kubelet/dockershim/helpers_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,3 +145,7 @@ func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {
}
return fmt.Sprintf(dockerNetNSFmt, c.State.Pid), nil
}

// applyExperimentalCreateConfig applys experimental configures from sandbox annotations.
func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) {
}
4 changes: 4 additions & 0 deletions pkg/kubelet/dockershim/helpers_unsupported.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,7 @@ func (ds *dockerService) determinePodIPBySandboxID(uid string) string {
func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {
return "", fmt.Errorf("unsupported platform")
}

// applyExperimentalCreateConfig applys experimental configures from sandbox annotations.
func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) {
}
47 changes: 44 additions & 3 deletions pkg/kubelet/dockershim/helpers_windows.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,19 @@ import (
dockercontainer "github.com/docker/docker/api/types/container"
dockerfilters "github.com/docker/docker/api/types/filters"
"github.com/golang/glog"

utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
)

const (
hypervIsolationAnnotationKey = "experimental.windows.kubernetes.io/isolation-type"

// Refer https://aka.ms/hyperv-container.
hypervIsolation = "hyperv"
)

func DefaultMemorySwap() int64 {
return 0
}
Expand All @@ -40,18 +50,40 @@ func (ds *dockerService) getSecurityOpts(seccompProfile string, separator rune)
return nil, nil
}

func shouldIsolatedByHyperV(annotations map[string]string) bool {
if !utilfeature.DefaultFeatureGate.Enabled(features.HyperVContainer) {
return false
}

v, ok := annotations[hypervIsolationAnnotationKey]
return ok && v == hypervIsolation
}

// applyExperimentalCreateConfig applys experimental configures from sandbox annotations.
func applyExperimentalCreateConfig(createConfig *dockertypes.ContainerCreateConfig, annotations map[string]string) {
if shouldIsolatedByHyperV(annotations) {
createConfig.HostConfig.Isolation = hypervIsolation

if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" {
createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode("none")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this mean that hyperV isolated containers will not have any networking by default?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no, hyperV container's network will be set by CNI plugins

}
}
}

func (ds *dockerService) updateCreateConfig(
createConfig *dockertypes.ContainerCreateConfig,
config *runtimeapi.ContainerConfig,
sandboxConfig *runtimeapi.PodSandboxConfig,
podSandboxID string, securityOptSep rune, apiVersion *semver.Version) error {
if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode != "" {
createConfig.HostConfig.NetworkMode = dockercontainer.NetworkMode(networkMode)
} else {
} else if !shouldIsolatedByHyperV(sandboxConfig.Annotations) {
// Todo: Refactor this call in future for calling methods directly in security_context.go
modifyHostNetworkOptionForContainer(false, podSandboxID, createConfig.HostConfig)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For Hyper-V container, no need to call modifyHostNetworkOptionForContainer.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ack, will fix

}

applyExperimentalCreateConfig(createConfig, sandboxConfig.Annotations)

return nil
}

Expand Down Expand Up @@ -87,8 +119,17 @@ func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string {
// Todo: Add a kernel version check for more validation

if networkMode := os.Getenv("CONTAINER_NETWORK"); networkMode == "" {
// Do not return any IP, so that we would continue and get the IP of the Sandbox
ds.getIP(sandboxID, r)
if r.HostConfig.Isolation == hypervIsolation {
// Hyper-V only supports one container per Pod yet and the container will have a different
// IP address from sandbox. Return the first non-sandbox container IP as POD IP.
// TODO(feiskyer): remove this workaround after Hyper-V supports multiple containers per Pod.
if containerIP := ds.getIP(c.ID, r); containerIP != "" {
return containerIP
}
} else {
// Do not return any IP, so that we would continue and get the IP of the Sandbox
ds.getIP(sandboxID, r)
}
} else {
// On Windows, every container that is created in a Sandbox, needs to invoke CNI plugin again for adding the Network,
// with the shared container name as NetNS info,
Expand Down