From b640a0e55fdebb6e2b3eb52fc404ecfee16ead98 Mon Sep 17 00:00:00 2001 From: Ravi Sankar Penta Date: Wed, 15 Feb 2017 12:02:32 -0800 Subject: [PATCH 1/3] Bug 1421643 - Use existing openshift/origin image instead of new openshift/diagnostics-deployer Any new image like 'openshift/diagnostics-deployer' incurs build/lifecycle costs to maintian and diagnostics-deployer image has only small block of shell code. To alleviate this problem, now the script is embedded into the pod definition and openshift/origin is used as diagnostics deployer image. On dev machines, currently openshift/origin is close to 800MB but we expect the size to be under 200MB when it is released (compressed, debug headers removed). --- hack/build-images.sh | 1 - images/diagnostics/Dockerfile | 9 ---- .../scripts/openshift-network-debug | 37 --------------- pkg/diagnostics/network/objects.go | 46 +++++++++++++++++-- 4 files changed, 42 insertions(+), 51 deletions(-) delete mode 100644 images/diagnostics/Dockerfile delete mode 100755 images/diagnostics/scripts/openshift-network-debug diff --git a/hack/build-images.sh b/hack/build-images.sh index 64af5cd6b59a..3e17945fcb3e 100755 --- a/hack/build-images.sh +++ b/hack/build-images.sh @@ -140,7 +140,6 @@ image "${tag_prefix}-docker-builder" images/builder/docker/docker-builder image "${tag_prefix}-sti-builder" images/builder/docker/sti-builder image "${tag_prefix}-f5-router" images/router/f5 image openshift/node images/node -image openshift/diagnostics-deployer images/diagnostics # extra images (not part of infrastructure) image openshift/hello-openshift examples/hello-openshift diff --git a/images/diagnostics/Dockerfile b/images/diagnostics/Dockerfile deleted file mode 100644 index d2141c846580..000000000000 --- a/images/diagnostics/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -# -# OpenShift diagnostics image -# Used by network diagnostics (oadm diagnostics NetworkCheck) -# -# The standard name for this image is openshift/diagnostics-deployer - -FROM openshift/origin-base - -COPY scripts/openshift-network-debug /usr/bin/ diff --git a/images/diagnostics/scripts/openshift-network-debug b/images/diagnostics/scripts/openshift-network-debug deleted file mode 100755 index eb386c20a8d2..000000000000 --- a/images/diagnostics/scripts/openshift-network-debug +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# -# This script is used by network diagnostics. -# Based on containerized/non-containerized openshift install, -# it sets the environment so that docker, openshift, iptables, etc. -# binaries are availble for network diagnostics. -# -set -o nounset -set -o pipefail - -node_rootfs=$1 -shift -cmd=$@ - -# Origin image: openshift/node, OSE image: openshift3/node -node_image_regex="^openshift.*/node" - -node_container_id="$(chroot "${node_rootfs}" docker ps --format='{{.Image}} {{.ID}}' | grep "${node_image_regex}" | cut -d' ' -f2)" - -if [[ -z "${node_container_id}" ]]; then # non-containerized openshift env - - chroot "${node_rootfs}" ${cmd} - -else # containerized env - - # On containerized install, docker on the host is used by node container, - # For the privileged network diagnostics pod to use all the binaries on the node: - # - Copy kubeconfig secret to node mount namespace - # - Run openshift under the mount namespace of node - - node_docker_pid="$(chroot "${node_rootfs}" docker inspect --format='{{.State.Pid}}' "${node_container_id}")" - kubeconfig="/etc/origin/node/kubeconfig" - cp "${node_rootfs}/secrets/kubeconfig" "${node_rootfs}/${kubeconfig}" - - chroot "${node_rootfs}" nsenter -m -t "${node_docker_pid}" -- sh -c 'KUBECONFIG='"${kubeconfig} ${cmd}"'' - -fi diff --git a/pkg/diagnostics/network/objects.go b/pkg/diagnostics/network/objects.go index edc887c4b2f3..c396b9b9689b 100644 --- a/pkg/diagnostics/network/objects.go +++ b/pkg/diagnostics/network/objects.go @@ -13,7 +13,7 @@ import ( ) const ( - diagnosticsImage = "openshift/diagnostics-deployer" + diagnosticsImage = "openshift/origin" networkDiagTestPodSelector = "network-diag-pod-name" testPodImage = "docker.io/openshift/hello-openshift" @@ -28,8 +28,6 @@ func GetNetworkDiagnosticsPod(command, podName, nodeName string) *kapi.Pod { secretDirBaseName := "secrets" gracePeriod := int64(0) - cmd := fmt.Sprintf("openshift-network-debug %s %s", util.NetworkDiagContainerMountPath, command) - pod := &kapi.Pod{ ObjectMeta: kapi.ObjectMeta{Name: podName}, Spec: kapi.PodSpec{ @@ -66,7 +64,8 @@ func GetNetworkDiagnosticsPod(command, podName, nodeName string) *kapi.Pod { ReadOnly: true, }, }, - Command: []string{"sh", "-c", cmd}, + Command: []string{"/bin/bash", "-c"}, + Args: []string{getNetworkDebugScript(util.NetworkDiagContainerMountPath, command)}, }, }, Volumes: []kapi.Volume{ @@ -135,3 +134,42 @@ func GetTestService(serviceName, podName, nodeName string) *kapi.Service { }, } } + +func getNetworkDebugScript(nodeRootFS, command string) string { + return fmt.Sprintf(` +#!/bin/bash +# +# Based on containerized/non-containerized openshift install, +# this script sets the environment so that docker, openshift, iptables, etc. +# binaries are availble for network diagnostics. +# +set -o nounset +set -o pipefail + +node_rootfs=%s +cmd="%s" + +# Origin image: openshift/node, OSE image: openshift3/node +node_image_regex="^openshift.*/node" + +node_container_id="$(chroot "${node_rootfs}" docker ps --format='{{.Image}} {{.ID}}' | grep "${node_image_regex}" | cut -d' ' -f2)" + +if [[ -z "${node_container_id}" ]]; then # non-containerized openshift env + + chroot "${node_rootfs}" ${cmd} + +else # containerized env + + # On containerized install, docker on the host is used by node container, + # For the privileged network diagnostics pod to use all the binaries on the node: + # - Copy kubeconfig secret to node mount namespace + # - Run openshift under the mount namespace of node + + node_docker_pid="$(chroot "${node_rootfs}" docker inspect --format='{{.State.Pid}}' "${node_container_id}")" + kubeconfig="/etc/origin/node/kubeconfig" + cp "${node_rootfs}/secrets/kubeconfig" "${node_rootfs}/${kubeconfig}" + + chroot "${node_rootfs}" nsenter -m -t "${node_docker_pid}" -- /bin/bash -c 'KUBECONFIG='"${kubeconfig} ${cmd}"'' + +fi`, nodeRootFS, command) +} From 37dccba87d085ed911fafbf0bab9bf281ba83f49 Mon Sep 17 00:00:00 2001 From: Ravi Sankar Penta Date: Sun, 19 Feb 2017 00:22:50 -0800 Subject: [PATCH 2/3] Make network diagnostic pod image configurable --- pkg/cmd/admin/diagnostics/client.go | 1 + pkg/cmd/admin/diagnostics/diagnostics.go | 3 +++ pkg/cmd/admin/diagnostics/options/flaginfo.go | 1 + pkg/diagnostics/network/objects.go | 3 +-- pkg/diagnostics/network/run_pod.go | 6 +++++- pkg/diagnostics/networkpod/util/util.go | 2 ++ 6 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/cmd/admin/diagnostics/client.go b/pkg/cmd/admin/diagnostics/client.go index b720b3e71e0f..71bc550aa0c0 100644 --- a/pkg/cmd/admin/diagnostics/client.go +++ b/pkg/cmd/admin/diagnostics/client.go @@ -62,6 +62,7 @@ func (o DiagnosticsOptions) buildClientDiagnostics(rawConfig *clientcmdapi.Confi Factory: o.Factory, PreventModification: o.PreventModification, LogDir: o.NetworkDiagLogDir, + PodImage: o.NetworkDiagPodImage, }) default: return nil, false, fmt.Errorf("unknown diagnostic: %v", diagnosticName) diff --git a/pkg/cmd/admin/diagnostics/diagnostics.go b/pkg/cmd/admin/diagnostics/diagnostics.go index e7c177fc111e..b3fa46987c4a 100644 --- a/pkg/cmd/admin/diagnostics/diagnostics.go +++ b/pkg/cmd/admin/diagnostics/diagnostics.go @@ -44,6 +44,8 @@ type DiagnosticsOptions struct { PreventModification bool // Path to store network diagnostic results in case of errors NetworkDiagLogDir string + // Image to use for network diagnostic pod + NetworkDiagPodImage string // We need a factory for creating clients. Creating a factory // creates flags as a byproduct, most of which we don't want. // The command creates these and binds only the flags we want. @@ -133,6 +135,7 @@ func NewCmdDiagnostics(name string, fullName string, out io.Writer) *cobra.Comma cmd.Flags().BoolVar(&o.ImageTemplate.Latest, options.FlagLatestImageName, false, "If true, when expanding the image template, use latest version, not release version") cmd.Flags().BoolVar(&o.PreventModification, options.FlagPreventModificationName, false, "If true, may be set to prevent diagnostics making any changes via the API") cmd.Flags().StringVar(&o.NetworkDiagLogDir, options.FlagNetworkDiagLogDir, netutil.NetworkDiagDefaultLogDir, "Path to store network diagnostic results in case of errors") + cmd.Flags().StringVar(&o.NetworkDiagPodImage, options.FlagNetworkDiagPodImage, netutil.NetworkDiagDefaultPodImage, "Image to use for network diagnostic pod") flagtypes.GLog(cmd.Flags()) options.BindLoggerOptionFlags(cmd.Flags(), o.LogOptions, options.RecommendedLoggerOptionFlags()) diff --git a/pkg/cmd/admin/diagnostics/options/flaginfo.go b/pkg/cmd/admin/diagnostics/options/flaginfo.go index 5f5f05f2f5cd..181754ae1cb5 100644 --- a/pkg/cmd/admin/diagnostics/options/flaginfo.go +++ b/pkg/cmd/admin/diagnostics/options/flaginfo.go @@ -54,4 +54,5 @@ const ( FlagLatestImageName = "latest-images" FlagPreventModificationName = "prevent-modification" FlagNetworkDiagLogDir = "network-logdir" + FlagNetworkDiagPodImage = "network-pod-image" ) diff --git a/pkg/diagnostics/network/objects.go b/pkg/diagnostics/network/objects.go index c396b9b9689b..faea9eaf2054 100644 --- a/pkg/diagnostics/network/objects.go +++ b/pkg/diagnostics/network/objects.go @@ -13,7 +13,6 @@ import ( ) const ( - diagnosticsImage = "openshift/origin" networkDiagTestPodSelector = "network-diag-pod-name" testPodImage = "docker.io/openshift/hello-openshift" @@ -21,7 +20,7 @@ const ( testTargetPort = 8080 ) -func GetNetworkDiagnosticsPod(command, podName, nodeName string) *kapi.Pod { +func GetNetworkDiagnosticsPod(diagnosticsImage, command, podName, nodeName string) *kapi.Pod { privileged := true hostRootVolName := "host-root-dir" secretVolName := "kconfig-secret" diff --git a/pkg/diagnostics/network/run_pod.go b/pkg/diagnostics/network/run_pod.go index 10248b6f800d..b24a659a9088 100644 --- a/pkg/diagnostics/network/run_pod.go +++ b/pkg/diagnostics/network/run_pod.go @@ -31,6 +31,7 @@ type NetworkDiagnostic struct { Factory *osclientcmd.Factory PreventModification bool LogDir string + PodImage string pluginName string nodes []kapi.Node @@ -94,6 +95,9 @@ func (d *NetworkDiagnostic) Check() types.DiagnosticResult { if len(d.LogDir) == 0 { d.LogDir = util.NetworkDiagDefaultLogDir } + if len(d.PodImage) == 0 { + d.PodImage = util.NetworkDiagDefaultPodImage + } d.runNetworkDiagnostic() return d.res } @@ -172,7 +176,7 @@ func (d *NetworkDiagnostic) runNetworkPod(command string) error { for _, node := range d.nodes { podName := kapi.SimpleNameGenerator.GenerateName(fmt.Sprintf("%s-", util.NetworkDiagPodNamePrefix)) - pod := GetNetworkDiagnosticsPod(command, podName, node.Name) + pod := GetNetworkDiagnosticsPod(d.PodImage, command, podName, node.Name) _, err := d.KubeClient.Core().Pods(d.nsName1).Create(pod) if err != nil { return fmt.Errorf("Creating network diagnostic pod %q on node %q with command %q failed: %v", podName, node.Name, command, err) diff --git a/pkg/diagnostics/networkpod/util/util.go b/pkg/diagnostics/networkpod/util/util.go index afe4b1e8e9c7..1ab5c4e5eac2 100644 --- a/pkg/diagnostics/networkpod/util/util.go +++ b/pkg/diagnostics/networkpod/util/util.go @@ -31,6 +31,8 @@ const ( NetworkDiagNodeLogDirPrefix = "/nodes" NetworkDiagMasterLogDirPrefix = "/master" NetworkDiagPodLogDirPrefix = "/pods" + + NetworkDiagDefaultPodImage = "openshift/origin" ) func GetOpenShiftNetworkPlugin(osClient *osclient.Client) (string, bool, error) { From cdc8ab9c4e0fc8edd91046ada7cfc5e697eb9c5b Mon Sep 17 00:00:00 2001 From: Ravi Sankar Penta Date: Mon, 20 Feb 2017 10:33:17 -0800 Subject: [PATCH 3/3] Auto generated: docs/bash completions for network diagnostic pod image option --- contrib/completions/bash/oadm | 2 ++ contrib/completions/bash/oc | 2 ++ contrib/completions/bash/openshift | 4 ++++ contrib/completions/zsh/oadm | 2 ++ contrib/completions/zsh/oc | 2 ++ contrib/completions/zsh/openshift | 4 ++++ docs/man/man1/oadm-diagnostics.1 | 4 ++++ docs/man/man1/oc-adm-diagnostics.1 | 4 ++++ docs/man/man1/openshift-admin-diagnostics.1 | 4 ++++ docs/man/man1/openshift-cli-adm-diagnostics.1 | 4 ++++ docs/man/man1/openshift-ex-diagnostics.1 | 4 ++++ 11 files changed, 36 insertions(+) diff --git a/contrib/completions/bash/oadm b/contrib/completions/bash/oadm index c44df02205f5..0206000c946b 100644 --- a/contrib/completions/bash/oadm +++ b/contrib/completions/bash/oadm @@ -2107,6 +2107,8 @@ _oadm_diagnostics() local_nonpersistent_flags+=("--master-config=") flags+=("--network-logdir=") local_nonpersistent_flags+=("--network-logdir=") + flags+=("--network-pod-image=") + local_nonpersistent_flags+=("--network-pod-image=") flags+=("--node-config=") local_nonpersistent_flags+=("--node-config=") flags+=("--prevent-modification") diff --git a/contrib/completions/bash/oc b/contrib/completions/bash/oc index decc1cce5345..20af2b9cd89d 100644 --- a/contrib/completions/bash/oc +++ b/contrib/completions/bash/oc @@ -2204,6 +2204,8 @@ _oc_adm_diagnostics() local_nonpersistent_flags+=("--master-config=") flags+=("--network-logdir=") local_nonpersistent_flags+=("--network-logdir=") + flags+=("--network-pod-image=") + local_nonpersistent_flags+=("--network-pod-image=") flags+=("--node-config=") local_nonpersistent_flags+=("--node-config=") flags+=("--prevent-modification") diff --git a/contrib/completions/bash/openshift b/contrib/completions/bash/openshift index 857000c74efa..be351978db78 100644 --- a/contrib/completions/bash/openshift +++ b/contrib/completions/bash/openshift @@ -2107,6 +2107,8 @@ _openshift_admin_diagnostics() local_nonpersistent_flags+=("--master-config=") flags+=("--network-logdir=") local_nonpersistent_flags+=("--network-logdir=") + flags+=("--network-pod-image=") + local_nonpersistent_flags+=("--network-pod-image=") flags+=("--node-config=") local_nonpersistent_flags+=("--node-config=") flags+=("--prevent-modification") @@ -7025,6 +7027,8 @@ _openshift_cli_adm_diagnostics() local_nonpersistent_flags+=("--master-config=") flags+=("--network-logdir=") local_nonpersistent_flags+=("--network-logdir=") + flags+=("--network-pod-image=") + local_nonpersistent_flags+=("--network-pod-image=") flags+=("--node-config=") local_nonpersistent_flags+=("--node-config=") flags+=("--prevent-modification") diff --git a/contrib/completions/zsh/oadm b/contrib/completions/zsh/oadm index c94063134deb..2413fd780b13 100644 --- a/contrib/completions/zsh/oadm +++ b/contrib/completions/zsh/oadm @@ -2255,6 +2255,8 @@ _oadm_diagnostics() local_nonpersistent_flags+=("--master-config=") flags+=("--network-logdir=") local_nonpersistent_flags+=("--network-logdir=") + flags+=("--network-pod-image=") + local_nonpersistent_flags+=("--network-pod-image=") flags+=("--node-config=") local_nonpersistent_flags+=("--node-config=") flags+=("--prevent-modification") diff --git a/contrib/completions/zsh/oc b/contrib/completions/zsh/oc index b57161dbaf3c..60eb41dd8a12 100644 --- a/contrib/completions/zsh/oc +++ b/contrib/completions/zsh/oc @@ -2352,6 +2352,8 @@ _oc_adm_diagnostics() local_nonpersistent_flags+=("--master-config=") flags+=("--network-logdir=") local_nonpersistent_flags+=("--network-logdir=") + flags+=("--network-pod-image=") + local_nonpersistent_flags+=("--network-pod-image=") flags+=("--node-config=") local_nonpersistent_flags+=("--node-config=") flags+=("--prevent-modification") diff --git a/contrib/completions/zsh/openshift b/contrib/completions/zsh/openshift index 11cdcdb1f8e6..e5cfb3377f98 100644 --- a/contrib/completions/zsh/openshift +++ b/contrib/completions/zsh/openshift @@ -2255,6 +2255,8 @@ _openshift_admin_diagnostics() local_nonpersistent_flags+=("--master-config=") flags+=("--network-logdir=") local_nonpersistent_flags+=("--network-logdir=") + flags+=("--network-pod-image=") + local_nonpersistent_flags+=("--network-pod-image=") flags+=("--node-config=") local_nonpersistent_flags+=("--node-config=") flags+=("--prevent-modification") @@ -7173,6 +7175,8 @@ _openshift_cli_adm_diagnostics() local_nonpersistent_flags+=("--master-config=") flags+=("--network-logdir=") local_nonpersistent_flags+=("--network-logdir=") + flags+=("--network-pod-image=") + local_nonpersistent_flags+=("--network-pod-image=") flags+=("--node-config=") local_nonpersistent_flags+=("--node-config=") flags+=("--prevent-modification") diff --git a/docs/man/man1/oadm-diagnostics.1 b/docs/man/man1/oadm-diagnostics.1 index cddb75c945ef..48f87efbf2f9 100644 --- a/docs/man/man1/oadm-diagnostics.1 +++ b/docs/man/man1/oadm-diagnostics.1 @@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis \fB\-\-network\-logdir\fP="/tmp/openshift/" Path to store network diagnostic results in case of errors +.PP +\fB\-\-network\-pod\-image\fP="openshift/origin" + Image to use for network diagnostic pod + .PP \fB\-\-node\-config\fP="" Path to node config file (implies \-\-host) diff --git a/docs/man/man1/oc-adm-diagnostics.1 b/docs/man/man1/oc-adm-diagnostics.1 index 25c535222bc3..87248cc461e5 100644 --- a/docs/man/man1/oc-adm-diagnostics.1 +++ b/docs/man/man1/oc-adm-diagnostics.1 @@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis \fB\-\-network\-logdir\fP="/tmp/openshift/" Path to store network diagnostic results in case of errors +.PP +\fB\-\-network\-pod\-image\fP="openshift/origin" + Image to use for network diagnostic pod + .PP \fB\-\-node\-config\fP="" Path to node config file (implies \-\-host) diff --git a/docs/man/man1/openshift-admin-diagnostics.1 b/docs/man/man1/openshift-admin-diagnostics.1 index 83dff08acafe..6cae9f392b40 100644 --- a/docs/man/man1/openshift-admin-diagnostics.1 +++ b/docs/man/man1/openshift-admin-diagnostics.1 @@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis \fB\-\-network\-logdir\fP="/tmp/openshift/" Path to store network diagnostic results in case of errors +.PP +\fB\-\-network\-pod\-image\fP="openshift/origin" + Image to use for network diagnostic pod + .PP \fB\-\-node\-config\fP="" Path to node config file (implies \-\-host) diff --git a/docs/man/man1/openshift-cli-adm-diagnostics.1 b/docs/man/man1/openshift-cli-adm-diagnostics.1 index 5538a62b2b69..859bd94d0a6a 100644 --- a/docs/man/man1/openshift-cli-adm-diagnostics.1 +++ b/docs/man/man1/openshift-cli-adm-diagnostics.1 @@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis \fB\-\-network\-logdir\fP="/tmp/openshift/" Path to store network diagnostic results in case of errors +.PP +\fB\-\-network\-pod\-image\fP="openshift/origin" + Image to use for network diagnostic pod + .PP \fB\-\-node\-config\fP="" Path to node config file (implies \-\-host) diff --git a/docs/man/man1/openshift-ex-diagnostics.1 b/docs/man/man1/openshift-ex-diagnostics.1 index 66634983911f..a390e7b63af7 100644 --- a/docs/man/man1/openshift-ex-diagnostics.1 +++ b/docs/man/man1/openshift-ex-diagnostics.1 @@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis \fB\-\-network\-logdir\fP="/tmp/openshift/" Path to store network diagnostic results in case of errors +.PP +\fB\-\-network\-pod\-image\fP="openshift/origin" + Image to use for network diagnostic pod + .PP \fB\-\-node\-config\fP="" Path to node config file (implies \-\-host)