Merge pull request #12982 from pravisankar/network-diags-fix

Merged by openshift-bot
openshift · Feb 23, 2017 · 1dc7754 · 1dc7754
2 parents 2ba2ff0 + cdc8ab9
commit 1dc7754
Show file tree

Hide file tree

Showing 20 changed files with 90 additions and 53 deletions.
diff --git a/contrib/completions/bash/oadm b/contrib/completions/bash/oadm
@@ -2107,6 +2107,8 @@ _oadm_diagnostics()
     local_nonpersistent_flags+=("--master-config=")
     flags+=("--network-logdir=")
     local_nonpersistent_flags+=("--network-logdir=")
+    flags+=("--network-pod-image=")
+    local_nonpersistent_flags+=("--network-pod-image=")
     flags+=("--node-config=")
     local_nonpersistent_flags+=("--node-config=")
     flags+=("--prevent-modification")

diff --git a/contrib/completions/bash/oc b/contrib/completions/bash/oc
@@ -2204,6 +2204,8 @@ _oc_adm_diagnostics()
     local_nonpersistent_flags+=("--master-config=")
     flags+=("--network-logdir=")
     local_nonpersistent_flags+=("--network-logdir=")
+    flags+=("--network-pod-image=")
+    local_nonpersistent_flags+=("--network-pod-image=")
     flags+=("--node-config=")
     local_nonpersistent_flags+=("--node-config=")
     flags+=("--prevent-modification")

diff --git a/contrib/completions/bash/openshift b/contrib/completions/bash/openshift
@@ -2107,6 +2107,8 @@ _openshift_admin_diagnostics()
     local_nonpersistent_flags+=("--master-config=")
     flags+=("--network-logdir=")
     local_nonpersistent_flags+=("--network-logdir=")
+    flags+=("--network-pod-image=")
+    local_nonpersistent_flags+=("--network-pod-image=")
     flags+=("--node-config=")
     local_nonpersistent_flags+=("--node-config=")
     flags+=("--prevent-modification")
@@ -7188,6 +7190,8 @@ _openshift_cli_adm_diagnostics()
     local_nonpersistent_flags+=("--master-config=")
     flags+=("--network-logdir=")
     local_nonpersistent_flags+=("--network-logdir=")
+    flags+=("--network-pod-image=")
+    local_nonpersistent_flags+=("--network-pod-image=")
     flags+=("--node-config=")
     local_nonpersistent_flags+=("--node-config=")
     flags+=("--prevent-modification")

diff --git a/contrib/completions/zsh/oadm b/contrib/completions/zsh/oadm
@@ -2255,6 +2255,8 @@ _oadm_diagnostics()
     local_nonpersistent_flags+=("--master-config=")
     flags+=("--network-logdir=")
     local_nonpersistent_flags+=("--network-logdir=")
+    flags+=("--network-pod-image=")
+    local_nonpersistent_flags+=("--network-pod-image=")
     flags+=("--node-config=")
     local_nonpersistent_flags+=("--node-config=")
     flags+=("--prevent-modification")

diff --git a/contrib/completions/zsh/oc b/contrib/completions/zsh/oc
@@ -2352,6 +2352,8 @@ _oc_adm_diagnostics()
     local_nonpersistent_flags+=("--master-config=")
     flags+=("--network-logdir=")
     local_nonpersistent_flags+=("--network-logdir=")
+    flags+=("--network-pod-image=")
+    local_nonpersistent_flags+=("--network-pod-image=")
     flags+=("--node-config=")
     local_nonpersistent_flags+=("--node-config=")
     flags+=("--prevent-modification")

diff --git a/contrib/completions/zsh/openshift b/contrib/completions/zsh/openshift
@@ -2255,6 +2255,8 @@ _openshift_admin_diagnostics()
     local_nonpersistent_flags+=("--master-config=")
     flags+=("--network-logdir=")
     local_nonpersistent_flags+=("--network-logdir=")
+    flags+=("--network-pod-image=")
+    local_nonpersistent_flags+=("--network-pod-image=")
     flags+=("--node-config=")
     local_nonpersistent_flags+=("--node-config=")
     flags+=("--prevent-modification")
@@ -7336,6 +7338,8 @@ _openshift_cli_adm_diagnostics()
     local_nonpersistent_flags+=("--master-config=")
     flags+=("--network-logdir=")
     local_nonpersistent_flags+=("--network-logdir=")
+    flags+=("--network-pod-image=")
+    local_nonpersistent_flags+=("--network-pod-image=")
     flags+=("--node-config=")
     local_nonpersistent_flags+=("--node-config=")
     flags+=("--prevent-modification")

diff --git a/docs/man/man1/oadm-diagnostics.1 b/docs/man/man1/oadm-diagnostics.1
@@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis
 \fB\-\-network\-logdir\fP="/tmp/openshift/"
     Path to store network diagnostic results in case of errors
 
+.PP
+\fB\-\-network\-pod\-image\fP="openshift/origin"
+    Image to use for network diagnostic pod
+
 .PP
 \fB\-\-node\-config\fP=""
     Path to node config file (implies \-\-host)

diff --git a/docs/man/man1/oc-adm-diagnostics.1 b/docs/man/man1/oc-adm-diagnostics.1
@@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis
 \fB\-\-network\-logdir\fP="/tmp/openshift/"
     Path to store network diagnostic results in case of errors
 
+.PP
+\fB\-\-network\-pod\-image\fP="openshift/origin"
+    Image to use for network diagnostic pod
+
 .PP
 \fB\-\-node\-config\fP=""
     Path to node config file (implies \-\-host)

diff --git a/docs/man/man1/openshift-admin-diagnostics.1 b/docs/man/man1/openshift-admin-diagnostics.1
@@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis
 \fB\-\-network\-logdir\fP="/tmp/openshift/"
     Path to store network diagnostic results in case of errors
 
+.PP
+\fB\-\-network\-pod\-image\fP="openshift/origin"
+    Image to use for network diagnostic pod
+
 .PP
 \fB\-\-node\-config\fP=""
     Path to node config file (implies \-\-host)

diff --git a/docs/man/man1/openshift-cli-adm-diagnostics.1 b/docs/man/man1/openshift-cli-adm-diagnostics.1
@@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis
 \fB\-\-network\-logdir\fP="/tmp/openshift/"
     Path to store network diagnostic results in case of errors
 
+.PP
+\fB\-\-network\-pod\-image\fP="openshift/origin"
+    Image to use for network diagnostic pod
+
 .PP
 \fB\-\-node\-config\fP=""
     Path to node config file (implies \-\-host)

diff --git a/docs/man/man1/openshift-ex-diagnostics.1 b/docs/man/man1/openshift-ex-diagnostics.1
@@ -89,6 +89,10 @@ The available diagnostic names are: AggregatedLogging, AnalyzeLogs, ClusterRegis
 \fB\-\-network\-logdir\fP="/tmp/openshift/"
     Path to store network diagnostic results in case of errors
 
+.PP
+\fB\-\-network\-pod\-image\fP="openshift/origin"
+    Image to use for network diagnostic pod
+
 .PP
 \fB\-\-node\-config\fP=""
     Path to node config file (implies \-\-host)

diff --git a/hack/build-images.sh b/hack/build-images.sh
@@ -140,7 +140,6 @@ image "${tag_prefix}-docker-builder"        images/builder/docker/docker-builder
 image "${tag_prefix}-sti-builder"           images/builder/docker/sti-builder
 image "${tag_prefix}-f5-router"             images/router/f5
 image openshift/node                        images/node
-image openshift/diagnostics-deployer        images/diagnostics
 
 # extra images (not part of infrastructure)
 image openshift/hello-openshift       examples/hello-openshift

diff --git a/images/diagnostics/Dockerfile b/images/diagnostics/Dockerfile
diff --git a/images/diagnostics/scripts/openshift-network-debug b/images/diagnostics/scripts/openshift-network-debug
diff --git a/pkg/cmd/admin/diagnostics/client.go b/pkg/cmd/admin/diagnostics/client.go
@@ -62,6 +62,7 @@ func (o DiagnosticsOptions) buildClientDiagnostics(rawConfig *clientcmdapi.Confi
 				Factory:             o.Factory,
 				PreventModification: o.PreventModification,
 				LogDir:              o.NetworkDiagLogDir,
+				PodImage:            o.NetworkDiagPodImage,
 			})
 		default:
 			return nil, false, fmt.Errorf("unknown diagnostic: %v", diagnosticName)

diff --git a/pkg/cmd/admin/diagnostics/diagnostics.go b/pkg/cmd/admin/diagnostics/diagnostics.go
@@ -44,6 +44,8 @@ type DiagnosticsOptions struct {
 	PreventModification bool
 	// Path to store network diagnostic results in case of errors
 	NetworkDiagLogDir string
+	// Image to use for network diagnostic pod
+	NetworkDiagPodImage string
 	// We need a factory for creating clients. Creating a factory
 	// creates flags as a byproduct, most of which we don't want.
 	// The command creates these and binds only the flags we want.
@@ -133,6 +135,7 @@ func NewCmdDiagnostics(name string, fullName string, out io.Writer) *cobra.Comma
 	cmd.Flags().BoolVar(&o.ImageTemplate.Latest, options.FlagLatestImageName, false, "If true, when expanding the image template, use latest version, not release version")
 	cmd.Flags().BoolVar(&o.PreventModification, options.FlagPreventModificationName, false, "If true, may be set to prevent diagnostics making any changes via the API")
 	cmd.Flags().StringVar(&o.NetworkDiagLogDir, options.FlagNetworkDiagLogDir, netutil.NetworkDiagDefaultLogDir, "Path to store network diagnostic results in case of errors")
+	cmd.Flags().StringVar(&o.NetworkDiagPodImage, options.FlagNetworkDiagPodImage, netutil.NetworkDiagDefaultPodImage, "Image to use for network diagnostic pod")
 	flagtypes.GLog(cmd.Flags())
 	options.BindLoggerOptionFlags(cmd.Flags(), o.LogOptions, options.RecommendedLoggerOptionFlags())
 

diff --git a/pkg/cmd/admin/diagnostics/options/flaginfo.go b/pkg/cmd/admin/diagnostics/options/flaginfo.go
@@ -54,4 +54,5 @@ const (
 	FlagLatestImageName         = "latest-images"
 	FlagPreventModificationName = "prevent-modification"
 	FlagNetworkDiagLogDir       = "network-logdir"
+	FlagNetworkDiagPodImage     = "network-pod-image"
 )
diff --git a/pkg/diagnostics/network/objects.go b/pkg/diagnostics/network/objects.go
@@ -13,23 +13,20 @@ import (
 )
 
 const (
-	diagnosticsImage           = "openshift/diagnostics-deployer"
 	networkDiagTestPodSelector = "network-diag-pod-name"
 
 	testPodImage   = "docker.io/openshift/hello-openshift"
 	testPodPort    = 9876
 	testTargetPort = 8080
 )
 
-func GetNetworkDiagnosticsPod(command, podName, nodeName string) *kapi.Pod {
+func GetNetworkDiagnosticsPod(diagnosticsImage, command, podName, nodeName string) *kapi.Pod {
 	privileged := true
 	hostRootVolName := "host-root-dir"
 	secretVolName := "kconfig-secret"
 	secretDirBaseName := "secrets"
 	gracePeriod := int64(0)
 
-	cmd := fmt.Sprintf("openshift-network-debug %s %s", util.NetworkDiagContainerMountPath, command)
-
 	pod := &kapi.Pod{
 		ObjectMeta: kapi.ObjectMeta{Name: podName},
 		Spec: kapi.PodSpec{
@@ -66,7 +63,8 @@ func GetNetworkDiagnosticsPod(command, podName, nodeName string) *kapi.Pod {
 							ReadOnly:  true,
 						},
 					},
-					Command: []string{"sh", "-c", cmd},
+					Command: []string{"/bin/bash", "-c"},
+					Args:    []string{getNetworkDebugScript(util.NetworkDiagContainerMountPath, command)},
 				},
 			},
 			Volumes: []kapi.Volume{
@@ -135,3 +133,42 @@ func GetTestService(serviceName, podName, nodeName string) *kapi.Service {
 		},
 	}
 }
+
+func getNetworkDebugScript(nodeRootFS, command string) string {
+	return fmt.Sprintf(`
+#!/bin/bash
+#
+# Based on containerized/non-containerized openshift install,
+# this script sets the environment so that docker, openshift, iptables, etc.
+# binaries are availble for network diagnostics.
+#
+set -o nounset
+set -o pipefail
+
+node_rootfs=%s
+cmd="%s"
+
+# Origin image: openshift/node, OSE image: openshift3/node
+node_image_regex="^openshift.*/node"
+
+node_container_id="$(chroot "${node_rootfs}" docker ps --format='{{.Image}} {{.ID}}' | grep "${node_image_regex}" | cut -d' ' -f2)"
+
+if [[ -z "${node_container_id}" ]]; then # non-containerized openshift env
+
+    chroot "${node_rootfs}" ${cmd}
+
+else # containerized env
+
+    # On containerized install, docker on the host is used by node container,
+    # For the privileged network diagnostics pod to use all the binaries on the node:
+    # - Copy kubeconfig secret to node mount namespace
+    # - Run openshift under the mount namespace of node
+
+    node_docker_pid="$(chroot "${node_rootfs}" docker inspect --format='{{.State.Pid}}' "${node_container_id}")"
+    kubeconfig="/etc/origin/node/kubeconfig"
+    cp "${node_rootfs}/secrets/kubeconfig" "${node_rootfs}/${kubeconfig}"
+
+    chroot "${node_rootfs}" nsenter -m -t "${node_docker_pid}" -- /bin/bash -c 'KUBECONFIG='"${kubeconfig} ${cmd}"''
+
+fi`, nodeRootFS, command)
+}
diff --git a/pkg/diagnostics/network/run_pod.go b/pkg/diagnostics/network/run_pod.go
@@ -31,6 +31,7 @@ type NetworkDiagnostic struct {
 	Factory             *osclientcmd.Factory
 	PreventModification bool
 	LogDir              string
+	PodImage            string
 
 	pluginName    string
 	nodes         []kapi.Node
@@ -94,6 +95,9 @@ func (d *NetworkDiagnostic) Check() types.DiagnosticResult {
 	if len(d.LogDir) == 0 {
 		d.LogDir = util.NetworkDiagDefaultLogDir
 	}
+	if len(d.PodImage) == 0 {
+		d.PodImage = util.NetworkDiagDefaultPodImage
+	}
 	d.runNetworkDiagnostic()
 	return d.res
 }
@@ -172,7 +176,7 @@ func (d *NetworkDiagnostic) runNetworkPod(command string) error {
 	for _, node := range d.nodes {
 		podName := kapi.SimpleNameGenerator.GenerateName(fmt.Sprintf("%s-", util.NetworkDiagPodNamePrefix))
 
-		pod := GetNetworkDiagnosticsPod(command, podName, node.Name)
+		pod := GetNetworkDiagnosticsPod(d.PodImage, command, podName, node.Name)
 		_, err := d.KubeClient.Core().Pods(d.nsName1).Create(pod)
 		if err != nil {
 			return fmt.Errorf("Creating network diagnostic pod %q on node %q with command %q failed: %v", podName, node.Name, command, err)

diff --git a/pkg/diagnostics/networkpod/util/util.go b/pkg/diagnostics/networkpod/util/util.go
@@ -31,6 +31,8 @@ const (
 	NetworkDiagNodeLogDirPrefix      = "/nodes"
 	NetworkDiagMasterLogDirPrefix    = "/master"
 	NetworkDiagPodLogDirPrefix       = "/pods"
+
+	NetworkDiagDefaultPodImage = "openshift/origin"
 )
 
 func GetOpenShiftNetworkPlugin(osClient *osclient.Client) (string, bool, error) {