From 592506686ed9d88ee09f991e55ab313f0d4a847f Mon Sep 17 00:00:00 2001 From: Siyi Wang Date: Wed, 28 Jun 2023 17:22:37 +0000 Subject: [PATCH 1/3] Add cni logs --- tools/bug-report/pkg/archive/archive.go | 5 +++ tools/bug-report/pkg/bugreport/bugreport.go | 42 ++++++++++++++++----- tools/bug-report/pkg/cluster/cluster.go | 19 +++++++++- tools/bug-report/pkg/common/common.go | 10 ++++- 4 files changed, 63 insertions(+), 13 deletions(-) diff --git a/tools/bug-report/pkg/archive/archive.go b/tools/bug-report/pkg/archive/archive.go index 3046e752b398..13357c44f75d 100644 --- a/tools/bug-report/pkg/archive/archive.go +++ b/tools/bug-report/pkg/archive/archive.go @@ -31,6 +31,7 @@ const ( clusterInfoSubdir = "cluster" analyzeSubdir = "analyze" operatorLogsPathSubdir = "operator" + cniLogsPathSubdir = "cni" ) var ( @@ -68,6 +69,10 @@ func ClusterInfoPath(rootDir string) string { return filepath.Join(getRootDir(rootDir), clusterInfoSubdir) } +func CniPath(rootDir, pod string) string { + return filepath.Join(getRootDir(rootDir), cniLogsPathSubdir, pod) +} + // Create creates a gzipped tar file from srcDir and writes it to outPath. func Create(srcDir, outPath string) error { mw, err := os.Create(outPath) diff --git a/tools/bug-report/pkg/bugreport/bugreport.go b/tools/bug-report/pkg/bugreport/bugreport.go index 1b4f4f0b8416..1603b320f46d 100644 --- a/tools/bug-report/pkg/bugreport/bugreport.go +++ b/tools/bug-report/pkg/bugreport/bugreport.go @@ -28,7 +28,6 @@ import ( "github.com/kr/pretty" "github.com/spf13/cobra" - label2 "istio.io/api/label" "istio.io/istio/istioctl/pkg/cli" "istio.io/istio/istioctl/pkg/util/ambient" @@ -158,7 +157,7 @@ func runBugReportCommand(ctx cli.Context, _ *cobra.Command, logOpts *log.Options return err } - common.LogAndPrintf("\n\nFetching proxy logs for the following containers:\n\n%s\n", strings.Join(paths, "\n")) + common.LogAndPrintf("\n\nFetching logs for the following containers:\n\n%s\n", strings.Join(paths, "\n")) gatherInfo(runner, config, resources, paths) if len(gErrors) != 0 { @@ -312,6 +311,11 @@ func gatherInfo(runner *kubectlcmd.Runner, config *config.BugReportConfig, resou getFromCluster(content.GetSecrets, params.SetVerbose(config.FullSecrets), clusterDir, &mandatoryWg) getFromCluster(content.GetDescribePods, params.SetIstioNamespace(config.IstioNamespace), clusterDir, &mandatoryWg) + common.LogAndPrintf("\nFetching CNI logs from cluster.\n\n") + for _, cniPod := range resources.CniPod { + getCniLogs(runner, config, resources, cniPod.Name, &mandatoryWg) + } + // optionalWg is subject to timer. var optionalWg sync.WaitGroup for _, p := range paths { @@ -390,7 +394,7 @@ func getProxyLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, res path, namespace, pod, container string, wg *sync.WaitGroup, ) { wg.Add(1) - log.Infof("Waiting on logs %s", pod) + log.Infof("Waiting on proxy logs %v/%v/%v", namespace, pod, container) go func() { defer func() { wg.Done() @@ -404,7 +408,7 @@ func getProxyLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, res logs[path], stats[path], importance[path] = clog, cstat, imp } lock.Unlock() - log.Infof("Done with logs %s", pod) + log.Infof("Done with proxy logs %v/%v/%v", namespace, pod, container) }() } @@ -414,7 +418,7 @@ func getIstiodLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, re namespace, pod string, wg *sync.WaitGroup, ) { wg.Add(1) - log.Infof("Waiting on logs %s", pod) + log.Infof("Waiting on Istiod logs for %v/%v", namespace, pod) go func() { defer func() { wg.Done() @@ -424,7 +428,7 @@ func getIstiodLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, re clog, _, _, err := getLog(runner, resources, config, namespace, pod, common.DiscoveryContainerName) appendGlobalErr(err) writeFile(filepath.Join(archive.IstiodPath(tempDir, namespace, pod), "discovery.log"), clog, config.DryRun) - log.Infof("Done with logs %s", pod) + log.Infof("Done with Istiod logs for %v/%v", namespace, pod) }() } @@ -433,7 +437,7 @@ func getOperatorLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, namespace, pod string, wg *sync.WaitGroup, ) { wg.Add(1) - log.Infof("Waiting on logs %s", pod) + log.Infof("Waiting on operator logs for %v/%v", namespace, pod) go func() { defer func() { wg.Done() @@ -443,7 +447,27 @@ func getOperatorLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, clog, _, _, err := getLog(runner, resources, config, namespace, pod, common.OperatorContainerName) appendGlobalErr(err) writeFile(filepath.Join(archive.OperatorPath(tempDir, namespace, pod), "operator.log"), clog, config.DryRun) - log.Infof("Done with logs %s", pod) + log.Infof("Done with operator logs for %v/%v", namespace, pod) + }() +} + +// getCniLogs fetches Cni logs from istio-cni-node daemonsets inside namespace kube-system and writes the output +// Runs if a goroutine, with errors reported through gErrors +func getCniLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, resources *cluster2.Resources, + pod string, wg *sync.WaitGroup, +) { + wg.Add(1) + log.Infof("Waiting on CNI logs for %v", pod) + go func() { + defer func() { + wg.Done() + logRuntime(time.Now(), "Done getting CNI logs for %v", pod) + }() + + clog, _, _, err := getLog(runner, resources, config, common.KubeSystemNamespace, pod, "") + appendGlobalErr(err) + writeFile(filepath.Join(archive.CniPath(tempDir, pod), "cni.log"), clog, config.DryRun) + log.Infof("Done with CNI logs %v", pod) }() } @@ -458,7 +482,7 @@ func getLog(runner *kubectlcmd.Runner, resources *cluster2.Resources, config *co if err != nil { return "", nil, 0, err } - if resources.ContainerRestarts(namespace, pod, container) > 0 { + if resources.ContainerRestarts(namespace, pod, container, common.IsCniPod(pod)) > 0 { pclog, err := runner.Logs(namespace, pod, container, true, config.DryRun) if err != nil { return "", nil, 0, err diff --git a/tools/bug-report/pkg/cluster/cluster.go b/tools/bug-report/pkg/cluster/cluster.go index dff520c9f845..722158a4eb3e 100644 --- a/tools/bug-report/pkg/cluster/cluster.go +++ b/tools/bug-report/pkg/cluster/cluster.go @@ -212,6 +212,7 @@ func GetClusterResources(ctx context.Context, clientset *kubernetes.Clientset, c Labels: make(map[string]map[string]string), Annotations: make(map[string]map[string]string), Pod: make(map[string]*corev1.Pod), + CniPod: make(map[string]*corev1.Pod), } pods, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) @@ -230,6 +231,12 @@ func GetClusterResources(ctx context.Context, clientset *kubernetes.Clientset, c } for i, p := range pods.Items { + if p.Namespace == "kube-system" { + if p.Labels["k8s-app"] == "istio-cni-node" { + out.CniPod[PodKey(p.Namespace, p.Name)] = &pods.Items[i] + } + } + if analyzer_util.IsSystemNamespace(resource.Namespace(p.Namespace)) { continue } @@ -276,6 +283,8 @@ type Resources struct { Annotations map[string]map[string]string // Pod maps a pod name to its Pod info. The key is namespace/pod-name. Pod map[string]*corev1.Pod + // CniPod + CniPod map[string]*corev1.Pod } func (r *Resources) insertContainer(namespace, deployment, pod, container string) { @@ -298,8 +307,14 @@ func (r *Resources) insertContainer(namespace, deployment, pod, container string } // ContainerRestarts returns the number of container restarts for the given container. -func (r *Resources) ContainerRestarts(namespace, pod, container string) int { - for _, cs := range r.Pod[PodKey(namespace, pod)].Status.ContainerStatuses { +func (r *Resources) ContainerRestarts(namespace, pod, container string, isCniPod bool) int { + var podItem *corev1.Pod + if isCniPod { + podItem = r.CniPod[PodKey(namespace, pod)] + } else { + podItem = r.Pod[PodKey(namespace, pod)] + } + for _, cs := range podItem.Status.ContainerStatuses { if cs.Name == container { return int(cs.RestartCount) } diff --git a/tools/bug-report/pkg/common/common.go b/tools/bug-report/pkg/common/common.go index 3b4cfb63978c..a3fca8fd8afa 100644 --- a/tools/bug-report/pkg/common/common.go +++ b/tools/bug-report/pkg/common/common.go @@ -17,6 +17,7 @@ package common import ( "fmt" + "strings" "istio.io/istio/pkg/log" ) @@ -30,8 +31,9 @@ const ( OperatorContainerName = "istio-operator" // namespaceAll is the default argument of across all namespaces - NamespaceAll = "" - StrNamespaceAll = "allNamespaces" + NamespaceAll = "" + StrNamespaceAll = "allNamespaces" + KubeSystemNamespace = "kube-system" ) type kv struct { @@ -127,6 +129,10 @@ func IsOperatorContainer(_, container string) bool { return container == OperatorContainerName } +func IsCniPod(pod string) bool { + return strings.HasPrefix(pod, "istio-cni-node") +} + func getVersionKey(clusterVersion string) string { if versionMap[clusterVersion] == nil { return latestKey From d435090e722b3389330bfd744294537e14236da4 Mon Sep 17 00:00:00 2001 From: Siyi Wang Date: Wed, 28 Jun 2023 22:40:39 +0000 Subject: [PATCH 2/3] resolve reviews --- tools/bug-report/pkg/cluster/cluster.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/bug-report/pkg/cluster/cluster.go b/tools/bug-report/pkg/cluster/cluster.go index 722158a4eb3e..ec1695b4aa5d 100644 --- a/tools/bug-report/pkg/cluster/cluster.go +++ b/tools/bug-report/pkg/cluster/cluster.go @@ -231,10 +231,8 @@ func GetClusterResources(ctx context.Context, clientset *kubernetes.Clientset, c } for i, p := range pods.Items { - if p.Namespace == "kube-system" { - if p.Labels["k8s-app"] == "istio-cni-node" { - out.CniPod[PodKey(p.Namespace, p.Name)] = &pods.Items[i] - } + if p.Labels["k8s-app"] == "istio-cni-node" { + out.CniPod[PodKey(p.Namespace, p.Name)] = &pods.Items[i] } if analyzer_util.IsSystemNamespace(resource.Namespace(p.Namespace)) { From 877d7861e24a2c402abd679072dad793989c2b42 Mon Sep 17 00:00:00 2001 From: Siyi Wang Date: Thu, 29 Jun 2023 23:28:08 +0000 Subject: [PATCH 3/3] Fix namespace issue --- tools/bug-report/pkg/bugreport/bugreport.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/bug-report/pkg/bugreport/bugreport.go b/tools/bug-report/pkg/bugreport/bugreport.go index 1603b320f46d..6019720f18ee 100644 --- a/tools/bug-report/pkg/bugreport/bugreport.go +++ b/tools/bug-report/pkg/bugreport/bugreport.go @@ -28,6 +28,7 @@ import ( "github.com/kr/pretty" "github.com/spf13/cobra" + label2 "istio.io/api/label" "istio.io/istio/istioctl/pkg/cli" "istio.io/istio/istioctl/pkg/util/ambient" @@ -313,7 +314,7 @@ func gatherInfo(runner *kubectlcmd.Runner, config *config.BugReportConfig, resou common.LogAndPrintf("\nFetching CNI logs from cluster.\n\n") for _, cniPod := range resources.CniPod { - getCniLogs(runner, config, resources, cniPod.Name, &mandatoryWg) + getCniLogs(runner, config, resources, cniPod.Namespace, cniPod.Name, &mandatoryWg) } // optionalWg is subject to timer. @@ -454,7 +455,7 @@ func getOperatorLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, // getCniLogs fetches Cni logs from istio-cni-node daemonsets inside namespace kube-system and writes the output // Runs if a goroutine, with errors reported through gErrors func getCniLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, resources *cluster2.Resources, - pod string, wg *sync.WaitGroup, + namespace, pod string, wg *sync.WaitGroup, ) { wg.Add(1) log.Infof("Waiting on CNI logs for %v", pod) @@ -464,7 +465,7 @@ func getCniLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, resou logRuntime(time.Now(), "Done getting CNI logs for %v", pod) }() - clog, _, _, err := getLog(runner, resources, config, common.KubeSystemNamespace, pod, "") + clog, _, _, err := getLog(runner, resources, config, namespace, pod, "") appendGlobalErr(err) writeFile(filepath.Join(archive.CniPath(tempDir, pod), "cni.log"), clog, config.DryRun) log.Infof("Done with CNI logs %v", pod)