read gluster log to surface glusterfs plugin errors properly

kubernetes · Jun 2, 2016 · a36cd3d · a36cd3d
1 parent 9db4559
commit a36cd3d
Show file tree

Hide file tree

Showing 2 changed files with 87 additions and 1 deletion.
diff --git a/pkg/volume/glusterfs/glusterfs.go b/pkg/volume/glusterfs/glusterfs.go
@@ -240,7 +240,12 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error {
 	if err := os.MkdirAll(p, 0750); err != nil {
 		return fmt.Errorf("glusterfs: mkdir failed: %v", err)
 	}
-	log := path.Join(p, "glusterfs.log")
+
+	// adding log-level ERROR to remove noise
+	// and more specific log path so each pod has
+	// it's own log based on PV + Pod
+	log := path.Join(p, b.pod.Name+"-glusterfs.log")
+	options = append(options, "log-level=ERROR")
 	options = append(options, "log-file="+log)
 
 	addr := make(map[string]struct{})
@@ -255,8 +260,18 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error {
 	for hostIP := range addr {
 		errs = b.mounter.Mount(hostIP+":"+b.path, dir, "glusterfs", options)
 		if errs == nil {
+			glog.Infof("glusterfs: successfully mounted %s", dir)
 			return nil
 		}
 	}
+
+	// Failed mount scenario.
+	// Since gluster does not return eror text
+	// it all goes in a log file, we will read the log file
+	logerror := readGlusterLog(log, b.pod.Name)
+	if logerror != nil {
+		// return fmt.Errorf("glusterfs: mount failed: %v", logerror)
+		return fmt.Errorf("glusterfs: mount failed: %v the following error information was pulled from the glusterfs log to help diagnose this issue: %v", errs, logerror)
+	}
 	return fmt.Errorf("glusterfs: mount failed: %v", errs)
 }
diff --git a/pkg/volume/glusterfs/glusterfs_util.go b/pkg/volume/glusterfs/glusterfs_util.go
@@ -0,0 +1,71 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package glusterfs
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+
+	"github.com/golang/glog"
+)
+
+// readGlusterLog will take the last 2 lines of the log file
+// on failure of gluster SetUp and return those so kubelet can
+// properly expose them
+// return nil on any failure
+func readGlusterLog(path string, podName string) error {
+
+	var line1 string
+	var line2 string
+	linecount := 0
+
+	glog.Infof("glusterfs: failure, now attempting to read the gluster log for pod %s", podName)
+
+	// Check and make sure path exists
+	if len(path) == 0 {
+		return fmt.Errorf("glusterfs: log file does not exist for pod: %s", podName)
+	}
+
+	// open the log file
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("glusterfs: could not open log file for pod: %s", podName)
+	}
+	defer file.Close()
+
+	// read in and scan the file using scanner
+	// from stdlib
+	fscan := bufio.NewScanner(file)
+
+	// rather than guessing on bytes or using Seek
+	// going to scan entire file and take the last two lines
+	// generally the file should be small since it is pod specific
+	for fscan.Scan() {
+		if linecount > 0 {
+			line1 = line2
+		}
+		line2 = "\n" + fscan.Text()
+
+		linecount++
+	}
+
+	if linecount > 0 {
+		return fmt.Errorf("%v", line1+line2+"\n")
+	}
+	return nil
+}