Add volume operation metrics to operation executor and PV controller

kubernetes · Aug 14, 2017 · 0d65762 · 0d65762
1 parent bc1a58a
commit 0d65762
Show file tree

Hide file tree

Showing 10 changed files with 198 additions and 94 deletions.
diff --git a/pkg/controller/volume/persistentvolume/BUILD b/pkg/controller/volume/persistentvolume/BUILD
@@ -24,6 +24,7 @@ go_library(
         "//pkg/util/io:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/volume:go_default_library",
+        "//pkg/volume/util:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/storage/v1:go_default_library",

diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go
@@ -39,6 +39,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/goroutinemap"
 	"k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff"
 	vol "k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/util"
 
 	"github.com/golang/glog"
 )
@@ -1216,7 +1217,10 @@ func (ctrl *PersistentVolumeController) doDeleteVolume(volume *v1.PersistentVolu
 		return false, fmt.Errorf("Failed to create deleter for volume %q: %v", volume.Name, err)
 	}
 
-	if err = deleter.Delete(); err != nil {
+	opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_delete")
+	err = deleter.Delete()
+	opComplete(err)
+	if err != nil {
 		// Deleter failed
 		return false, err
 	}
@@ -1326,7 +1330,9 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
 		return
 	}
 
+	opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_provision")
 	volume, err = provisioner.Provision()
+	opComplete(err)
 	if err != nil {
 		strerr := fmt.Sprintf("Failed to provision volume with StorageClass %q: %v", storageClass.Name, err)
 		glog.V(2).Infof("failed to provision volume for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err)

diff --git a/pkg/controller/volume/persistentvolume/pv_controller_base.go b/pkg/controller/volume/persistentvolume/pv_controller_base.go
@@ -42,6 +42,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/util/goroutinemap"
 	vol "k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/util"
 
 	"github.com/golang/glog"
 )
@@ -74,6 +75,8 @@ func NewController(p ControllerParameters) (*PersistentVolumeController, error)
 		eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "persistentvolume-controller"})
 	}
 
+	util.RegisterMetrics()
+
 	controller := &PersistentVolumeController{
 		volumes:           newPersistentVolumeOrderedIndex(),
 		claims:            cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),

diff --git a/pkg/volume/util/BUILD b/pkg/volume/util/BUILD
@@ -15,6 +15,7 @@ go_library(
         "doc.go",
         "fs_unsupported.go",
         "io_util.go",
+        "metrics.go",
         "util.go",
     ] + select({
         "@io_bazel_rules_go//go/platform:darwin_amd64": [
@@ -31,6 +32,7 @@ go_library(
         "//pkg/api/v1/helper:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/storage/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",

diff --git a/pkg/volume/util/metrics.go b/pkg/volume/util/metrics.go
@@ -0,0 +1,64 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var StorageOperationMetric = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Name: "storage_operation_duration_seconds",
+		Help: "Storage operation duration",
+	},
+	[]string{"volume_plugin", "operation_name"},
+)
+
+var StorageOperationErrorMetric = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "storage_operation_errors_total",
+		Help: "Storage operation errors",
+	},
+	[]string{"volume_plugin", "operation_name"},
+)
+
+var registerMetrics sync.Once
+
+func RegisterMetrics() {
+	registerMetrics.Do(func() {
+		prometheus.MustRegister(StorageOperationMetric)
+		prometheus.MustRegister(StorageOperationErrorMetric)
+	})
+}
+
+// OperationCompleteHook returns a hook to call when an operation is completed
+func OperationCompleteHook(plugin, operationName string) func(error) {
+	requestTime := time.Now()
+	opComplete := func(err error) {
+		timeTaken := time.Since(requestTime).Seconds()
+		// Create metric with operation name and plugin name
+		if err != nil {
+			StorageOperationErrorMetric.WithLabelValues(plugin, operationName).Inc()
+		} else {
+			StorageOperationMetric.WithLabelValues(plugin, operationName).Observe(timeTaken)
+		}
+	}
+	return opComplete
+}
diff --git a/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go b/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go
@@ -55,7 +55,7 @@ type NestedPendingOperations interface {
 	// concatenation of volumeName and podName is removed from the list of
 	// executing operations allowing a new operation to be started with the
 	// volumeName without error.
-	Run(volumeName v1.UniqueVolumeName, podName types.UniquePodName, operationFunc func() error) error
+	Run(volumeName v1.UniqueVolumeName, podName types.UniquePodName, operationFunc func() error, operationCompleteFunc func(error)) error
 
 	// Wait blocks until all operations are completed. This is typically
 	// necessary during tests - the test should wait until all operations finish
@@ -94,7 +94,8 @@ type operation struct {
 func (grm *nestedPendingOperations) Run(
 	volumeName v1.UniqueVolumeName,
 	podName types.UniquePodName,
-	operationFunc func() error) error {
+	operationFunc func() error,
+	operationCompleteFunc func(error)) error {
 	grm.lock.Lock()
 	defer grm.lock.Unlock()
 	opExists, previousOpIndex := grm.isOperationExists(volumeName, podName)
@@ -132,6 +133,7 @@ func (grm *nestedPendingOperations) Run(
 		defer k8sRuntime.HandleCrash()
 		// Handle completion of and error, if any, from operationFunc()
 		defer grm.operationComplete(volumeName, podName, &err)
+		defer operationCompleteFunc(err)
 		// Handle panic, if any, from operationFunc()
 		defer k8sRuntime.RecoverFromPanic(&err)
 		return operationFunc()