Add job namespace to tf_operator_jobs_* counters (#1283)

* Add job namespace to 'tf_operator_jobs_*' counters * Update `tfJobsDeletedCount`
kubeflow · Jun 28, 2021 · af5bdd5 · af5bdd5
1 parent 6fd9489
commit af5bdd5
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 28 deletions.
diff --git a/pkg/controller.v1/tensorflow/controller.go b/pkg/controller.v1/tensorflow/controller.go
@@ -65,10 +65,13 @@ var (
 	// key function but it should be just fine for non delete events.
 	KeyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
 
-	tfJobsDeletedCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "tf_operator_jobs_deleted_total",
-		Help: "Counts number of TF jobs deleted",
-	})
+	tfJobsDeletedCount = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "tf_operator_jobs_deleted_total",
+			Help: "Counts number of TF jobs deleted",
+		},
+		[]string{"job_namespace"},
+	)
 )
 
 // TFController is the type for TFJob Controller, which manages
@@ -245,7 +248,7 @@ func (tc *TFController) processNextWorkItem() bool {
 	if err != nil {
 		if err == errNotExists {
 			logger.Infof("TFJob has been deleted: %v", key)
-			tfJobsDeletedCount.Inc()
+			tfJobsDeletedCount.WithLabelValues(tfJob.Namespace).Inc()
 			return true
 		}
 
@@ -308,7 +311,7 @@ func (tc *TFController) syncTFJob(key string) (bool, error) {
 	if err != nil {
 		if err == errNotExists {
 			logger.Infof("TFJob has been deleted: %v", key)
-			tfJobsDeletedCount.Inc()
+			tfJobsDeletedCount.WithLabelValues(namespace).Inc()
 			return true, nil
 		}
 		return false, err

diff --git a/pkg/controller.v1/tensorflow/job.go b/pkg/controller.v1/tensorflow/job.go
@@ -26,10 +26,13 @@ const (
 )
 
 var (
-	tfJobsCreatedCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "tf_operator_jobs_created_total",
-		Help: "Counts number of TF jobs created",
-	})
+	tfJobsCreatedCount = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "tf_operator_jobs_created_total",
+			Help: "Counts number of TF jobs created",
+		},
+		[]string{"job_namespace"},
+	)
 )
 
 // DeleteJob implements ControllerInterface interface.
@@ -128,7 +131,7 @@ func (tc *TFController) addTFJob(obj interface{}) {
 		return
 	}
 	tc.enqueueTFJob(obj)
-	tfJobsCreatedCount.Inc()
+	tfJobsCreatedCount.WithLabelValues(tfJob.Namespace).Inc()
 }
 
 // updateTFJob enqueues the current tfjob.

diff --git a/pkg/controller.v1/tensorflow/pod.go b/pkg/controller.v1/tensorflow/pod.go
@@ -53,10 +53,13 @@ const (
 )
 
 var (
-	tfJobsRestartCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "tf_operator_jobs_restarted_total",
-		Help: "Counts number of TF jobs restarted",
-	})
+	tfJobsRestartCount = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "tf_operator_jobs_restarted_total",
+			Help: "Counts number of TF jobs restarted",
+		},
+		[]string{"job_namespace"},
+	)
 )
 
 // reconcilePods checks and updates pods for each given TFReplicaSpec.
@@ -149,7 +152,7 @@ func (tc *TFController) ReconcilePods(
 						commonutil.LoggerForJob(tfJob).Infof("Append tfjob condition error: %v", err)
 						return err
 					}
-					tfJobsRestartCount.Inc()
+					tfJobsRestartCount.WithLabelValues(tfJob.Namespace).Inc()
 				}
 			}
 

diff --git a/pkg/controller.v1/tensorflow/status.go b/pkg/controller.v1/tensorflow/status.go
@@ -44,14 +44,20 @@ const (
 )
 
 var (
-	tfJobsSuccessCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "tf_operator_jobs_successful_total",
-		Help: "Counts number of TF jobs successful",
-	})
-	tfJobsFailureCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "tf_operator_jobs_failed_total",
-		Help: "Counts number of TF jobs failed",
-	})
+	tfJobsSuccessCount = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "tf_operator_jobs_successful_total",
+			Help: "Counts number of TF jobs successful",
+		},
+		[]string{"job_namespace"},
+	)
+	tfJobsFailureCount = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "tf_operator_jobs_failed_total",
+			Help: "Counts number of TF jobs failed",
+		},
+		[]string{"job_namespace"},
+	)
 )
 
 func (tc *TFController) UpdateJobStatus(job interface{}, replicas map[commonv1.ReplicaType]*commonv1.ReplicaSpec, jobStatus *commonv1.JobStatus) error {
@@ -137,7 +143,7 @@ func (tc *TFController) UpdateJobStatus(job interface{}, replicas map[commonv1.R
 						commonutil.LoggerForJob(tfJob).Infof("Append tfjob condition error: %v", err)
 						return err
 					}
-					tfJobsSuccessCount.Inc()
+					tfJobsSuccessCount.WithLabelValues(tfJob.Namespace).Inc()
 				}
 			}
 		} else {
@@ -159,7 +165,7 @@ func (tc *TFController) UpdateJobStatus(job interface{}, replicas map[commonv1.R
 						commonutil.LoggerForJob(tfJob).Infof("Append tfjob condition error: %v", err)
 						return err
 					}
-					tfJobsSuccessCount.Inc()
+					tfJobsSuccessCount.WithLabelValues(tfJob.Namespace).Inc()
 				} else if running > 0 {
 					// Some workers are still running, leave a running condition.
 					msg := fmt.Sprintf("TFJob %s/%s is running.",
@@ -184,7 +190,7 @@ func (tc *TFController) UpdateJobStatus(job interface{}, replicas map[commonv1.R
 			if restart {
 				// job is restarting, no need to set it failed
 				// we know it because we update the status condition when reconciling the replicas
-				tfJobsFailureCount.Inc()
+				tfJobsFailureCount.WithLabelValues(tfJob.Namespace).Inc()
 			} else {
 				msg := fmt.Sprintf("TFJob %s/%s has failed because %d %s replica(s) failed.",
 					tfJob.Namespace, tfJob.Name, failed, rtype)
@@ -199,7 +205,7 @@ func (tc *TFController) UpdateJobStatus(job interface{}, replicas map[commonv1.R
 					commonutil.LoggerForJob(tfJob).Infof("Append tfjob condition error: %v", err)
 					return err
 				}
-				tfJobsFailureCount.Inc()
+				tfJobsFailureCount.WithLabelValues(tfJob.Namespace).Inc()
 			}
 		}
 	}