[RayJob]: Disable job submission retry for now

ray-project · Oct 18, 2023 · 62519ac · 62519ac
1 parent f2b7ce0
commit 62519ac
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/ray-operator/controllers/ray/rayjob_controller.go b/ray-operator/controllers/ray/rayjob_controller.go
@@ -14,6 +14,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -412,7 +413,10 @@ func (r *RayJobReconciler) createNewK8sJob(ctx context.Context, rayJobInstance *
 			},
 		},
 		Spec: batchv1.JobSpec{
-			Template: submitterTemplate,
+			// The `ray job submit` command fails when submission is retried with the same job ID.
+			// Let's disable backoff retry, until job submission retry effectively works.
+			BackoffLimit: pointer.Int32(0),
+			Template:     submitterTemplate,
 		},
 	}