Merge pull request kubernetes#30872 from mwielgus/fed-informer-deadlock

Automatic merge from submit-queue Fix deadlock possibility in federated informer On cluster add subinformer locks and tries to add cluster to federated informer. When someone checks if everything is in sync federated informer is locked and then subinformer is inspected what apparently requires a lock. With really bad timing this can create a deadlock. This PR ensures that there is always at most 1 lock taken in federated informer. cc: @quinton-hoole @kubernetes/sig-cluster-federation Fixes: kubernetes#30855
lavalamp · Aug 18, 2016 · b15c2d6 · b15c2d6
2 parents 7c1d59b + 681d153
commit b15c2d6
Showing 1 changed file with 22 additions and 11 deletions.
diff --git a/federation/pkg/federation-controller/util/federated_informer.go b/federation/pkg/federation-controller/util/federated_informer.go
@@ -344,8 +344,6 @@ func (f *federatedInformerImpl) getReadyClusterUnlocked(name string) (*federatio
 
 // Synced returns true if the view is synced (for the first time)
 func (f *federatedInformerImpl) ClustersSynced() bool {
-	f.Lock()
-	defer f.Unlock()
 	return f.clusterInformer.controller.HasSynced()
 }
 
@@ -452,18 +450,31 @@ func (fs *federatedStoreImpl) GetKeyFor(item interface{}) string {
 // Checks whether stores for all clusters form the lists (and only these) are there and
 // are synced.
 func (fs *federatedStoreImpl) ClustersSynced(clusters []*federation_api.Cluster) bool {
-	fs.federatedInformer.Lock()
-	defer fs.federatedInformer.Unlock()
 
-	if len(fs.federatedInformer.targetInformers) != len(clusters) {
+	// Get the list of informers to check under a lock and check it outside.
+	okSoFar, informersToCheck := func() (bool, []informer) {
+		fs.federatedInformer.Lock()
+		defer fs.federatedInformer.Unlock()
+
+		if len(fs.federatedInformer.targetInformers) != len(clusters) {
+			return false, []informer{}
+		}
+		informersToCheck := make([]informer, 0, len(clusters))
+		for _, cluster := range clusters {
+			if targetInformer, found := fs.federatedInformer.targetInformers[cluster.Name]; found {
+				informersToCheck = append(informersToCheck, targetInformer)
+			} else {
+				return false, []informer{}
+			}
+		}
+		return true, informersToCheck
+	}()
+
+	if !okSoFar {
 		return false
 	}
-	for _, cluster := range clusters {
-		if targetInformer, found := fs.federatedInformer.targetInformers[cluster.Name]; found {
-			if !targetInformer.controller.HasSynced() {
-				return false
-			}
-		} else {
+	for _, informerToCheck := range informersToCheck {
+		if !informerToCheck.controller.HasSynced() {
 			return false
 		}
 	}