fix(provider/cf): do service creation asynchronously so it cannot tim…

…eout * backfill tests for other service-related operations Co-Authored-By: Stu Pollock <spollock@pivotal.io>
spinnaker · Nov 6, 2018 · 9ec3207 · 9ec3207
1 parent 2d88d3a
commit 9ec3207
Showing 5 changed files with 530 additions and 17 deletions.
diff --git a/...src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/ServiceInstances.java b/...src/main/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/ServiceInstances.java
@@ -17,29 +17,32 @@
 package com.netflix.spinnaker.clouddriver.cloudfoundry.client;
 
 import com.netflix.spinnaker.clouddriver.cloudfoundry.client.api.ServiceInstanceService;
-import com.netflix.spinnaker.clouddriver.cloudfoundry.client.model.ErrorDescription;
 import com.netflix.spinnaker.clouddriver.cloudfoundry.client.model.v2.*;
 import com.netflix.spinnaker.clouddriver.cloudfoundry.model.*;
+import io.github.resilience4j.retry.Retry;
+import io.github.resilience4j.retry.RetryConfig;
+import lombok.AllArgsConstructor;
 import lombok.RequiredArgsConstructor;
 
 import javax.annotation.Nullable;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
+import java.time.Duration;
+import java.util.*;
 
 import static com.netflix.spinnaker.clouddriver.cloudfoundry.client.CloudFoundryClientUtils.collectPageResources;
 import static com.netflix.spinnaker.clouddriver.cloudfoundry.client.CloudFoundryClientUtils.safelyCall;
+import static com.netflix.spinnaker.clouddriver.cloudfoundry.client.model.ErrorDescription.Code.SERVICE_ALREADY_EXISTS;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toList;
 
+@AllArgsConstructor
 @RequiredArgsConstructor
 public class ServiceInstances {
   private final ServiceInstanceService api;
   private final Organizations orgs;
   private final Spaces spaces;
+  private Duration timeout = Duration.ofSeconds(450);
+  private Duration pollingInterval = Duration.ofSeconds(10);
 
   public void createServiceBindingsByName(CloudFoundryServerGroup cloudFoundryServerGroup, @Nullable List<String> serviceNames) throws CloudFoundryApiException {
     if (serviceNames != null && !serviceNames.isEmpty()) {
@@ -155,19 +158,56 @@ public void createServiceInstance(String newServiceInstanceName, String serviceN
     command.setTags(tags);
     command.setParameters(parameters);
 
+    Optional<Resource<ServiceInstance>> serviceInstance;
     try {
-      safelyCall(() -> api.createServiceInstance(command));
-    } catch (CloudFoundryApiException e) {
-      if (ErrorDescription.Code.SERVICE_ALREADY_EXISTS == e.getErrorCode()) {
+      serviceInstance = safelyCall(() -> api.createServiceInstance(command));
+    } catch (CloudFoundryApiException cfe) {
+      if (cfe.getErrorCode() == SERVICE_ALREADY_EXISTS) {
         Resource<ServiceInstance> serviceInstanceResource = getServiceInstances(space, newServiceInstanceName);
-        if (serviceInstanceResource.getEntity().getServicePlanGuid().equals(command.getServicePlanGuid())) {
-          safelyCall(() -> api.updateServiceInstance(serviceInstanceResource.getMetadata().getGuid(), command));
-        } else {
+        if (!serviceInstanceResource.getEntity().getServicePlanGuid().equals(command.getServicePlanGuid())) {
           throw new CloudFoundryApiException("A service with name '" + serviceName + "' exists but has a different plan");
         }
+
+        serviceInstance = safelyCall(() -> api.updateServiceInstance(serviceInstanceResource.getMetadata().getGuid(), command));
       } else {
-        throw e;
+        throw cfe;
       }
     }
+
+    String guid = serviceInstance.map(res -> res.getMetadata().getGuid())
+      .orElseThrow(() -> new CloudFoundryApiException("Service instance '" + newServiceInstanceName + "' not found"));
+
+    RetryConfig retryConfig = RetryConfig.custom()
+      .waitDuration(pollingInterval)
+      .maxAttempts((int) (timeout.getSeconds() / pollingInterval.getSeconds()))
+      .retryExceptions(OperationInProgressException.class)
+      .build();
+
+    try {
+      Retry.of("async-create-service", retryConfig).executeCallable(() -> {
+          LastOperation.State state = safelyCall(() -> api.getServiceInstanceById(guid))
+            .map(res -> res.getEntity().getLastOperation().getState())
+            .orElseThrow(() -> new CloudFoundryApiException("Service instance '" + newServiceInstanceName + "' not found"));
+
+          switch (state) {
+            case FAILED:
+              throw new CloudFoundryApiException("Service instance '" + newServiceInstanceName + "' creation failed");
+            case IN_PROGRESS:
+              throw new OperationInProgressException();
+            case SUCCEEDED:
+              break;
+          }
+          return state;
+        });
+      } catch (CloudFoundryApiException e) {
+        throw e;
+      } catch (OperationInProgressException ignored) {
+        throw new CloudFoundryApiException("Service instance '" + newServiceInstanceName + "' creation did not complete");
+      } catch (Exception unknown) {
+        throw new CloudFoundryApiException(unknown);
+      }
+  }
+
+  private static class OperationInProgressException extends RuntimeException {
   }
 }
diff --git a/...ava/com/netflix/spinnaker/clouddriver/cloudfoundry/client/api/ServiceInstanceService.java b/...ava/com/netflix/spinnaker/clouddriver/cloudfoundry/client/api/ServiceInstanceService.java
@@ -41,15 +41,18 @@ public interface ServiceInstanceService {
   @GET("/v2/service_plans")
   Page<ServicePlan> findServicePlans(@Query("page") Integer page, @Query("q") List<String> queryParams);
 
-  @POST("/v2/service_instances?accepts_incomplete=false")
-  Response createServiceInstance(@Body CreateServiceInstance body);
+  @POST("/v2/service_instances?accepts_incomplete=true")
+  Resource<ServiceInstance> createServiceInstance(@Body CreateServiceInstance body);
 
-  @PUT("/v2/service_instances/{guid}?accepts_incomplete=false")
-  Response updateServiceInstance(@Path("guid") String serviceInstanceGuid, @Body CreateServiceInstance body);
+  @PUT("/v2/service_instances/{guid}?accepts_incomplete=true")
+  Resource<ServiceInstance> updateServiceInstance(@Path("guid") String serviceInstanceGuid, @Body CreateServiceInstance body);
 
   @GET("/v2/service_instances/{guid}/service_bindings")
   Page<ServiceBinding> getBindingsForServiceInstance(@Path("guid") String serviceInstanceGuid, @Query("page") Integer page, @Query("q") List<String> queryParams);
 
   @DELETE("/v2/service_instances/{guid}?accepts_incomplete=false")
   Response destroyServiceInstance(@Path("guid") String serviceInstanceGuid);
+
+  @GET("/v2/service_instances/{guid}")
+  Resource<ServiceInstance> getServiceInstanceById(@Path("guid") String serviceInstanceGuid);
 }
diff --git a/...in/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/model/v2/LastOperation.java b/...in/java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/model/v2/LastOperation.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2018 Pivotal, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.cloudfoundry.client.model.v2;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import lombok.Data;
+
+import javax.annotation.Nullable;
+
+import static java.util.Arrays.stream;
+
+@Data
+public class LastOperation {
+  private LastOperation.Type type;
+  private LastOperation.State state;
+
+  public enum Type {
+    CREATE("create"),
+    DELETE("delete"),
+    UPDATE("update");
+
+    private final String type;
+
+    Type(String type) {
+      this.type = type;
+    }
+
+    @Nullable
+    @JsonCreator
+    public static Type fromType(String type) {
+      return stream(Type.values()).filter(st -> st.type.equals(type)).findFirst().orElse(null);
+    }
+  }
+
+  public enum State {
+    FAILED("failed"),
+    IN_PROGRESS("in progress"),
+    SUCCEEDED("succeeded");
+
+    private final String state;
+
+    State(String state) {
+      this.state = state;
+    }
+
+    @Nullable
+    @JsonCreator
+    public static State fromState(String state) {
+      return stream(State.values()).filter(st -> st.state.equals(state)).findFirst().orElse(null);
+    }
+  }
+}
diff --git a/.../java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/model/v2/ServiceInstance.java b/.../java/com/netflix/spinnaker/clouddriver/cloudfoundry/client/model/v2/ServiceInstance.java
@@ -23,4 +23,5 @@ public class ServiceInstance {
   private String name;
   private String plan;
   private String servicePlanGuid;
+  private LastOperation lastOperation;
 }