feat(artifacts): GCE deploy can consume image artifacts (spinnaker#2477)

feat(artifacts): GCE deploy can consume image artifacts With this change, a BaseGoogleInstanceDescription can now optionally contain a reference to an artifact, containing a gce/image artifact that should be deployed to that image.
spinnakerbot · Apr 4, 2018 · 935e4d2 · 935e4d2
1 parent 79cb92e
commit 935e4d2
Show file tree

Hide file tree

Showing 8 changed files with 239 additions and 19 deletions.
diff --git a/...er-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ArtifactUtils.java b/...er-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ArtifactUtils.java
@@ -28,6 +28,8 @@
 import java.util.Stack;
 
 public class ArtifactUtils {
+  public static final String GCE_IMAGE_TYPE = "gce/image";
+
   public static void untarStreamToPath(InputStream inputStream, String basePath) throws IOException {
     class DirectoryTimestamp {
       public DirectoryTimestamp(File d, long m) {

diff --git a/clouddriver-google/clouddriver-google.gradle b/clouddriver-google/clouddriver-google.gradle
@@ -1,4 +1,5 @@
 dependencies {
+  compile project(":clouddriver-artifacts")
   compile project(":clouddriver-core")
   compile project(":clouddriver-consul")
   compile project(":clouddriver-google-common")

diff --git a/...ver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/deploy/GCEUtil.groovy b/...ver-google/src/main/groovy/com/netflix/spinnaker/clouddriver/google/deploy/GCEUtil.groovy
@@ -21,13 +21,18 @@ import com.google.api.client.googleapis.batch.BatchRequest
 import com.google.api.client.googleapis.batch.json.JsonBatchCallback
 import com.google.api.client.googleapis.json.GoogleJsonError
 import com.google.api.client.googleapis.json.GoogleJsonResponseException
+import com.google.api.client.http.GenericUrl
 import com.google.api.client.http.HttpHeaders
 import com.google.api.client.http.HttpRequest
 import com.google.api.client.http.HttpRequestInitializer
+import com.google.api.client.http.HttpResponse
+import com.google.api.client.json.JsonObjectParser
+import com.google.api.client.json.jackson2.JacksonFactory
 import com.google.api.services.compute.Compute
 import com.google.api.services.compute.model.*
 import com.netflix.spinnaker.cats.cache.Cache
 import com.netflix.spinnaker.cats.cache.RelationshipCacheFilter
+import com.netflix.spinnaker.clouddriver.artifacts.ArtifactUtils
 import com.netflix.spinnaker.clouddriver.consul.provider.ConsulProviderUtils
 import com.netflix.spinnaker.clouddriver.data.task.Task
 import com.netflix.spinnaker.clouddriver.google.GoogleConfiguration
@@ -49,6 +54,7 @@ import com.netflix.spinnaker.clouddriver.google.provider.view.GoogleLoadBalancer
 import com.netflix.spinnaker.clouddriver.google.provider.view.GoogleNetworkProvider
 import com.netflix.spinnaker.clouddriver.google.provider.view.GoogleSubnetProvider
 import com.netflix.spinnaker.clouddriver.google.security.GoogleNamedAccountCredentials
+import com.netflix.spinnaker.kork.artifacts.model.Artifact
 import groovy.util.logging.Slf4j
 
 import static com.netflix.spinnaker.clouddriver.google.cache.Keys.Namespace.HTTP_HEALTH_CHECKS
@@ -118,6 +124,66 @@ class GCEUtil {
     }
   }
 
+  static Image getImageFromArtifact(Artifact artifact,
+                                    Compute compute,
+                                    Task task,
+                                    String phase,
+                                    SafeRetry safeRetry,
+                                    GoogleExecutorTraits executor) {
+    if (artifact.getType() != ArtifactUtils.GCE_IMAGE_TYPE) {
+      throw new GoogleOperationException("Artifact to deploy to GCE must be of type ${ArtifactUtils.GCE_IMAGE_TYPE}")
+    }
+
+    def reference = artifact.getReference()
+    task.updateStatus phase, "Looking up image $reference..."
+
+    def result = safeRetry.doRetry(
+      {
+        return compute.getRequestFactory()
+          .buildGetRequest(new GenericUrl(reference))
+          .setParser(new JsonObjectParser(JacksonFactory.getDefaultInstance()))
+          .execute()
+      },
+      "gce/image",
+      task,
+      RETRY_ERROR_CODES,
+      [],
+      [action: "get", phase: phase, operation: "compute.buildGetRequest.execute", (executor.TAG_SCOPE): executor.SCOPE_GLOBAL],
+      executor.registry
+    ) as HttpResponse
+
+    return result.parseAs(Image)
+  }
+
+  static Image getBootImage(BaseGoogleInstanceDescription description,
+                            Task task,
+                            String phase,
+                            String clouddriverUserAgentApplicationName,
+                            List<String> baseImageProjects,
+                            SafeRetry safeRetry,
+                            GoogleExecutorTraits executor) {
+    if (description.imageSource == BaseGoogleInstanceDescription.ImageSource.ARTIFACT) {
+      return getImageFromArtifact(
+        description.imageArtifact,
+        description.credentials.compute,
+        task,
+        phase,
+        safeRetry,
+        executor
+      )
+    } else {
+      return queryImage(description.credentials.project,
+        description.image,
+        description.credentials,
+        description.credentials.compute,
+        task,
+        phase,
+        clouddriverUserAgentApplicationName,
+        baseImageProjects,
+        executor)
+    }
+  }
+
   private static BatchRequest buildBatchRequest(def compute, String clouddriverUserAgentApplicationName) {
     return compute.batch(
       new HttpRequestInitializer() {
@@ -630,10 +696,9 @@ class GCEUtil {
                                                String phase,
                                                String clouddriverUserAgentApplicationName,
                                                List<String> baseImageProjects,
+                                               SafeRetry safeRetry,
                                                GoogleExecutorTraits executor) {
     def credentials = description.credentials
-    def compute = credentials.compute
-    def project = credentials.project
     def disks = description.disks
     def instanceType = description.instanceType
 
@@ -645,22 +710,33 @@ class GCEUtil {
       throw new GoogleOperationException("Unable to determine disks for instance type $instanceType.")
     }
 
-    def firstPersistentDisk = disks.find { it.persistent }
+    def bootImage = getBootImage(description,
+                                 task,
+                                 phase,
+                                 clouddriverUserAgentApplicationName,
+                                 baseImageProjects,
+                                 safeRetry,
+                                 executor)
 
+    def firstPersistentDisk = disks.find { it.persistent }
     return disks.collect { disk ->
-      def diskType = useDiskTypeUrl ? buildDiskTypeUrl(project, zone, disk.type) : disk.type
-      def sourceImage =
-        disk.persistent
-        ? queryImage(project,
-                     disk.is(firstPersistentDisk) ? description.image : disk.sourceImage,
-                     credentials,
-                     compute,
-                     task,
-                     phase,
-                     clouddriverUserAgentApplicationName,
-                     baseImageProjects,
-                     executor)
-        : null
+      def diskType = useDiskTypeUrl ? buildDiskTypeUrl(credentials.project, zone, disk.type) : disk.type
+
+      def sourceImage
+      if (disk.persistent) {
+        sourceImage =
+          disk.is(firstPersistentDisk)
+          ? bootImage
+          : queryImage(credentials.project,
+                       disk.sourceImage,
+                       credentials,
+                       credentials.compute,
+                       task,
+                       phase,
+                       clouddriverUserAgentApplicationName,
+                       baseImageProjects,
+                       executor)
+      }
 
       if (sourceImage && sourceImage.diskSizeGb > disk.sizeGb) {
         disk.sizeGb = sourceImage.diskSizeGb

diff --git a/...flix/spinnaker/clouddriver/google/deploy/description/BaseGoogleInstanceDescription.groovy b/...flix/spinnaker/clouddriver/google/deploy/description/BaseGoogleInstanceDescription.groovy
@@ -17,6 +17,7 @@
 package com.netflix.spinnaker.clouddriver.google.deploy.description
 
 import com.netflix.spinnaker.clouddriver.google.model.GoogleDisk
+import com.netflix.spinnaker.kork.artifacts.model.Artifact
 import groovy.transform.AutoClone
 import groovy.transform.Canonical
 import groovy.transform.ToString
@@ -25,7 +26,6 @@ import groovy.transform.ToString
 @Canonical
 @ToString(includeNames = true)
 class BaseGoogleInstanceDescription extends AbstractGoogleCredentialsDescription {
-  String image
   String instanceType
   String minCpuPlatform
   List<GoogleDisk> disks
@@ -43,8 +43,22 @@ class BaseGoogleInstanceDescription extends AbstractGoogleCredentialsDescription
   Boolean automaticRestart
   OnHostMaintenance onHostMaintenance
 
+  // We support passing the image to deploy as either a string or an artifact, but default to
+  // the string for backwards-compatibility
+  ImageSource imageSource = ImageSource.STRING
+  String image
+  Artifact imageArtifact
+
   String accountName
 
+  // The source of the image to deploy
+  // ARTIFACT: An artifact of type gce/image stored in imageArtifact
+  // STRING:   A string representing a GCE image name in the current
+  //           project, stored in image
+  enum ImageSource {
+    ARTIFACT, STRING
+  }
+
   enum OnHostMaintenance {
     MIGRATE, TERMINATE
   }

diff --git a/.../com/netflix/spinnaker/clouddriver/google/deploy/handlers/BasicGoogleDeployHandler.groovy b/.../com/netflix/spinnaker/clouddriver/google/deploy/handlers/BasicGoogleDeployHandler.groovy
@@ -200,6 +200,7 @@ class BasicGoogleDeployHandler implements DeployHandler<BasicGoogleDeployDescrip
                                                    BASE_PHASE,
                                                    clouddriverUserAgentApplicationName,
                                                    googleConfigurationProperties.baseImageProjects,
+                                                   safeRetry,
                                                    this)
 
     def networkInterface = GCEUtil.buildNetworkInterface(network,

diff --git a/...etflix/spinnaker/clouddriver/google/deploy/ops/CreateGoogleInstanceAtomicOperation.groovy b/...etflix/spinnaker/clouddriver/google/deploy/ops/CreateGoogleInstanceAtomicOperation.groovy
@@ -23,6 +23,7 @@ import com.netflix.spinnaker.clouddriver.deploy.DeploymentResult
 import com.netflix.spinnaker.clouddriver.google.GoogleConfiguration
 import com.netflix.spinnaker.clouddriver.google.config.GoogleConfigurationProperties
 import com.netflix.spinnaker.clouddriver.google.deploy.GCEUtil
+import com.netflix.spinnaker.clouddriver.google.deploy.SafeRetry
 import com.netflix.spinnaker.clouddriver.google.deploy.description.CreateGoogleInstanceDescription
 import com.netflix.spinnaker.clouddriver.google.provider.view.GoogleNetworkProvider
 import com.netflix.spinnaker.clouddriver.google.provider.view.GoogleSubnetProvider
@@ -51,6 +52,9 @@ class CreateGoogleInstanceAtomicOperation extends GoogleAtomicOperation<Deployme
   @Autowired
   String clouddriverUserAgentApplicationName
 
+  @Autowired
+  SafeRetry safeRetry
+
   private static Task getTask() {
     TaskRepository.threadLocalTask.get()
   }
@@ -102,6 +106,7 @@ class CreateGoogleInstanceAtomicOperation extends GoogleAtomicOperation<Deployme
                                                    BASE_PHASE,
                                                    clouddriverUserAgentApplicationName,
                                                    googleConfigurationProperties.baseImageProjects,
+                                                   safeRetry,
                                                    this)
 
     def networkInterface = GCEUtil.buildNetworkInterface(network,

diff --git a/...ouddriver/google/deploy/ops/ModifyGoogleServerGroupInstanceTemplateAtomicOperation.groovy b/...ouddriver/google/deploy/ops/ModifyGoogleServerGroupInstanceTemplateAtomicOperation.groovy
@@ -24,6 +24,7 @@ import com.netflix.spinnaker.clouddriver.google.GoogleConfiguration
 import com.netflix.spinnaker.clouddriver.google.config.GoogleConfigurationProperties
 import com.netflix.spinnaker.clouddriver.google.deploy.GCEUtil
 import com.netflix.spinnaker.clouddriver.google.deploy.GoogleOperationPoller
+import com.netflix.spinnaker.clouddriver.google.deploy.SafeRetry
 import com.netflix.spinnaker.clouddriver.google.deploy.description.BaseGoogleInstanceDescription
 import com.netflix.spinnaker.clouddriver.google.deploy.description.ModifyGoogleServerGroupInstanceTemplateDescription
 import com.netflix.spinnaker.clouddriver.google.deploy.exception.GoogleResourceNotFoundException
@@ -72,6 +73,9 @@ class ModifyGoogleServerGroupInstanceTemplateAtomicOperation extends GoogleAtomi
   @Autowired
   String clouddriverUserAgentApplicationName
 
+  @Autowired
+  SafeRetry safeRetry
+
   ModifyGoogleServerGroupInstanceTemplateAtomicOperation(ModifyGoogleServerGroupInstanceTemplateDescription description) {
     this.description = description
   }
@@ -171,6 +175,7 @@ class ModifyGoogleServerGroupInstanceTemplateAtomicOperation extends GoogleAtomi
                                                        BASE_PHASE,
                                                        clouddriverUserAgentApplicationName,
                                                        googleConfigurationProperties.baseImageProjects,
+                                                       safeRetry,
                                                        this)
 
         instanceTemplateProperties.setDisks(attachedDisks)