feat(artifacts): Download artifacts from GitHub (spinnaker#2231)

spinnakerbot · Dec 20, 2017 · 3031067 · 3031067
1 parent de9bc20
commit 3031067
Show file tree

Hide file tree

Showing 5 changed files with 279 additions and 1 deletion.
diff --git a/...acts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ArtifactConfiguration.java b/...acts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ArtifactConfiguration.java
@@ -18,6 +18,7 @@
 package com.netflix.spinnaker.clouddriver.artifacts;
 
 import com.netflix.spinnaker.clouddriver.artifacts.gcs.GcsArtifactConfiguration;
+import com.netflix.spinnaker.clouddriver.artifacts.github.GitHubArtifactConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
@@ -31,7 +32,10 @@
 @EnableScheduling
 @Component
 @ComponentScan({"com.netflix.spinnaker.clouddriver.artifacts"})
-@Import({ GcsArtifactConfiguration.class })
+@Import({
+  GcsArtifactConfiguration.class,
+  GitHubArtifactConfiguration.class
+})
 public class ArtifactConfiguration {
   @Bean
   ArtifactCredentialsRepository artifactCredentialsRepository() {

diff --git a/...c/main/java/com/netflix/spinnaker/clouddriver/artifacts/github/GitHubArtifactAccount.java b/...c/main/java/com/netflix/spinnaker/clouddriver/artifacts/github/GitHubArtifactAccount.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2017 Armory, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.github;
+
+
+import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactAccount;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class GitHubArtifactAccount extends ArtifactAccount {
+  private String name;
+  /*
+    One of the following are required for auth:
+     - username and password
+     - usernamePasswordFile : path to file containing "username:password"
+     - token
+     - tokenFile : path to file containing token
+   */
+  private String username;
+  private String password;
+  private String usernamePasswordFile;
+  private String token;
+  private String tokenFile;
+}
diff --git a/.../java/com/netflix/spinnaker/clouddriver/artifacts/github/GitHubArtifactConfiguration.java b/.../java/com/netflix/spinnaker/clouddriver/artifacts/github/GitHubArtifactConfiguration.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2017 Armory, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.github;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.spinnaker.clouddriver.artifacts.ArtifactCredentialsRepository;
+import com.squareup.okhttp.OkHttpClient;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Scope;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+@Configuration
+@ConditionalOnProperty("artifacts.github.enabled")
+@EnableScheduling
+@Slf4j
+public class GitHubArtifactConfiguration {
+  @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+  @Bean
+  @ConfigurationProperties("artifacts.github")
+  GitHubArtifactProviderProperties githubArtifactProviderProperties() {
+    return new GitHubArtifactProviderProperties();
+  }
+
+  @Autowired
+  GitHubArtifactProviderProperties gitHubArtifactProviderProperties;
+
+  @Autowired
+  ArtifactCredentialsRepository artifactCredentialsRepository;
+
+  @Autowired
+  OkHttpClient okHttpClient;
+
+  @Autowired
+  ObjectMapper objectMapper;
+
+  @Bean
+  OkHttpClient okHttpClient() {
+    return new OkHttpClient();
+  }
+
+  @Bean
+  List<? extends GitHubArtifactCredentials> gitHubArtifactCredentials() {
+    return gitHubArtifactProviderProperties.getAccounts()
+      .stream()
+      .map(a -> {
+        try {
+          GitHubArtifactCredentials c = new GitHubArtifactCredentials(a, okHttpClient, objectMapper);
+          artifactCredentialsRepository.save(c);
+          return c;
+        } catch (Exception e) {
+          log.warn("Failure instantiating GitHub artifact account {}: ", a, e);
+          return null;
+        }
+      })
+      .filter(Objects::nonNull)
+      .collect(Collectors.toList());
+  }
+}
diff --git a/...in/java/com/netflix/spinnaker/clouddriver/artifacts/github/GitHubArtifactCredentials.java b/...in/java/com/netflix/spinnaker/clouddriver/artifacts/github/GitHubArtifactCredentials.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2017 Armory, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.github;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactCredentials;
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import com.squareup.okhttp.HttpUrl;
+import com.squareup.okhttp.OkHttpClient;
+import com.squareup.okhttp.Request;
+import com.squareup.okhttp.Request.Builder;
+import com.squareup.okhttp.Response;
+import lombok.extern.slf4j.Slf4j;
+import lombok.Data;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
+@Slf4j
+@Data
+public class GitHubArtifactCredentials implements ArtifactCredentials {
+  private final String name;
+
+  @JsonIgnore
+  private final Builder requestBuilder;
+
+  @JsonIgnore
+  OkHttpClient okHttpClient;
+
+  @JsonIgnore
+  ObjectMapper objectMapper;
+
+  public GitHubArtifactCredentials(GitHubArtifactAccount account, OkHttpClient okHttpClient, ObjectMapper objectMapper) {
+    this.name = account.getName();
+    this.okHttpClient = okHttpClient;
+    this.objectMapper = objectMapper;
+    Builder builder = new Request.Builder();
+    boolean useLogin = !StringUtils.isEmpty(account.getUsername()) && !StringUtils.isEmpty(account.getPassword());
+    boolean useUsernamePasswordFile = !StringUtils.isEmpty(account.getUsernamePasswordFile());
+    boolean useToken = !StringUtils.isEmpty(account.getToken());
+    boolean useTokenFile = !StringUtils.isEmpty(account.getTokenFile());
+    boolean useAuth = useLogin || useToken || useUsernamePasswordFile || useTokenFile;
+    if (useAuth) {
+      String authHeader = "";
+      if (useTokenFile) {
+        authHeader = "token " + credentialsFromFile(account.getTokenFile());
+      } else if (useUsernamePasswordFile) {
+        authHeader = "Basic " + Base64.encodeBase64String((credentialsFromFile(account.getUsernamePasswordFile())).getBytes());
+      }  else if (useToken) {
+        authHeader = "token " + account.getToken();
+      } else if (useLogin) {
+        authHeader = "Basic " + Base64.encodeBase64String((account.getUsername() + ":" + account.getPassword()).getBytes());
+      }
+      builder.header("Authorization", authHeader);
+      log.info("Loaded credentials for GitHub Artifact Account {}", account.getName());
+    } else {
+      log.info("No credentials included with GitHub Artifact Account {}", account.getName());
+    }
+    requestBuilder = builder;
+  }
+
+  private String credentialsFromFile(String filename) {
+    try {
+      String credentials = FileUtils.readFileToString(new File(filename));
+      return credentials.replace("\n", "");
+    } catch (IOException e) {
+      log.error("Could not read GitHub credentials file {}", filename);
+      return null;
+    }
+  }
+
+  public InputStream download(Artifact artifact) throws IOException {
+    HttpUrl.Builder metadataUrlBuilder = HttpUrl.parse(artifact.getReference()).newBuilder();
+    metadataUrlBuilder.addQueryParameter("ref", artifact.getVersion());
+    Request metadataRequest = requestBuilder
+      .url(metadataUrlBuilder.build().toString())
+      .build();
+    Response metadataResponse = okHttpClient.newCall(metadataRequest).execute();
+    String body = metadataResponse.body().string();
+    ContentMetadata metadata = objectMapper.readValue(body, ContentMetadata.class);
+    Request downloadRequest = requestBuilder
+      .url(metadata.getDownloadUrl())
+      .build();
+    Response downloadResponse = okHttpClient.newCall(downloadRequest).execute();
+    return downloadResponse.body().byteStream();
+  }
+
+  @Override
+  public boolean handlesType(String type) {
+    return type.equals("github/file");
+  }
+
+  @Data
+  public static class ContentMetadata {
+    @JsonProperty("download_url")
+    private String downloadUrl;
+  }
+}
diff --git a/.../com/netflix/spinnaker/clouddriver/artifacts/github/GitHubArtifactProviderProperties.java b/.../com/netflix/spinnaker/clouddriver/artifacts/github/GitHubArtifactProviderProperties.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2017 Armory, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.github;
+
+import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactProvider;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+public class GitHubArtifactProviderProperties extends ArtifactProvider<GitHubArtifactAccount> {
+  private boolean enabled;
+  private List<GitHubArtifactAccount> accounts = new ArrayList<>();
+}