feat(artifacts): add ivy/maven artifact account support

spinnaker · Nov 23, 2018 · d24e722 · d24e722
1 parent 6264624
commit d24e722
Showing 31 changed files with 1,622 additions and 11 deletions.
diff --git a/clouddriver-artifacts/clouddriver-artifacts.gradle b/clouddriver-artifacts/clouddriver-artifacts.gradle
@@ -1,3 +1,6 @@
+test {
+  useJUnitPlatform()
+}
 
 class DownloadTask extends DefaultTask {
   @Input
@@ -67,7 +70,17 @@ dependencies {
   compile spinnaker.dependency("googleStorage")
   compile spinnaker.dependency("awsS3")
   compile "org.apache.commons:commons-compress:1.14"
-  compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.6"
+  compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.7"
+  compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.9.7'
+  compile "org.apache.ivy:ivy:2.4.0"
 
   compile fileTree(sdkLocation)
+
+  testCompile('org.junit.jupiter:junit-jupiter-api:5.2.0')
+  testRuntime('org.junit.jupiter:junit-jupiter-engine:5.2.0')
+  testCompile 'org.assertj:assertj-core:3.8.0'
+  testCompile 'org.junit-pioneer:junit-pioneer:latest.release'
+
+  testCompile 'ru.lanwen.wiremock:wiremock-junit5:1.2.0'
+  testCompile 'com.github.tomakehurst:wiremock:latest.release'
 }
diff --git a/...src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/DiskFreeingInputStream.java b/...src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/DiskFreeingInputStream.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2018 Pivotal, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.ivy;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+
+/**
+ * An {@link java.io.InputStream} that frees local disk resources when closed.
+ */
+public class DiskFreeingInputStream extends InputStream {
+  private final InputStream delegate;
+  private final Path deleteOnClose;
+
+  public DiskFreeingInputStream(InputStream delegate, Path deleteOnClose) {
+    this.delegate = delegate;
+    this.deleteOnClose = deleteOnClose;
+  }
+
+  @Override
+  public int read() throws IOException {
+    return delegate.read();
+  }
+
+  @Override
+  public void close() throws IOException {
+    super.close();
+    if (Files.exists(deleteOnClose)) {
+      Files.walk(deleteOnClose)
+        .map(Path::toFile)
+        .sorted(Comparator.reverseOrder())
+        .forEach(File::delete);
+    }
+  }
+}
diff --git a/...cts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/IvyArtifactAccount.java b/...cts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/IvyArtifactAccount.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2018 Pivotal, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.ivy;
+
+import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactAccount;
+import com.netflix.spinnaker.clouddriver.artifacts.ivy.settings.IvySettings;
+import lombok.Data;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+
+@Data
+public class IvyArtifactAccount implements ArtifactAccount {
+  private String name;
+  private IvySettings settings;
+  private List<String> resolveConfigurations = singletonList("master");
+}
diff --git a/...c/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/IvyArtifactConfiguration.java b/...c/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/IvyArtifactConfiguration.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2018 Pivotal, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.ivy;
+
+import com.netflix.spinnaker.clouddriver.artifacts.ArtifactCredentialsRepository;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+@Configuration
+@ConditionalOnProperty("artifacts.ivy.enabled")
+@EnableConfigurationProperties(IvyArtifactProviderProperties.class)
+@RequiredArgsConstructor
+@Slf4j
+public class IvyArtifactConfiguration {
+  private final IvyArtifactProviderProperties ivyArtifactProviderProperties;
+  private final ArtifactCredentialsRepository artifactCredentialsRepository;
+
+  @Bean
+  List<? extends IvyArtifactCredentials> ivyArtifactCredentials() {
+    return ivyArtifactProviderProperties.getAccounts()
+      .stream()
+      .map(a -> {
+        try {
+          IvyArtifactCredentials c = new IvyArtifactCredentials(a);
+          artifactCredentialsRepository.save(c);
+          return c;
+        } catch (Exception e) {
+          log.warn("Failure instantiating ivy artifact account {}: ", a, e);
+          return null;
+        }
+      })
+      .filter(Objects::nonNull)
+      .collect(Collectors.toList());
+  }
+}
diff --git a/...src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/IvyArtifactCredentials.java b/...src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/IvyArtifactCredentials.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2018 Pivotal, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.ivy;
+
+import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactCredentials;
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.ivy.Ivy;
+import org.apache.ivy.core.module.id.ModuleId;
+import org.apache.ivy.core.module.id.ModuleRevisionId;
+import org.apache.ivy.core.report.ResolveReport;
+import org.apache.ivy.core.resolve.ResolveOptions;
+import org.apache.ivy.util.AbstractMessageLogger;
+import org.apache.ivy.util.Message;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.ParseException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.function.Supplier;
+
+@Slf4j
+@Data
+public class IvyArtifactCredentials implements ArtifactCredentials {
+  private final List<String> types = Collections.singletonList("ivy/file");
+  private final IvyArtifactAccount account;
+  private final Supplier<Path> cacheBuilder;
+
+  public IvyArtifactCredentials(IvyArtifactAccount account) {
+    this(account, () -> Paths.get(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString()));
+  }
+
+  public IvyArtifactCredentials(IvyArtifactAccount account, Supplier<Path> cacheBuilder) {
+    this.cacheBuilder = cacheBuilder;
+    redirectIvyLogsToSlf4j();
+    this.account = account;
+  }
+
+  private static void redirectIvyLogsToSlf4j() {
+    Message.setDefaultLogger(new AbstractMessageLogger() {
+      private final Logger logger = LoggerFactory.getLogger("org.apache.ivy");
+
+      @Override
+      protected void doProgress() {
+      }
+
+      @Override
+      protected void doEndProgress(String msg) {
+        log(msg, Message.MSG_INFO);
+      }
+
+      @Override
+      public void log(String msg, int level) {
+        switch (level) {
+          case Message.MSG_ERR:
+            logger.error(msg);
+            break;
+          case Message.MSG_WARN:
+            logger.warn(msg);
+            break;
+          case Message.MSG_INFO:
+            logger.info(msg);
+            break;
+          case Message.MSG_DEBUG:
+            logger.debug(msg);
+            break;
+          case Message.MSG_VERBOSE:
+            logger.trace(msg);
+          default:
+            // do nothing
+        }
+      }
+
+      @Override
+      public void rawlog(String msg, int level) {
+        log(msg, level);
+      }
+    });
+  }
+
+  public InputStream download(Artifact artifact) {
+    Path cacheDir = cacheBuilder.get();
+    Ivy ivy = account.getSettings().toIvy(cacheDir);
+
+    String[] parts = artifact.getReference().split(":");
+    if (parts.length < 3) {
+      throw new IllegalArgumentException("Ivy artifact reference must have a group, artifact, and version separated by ':'");
+    }
+
+    ModuleRevisionId mrid = new ModuleRevisionId(new ModuleId(parts[0], parts[1]), parts[2]);
+
+    try {
+      ResolveReport report = ivy.resolve(mrid, (ResolveOptions) new ResolveOptions()
+        .setTransitive(false)
+        .setConfs(account.getResolveConfigurations().toArray(new String[0]))
+        .setLog("download-only"), true);
+      return Arrays.stream(report.getAllArtifactsReports())
+        .findFirst()
+        .map(rep -> {
+          try {
+            return new DiskFreeingInputStream(new FileInputStream(rep.getLocalFile()), cacheDir);
+          } catch (FileNotFoundException e) {
+            throw new UncheckedIOException(e);
+          }
+        })
+        .orElseThrow(() -> new IllegalArgumentException("Unable to resolve artifact for reference '" + artifact.getReference() + "'"));
+    } catch (ParseException | IOException e) {
+      throw new IllegalArgumentException(e);
+    }
+  }
+
+  @Override
+  public String getName() {
+    return account.getName();
+  }
+}
diff --git a/...n/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/IvyArtifactProviderProperties.java b/...n/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/IvyArtifactProviderProperties.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2018 Pivotal, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.ivy;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+@ConfigurationProperties("artifacts.ivy")
+public class IvyArtifactProviderProperties {
+  private boolean enabled;
+  private List<IvyArtifactAccount> accounts = new ArrayList<>();
+}
diff --git a/...c/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/settings/BintrayResolver.java b/...c/main/java/com/netflix/spinnaker/clouddriver/artifacts/ivy/settings/BintrayResolver.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2018 Pivotal, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.ivy.settings;
+
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import javax.annotation.Nullable;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class BintrayResolver extends Resolver<org.apache.ivy.plugins.resolver.BintrayResolver> {
+  /**
+   * Bintray username of a repository owner.
+   */
+  @JacksonXmlProperty(isAttribute = true)
+  @Nullable
+  private String subject;
+
+  /**
+   * User’s repository name.
+   */
+  @JacksonXmlProperty(isAttribute = true)
+  @Nullable
+  private String repo;
+
+  @Override
+  public org.apache.ivy.plugins.resolver.BintrayResolver toIvyModel() {
+    org.apache.ivy.plugins.resolver.BintrayResolver bintrayResolver = new org.apache.ivy.plugins.resolver.BintrayResolver();
+    bintrayResolver.setRepo(repo);
+    bintrayResolver.setSubject(subject);
+    return super.toIvyModel(bintrayResolver);
+  }
+}