poc(provider/aws): Index a subset of server group / instance details (#…

…2204) Mo bettah search! Specifically this PR introduces a brand new agent that fetches all server groups and instances in each aws account / region. The current breakdown of caching agents (per account and per region) are potentially too granular to create useful elastic search indexes off of.
spinnaker · Jan 15, 2018 · 8b3fe89 · 8b3fe89
1 parent c2e4a60
commit 8b3fe89
Show file tree

Hide file tree

Showing 12 changed files with 545 additions and 1 deletion.
diff --git a/build.gradle b/build.gradle
@@ -17,6 +17,8 @@
 buildscript {
   ext {
     springBootVersion = "1.5.7.RELEASE"
+    kotlinVersion = "1.2.0"
+    junitPlatformVersion = "1.0.2"
   }
   repositories {
     jcenter()
@@ -26,6 +28,8 @@ buildscript {
   dependencies {
     classpath 'com.netflix.spinnaker.gradle:spinnaker-gradle-project:3.17.0'
     classpath "org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}"
+    classpath "org.junit.platform:junit-platform-gradle-plugin:${junitPlatformVersion}"
+    classpath "com.netflix.nebula:nebula-kotlin-plugin:${kotlinVersion}"
   }
 }
 
@@ -35,7 +39,7 @@ allprojects {
   apply plugin: 'groovy'
 
   ext {
-    spinnakerDependenciesVersion = project.hasProperty('spinnakerDependenciesVersion') ? project.property('spinnakerDependenciesVersion') : '0.131.0'
+    spinnakerDependenciesVersion = project.hasProperty('spinnakerDependenciesVersion') ? project.property('spinnakerDependenciesVersion') : '0.134.0'
   }
 
   def checkLocalVersions = [spinnakerDependenciesVersion: spinnakerDependenciesVersion]

diff --git a/clouddriver-elasticsearch-aws/clouddriver-elasticsearch-aws.gradle b/clouddriver-elasticsearch-aws/clouddriver-elasticsearch-aws.gradle
@@ -0,0 +1,10 @@
+apply from: "$rootDir/gradle/kotlin.gradle"
+
+repositories {
+  jcenter()
+}
+
+dependencies {
+  compile project(":clouddriver-aws")
+  compile project(":clouddriver-elasticsearch")
+}
diff --git a/...ws/src/main/kotlin/com/netflix/spinnaker/clouddriver/elasticsearch/ElasticSearchClient.kt b/...ws/src/main/kotlin/com/netflix/spinnaker/clouddriver/elasticsearch/ElasticSearchClient.kt
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.elasticsearch
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.netflix.spinnaker.clouddriver.elasticsearch.model.ElasticSearchException
+import com.netflix.spinnaker.clouddriver.elasticsearch.model.ServerGroupModel
+import io.searchbox.client.JestClient
+import io.searchbox.core.Bulk
+import io.searchbox.core.BulkResult
+import io.searchbox.core.Index
+import io.searchbox.indices.CreateIndex
+import io.searchbox.indices.DeleteIndex
+import io.searchbox.indices.aliases.AddAliasMapping
+import io.searchbox.indices.aliases.GetAliases
+import io.searchbox.indices.aliases.ModifyAliases
+import java.io.IOException
+
+class ElasticSearchClient(private val objectMapper : ObjectMapper, private val jestClient: JestClient) {
+  fun getPreviousIndexes(prefix: String): Set<String> {
+    try {
+      val result = jestClient.execute(GetAliases.Builder().build())
+      val r = objectMapper.readValue(result.jsonString, Map::class.java) as Map<String, Any>
+      return r.keys.filter { k -> k.startsWith(prefix) }.toSet()
+    } catch (e: IOException) {
+      throw ElasticSearchException("Unable to fetch previous indexes (prefix: $prefix)", e)
+    }
+  }
+
+  fun createIndex(prefix: String): String {
+    val newIndexName = "${prefix}_${System.currentTimeMillis()}"
+
+    try {
+      jestClient.execute(CreateIndex.Builder(newIndexName).build())
+      return newIndexName
+    } catch (e: IOException) {
+      throw ElasticSearchException("Unable to create index (index: $newIndexName)", e)
+    }
+  }
+
+  fun createAlias(index: String, alias: String) {
+    try {
+      jestClient.execute(
+        ModifyAliases.Builder(
+          AddAliasMapping.Builder(index, alias).build()
+        ).build()
+      )
+    } catch (e: IOException) {
+      throw ElasticSearchException("Unable to create alias (index: $index, alias: $alias)", e)
+    }
+  }
+
+  fun deleteIndex(index: String) {
+    try {
+      jestClient.execute(
+        DeleteIndex.Builder(index).build()
+      )
+    } catch (e: IOException) {
+      throw ElasticSearchException("Unable to delete index (index: $index)", e)
+    }
+  }
+
+  fun store(index: String, partition: List<ServerGroupModel>) {
+    var builder: Bulk.Builder = Bulk.Builder().defaultIndex(index)
+
+    for (serverGroupModel in partition) {
+      builder = builder.addAction(
+        Index.Builder(objectMapper.convertValue(serverGroupModel, Map::class.java))
+          .index(index)
+          .type("ServerGroup")
+          .id(serverGroupModel.id)
+          .build()
+      )
+    }
+
+    val bulk = builder.build()
+    try {
+      val jestResult = jestClient.execute<BulkResult>(bulk)
+      if (!jestResult.isSucceeded) {
+        throw ElasticSearchException(
+          java.lang.String.format("Failed to index server groups, reason: '%s'", jestResult.getErrorMessage())
+        )
+      }
+    } catch (e: IOException) {
+      throw ElasticSearchException(
+        java.lang.String.format("Failed to index server groups, reason: '%s'", e.message)
+      )
+    }
+  }
+}
diff --git a/...etflix/spinnaker/clouddriver/elasticsearch/aws/ElasticSearchAmazonCachingAgentProvider.kt b/...etflix/spinnaker/clouddriver/elasticsearch/aws/ElasticSearchAmazonCachingAgentProvider.kt
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.elasticsearch.aws
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.SerializationFeature
+import com.netflix.spectator.api.Registry
+import com.netflix.spinnaker.cats.agent.Agent
+import com.netflix.spinnaker.cats.agent.AgentProvider
+import com.netflix.spinnaker.clouddriver.aws.provider.AwsProvider
+import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider
+import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials
+import com.netflix.spinnaker.clouddriver.elasticsearch.ElasticSearchClient
+import com.netflix.spinnaker.clouddriver.security.AccountCredentialsProvider
+import com.netflix.spinnaker.kork.core.RetrySupport
+import io.searchbox.client.JestClient
+
+open class ElasticSearchAmazonCachingAgentProvider(
+  private val objectMapper: ObjectMapper,
+  private val jestClient: JestClient,
+  private val retrySupport: RetrySupport,
+  private val registry: Registry,
+  private val amazonClientProvider: AmazonClientProvider,
+  private val accountCredentialsProvider: AccountCredentialsProvider
+) : AgentProvider {
+
+  override fun supports(providerName: String): Boolean {
+    return providerName.equals(AwsProvider.PROVIDER_NAME, ignoreCase = true)
+  }
+
+  override fun agents(): Collection<Agent> {
+    val credentials = accountCredentialsProvider
+      .all
+      .filter { NetflixAmazonCredentials::class.java.isInstance(it) }
+      .map { c -> c as NetflixAmazonCredentials }
+
+    val elasticSearchClient = ElasticSearchClient(
+      objectMapper.copy().enable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS),
+      jestClient
+    )
+
+    return listOf<Agent>(ElasticSearchAmazonServerGroupCachingAgent(
+      retrySupport,
+      registry,
+      amazonClientProvider,
+      credentials,
+      elasticSearchClient
+    ))
+  }
+}