Fix issue with K8s log in with custom Vault path (lensesio#51)

* Fix issue with K8s log in with custom Vault path Signed-off-by: dttung2905 <ttdao.2015@accountancy.smu.edu.sg> * Remove extra lines added from the config Signed-off-by: dttung2905 <ttdao.2015@accountancy.smu.edu.sg> * Replace bettercloud with jopenlibs Signed-off-by: dttung2905 <ttdao.2015@accountancy.smu.edu.sg> * Fix vault container import Signed-off-by: dttung2905 <ttdao.2015@accountancy.smu.edu.sg> * Add changes from code review Signed-off-by: dttung2905 <ttdao.2015@accountancy.smu.edu.sg> * More changes from code review suggestion Signed-off-by: dttung2905 <ttdao.2015@accountancy.smu.edu.sg> * code formatting Signed-off-by: dttung2905 <ttdao.2015@accountancy.smu.edu.sg> --------- Signed-off-by: dttung2905 <ttdao.2015@accountancy.smu.edu.sg>
caltangelo · Apr 26, 2023 · e11640a · e11640a
1 parent e980442
commit e11640a
Show file tree

Hide file tree

Showing 12 changed files with 56 additions and 26 deletions.
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -10,7 +10,7 @@ trait Dependencies {
 
     val scalaLoggingVersion  = "3.9.5"
     val kafkaVersion         = "3.4.0"
-    val vaultVersion         = "5.1.0"
+    val vaultVersion         = "5.3.0"
     val azureKeyVaultVersion = "4.5.2"
     val azureIdentityVersion = "1.8.1"
 
@@ -41,7 +41,7 @@ trait Dependencies {
       "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion
     val `kafka-connect-api` = "org.apache.kafka" % "connect-api" % kafkaVersion
     val `vault-java-driver` =
-      "com.bettercloud" % "vault-java-driver" % vaultVersion
+      "io.github.jopenlibs" % "vault-java-driver" % vaultVersion
     val `azure-key-vault` =
       "com.azure" % "azure-security-keyvault-secrets" % azureKeyVaultVersion
     val `azure-identity` = "com.azure" % "azure-identity" % azureIdentityVersion

diff --git a/...vider/src/it/scala/io/lenses/connect/secrets/integration/SecretProviderRestResponse.scala b/...vider/src/it/scala/io/lenses/connect/secrets/integration/SecretProviderRestResponse.scala
@@ -1,6 +1,6 @@
 package io.lenses.connect.secrets.integration
 
-import com.bettercloud.vault.rest.RestResponse
+import io.github.jopenlibs.vault.rest.RestResponse
 import org.json4s.DefaultFormats
 import org.json4s._
 import org.json4s.native.JsonMethods._

diff --git a/secret-provider/src/it/scala/io/lenses/connect/secrets/testcontainers/VaultContainer.scala b/secret-provider/src/it/scala/io/lenses/connect/secrets/testcontainers/VaultContainer.scala
@@ -2,12 +2,12 @@ package io.lenses.connect.secrets.testcontainers
 
 import cats.effect.IO
 import cats.effect.Resource
-import com.bettercloud.vault.Vault
-import com.bettercloud.vault.VaultConfig
-import com.bettercloud.vault.api.mounts.MountPayload
-import com.bettercloud.vault.api.mounts.MountType
-import com.bettercloud.vault.api.mounts.TimeToLive
-import com.bettercloud.vault.response.MountResponse
+import io.github.jopenlibs.vault.Vault
+import io.github.jopenlibs.vault.VaultConfig
+import io.github.jopenlibs.vault.api.sys.mounts.MountPayload
+import io.github.jopenlibs.vault.api.sys.mounts.MountType
+import io.github.jopenlibs.vault.api.sys.mounts.TimeToLive
+import io.github.jopenlibs.vault.response.MountResponse
 import com.typesafe.scalalogging.LazyLogging
 import io.lenses.connect.secrets.testcontainers.VaultContainer._
 import org.scalatest.matchers.should.Matchers

diff --git a/secret-provider/src/main/scala/io/lenses/connect/secrets/config/VaultProviderConfig.scala b/secret-provider/src/main/scala/io/lenses/connect/secrets/config/VaultProviderConfig.scala
@@ -47,6 +47,8 @@ object VaultProviderConfig {
   val KUBERNETES_TOKEN_PATH: String = "kubernetes.token.path"
   val KUBERNETES_DEFAULT_TOKEN_PATH: String =
     "/var/run/secrets/kubernetes.io/serviceaccount/token"
+  val KUBERNETES_AUTH_PATH:         String = "kubernetes.auth.path"
+  val KUBERNETES_AUTH_PATH_DEFAULT: String = "auth/kubernetes"
 
   val APP_ROLE_PATH:      String = "app.role.path"
   val APP_ROLE:           String = "app.role.id"
@@ -379,6 +381,12 @@ object VaultProviderConfig {
       TOKEN_RENEWAL_DEFAULT,
       Importance.MEDIUM,
       "The time in milliseconds to renew the Vault token",
+    ).define(
+      KUBERNETES_AUTH_PATH,
+      Type.STRING,
+      KUBERNETES_AUTH_PATH_DEFAULT,
+      Importance.MEDIUM,
+      "The mount path of the Vault Kubernetes Auth Method",
     )
     .define(
       SECRET_DEFAULT_TTL,

diff --git a/secret-provider/src/main/scala/io/lenses/connect/secrets/config/VaultProviderSettings.scala b/secret-provider/src/main/scala/io/lenses/connect/secrets/config/VaultProviderSettings.scala
@@ -35,7 +35,7 @@ case class Jwt(role: String, provider: String, jwt: Password)
 case class UserPass(username: String, password: Password, mount: String)
 case class Ldap(username: String, password: Password, mount: String)
 case class AppRole(path: String, role: String, secretId: Password)
-case class K8s(role: String, jwt: Password)
+case class K8s(role: String, jwt: Password, authPath: String)
 case class Cert(mount: String)
 case class Github(token: Password, mount: String)
 
@@ -180,14 +180,15 @@ object VaultSettings extends StrictLogging {
       config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_ROLE)
     val path =
       config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_TOKEN_PATH)
+    val authPath = config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_AUTH_PATH)
     Using(Source.fromFile(path))(_.getLines().mkString) match {
       case Failure(exception) =>
         throw new ConnectException(
           s"Failed to load kubernetes token file [$path]",
           exception,
         )
       case Success(fileContents) =>
-        K8s(role = role, jwt = new Password(fileContents))
+        K8s(role = role, jwt = new Password(fileContents), authPath)
     }
   }
 

diff --git a/secret-provider/src/main/scala/io/lenses/connect/secrets/providers/VaultHelper.scala b/secret-provider/src/main/scala/io/lenses/connect/secrets/providers/VaultHelper.scala
@@ -6,10 +6,10 @@
 
 package io.lenses.connect.secrets.providers
 
-import com.bettercloud.vault.SslConfig
-import com.bettercloud.vault.Vault
-import com.bettercloud.vault.VaultConfig
-import com.bettercloud.vault.response.LogicalResponse
+import io.github.jopenlibs.vault.SslConfig
+import io.github.jopenlibs.vault.Vault
+import io.github.jopenlibs.vault.VaultConfig
+import io.github.jopenlibs.vault.response.LogicalResponse
 import com.typesafe.scalalogging.LazyLogging
 import com.typesafe.scalalogging.StrictLogging
 import io.lenses.connect.secrets.cache.ValueWithTtl
@@ -147,7 +147,7 @@ object VaultHelper extends StrictLogging {
           .map(k8s =>
             vault
               .auth()
-              .loginByKubernetes(k8s.role, k8s.jwt.value())
+              .loginByJwt("kubernetes", k8s.role, k8s.jwt.value(), k8s.authPath)
               .getAuthClientToken,
           )
       case VaultAuthMethod.GCP =>

diff --git a/secret-provider/src/main/scala/io/lenses/connect/secrets/providers/VaultSecretProvider.scala b/secret-provider/src/main/scala/io/lenses/connect/secrets/providers/VaultSecretProvider.scala
@@ -6,7 +6,7 @@
 
 package io.lenses.connect.secrets.providers
 
-import com.bettercloud.vault.Vault
+import io.github.jopenlibs.vault.Vault
 import io.lenses.connect.secrets.async.AsyncFunctionLoop
 import io.lenses.connect.secrets.config.VaultProviderConfig
 import io.lenses.connect.secrets.config.VaultSettings

diff --git a/secret-provider/src/test/java/io/lenses/connect/secrets/vault/MockVault.java b/secret-provider/src/test/java/io/lenses/connect/secrets/vault/MockVault.java
@@ -6,7 +6,7 @@
 
 package io.lenses.connect.secrets.vault;
 
-import com.bettercloud.vault.json.JsonObject;
+import io.github.jopenlibs.vault.json.JsonObject;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 

diff --git a/secret-provider/src/test/java/io/lenses/connect/secrets/vault/VaultTestUtils.java b/secret-provider/src/test/java/io/lenses/connect/secrets/vault/VaultTestUtils.java
@@ -6,8 +6,8 @@
 
 package io.lenses.connect.secrets.vault;
 
-import com.bettercloud.vault.json.Json;
-import com.bettercloud.vault.json.JsonObject;
+import io.github.jopenlibs.vault.json.Json;
+import io.github.jopenlibs.vault.json.JsonObject;
 import org.apache.commons.io.IOUtils;
 import org.eclipse.jetty.server.*;
 import org.eclipse.jetty.util.ssl.SslContextFactory;

diff --git a/...t-provider/src/test/scala/io/lenses/connect/secrets/providers/AWSSecretProviderTest.scala b/...t-provider/src/test/scala/io/lenses/connect/secrets/providers/AWSSecretProviderTest.scala
@@ -6,7 +6,7 @@
 
 package io.lenses.connect.secrets.providers
 
-import com.bettercloud.vault.json.JsonObject
+import io.github.jopenlibs.vault.json.JsonObject
 import io.lenses.connect.secrets.TmpDirUtil.getTempDir
 import io.lenses.connect.secrets.config.AWSProviderConfig
 import io.lenses.connect.secrets.config.AWSProviderSettings

diff --git a/...provider/src/test/scala/io/lenses/connect/secrets/providers/VaultSecretProviderTest.scala b/...provider/src/test/scala/io/lenses/connect/secrets/providers/VaultSecretProviderTest.scala
@@ -6,10 +6,12 @@
 
 package io.lenses.connect.secrets.providers
 
-import com.bettercloud.vault.json.JsonArray
-import com.bettercloud.vault.json.JsonObject
+import io.github.jopenlibs.vault.json.JsonArray
+import io.github.jopenlibs.vault.json.JsonObject
 import io.lenses.connect.secrets.TmpDirUtil.getTempDir
 import io.lenses.connect.secrets.TmpDirUtil.separator
+import io.lenses.connect.secrets.config.VaultProviderConfig.KUBERNETES_AUTH_PATH_DEFAULT
+import io.lenses.connect.secrets.config.K8s
 import io.lenses.connect.secrets.config.VaultAuthMethod
 import io.lenses.connect.secrets.config.VaultProviderConfig
 import io.lenses.connect.secrets.config.VaultSettings
@@ -19,8 +21,10 @@ import io.lenses.connect.secrets.vault.VaultTestUtils
 import org.apache.kafka.common.config.provider.ConfigProvider
 import org.apache.kafka.common.config.ConfigData
 import org.apache.kafka.common.config.ConfigTransformer
+import org.apache.kafka.common.config.types.Password
 import org.eclipse.jetty.server.Server
 import org.scalatest.BeforeAndAfterAll
+import org.scalatest.OptionValues.convertOptionToValuable
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 
@@ -233,8 +237,25 @@ class VaultSecretProviderTest extends AnyWordSpec with Matchers with BeforeAndAf
     ).asJava
 
     val settings = VaultSettings(VaultProviderConfig(props))
-    settings.k8s.isDefined shouldBe true
+    settings.k8s.value shouldBe K8s("role", new Password(fileToString(k8sToken)), KUBERNETES_AUTH_PATH_DEFAULT)
   }
+  "should be configured for kubernetes auth custom path" in {
+    val props = Map(
+      VaultProviderConfig.VAULT_ADDR            -> "https://127.0.0.1:9998",
+      VaultProviderConfig.VAULT_TOKEN           -> "mock_token",
+      VaultProviderConfig.VAULT_PEM             -> pemFile,
+      VaultProviderConfig.AUTH_METHOD           -> VaultAuthMethod.KUBERNETES.toString,
+      VaultProviderConfig.KUBERNETES_TOKEN_PATH -> k8sToken,
+      VaultProviderConfig.KUBERNETES_ROLE       -> "role",
+      VaultProviderConfig.KUBERNETES_AUTH_PATH  -> "custom/path",
+    ).asJava
+
+    val settings = VaultSettings(VaultProviderConfig(props))
+    settings.k8s.value shouldBe K8s("role", new Password(fileToString(k8sToken)), "custom/path")
+  }
+
+  private def fileToString(k8sTokenFile: String): String =
+    Using(Source.fromFile(k8sTokenFile))(_.getLines().mkString).getOrElse(fail("Unable to load token"))
 
   "should be configured for approle auth" in {
     val props = Map(

diff --git a/test-sink/src/main/scala/io/lenses/connect/secrets/test/vault/VaultState.scala b/test-sink/src/main/scala/io/lenses/connect/secrets/test/vault/VaultState.scala
@@ -1,7 +1,7 @@
 package io.lenses.connect.secrets.test.vault
 
-import com.bettercloud.vault.Vault
-import com.bettercloud.vault.VaultConfig
+import io.github.jopenlibs.vault.Vault
+import io.github.jopenlibs.vault.VaultConfig
 
 case class VaultState(
   vault:       Vault,