* Expose plugiDownloadURL from provider schemas to plugin.json to mak…

…e community packages automatically installable * Avoid clashes on plugin.json resources * Enable building language plugin for different platforms
VirtusLab · Sep 18, 2023 · 3b395cf · 3b395cf
1 parent 0e51201
commit 3b395cf
Show file tree

Hide file tree

Showing 15 changed files with 140 additions and 49 deletions.
diff --git a/Justfile b/Justfile
@@ -1,6 +1,6 @@
 # Big idea behind using a Justfile is so that we can have modules like in sbt.
 
-publish-version := "0.0.1-SNAPSHOT"
+besom-version := "0.0.1-SNAPSHOT"
 language-plugin-output-dir := justfile_directory() + "/.out/language-plugin"
 codegen-output-dir := justfile_directory() + "/.out/codegen"
 schemas-output-dir := justfile_directory() + "/.out/schemas"
@@ -76,22 +76,22 @@ test-sdk: compile-sdk test-core test-cats test-zio
 
 # Publishes locally core besom SDK
 publish-local-core:
-  scala-cli --power publish local core --project-version {{publish-version}} --doc=false
+  scala-cli --power publish local core --project-version {{besom-version}} --doc=false
 
 # Publishes locally besom cats-effect extension
 publish-local-cats:
-	scala-cli --power publish local besom-cats --project-version {{publish-version}} --doc=false
+	scala-cli --power publish local besom-cats --project-version {{besom-version}} --doc=false
 
 # Publishes locally besom zio extension
 publish-local-zio:
-	scala-cli --power publish local besom-zio --project-version {{publish-version}} --doc=false
+	scala-cli --power publish local besom-zio --project-version {{besom-version}} --doc=false
 
 # Publishes locally all SDK modules: core, cats-effect extension, zio extension
 publish-local-sdk: publish-local-core publish-local-cats publish-local-zio
 
 # Publishes locally besom compiler plugin
 publish-local-compiler-plugin:
-	scala-cli --power publish local compiler-plugin --project-version {{publish-version}} --doc=false
+	scala-cli --power publish local compiler-plugin --project-version {{besom-version}} --doc=false
 
 # Cleans core build
 clean-core: 
@@ -128,7 +128,7 @@ clean-coverage: clean-sdk
 ####################
 
 # Builds .jar file with language plugin bootstrap library
-build-bootstrap:
+build-language-plugin-bootstrap:
 	mkdir -p {{language-plugin-output-dir}} && \
 	scala-cli --power package language-plugin/bootstrap --assembly -o {{language-plugin-output-dir}}/bootstrap.jar -f
 
@@ -139,21 +139,49 @@ build-language-host:
 	go build -o {{language-plugin-output-dir}}/pulumi-language-scala
 
 # Builds the entire scala language plugin
-build-language-plugin: build-bootstrap build-language-host
+build-language-plugin: build-language-plugin-bootstrap build-language-host
 
 # Runs the tests for the language plugin assuming it has already been built
 run-language-plugin-tests:
-	PULUMI_SCALA_PLUGIN_VERSION={{publish-version}} \
 	PULUMI_SCALA_PLUGIN_LOCAL_PATH={{language-plugin-output-dir}} \
 	scala-cli test language-plugin/tests/src
 
 # Builds and tests the language plugin
 test-language-plugin: build-language-plugin run-language-plugin-tests
 
 # Installs the scala language plugin locally
-install-language-plugin: build-language-plugin
+install-language-plugin:
+	#!/usr/bin/env sh
+	just build-language-plugin-bootstrap
+	just build-language-host
+	output_dir={{language-plugin-output-dir}}/local
+	rf -rf $output_dir
+	mkdir -p $output_dir
+	cp {{language-plugin-output-dir}}/bootstrap.jar $output_dir/
+	cp {{language-plugin-output-dir}}/pulumi-language-scala $output_dir/
 	pulumi plugin rm language scala -y
-	pulumi plugin install language scala {{publish-version}} --file {{language-plugin-output-dir}}
+	pulumi plugin install language scala {{besom-version}} --file {{language-plugin-output-dir}}/local
+
+# Package the scala language plugin for a given architecture
+package-language-plugin GOOS GOARCH:
+	#!/usr/bin/env sh
+	subdir={{ "dist/" + GOOS + "-" + GOARCH }}
+	output_dir={{language-plugin-output-dir}}/$subdir
+	mkdir -p $output_dir
+	just build-language-host $GOOS $GOARCH
+	cp {{language-plugin-output-dir}}/bootstrap.jar $output_dir/
+	cp {{language-plugin-output-dir}}/pulumi-language-scala $output_dir/
+	cd $output_dir
+	tar czvf pulumi-language-scala-v{{besom-version}}-{{GOOS}}-{{GOARCH}}.tar.gz *
+
+# Package the scala language plugin for all supported architectures
+package-language-plugins-all: build-language-plugin-bootstrap
+	just package-language-plugin darwin arm64
+	just package-language-plugin darwin amd64
+	just package-language-plugin linux arm64
+	just package-language-plugin linux amd64
+	just package-language-plugin windows arm64
+	just package-language-plugin windows amd64
 
 
 ####################
@@ -175,7 +203,7 @@ get-schema schema-name schema-version:
 
 # Generate scala API code for the given provider, e.g. `just generate-provider-sdk kubernetes`
 generate-provider-sdk schema-name schema-version:
-	scala-cli run codegen -- {{schemas-output-dir}} {{codegen-output-dir}} {{schema-name}} {{schema-version}}
+	scala-cli run codegen -- {{schemas-output-dir}} {{codegen-output-dir}} {{schema-name}} {{schema-version}} {{besom-version}}
 
 # Compiles the previously generated scala API code for the given provider, e.g. `just compile-provider-sdk kubernetes`
 compile-provider-sdk schema-name:

diff --git a/codegen/project.scala b/codegen/project.scala
@@ -1,4 +1,4 @@
 //> using scala "2.13.10"
 //> using lib "org.scalameta::scalameta:4.8.10"
-//> using lib "com.lihaoyi::upickle:3.1.2"
+//> using lib "com.lihaoyi::upickle:3.1.3"
 //> using lib "com.lihaoyi::os-lib:0.9.1"
diff --git a/codegen/src/CodeGen.scala b/codegen/src/CodeGen.scala
@@ -14,7 +14,7 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa
     "besom.types.Context"
   )
 
-  def sourcesFromPulumiPackage(pulumiPackage: PulumiPackage, besomVersion: String): Seq[SourceFile] = {
+  def sourcesFromPulumiPackage(pulumiPackage: PulumiPackage, schemaVersion: String, besomVersion: String): Seq[SourceFile] = {
     val scalaSources = (
       sourceFilesForProviderResource(pulumiPackage) ++
       sourceFilesForNonResourceTypes(pulumiPackage) ++
@@ -28,11 +28,8 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa
       ),
       resourcePluginMetadataFile(
         pluginName = pulumiPackage.name,
-        pluginVersion = pulumiPackage.version.getOrElse {
-          val defaultVersion = "0.0.1-SNAPSHOT"
-          logger.warn(s"Pulumi package version missing - defaulting to ${defaultVersion}")
-          defaultVersion
-        }
+        pluginVersion = schemaVersion,
+        pluginDownloadUrl = pulumiPackage.pluginDownloadURL
       ),
     )
   }
@@ -58,15 +55,20 @@ class CodeGen(implicit providerConfig: Config.ProviderConfig, typeMapper: TypeMa
     SourceFile(filePath = filePath, sourceCode = fileContent)
   }
 
-  def resourcePluginMetadataFile(pluginName: String, pluginVersion: String): SourceFile = {
+  def resourcePluginMetadataFile(pluginName: String, pluginVersion: String, pluginDownloadUrl: Option[String]): SourceFile = {
+    val pluginDownloadUrlJsonValue = pluginDownloadUrl match {
+      case Some(url) => s"\"${url}\""
+      case None => "null"
+    }
     val fileContent =
       s"""|{
           |  "resource": true,
           |  "name": "${pluginName}",
-          |  "version": "${pluginVersion}"
+          |  "version": "${pluginVersion}",
+          |  "server": ${pluginDownloadUrlJsonValue}
           |}
           |""".stripMargin
-    val filePath = FilePath(Seq("resources", "plugin.json"))
+    val filePath = FilePath(Seq("resources", "besom", "api", pluginName, "plugin.json"))
 
     SourceFile(filePath = filePath, sourceCode = fileContent)
   } 

diff --git a/codegen/src/Main.scala b/codegen/src/Main.scala
@@ -5,20 +5,21 @@ import java.util.Arrays
 object Main {
   def main(args: Array[String]): Unit = {
     args.toList match {
-      case schemasDirPath :: outputDirBasePath :: providerName :: schemaVersion :: Nil =>
+      case schemasDirPath :: outputDirBasePath :: providerName :: schemaVersion :: besomVersion :: Nil =>
         generatePackageSources(
           schemasDirPath = os.Path(schemasDirPath),
           outputDirBasePath = os.Path(outputDirBasePath),
           providerName = providerName,
-          schemaVersion = schemaVersion
+          schemaVersion = schemaVersion,
+          besomVersion = besomVersion
         )
       case _ =>
-        System.err.println("Codegen's expected arguments: <schemasDirPath> <outputDirBasePath> <providerName> <schemaVersion>")
+        System.err.println("Codegen's expected arguments: <schemasDirPath> <outputDirBasePath> <providerName> <schemaVersion> <besomVersion>")
         sys.exit(1)
     }
   }
 
-  def generatePackageSources(schemasDirPath: os.Path, outputDirBasePath: os.Path, providerName: String, schemaVersion: String): Unit = {
+  def generatePackageSources(schemasDirPath: os.Path, outputDirBasePath: os.Path, providerName: String, schemaVersion: String, besomVersion: String): Unit = {
     println(s"Generating provider SDK for $providerName")
 
     val schemaProvider = new SchemaProvider(schemaCacheDirPath = schemasDirPath)
@@ -44,7 +45,8 @@ object Main {
     try {
       codeGen.sourcesFromPulumiPackage(
         pulumiPackage,
-        besomVersion = "0.0.1-SNAPSHOT"
+        schemaVersion = schemaVersion,
+        besomVersion = besomVersion
       ).foreach { sourceFile =>
         val filePath = destinationDir / sourceFile.filePath.osSubPath
         os.makeDir.all(filePath / os.up)

diff --git a/codegen/src/PulumiPackage.scala b/codegen/src/PulumiPackage.scala
@@ -6,7 +6,16 @@ import upickle.implicits.{key => fieldKey}
 import besom.codegen.UpickleApi
 import besom.codegen.UpickleApi._
 
-case class PulumiPackage(name: String, version: Option[String] = None, language: Language = Language(), meta: Meta = Meta(), types: Map[String, TypeDefinition] = Map.empty, provider: ResourceDefinition, resources: Map[String, ResourceDefinition] = Map.empty)
+case class PulumiPackage(
+  name: String,
+  version: Option[String] = None,
+  language: Language = Language(),
+  meta: Meta = Meta(),
+  pluginDownloadURL: Option[String] = None,
+  types: Map[String, TypeDefinition] = Map.empty,
+  provider: ResourceDefinition,
+  resources: Map[String, ResourceDefinition] = Map.empty
+)
 object PulumiPackage {
   implicit val reader: Reader[PulumiPackage] = macroR
 

diff --git a/language-plugin/bootstrap/project.scala b/language-plugin/bootstrap/project.scala
@@ -1,5 +1,5 @@
 //> using scala "3.3.0"
 
 //> using lib "io.spray::spray-json:1.3.6"
-//> using lib "io.github.classgraph:classgraph:4.8.161"
+//> using lib "io.github.classgraph:classgraph:4.8.162"
 //> using options "-java-output-version:11"
diff --git a/language-plugin/tests/resources/executors/sbt/build.sbt b/language-plugin/tests/resources/executors/sbt/build.sbt
@@ -2,5 +2,8 @@ lazy val root = project
   .in(file("."))
   .settings(
     scalaVersion := "3.3.0",
-    libraryDependencies += "org.virtuslab" %% "besom-custom-resource-plugin" % "0.0.1-SNAPSHOT"
+    libraryDependencies ++= Seq(
+      "org.virtuslab" %% "besom-fake-standard-resource" % "1.2.3-TEST",
+      "org.virtuslab" %% "besom-fake-external-resource" % "2.3.4-TEST"
+    )
   )
diff --git a/language-plugin/tests/resources/executors/sbt/src/main/scala/Main.scala b/language-plugin/tests/resources/executors/sbt/src/main/scala/Main.scala
@@ -3,7 +3,8 @@ package besom.languageplugin.test.pulumiapp
 object Main {
   def main(args: Array[String]): Unit = {
     // Making sure the plugin is actually on the classpath
-    val x = besom.languageplugin.test.resourceplugin.customVal
+    val x = besom.languageplugin.test.resourceplugin.standard.customVal
+    val y = besom.languageplugin.test.resourceplugin.external.customVal
 
     // Throwing exception instead of printing because 'pulumi up' swallows stdout 
     throw new Exception("scala executor test got executed")

diff --git a/language-plugin/tests/resources/executors/scala-cli/Main.scala b/language-plugin/tests/resources/executors/scala-cli/Main.scala
@@ -3,7 +3,8 @@ package besom.languageplugin.test.pulumiapp
 object Main {
   def main(args: Array[String]): Unit = {
     // Making sure the plugin is actually on the classpath
-    val x = besom.languageplugin.test.resourceplugin.customVal
+    val x = besom.languageplugin.test.resourceplugin.standard.customVal
+    val y = besom.languageplugin.test.resourceplugin.external.customVal
 
     // Throwing exception instead of printing because 'pulumi up' swallows stdout 
     throw new Exception("scala executor test got executed")

diff --git a/language-plugin/tests/resources/executors/scala-cli/project.scala b/language-plugin/tests/resources/executors/scala-cli/project.scala
@@ -1,3 +1,4 @@
 //> using scala "3.3.0"
-//> using dep "org.virtuslab::besom-custom-resource-plugin:0.0.1-SNAPSHOT"
+//> using dep "org.virtuslab::besom-fake-standard-resource:1.2.3-TEST"
+//> using dep "org.virtuslab::besom-fake-external-resource:2.3.4-TEST"
 //> using options "-java-output-version:11"
diff --git a/...esources/custom-resource-plugin/Lib.scala → ...s/fake-external-resource-plugin/Lib.scala b/...esources/custom-resource-plugin/Lib.scala → ...s/fake-external-resource-plugin/Lib.scala
@@ -3,11 +3,11 @@
 //> using resourceDir "resources"
 
 //> using publish.organization "org.virtuslab"
-//> using publish.name "besom-custom-resource-plugin"
-//> using publish.version "0.0.1-SNAPSHOT"
+//> using publish.name "besom-fake-external-resource"
+//> using publish.version "2.3.4-TEST"
 
 //> using options -java-output-version:11
 
-package besom.languageplugin.test.resourceplugin
+package besom.languageplugin.test.resourceplugin.external
 
 val customVal = 123
diff --git a/...rces/fake-external-resource-plugin/resources/besom/api/fake-external-resource/plugin.json b/...rces/fake-external-resource-plugin/resources/besom/api/fake-external-resource/plugin.json
@@ -0,0 +1,6 @@
+{
+  "resource": true,
+  "name": "aci",
+  "version": "0.0.6",
+  "server": "github://api.github.com/netascode/pulumi-aci"
+}
diff --git a/language-plugin/tests/resources/fake-standard-resource-plugin/Lib.scala b/language-plugin/tests/resources/fake-standard-resource-plugin/Lib.scala
@@ -0,0 +1,15 @@
+//> using scala "3.3.0"
+
+//> using resourceDir "resources"
+
+//> using publish.organization "org.virtuslab"
+//> using publish.name "besom-fake-standard-resource"
+//> using publish.version "1.2.3-TEST"
+
+//> using dep "org.virtuslab::besom-core:0.0.1-beta"
+
+//> using options -java-output-version:11
+
+package besom.languageplugin.test.resourceplugin.standard
+
+val customVal = 123
diff --git a/...sources/besom/custom-resource/plugin.json → ...om/api/fake-standard-resource/plugin.json b/...sources/besom/custom-resource/plugin.json → ...om/api/fake-standard-resource/plugin.json
@@ -1,5 +1,6 @@
 {
   "resource": true,
   "name": "random",
-  "version": "4.3.1"
+  "version": "4.3.1",
+  "server": null
 }
diff --git a/language-plugin/tests/src/LanguagePluginExecutorTest.scala b/language-plugin/tests/src/LanguagePluginExecutorTest.scala
@@ -12,19 +12,26 @@ class LanguagePluginExecutorTest extends munit.FunSuite {
   def getEnvVar(name: String) =
     sys.env.get(name).getOrElse(throw new Exception(s"Environment variable $name is not set"))
 
-  def publishLocalCustomResourcePlugin() =
+  def publishLocalResourcePlugin(pluginName: String) =
     val tmpBuildDir = os.temp.dir()
-    os.list(resourcesDir / "custom-resource-plugin").foreach(file => os.copy.into(file, tmpBuildDir))
+    os.list(resourcesDir / pluginName).foreach(file => os.copy.into(file, tmpBuildDir))
     os.proc("scala-cli", "--power", "publish", "local", ".").call(cwd = tmpBuildDir)
 
+  def publishLocalResourcePlugins() =
+    publishLocalResourcePlugin("fake-standard-resource-plugin")
+    publishLocalResourcePlugin("fake-external-resource-plugin")
+
   def testExecutor(executorDir: os.Path) =
-    val tmpPulumiHome        = os.temp.dir()
-    val env                  = Map("PULUMI_CONFIG_PASSPHRASE" -> "")
+    val tmpPulumiHome        = os.temp.dir().toString
+    val env                  = Map(
+      "PULUMI_CONFIG_PASSPHRASE" -> "",
+      "PULUMI_HOME" -> tmpPulumiHome
+    )
     val stackName            = "organization/language-plugin-test/executor-test"
-    val scalaPluginVersion   = getEnvVar("PULUMI_SCALA_PLUGIN_VERSION")
+    val scalaPluginVersion   = "0.0.1-TEST"
     val scalaPluginLocalPath = getEnvVar("PULUMI_SCALA_PLUGIN_LOCAL_PATH")
 
-    os.proc("pulumi", "login", s"file://${tmpPulumiHome}").call()
+    os.proc("pulumi", "login", s"file://${tmpPulumiHome}").call(env = env)
     os.proc(
       "pulumi",
       "plugin",
@@ -35,7 +42,7 @@ class LanguagePluginExecutorTest extends munit.FunSuite {
       "--file",
       scalaPluginLocalPath,
       "--reinstall"
-    ).call()
+    ).call(env = env)
     os.proc("pulumi", "stack", "--non-interactive", "init", stackName).call(cwd = executorDir, env = env)
     val pulumiUpOutput = os
       .proc("pulumi", "up", "--non-interactive", "--stack", stackName, "--skip-preview")
@@ -47,23 +54,38 @@ class LanguagePluginExecutorTest extends munit.FunSuite {
 
     assert(clue(pulumiUpOutput).contains(expectedError))
 
-    val aboutInfoLines = os.proc("pulumi", "about").call(cwd = executorDir, env = env).out.lines()
-    val pluginsInfo = aboutInfoLines
+    val aboutInfoLines = os.proc("pulumi", "about").call(cwd = executorDir, env = env).out.lines().toList
+
+    val aboutPluginsVersions = aboutInfoLines
       .dropWhile(_ != "Plugins") // Find the plugins section
       .drop(2) // Drop headers
       .takeWhile(_.nonEmpty) // Empty line separates sections
       .map { line =>
         val lineParts = line.split("""\s+""") // Parse each plugin line splitting on whitespace
-        lineParts(0) -> lineParts(1) // plugin name -> plugin version
+        lineParts(0) -> lineParts(1) // plugin_name -> plugin_version from extected format:   NAME    VERSION
+      }
+      .toMap
+
+    val expectedAboutPluginsVersions = Map("scala" -> "unknown", "random" -> "4.3.1", "aci" -> "0.0.6")
+
+    assert(clue(aboutPluginsVersions) == expectedAboutPluginsVersions)
+
+    val pluginsLsLines = os.proc("pulumi", "plugin", "ls").call(cwd = executorDir, env = env).out.lines().toList
+    val installedPluginsVersions = pluginsLsLines
+      .drop(1) // Drop headers
+      .takeWhile(_.nonEmpty) // Empty line separates sections
+      .map { line =>
+        val lineParts = line.split("""\s+""") // Parse each plugin line splitting on whitespace
+        lineParts(0) -> lineParts(2) // plugin_name -> plugin_version from extected format:   NAME   KIND      VERSION     SIZE   INSTALLED   LAST USED
       }
       .toMap
 
-    val expectedPluginsInfo = Map("scala" -> "unknown", "random" -> "4.3.1")
+    val expectedInstalledPluginsVersions = Map("scala" -> scalaPluginVersion, "random" -> "4.3.1", "aci" -> "0.0.6")
 
-    assert(clue(pluginsInfo) == expectedPluginsInfo)
+    assert(clue(installedPluginsVersions) == expectedInstalledPluginsVersions)
 
   override def beforeAll() =
-    publishLocalCustomResourcePlugin()
+    publishLocalResourcePlugins()
 
   override def afterAll() =
     os.proc("pulumi", "logout").call()