Change over to using scala.BigDecimal to back the Decimal type

finos · DamianReeves · Feb 14, 2021 · Feb 14, 2021 · Feb 14, 2021 · Feb 14, 2021
commit f6f8dec0c68b37c4e48afd29a9a73e8dbda7bfe3
diff --git a/build.sc b/build.sc
@@ -36,7 +36,7 @@ object Deps {
     val zioLogging             = "0.5.6"
     val zioMagic               = "0.1.8"
     val zioNio                 = "1.0.0-RC10"
-    val zioPrelude             = "1.0.0-RC1"
+    val zioPrelude             = "1.0.0-RC2"
     val zioProcess             = "0.2.0"
     val newtype                = "0.4.4"
     def decline(scalaVersion: String) = scalaVersion match {
@@ -52,6 +52,7 @@ object Deps {
     val slf4zio       = "1.0.0"
     val scalactic     = "3.1.2"
     val scalaUri      = "2.2.2"
+    val spark         = "2.4.7"
     val oslib         = "0.6.2"
     val quill         = "3.6.0-RC3"
   }
@@ -220,28 +221,28 @@ object morphir extends Module {
       }
     }
   }
-
-  object scala extends Module {
-
-    object jvm extends Cross[JvmMorphirScalaModule](Versions.scala213)
-    class JvmMorphirScalaModule(val crossScalaVersion: String)
-        extends CrossScalaModule
-        with CommonJvmModule
-        with ScalaMacroModule
-        with MorphirPublishModule { self =>
-      def artifactName = "morphir-scala"
-      def moduleDeps   = Seq(morphir.ir.jvm(crossScalaVersion))
-
-      def ivyDeps = Agg(
-        ivy"org.scalameta::scalameta:${Versions.scalameta}"
-      )
-
-      object test extends Tests {
-        def platformSegment: String = self.platformSegment
-        def crossScalaVersion       = JvmMorphirScalaModule.this.crossScalaVersion
-      }
-    }
-  }
+//
+//  object scala extends Module {
+//
+//    object jvm extends Cross[JvmMorphirScalaModule](Versions.scala213)
+//    class JvmMorphirScalaModule(val crossScalaVersion: String)
+//        extends CrossScalaModule
+//        with CommonJvmModule
+//        with ScalaMacroModule
+//        with MorphirPublishModule { self =>
+//      def artifactName = "morphir-scala"
+//      def moduleDeps   = Seq(morphir.ir.jvm(crossScalaVersion))
+//
+//      def ivyDeps = Agg(
+//        ivy"org.scalameta::scalameta:${Versions.scalameta}"
+//      )
+//
+//      object test extends Tests {
+//        def platformSegment: String = self.platformSegment
+//        def crossScalaVersion       = JvmMorphirScalaModule.this.crossScalaVersion
+//      }
+//    }
+//  }
   object sdk extends Module {
 
     object core extends Module {
@@ -263,6 +264,42 @@ object morphir extends Module {
         }
       }
     }
+
+    object spark extends Module {
+      object jvm
+          extends Cross[JvmMorphirSdkSpark](
+            Versions.scala212,
+            Versions.scala211
+          )
+      class JvmMorphirSdkSpark(val crossScalaVersion: String)
+          extends CrossScalaModule
+          with CommonJvmModule
+          with MorphirPublishModule { self =>
+
+        def artifactName = "morphir-sdk-spark"
+        def compileIvyDeps = Agg(
+          ivy"org.apache.spark::spark-sql:2.4.7",
+          ivy"com.github.ghik:::silencer-lib:${Versions.silencer}"
+        )
+        def ivyDeps = Agg(
+          ivy"dev.zio::zio-prelude:${Versions.zioPrelude}"
+        )
+        def scalacPluginIvyDeps = Agg(ivy"com.github.ghik:::silencer-plugin:${Versions.silencer}")
+        def moduleDeps          = Seq(morphir.sdk.core.jvm(crossScalaVersion))
+
+        object test extends Tests {
+          def platformSegment: String = self.platformSegment
+          def crossScalaVersion       = JvmMorphirSdkSpark.this.crossScalaVersion
+
+          override def ivyDeps = super.ivyDeps() ++
+            Agg(
+              ivy"dev.zio::zio-logging:${Versions.zioLogging}",
+              ivy"dev.zio::zio-logging-slf4j:${Versions.zioLogging}",
+              ivy"org.apache.spark::spark-sql:2.4.7"
+            )
+        }
+      }
+    }
   }
 
   object flowz extends Module {

diff --git a/morphir/sdk/core/src-2.11/morphir/sdk/DecimalModuleCompat.scala b/morphir/sdk/core/src-2.11/morphir/sdk/DecimalModuleCompat.scala
@@ -0,0 +1,14 @@
+package morphir.sdk
+
+trait DecimalModuleCompat {
+  import DecimalModuleCompat._
+  implicit def toBigDecimalOps(value: BigDecimal): BigDecimalOps =
+    new BigDecimalOps(value)
+}
+
+object DecimalModuleCompat {
+  class BigDecimalOps(private val self: BigDecimal) extends AnyVal {
+    def compareTo(that: BigDecimal): Int =
+      self.compare(that)
+  }
+}
diff --git a/morphir/sdk/core/src-2.12/morphir/sdk/DecimalModuleCompat.scala b/morphir/sdk/core/src-2.12/morphir/sdk/DecimalModuleCompat.scala
@@ -0,0 +1,3 @@
+package morphir.sdk
+
+trait DecimalModuleCompat {}
diff --git a/morphir/sdk/core/src-2.13/morphir/sdk/DecimalModuleCompat.scala b/morphir/sdk/core/src-2.13/morphir/sdk/DecimalModuleCompat.scala
@@ -0,0 +1,3 @@
+package morphir.sdk
+
+trait DecimalModuleCompat {}
diff --git a/morphir/sdk/core/src/morphir/sdk/Decimal.scala b/morphir/sdk/core/src/morphir/sdk/Decimal.scala
@@ -2,16 +2,16 @@ package morphir.sdk
 import morphir.sdk.Maybe.Maybe
 import morphir.sdk.Basics.Order
 
-import java.math.{ BigDecimal => BigDec, RoundingMode }
+import java.math.{ BigDecimal => BigDec }
 import scala.util.control.NonFatal
 
-object Decimal {
+object Decimal extends DecimalModuleCompat {
 
-  type Decimal = BigDec
+  type Decimal = BigDecimal
 
   object Decimal {
-    def apply(value: BigDec): Decimal                  = value
-    def apply(value: scala.BigDecimal): Decimal        = value.bigDecimal
+    def apply(value: BigDec): Decimal                  = BigDecimal(value)
+    def apply(value: scala.BigDecimal): Decimal        = value
     def apply(value: morphir.sdk.Float.Float): Decimal = BigDecimal.exact(value).bigDecimal
     def apply(value: morphir.sdk.Int.Int): Decimal     = BigDecimal(value).bigDecimal
 
@@ -20,9 +20,9 @@ object Decimal {
   /**
    * Absolute value (sets the sign as positive)
    */
-  def abs(value: Decimal): Decimal = value.abs()
+  def abs(value: Decimal): Decimal = value.abs
 
-  def add(a: Decimal)(b: Decimal): Decimal = a.add(b)
+  def add(a: Decimal)(b: Decimal): Decimal = a + b
 
   def bps(n: morphir.sdk.Int.Int): Decimal = Decimal(n * 0.0001)
 
@@ -37,7 +37,7 @@ object Decimal {
     if (b.compareTo(zero) == 0) Maybe.nothing
     else
       try {
-        Maybe.just(a.divide(b))
+        Maybe.just(a / b)
       } catch {
         case NonFatal(_) => Maybe.nothing
       }
@@ -83,39 +83,39 @@ object Decimal {
   def millionth(n: morphir.sdk.Int.Int): Decimal =
     Decimal(n * 0.000001)
 
-  def mul(a: Decimal)(b: Decimal): Decimal = a.multiply(b)
+  def mul(a: Decimal)(b: Decimal): Decimal = a * b
 
   @inline def ne(a: Decimal)(b: Decimal): morphir.sdk.Bool.Bool = neq(a)(b)
   def neq(a: Decimal)(b: Decimal): morphir.sdk.Bool.Bool        = a.compareTo(b) != 0
 
-  def negate(value: Decimal): Decimal = value.negate()
+  def negate(value: Decimal): Decimal = -value
 
   def round(decimal: Decimal): Decimal = {
-    val scale = decimal.scale()
-    decimal.setScale(scale, RoundingMode.HALF_EVEN)
+    val scale = decimal.scale
+    decimal.setScale(scale, BigDecimal.RoundingMode.HALF_EVEN)
   }
 
   def shiftDecimalLeft(n: morphir.sdk.Int.Int)(value: Decimal): Decimal =
-    value.scaleByPowerOfTen(-n.intValue()) //TODO: When we align Int to Int this should settle in correctly
+    value.bigDecimal.scaleByPowerOfTen(-n.intValue()) //TODO: When we align Int to Int this should settle in correctly
 
   def shiftDecimalRight(n: morphir.sdk.Int.Int)(value: Decimal): Decimal =
-    value.scaleByPowerOfTen(n.intValue()) //TODO: When we align Int to Int this should settle in correctly
+    value.bigDecimal.scaleByPowerOfTen(n.intValue()) //TODO: When we align Int to Int this should settle in correctly
 
-  def sub(a: Decimal)(b: Decimal): Decimal = a.subtract(b)
+  def sub(a: Decimal)(b: Decimal): Decimal = a - b
 
   def thousand(n: morphir.sdk.Int.Int): Decimal =
     Decimal(n * 1000)
 
   def toFloat(value: Decimal): morphir.sdk.Float.Float =
-    morphir.sdk.Float.Float(value.doubleValue())
+    morphir.sdk.Float.Float(value.toDouble)
 
   //TODO: Make sure the Elm call and this call return the same value
   def toString(value: Decimal): morphir.sdk.String.String = value.toString
 
   def truncate(decimal: Decimal): Decimal = {
     // Since morphir's Int is actually a Long this isn't really safe
-    val scale = decimal.scale()
-    decimal.setScale(scale, RoundingMode.DOWN)
+    val scale = decimal.scale
+    decimal.setScale(scale, BigDecimal.RoundingMode.DOWN)
   }
 
   /**

diff --git a/morphir/sdk/spark/test/src/morphir/sdk/SdkTypesEncodingSpec.scala b/morphir/sdk/spark/test/src/morphir/sdk/SdkTypesEncodingSpec.scala
@@ -0,0 +1,33 @@
+package morphir.sdk
+
+import zio.test._
+import zio.test.Assertion._
+import zio.{ console, ZIO }
+import morphir.sdk.spark.testing.SparkSpec
+import morphir.sdk.spark.testing.sparkModule
+object SdkTypesEncodingSpec extends SparkSpec {
+  def spec = suite("SdkTypesEncoding Spec")(
+    testM("Encoding should work for a row with a Decimal")(
+      for {
+        data    <- ZIO.succeed(List(MyRow(MyDecimal(BigDecimal(1.1))), MyRow(MyDecimal(BigDecimal(3.14)))))
+        dataset <- sparkModule.createDataset(data)
+        actual <- sparkModule { _ =>
+                    dataset.collect().toList
+                  }
+        _ <- console.putStrLn(s"Actual: $actual")
+      } yield assert(actual)(equalTo(data))
+    )
+  )
+
+  //import zio.prelude._
+
+  //object MyDecimal extends Subtype[scala.BigDecimal]
+  //type MyDecimal = MyDecimal.Type
+
+  object MyDecimal {
+    def apply(value: BigDecimal): MyDecimal = value
+  }
+  type MyDecimal = BigDecimal
+
+  final case class MyRow(amount: MyDecimal)
+}
diff --git a/morphir/sdk/spark/test/src/morphir/sdk/spark/testing/SparkSpec.scala b/morphir/sdk/spark/test/src/morphir/sdk/spark/testing/SparkSpec.scala
@@ -0,0 +1,50 @@
+package morphir.sdk.spark.testing
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.SparkSession
+import zio.duration.durationInt
+import zio.logging._
+import zio.logging.slf4j.Slf4jLogger
+import zio.test.{ RunnableSpec, TestAspect, TestExecutor, TestRunner }
+import zio.test.environment.{ TestEnvironment, testEnvironment }
+import zio.{ ULayer, ZLayer }
+import morphir.sdk.spark.testing.sparkModule.SparkModule
+import SparkSpec.SparkTestingEnv
+
+abstract class SparkSpec extends RunnableSpec[SparkTestingEnv, Any] {
+  val sparkSessionBuilder: SparkSession.Builder = SparkSpec.sparkSessionBuilder
+
+  val sparkTestingLayer: ULayer[SparkTestingEnv] = {
+    val logFormat = "[correlation-id = %s] %s"
+    val logging = Slf4jLogger.make { (context, message) =>
+      val correlationId = LogAnnotation.CorrelationId.render(
+        context.get(LogAnnotation.CorrelationId)
+      )
+      logFormat.format(correlationId, message)
+    }
+
+    val spark =
+      SparkModule.buildLayer(sparkSessionBuilder, newSession = true)
+    ZLayer.succeed(sparkSessionBuilder) >>> SparkModule.fromSparkSessionBuilder
+
+    testEnvironment ++ logging ++ spark
+  }.orDie
+
+  override def aspects: List[TestAspect[Nothing, SparkTestingEnv, Nothing, Any]] =
+    List(TestAspect.timeout(60.seconds))
+
+  override def runner: TestRunner[SparkTestingEnv, Any] =
+    TestRunner(TestExecutor.default(sparkTestingLayer))
+}
+object SparkSpec {
+  val sparkSessionBuilder: SparkSession.Builder = {
+    val sparkConf = new SparkConf()
+      .setMaster("local[*]")
+      .setAppName("spark-spec")
+      .set("spark.ui.enabled", "false")
+      .set("spark.driver.host", "localhost")
+    SparkSession.builder().config(sparkConf)
+  }
+  type SparkTestingEnv = TestEnvironment with Logging with SparkModule
+
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		package morphir.sdk

		trait DecimalModuleCompat {}