Skip to content

Commit

Permalink
Refactoring of code responsible for translating to/from CellType defi…
Browse files Browse the repository at this point in the history 
…nitions

and string representations, particularly for JSON serialization. Addresses issues
outlined in #2166, where specific NoData values that are negative, or ones
for unsigned CellTypes that overflow JVM native encodings (e.g. uint8ud255).
As a part of the refactoring, expanded the tests around edge cases associated
with numeric bounds. The only public API change is the addition of the
method `UserDefinedNoData.widenedNoData`.

Signed-off-by: Simeon H.K. fitch <fitch@astraea.io>
  • Loading branch information
@metasim
metasim committed Apr 28, 2017
1 parent 261aa3c commit 405ae64
Show file tree
Hide file tree
Showing 4 changed files with 222 additions and 144 deletions.
97 changes: 44 additions & 53 deletions raster-test/src/test/scala/geotrellis/raster/CellTypeSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,111 +134,102 @@ class CellTypeSpec extends FunSpec with Matchers with Inspectors {

describe("CellType Bounds checking") {

// it("should serialize uint8ud with various ND values") {
//
// val ndVals = Seq(0, 1, Byte.MaxValue/2, Byte.MaxValue - 1, Byte.MaxValue, Byte.MaxValue * 2)
// forEvery(ndVals) { nd ⇒
// roundTrip(UByteUserDefinedNoDataCellType(nd.toByte))
// }
// }
//
// it("should serialize int8ud with various ND values") {
//
// val ndVals = Seq(0, 1, 127, 128, 255)
// forEvery(ndVals) { nd ⇒
// roundTrip(ByteUserDefinedNoDataCellType(nd.toByte))
// }
// }



implicit val doubleAsIntegral = scala.math.Numeric.DoubleAsIfIntegral
implicit val floatAsIntegral = scala.math.Numeric.FloatAsIfIntegral


it("should handle encoding no data types across valid bounds") {
forEvery(CellDef.all) { cellDef ⇒
val cd = cellDef.asInstanceOf[CellDef[AnyVal]]
forEvery(cd.testPoints) { nd ⇒
val ct = cd(nd)
assert(cd.toCode(nd) === ct.name)
roundTrip(ct)
type PhantomCell = AnyVal
type PhantomNoData = AnyVal

forEvery(CellDef.all) { cd ⇒
val cellDef = cd.asInstanceOf[CellDef[PhantomCell, PhantomNoData]]
withClue("Cell type " + cellDef) {
val range = cellDef.range
forEvery(cellDef.range.testPoints) { nd ⇒
withClue("No data " + nd) {
val ct = cellDef(nd)
roundTrip(ct)
// assert(cellDef.toCode(nd) === ct.name)
// println(ct.widenedNoData(cellDef.alg))
}
}
}
}
}

abstract class RangeAlgebra[T: Integral] {
protected val alg = implicitly[Integral[T]]
val alg = implicitly[Integral[T]]
import alg._
val one = alg.one
val twice = one + one
val half = one / twice
}

case class TestRange[Encoding: Integral](min: Encoding, max: Encoding) extends RangeAlgebra[Encoding]{
import alg._
def width = max - min
def middle = width * half
def middle = width / twice
def testPoints = Seq(
min, min + one, middle - one, middle, middle + one, max - one, max
)
}

abstract class CellDef[T: Integral] extends RangeAlgebra[T] {
import alg._
type Encoding = T
val rng: TestRange[Encoding]
abstract class CellDef[CellEncoding: Integral, NoDataEncoding: Integral] extends RangeAlgebra[CellEncoding] {
val range: TestRange[NoDataEncoding]
val baseCode: String
def apply(noData: Encoding): CellType with UserDefinedNoData[_]
def toCode(noData: Encoding): String = {
def apply(noData: NoDataEncoding): CellType with UserDefinedNoData[CellEncoding]
def toCode(noData: NoDataEncoding): String = {
s"${baseCode}ud${noData}"
}
def testPoints = Seq(
rng.min, rng.min + one, rng.middle - one, rng.middle, rng.middle + one, rng.max - one, rng.max
)
override def toString = getClass.getSimpleName

override def toString = baseCode
}
object CellDef {
val all = Seq(UByteDef, ByteDef, UShortDef, ShortDef, IntDef, FloatDef, DoubleDef)
}

object UByteDef extends CellDef[Short] {
object UByteDef extends CellDef[Byte, Short] {
val baseCode = "uint8"
def apply(noData: Short) = UByteUserDefinedNoDataCellType(noData.toByte)
val rng = TestRange(0.toShort, (Byte.MaxValue * 2).toShort)
val range = TestRange(0.toShort, (Byte.MaxValue * 2).toShort)
}

object ByteDef extends CellDef[Byte] {
object ByteDef extends CellDef[Byte, Byte] {
val baseCode = "int8"
def apply(noData: Byte) = ByteUserDefinedNoDataCellType(noData.toByte)
val rng = TestRange(Byte.MinValue, Byte.MaxValue)
val range = TestRange(Byte.MinValue, Byte.MaxValue)
}

object UShortDef extends CellDef[Int] {
object UShortDef extends CellDef[Short, Int] {
val baseCode = "uint16"
def apply(noData: Int) = UShortUserDefinedNoDataCellType(noData.toShort)
val rng = TestRange(0, Short.MaxValue * 2)
val range = TestRange(0, Short.MaxValue * 2)
}

object ShortDef extends CellDef[Short] {
object ShortDef extends CellDef[Short, Short] {
val baseCode = "int16"
def apply(noData: Short) = ShortUserDefinedNoDataCellType(noData)
val rng = TestRange(Short.MinValue, Short.MaxValue)
val range = TestRange(Short.MinValue, Short.MaxValue)
}

object IntDef extends CellDef[Int] {
object IntDef extends CellDef[Int, Int] {
val baseCode = "int32"
def apply(noData: Int) = IntUserDefinedNoDataCellType(noData)
val rng = TestRange(Int.MinValue, Int.MaxValue)
val range = TestRange(Int.MinValue, Int.MaxValue)
}

object FloatDef extends CellDef[Float] {
object FloatDef extends CellDef[Float, Float] {
val baseCode = "float32"
def apply(noData: Float) = FloatUserDefinedNoDataCellType(noData)
val rng = TestRange(Float.MinValue, Float.MaxValue)
val range = new TestRange(Float.MinValue, Float.MaxValue) {
override def middle = 0.0f
}
}

object DoubleDef extends CellDef[Double] {
object DoubleDef extends CellDef[Double, Double] {
val baseCode = "float64"
def apply(noData: Double) = DoubleUserDefinedNoDataCellType(noData)
val rng = TestRange(Double.MinValue, Double.MaxValue)
val range = new TestRange(Double.MinValue, Double.MaxValue) {
override def middle = 0.0
}
}
}

Expand Down
156 changes: 69 additions & 87 deletions raster/src/main/scala/geotrellis/raster/CellType.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,13 @@ sealed trait NoNoData extends NoDataHandling { cellType: CellType => }
* The [[UserDefinedNoData]] type, derived from [[NoDataHandling]].
*/
sealed trait UserDefinedNoData[@specialized(Byte, Short, Int, Float, Double) T] extends NoDataHandling { cellType: CellType =>
/** The no data value as represented in the JVM in the underlying cell. If unsigned types are involved then
* this value may be an overflow representation, and `widenedNoData` should be used.*/
val noDataValue: T

def widenedNoData(implicit ev: Numeric[T]): WidenedNoData =
if (cellType.isFloatingPoint) WideDoubleNoData(ev.toDouble(noDataValue))
else WideIntNoData(ev.toInt(noDataValue))
}

/**
Expand All @@ -344,7 +350,9 @@ case object UByteCellType
case object UByteConstantNoDataCellType
extends UByteCells with ConstantNoData
case class UByteUserDefinedNoDataCellType(noDataValue: Byte)
extends UByteCells with UserDefinedNoData[Byte]
extends UByteCells with UserDefinedNoData[Byte] {
override def widenedNoData(implicit ev: Numeric[Byte]) = WideIntNoData(noDataValue & 0xFF)
}

case object ShortCellType
extends ShortCells with NoNoData
Expand All @@ -358,7 +366,9 @@ case object UShortCellType
case object UShortConstantNoDataCellType
extends UShortCells with ConstantNoData
case class UShortUserDefinedNoDataCellType(noDataValue: Short)
extends UShortCells with UserDefinedNoData[Short]
extends UShortCells with UserDefinedNoData[Short] {
override def widenedNoData(implicit ev: Numeric[Short]) = WideIntNoData(noDataValue & 0xFFFF)
}

case object IntCellType
extends IntCells with NoNoData
Expand All @@ -381,9 +391,8 @@ case object DoubleConstantNoDataCellType
case class DoubleUserDefinedNoDataCellType(noDataValue: Double)
extends DoubleCells with UserDefinedNoData[Double]


// No NoData
object CellType {
import CellTypeEncoding._

/**
* Translate a string representing a cell type into a [[CellType]].
Expand All @@ -400,65 +409,37 @@ object CellType {
* @param name A string representing a cell type, as reported by [[DataType.name]] e.g. "uint32"
* @return The CellType corresponding to `name`
*/
def fromName(name: String): CellType = name match {
case "bool" | "boolraw" => BitCellType // No NoData values
case "int8raw" => ByteCellType
case "uint8raw" => UByteCellType
case "int16raw" => ShortCellType
case "uint16raw" => UShortCellType
case "float32raw" => FloatCellType
case "float64raw" => DoubleCellType
case "int8" => ByteConstantNoDataCellType // Constant NoData values
case "uint8" => UByteConstantNoDataCellType
case "int16" => ShortConstantNoDataCellType
case "uint16" => UShortConstantNoDataCellType
case "int32" => IntConstantNoDataCellType
case "int32raw" => IntCellType
case "float32" => FloatConstantNoDataCellType
case "float64" => DoubleConstantNoDataCellType
case ct if ct.startsWith("int8ud") =>
val ndVal = new Regex("\\d+$").findFirstIn(ct).getOrElse {
throw new IllegalArgumentException(s"Cell type $name is not supported")
}
ByteUserDefinedNoDataCellType(ndVal.toByte)
case ct if ct.startsWith("uint8ud") =>
val ndVal = new Regex("\\d+$").findFirstIn(ct).getOrElse {
throw new IllegalArgumentException(s"Cell type $name is not supported")
}
UByteUserDefinedNoDataCellType(ndVal.toByte)
case ct if ct.startsWith("int16ud") =>
val ndVal = new Regex("\\d+$").findFirstIn(ct).getOrElse {
def fromName(name: String): CellType = {
name match {
case bool() | boolraw() => BitCellType // No NoData values
case int8raw() => ByteCellType
case uint8raw() => UByteCellType
case int16raw() => ShortCellType
case uint16raw() => UShortCellType
case float32raw() => FloatCellType
case float64raw() => DoubleCellType
case int8() => ByteConstantNoDataCellType // Constant NoData values
case uint8() => UByteConstantNoDataCellType
case int16() => ShortConstantNoDataCellType
case uint16() => UShortConstantNoDataCellType
case int32() => IntConstantNoDataCellType
case int32raw() => IntCellType
case float32() => FloatConstantNoDataCellType
case float64() => DoubleConstantNoDataCellType
case int8ud(nd) => ByteUserDefinedNoDataCellType(nd.asInt.toByte)
case uint8ud(nd) => UByteUserDefinedNoDataCellType(nd.asInt.toByte)
case int16ud(nd) => ShortUserDefinedNoDataCellType(nd.asInt.toShort)
case uint16ud(nd) => UShortUserDefinedNoDataCellType(nd.asInt.toShort)
case int32ud(nd) => IntUserDefinedNoDataCellType(nd.asInt)
case float32ud(nd) =>
if (nd.asDouble.isNaN) FloatConstantNoDataCellType
else FloatUserDefinedNoDataCellType(nd.asDouble.toFloat)
case float64ud(nd) =>
if (nd.asDouble.isNaN) DoubleConstantNoDataCellType
else DoubleUserDefinedNoDataCellType(nd.asDouble)
case _ =>
throw new IllegalArgumentException(s"Cell type $name is not supported")
}
ShortUserDefinedNoDataCellType(ndVal.toShort)
case ct if ct.startsWith("uint16ud") =>
val ndVal = new Regex("\\d+$").findFirstIn(ct).getOrElse {
throw new IllegalArgumentException(s"Cell type $name is not supported")
}
UShortUserDefinedNoDataCellType(ndVal.toShort)
case ct if ct.startsWith("int32ud") =>
val ndVal = new Regex("\\d+$").findFirstIn(ct).getOrElse {
throw new IllegalArgumentException(s"Cell type $name is not supported")
}
IntUserDefinedNoDataCellType(ndVal.toInt)
case ct if ct.startsWith("float32ud") =>
try {
val ndVal = ct.stripPrefix("float32ud").toDouble.toFloat
if (ndVal.isNaN) FloatConstantNoDataCellType
else FloatUserDefinedNoDataCellType(ndVal)
} catch {
case _: NumberFormatException => throw new IllegalArgumentException(s"Cell type $name is not supported")
}
case ct if ct.startsWith("float64ud") =>
try {
val ndVal = ct.stripPrefix("float64ud").toDouble
if (ndVal.isNaN) DoubleConstantNoDataCellType
else DoubleUserDefinedNoDataCellType(ndVal)
} catch {
case _: NumberFormatException => throw new IllegalArgumentException(s"Cell type $name is not supported")
}
case str =>
throw new IllegalArgumentException(s"Cell type $name is not supported")
}
}

/**
Expand All @@ -468,32 +449,33 @@ object CellType {
* @return String representation
*/
def toName(cellType: CellType): String = {
def forUserDefined[T <: AnyVal](base: CellType, value: T) = base.name + "ud" + value.toString

cellType match {
case BitCellType => "bool"
case ByteCellType => "int8raw"
case UByteCellType => "uint8raw"
case ShortCellType => "int16raw"
case UShortCellType => "uint16raw"
case IntCellType => "int32raw"
case FloatCellType => "float32raw"
case DoubleCellType => "float64raw"
case ByteConstantNoDataCellType => "int8"
case UByteConstantNoDataCellType => "uint8"
case ShortConstantNoDataCellType => "int16"
case UShortConstantNoDataCellType => "uint16"
case IntConstantNoDataCellType => "int32"
case FloatConstantNoDataCellType => "float32"
case DoubleConstantNoDataCellType => "float64"
case ByteUserDefinedNoDataCellType(nd) => forUserDefined(ByteConstantNoDataCellType, nd)
case UByteUserDefinedNoDataCellType(nd) => forUserDefined(UByteConstantNoDataCellType, nd)
case ShortUserDefinedNoDataCellType(nd) => forUserDefined(ShortConstantNoDataCellType, nd)
case UShortUserDefinedNoDataCellType(nd) => forUserDefined(UShortConstantNoDataCellType, nd)
case IntUserDefinedNoDataCellType(nd) => forUserDefined(IntConstantNoDataCellType, nd)
case FloatUserDefinedNoDataCellType(nd) => forUserDefined(FloatConstantNoDataCellType, nd)
case DoubleUserDefinedNoDataCellType(nd) => forUserDefined(DoubleConstantNoDataCellType, nd)

val encoding = cellType match {
case BitCellType => bool
case ByteCellType => int8raw
case UByteCellType => uint8raw
case ShortCellType => int16raw
case UShortCellType => uint16raw
case IntCellType => int32raw
case FloatCellType => float32raw
case DoubleCellType => float64raw
case ByteConstantNoDataCellType => int8
case UByteConstantNoDataCellType => uint8
case ShortConstantNoDataCellType => int16
case UShortConstantNoDataCellType => uint16
case IntConstantNoDataCellType => int32
case FloatConstantNoDataCellType => float32
case DoubleConstantNoDataCellType => float64
case ct: ByteUserDefinedNoDataCellType => int8ud(ct.widenedNoData.asInt)
case ct: UByteUserDefinedNoDataCellType => uint8ud(ct.widenedNoData.asInt)
case ct: ShortUserDefinedNoDataCellType => int16ud(ct.widenedNoData.asInt)
case ct: UShortUserDefinedNoDataCellType => uint16ud(ct.widenedNoData.asInt)
case ct: IntUserDefinedNoDataCellType => int32ud(ct.widenedNoData.asInt)
case ct: FloatUserDefinedNoDataCellType => float32ud(ct.widenedNoData.asDouble)
case ct: DoubleUserDefinedNoDataCellType => float64ud(ct.widenedNoData.asDouble)
}

encoding.name
}

/**
Expand Down
Loading

0 comments on commit 405ae64

Please sign in to comment.