Skip to content

Commit

Permalink
Data2 implementation and Data3 fix (#14)
Browse files Browse the repository at this point in the history 
* Integration data for testing

* data2: empty test

* Data2 implementation

* shortcut for data

* fix spacial differenting

* enforce tests

* refacto data2 and data3
  • Loading branch information
@landru29 @nilsmagnus
landru29 authored and nilsmagnus committed Jan 11, 2019
1 parent 1901316 commit 45a796e
Show file tree
Hide file tree
Showing 26 changed files with 131,165 additions and 413 deletions.
119 changes: 119 additions & 0 deletions griblib/bitgroup.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
package griblib

import "fmt"

type bitGroupParameter struct {
Reference uint64
Width uint64
Length uint64
}

func (bitGroup bitGroupParameter) zeroGroup() []int64 {
result := []int64{}
for l := 0; l < int(bitGroup.Length); l++ {
result = append(result, 0)
}

return result
}

//
// Test to see if the group widths and lengths are consistent with number of
// values, and length of section 7.
//
func checkLengths(bitGroups []bitGroupParameter, dataLength int) error {
totBit := 0
totLen := 0

for _, param := range bitGroups {
totBit += int(param.Width) * int(param.Length)
totLen += int(param.Length)
}

if totBit/8 > int(dataLength) {
return fmt.Errorf("Checksum err %d - %d", dataLength, totBit/8)
}

return nil
}

//
// Extract Each Group's reference value
//
func (template *Data2) extractGroupReferences(bitReader *BitReader) ([]uint64, error) {
numberOfGroups := int64(template.NG)
return bitReader.readUintsBlock(int(template.Bits), numberOfGroups, true)
}

//
// Extract Each Group's bit width
//
func (template *Data2) extractGroupBitWidths(bitReader *BitReader) ([]uint64, error) {
numberOfGroups := int64(template.NG)
widths, err := bitReader.readUintsBlock(int(template.GroupWidthsBits), numberOfGroups, true)
if err != nil {
return widths, err
}

for j := range widths {
widths[j] += uint64(template.GroupWidths)
}

return widths, nil
}

//
// Extract Each Group's length (number of values in each group)
//
func (template *Data2) extractGroupLengths(bitReader *BitReader) ([]uint64, error) {
numberOfGroups := int64(template.NG)
lengths, err := bitReader.readUintsBlock(int(template.GroupScaledLengthsBits), numberOfGroups, true)
if err != nil {
return lengths, err
}

for j := range lengths {
lengths[j] = (lengths[j] * uint64(template.GroupLengthIncrement)) + uint64(template.GroupLengthsReference)
}
lengths[numberOfGroups-1] = uint64(template.GroupLastLength)
return lengths, nil
}

func (template *Data2) extractBitGroupParameters(bitReader *BitReader) ([]bitGroupParameter, error) {
result := []bitGroupParameter{}
//
// Extract Each Group's reference value
//
references, err := template.extractGroupReferences(bitReader)
if err != nil {
return result, err
}

//
// Extract Each Group's bit width
//
widths, err := template.extractGroupBitWidths(bitReader)
if err != nil {
return result, err
}

//
// Extract Each Group's length (number of values in each group)
//
lengths, err := template.extractGroupLengths(bitReader)
if err != nil {
return result, err
}

for index := range references {
result = append(result, bitGroupParameter{
Reference: references[index],
Width: widths[index],
Length: lengths[index],
})
}

bitReader.resetOffset()

return result, nil
}
91 changes: 51 additions & 40 deletions griblib/bitreader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ package griblib

import (
//"fmt"
"bufio"

"bytes"
"io"
)

Expand All @@ -13,21 +14,36 @@ type BitReader struct {
offset byte
}

func (r *BitReader) resetOffset() {
r.offset = 0
}

func (r *BitReader) currentBit() byte {
return (r.byte >> (7 - r.offset)) & 0x01
}

func newReader(r io.ByteReader) *BitReader {
return &BitReader{r, 0, 0}
}

func (r *BitReader) readBit() (bool, error) {
func makeBitReader(dataReader io.Reader, dataLength int) *BitReader {
rawData := make([]byte, dataLength)
dataReader.Read(rawData)
buffer := bytes.NewBuffer(rawData)
return newReader(buffer)
}

func (r *BitReader) readBit() (uint, error) {
if r.offset == 8 {
r.offset = 0
r.resetOffset()
}
if r.offset == 0 {
var err error
if r.byte, err = r.reader.ReadByte(); err != nil {
return false, err
return 0, err
}
}
bit := (r.byte & (0x80 >> r.offset)) != 0
bit := uint(r.currentBit())
r.offset++
return bit, nil
}
Expand All @@ -40,9 +56,7 @@ func (r *BitReader) readUint(nbits int) (uint64, error) {
if err != nil {
return 0, err
}
if bit {
result |= 1 << uint(i)
}
result |= uint64(bit << uint(i))
}

return result, nil
Expand All @@ -57,57 +71,53 @@ func (r *BitReader) readInt(nbits int) (int64, error) {
if err != nil {
return 0, err
}
if i == (nbits-1) && bit {
if i == (nbits-1) && bit == 1 {
negative = -1
} else if bit {
result |= 1 << uint(i)
continue
}
result |= int64(bit << uint(i))
}
return negative * result, nil
}

func (r *BitReader) readUintsBlock(bits int, count int) ([]uint64, error) {
//fmt.Println("Reading", bits, "bits", count, "x")
data := make([]uint64, count)
var err error
func (r *BitReader) readUintsBlock(bits int, count int64, resetOffset bool) ([]uint64, error) {
result := []uint64{}

if resetOffset {
r.resetOffset()
}

if bits != 0 {
for i := 0; i != count; i++ {
data[i], err = r.readUint(bits)
for i := int64(0); i != count; i++ {
data, err := r.readUint(bits)
if err != nil {
return data, err
return result, err
}

//fmt.Println(data[i])
result = append(result, data)
}

// if we are not fitting last byte seek to byte end
//rest := (bits * count) % 8
//if rest != 0 {
// r.offset += byte(8 - int64(rest))
//}
r.offset = 0

}
return data, nil

return result, nil
}

func (r *BitReader) readIntsBlock(bits int, count int) ([]int64, error) {
//fmt.Println("Reading", bits, "bits", count, "x")
data := make([]int64, count)
var err error
func (r *BitReader) readIntsBlock(bits int, count int64, resetOffset bool) ([]int64, error) {
result := []int64{}

if resetOffset {
r.resetOffset()
}

if bits != 0 {
for i := 0; i != count; i++ {
data[i], err = r.readInt(bits)
for i := int64(0); i != count; i++ {
data, err := r.readUint(bits)
if err != nil {
return data, err
return result, err
}
//fmt.Println(data[i])
result = append(result, int64(data))
}

}
return data, nil

return result, nil
}

////////////////////////////////////////////////////////////////////////////////////
Expand All @@ -116,7 +126,7 @@ func (r *BitReader) readIntsBlock(bits int, count int) ([]int64, error) {
// bit-by-bit, from it. Its Read* methods don't return the usual error
// because the error handling was verbose. Instead, any error is kept and can
// be checked afterwards.
type bitReader struct {
/*type bitReader struct {
r io.ByteReader
n uint64
bits uint
Expand Down Expand Up @@ -217,3 +227,4 @@ func (br *bitReader) tryReadBit() (bit byte, ok bool) {
func (br *bitReader) Err() error {
return br.err
}
*/
53 changes: 28 additions & 25 deletions griblib/data0.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package griblib

import (
"bytes"
"fmt"
"io"
"math"
)
Expand Down Expand Up @@ -30,44 +28,49 @@ type Data0 struct {
Type uint8 `json:"type"`
}

// ParseData0 parses data0 struct from the reader into the an array of floating-point values
func ParseData0(dataReader io.Reader, dataLength int, template *Data0) []float64 {
func (template Data0) getRefScale() (float64, float64) {
bscale := math.Pow(2.0, float64(template.BinaryScale))
dscale := math.Pow(10.0, -float64(template.DecimalScale))

fld := make([]float64, 0)
scale := bscale * dscale
ref := dscale * float64(template.Reference)

if dataLength == 0 {
return fld
}
return ref, scale
}

rawData := make([]byte, dataLength)
bytesRead, errRead := dataReader.Read(rawData)
if errRead != nil {
panic(errRead)
func (template Data0) scaleFunc() func(uintValue int64) float64 {
ref, scale := template.getRefScale()
return func(value int64) float64 {
signed := int64(value)
return ref + float64(signed)*scale
}
fmt.Printf("read: %d\n", bytesRead)
}

bscale := math.Pow(2.0, float64(template.BinaryScale))
dscale := math.Pow(10.0, -float64(template.DecimalScale))
// ParseData0 parses data0 struct from the reader into the an array of floating-point values
func ParseData0(dataReader io.Reader, dataLength int, template *Data0) ([]float64, error) {

scale := bscale * dscale
ref := dscale * float64(template.Reference)
fld := []float64{}

if dataLength == 0 {
return fld, nil
}

scaleStrategy := template.scaleFunc()

buffer := bytes.NewBuffer(rawData)
bitReader := newReader(buffer)
bitReader := makeBitReader(dataReader, dataLength)

dataSize := int(math.Floor(
dataSize := int64(math.Floor(
float64(8*dataLength) / float64(template.Bits),
))

uintDataSlice, errRead := bitReader.readUintsBlock(int(template.Bits), dataSize)
uintDataSlice, errRead := bitReader.readUintsBlock(int(template.Bits), dataSize, false)
if errRead != nil {
panic(errRead)
return []float64{}, errRead
}

for _, uintValue := range uintDataSlice {
signed := int64(uintValue)
fld = append(fld, ref+float64(signed)*scale)
fld = append(fld, scaleStrategy(int64(uintValue)))
}

return fld
return fld, nil
}
24 changes: 0 additions & 24 deletions griblib/data0_test.go
View file

This file was deleted.

Loading

0 comments on commit 45a796e

Please sign in to comment.