Skip to content

Commit

Permalink
moved to dataence, updated import path
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhenjl
zhenjl committed Dec 23, 2017
1 parent e31efcf commit b90e310
Show file tree
Hide file tree
Showing 21 changed files with 125 additions and 107 deletions.
23 changes: 12 additions & 11 deletions benchmark/benchmark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,6 @@ import (
"compress/gzip"
"flag"
"fmt"
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/bp32"
"github.com/reducedb/encoding/composition"
"github.com/reducedb/encoding/cursor"
dbp32 "github.com/reducedb/encoding/delta/bp32"
dfastpfor "github.com/reducedb/encoding/delta/fastpfor"
dvb "github.com/reducedb/encoding/delta/variablebyte"
"github.com/reducedb/encoding/fastpfor"
"github.com/reducedb/encoding/variablebyte"
zbp32 "github.com/reducedb/encoding/zigzag/bp32"
zfastpfor "github.com/reducedb/encoding/zigzag/fastpfor"
"io/ioutil"
"log"
"os"
Expand All @@ -30,6 +19,18 @@ import (
"strconv"
"strings"
"time"

"github.com/dataence/encoding"
"github.com/dataence/encoding/bp32"
"github.com/dataence/encoding/composition"
"github.com/dataence/encoding/cursor"
dbp32 "github.com/dataence/encoding/delta/bp32"
dfastpfor "github.com/dataence/encoding/delta/fastpfor"
dvb "github.com/dataence/encoding/delta/variablebyte"
"github.com/dataence/encoding/fastpfor"
"github.com/dataence/encoding/variablebyte"
zbp32 "github.com/dataence/encoding/zigzag/bp32"
zfastpfor "github.com/dataence/encoding/zigzag/fastpfor"
)

type paramList []string
Expand Down
7 changes: 4 additions & 3 deletions benchtools/benchtools.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,18 @@ package benchtools

import (
"bytes"
"code.google.com/p/snappy-go/snappy"
"compress/gzip"
"compress/lzw"
"fmt"
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/cursor"
"io"
"log"
"os"
"runtime/pprof"
"time"

"code.google.com/p/snappy-go/snappy"
"github.com/dataence/encoding"
"github.com/dataence/encoding/cursor"
)

func TestCodec(codec encoding.Integer, in []int32, sizes []int) {
Expand Down
10 changes: 5 additions & 5 deletions bp32/bp32.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@ package bp32

import (
"errors"
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/bitpacking"
"github.com/reducedb/encoding/cursor"

"github.com/dataence/encoding"
"github.com/dataence/encoding/bitpacking"
"github.com/dataence/encoding/cursor"
)

const (
Expand All @@ -44,15 +45,14 @@ func (this *BP32) Compress(in []int32, inpos *cursor.Cursor, inlength int, out [
return errors.New("BP32/Compress: block size less than 128. No work done.")
}


out[outpos.Get()] = int32(inlength)
outpos.Increment()

tmpoutpos := outpos.Get()
s := inpos.Get()
finalinpos := s + inlength

for ; s < finalinpos; s += DefaultBlockSize {
for ; s < finalinpos; s += DefaultBlockSize {
mbits1 := encoding.MaxBits(in[s : s+32])
mbits2 := encoding.MaxBits(in[s+32 : s+2*32])
mbits3 := encoding.MaxBits(in[s+2*32 : s+3*32])
Expand Down
37 changes: 19 additions & 18 deletions bp32/bp32_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@
package bp32

import (
"github.com/reducedb/encoding/benchtools"
"github.com/reducedb/encoding/generators"
"github.com/reducedb/encoding/cursor"
"log"
"testing"

"github.com/dataence/encoding/benchtools"
"github.com/dataence/encoding/cursor"
"github.com/dataence/encoding/generators"
)

var (
Expand All @@ -32,19 +33,19 @@ func TestCodec(t *testing.T) {

// go test -bench=Decode
func BenchmarkDecode(b *testing.B) {
b.StopTimer()
length := 128 * 1024
data := generators.GenerateClustered(length, 1<<24)
compdata := make([]int32, 2*length)
recov := make([]int32, length)
inpos := cursor.New()
outpos := cursor.New()
codec := New()
codec.Compress(data, inpos, len(data), compdata, outpos)
b.StartTimer()
for j := 0; j < b.N; j++ {
newinpos := cursor.New()
newoutpos := cursor.New()
codec.Uncompress(compdata, newinpos, outpos.Get()-newinpos.Get(), recov, newoutpos)
}
b.StopTimer()
length := 128 * 1024
data := generators.GenerateClustered(length, 1<<24)
compdata := make([]int32, 2*length)
recov := make([]int32, length)
inpos := cursor.New()
outpos := cursor.New()
codec := New()
codec.Compress(data, inpos, len(data), compdata, outpos)
b.StartTimer()
for j := 0; j < b.N; j++ {
newinpos := cursor.New()
newoutpos := cursor.New()
codec.Uncompress(compdata, newinpos, outpos.Get()-newinpos.Get(), recov, newoutpos)
}
}
5 changes: 3 additions & 2 deletions composition/composition.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ package composition

import (
"errors"
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/cursor"

"github.com/dataence/encoding"
"github.com/dataence/encoding/cursor"
)

type Composition struct {
Expand Down
15 changes: 8 additions & 7 deletions composition/composition_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,16 @@
package composition

import (
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/benchtools"
"github.com/reducedb/encoding/bp32"
dbp32 "github.com/reducedb/encoding/delta/bp32"
dvb "github.com/reducedb/encoding/delta/variablebyte"
"github.com/reducedb/encoding/generators"
"github.com/reducedb/encoding/variablebyte"
"log"
"testing"

"github.com/dataence/encoding"
"github.com/dataence/encoding/benchtools"
"github.com/dataence/encoding/bp32"
dbp32 "github.com/dataence/encoding/delta/bp32"
dvb "github.com/dataence/encoding/delta/variablebyte"
"github.com/dataence/encoding/generators"
"github.com/dataence/encoding/variablebyte"
)

var (
Expand Down
7 changes: 4 additions & 3 deletions delta/bp32/bp32.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ package bp32

import (
"errors"
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/bitpacking"
"github.com/reducedb/encoding/cursor"

"github.com/dataence/encoding"
"github.com/dataence/encoding/bitpacking"
"github.com/dataence/encoding/cursor"
)

const (
Expand Down
5 changes: 3 additions & 2 deletions delta/bp32/bp32_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@
package bp32

import (
"github.com/reducedb/encoding/benchtools"
"github.com/reducedb/encoding/generators"
"log"
"testing"

"github.com/dataence/encoding/benchtools"
"github.com/dataence/encoding/generators"
)

var (
Expand Down
9 changes: 5 additions & 4 deletions delta/fastpfor/fastpfor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@ package fastpfor

import (
"errors"
"github.com/reducedb/bytebuffer"
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/bitpacking"
"github.com/reducedb/encoding/cursor"
"math"

"github.com/dataence/bytebuffer"
"github.com/dataence/encoding"
"github.com/dataence/encoding/bitpacking"
"github.com/dataence/encoding/cursor"
)

const (
Expand Down
5 changes: 3 additions & 2 deletions delta/fastpfor/fastpfor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@
package fastpfor

import (
"github.com/reducedb/encoding/benchtools"
"github.com/reducedb/encoding/generators"
"log"
"testing"

"github.com/dataence/encoding/benchtools"
"github.com/dataence/encoding/generators"
)

var (
Expand Down
7 changes: 4 additions & 3 deletions delta/variablebyte/variablebyte.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ package variablebyte

import (
"errors"
"github.com/reducedb/bytebuffer"
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/cursor"

"github.com/dataence/bytebuffer"
"github.com/dataence/encoding"
"github.com/dataence/encoding/cursor"
)

type VariableByte struct {
Expand Down
5 changes: 3 additions & 2 deletions delta/variablebyte/variablebyte_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@
package variablebyte

import (
"github.com/reducedb/encoding/benchtools"
"github.com/reducedb/encoding/generators"
"log"
"testing"

"github.com/dataence/encoding/benchtools"
"github.com/dataence/encoding/generators"
)

var (
Expand Down
17 changes: 9 additions & 8 deletions fastpfor/fastpfor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ package fastpfor

import (
"errors"
"github.com/reducedb/bytebuffer"
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/bitpacking"
"github.com/reducedb/encoding/cursor"
"math"

"github.com/dataence/bytebuffer"
"github.com/dataence/encoding"
"github.com/dataence/encoding/bitpacking"
"github.com/dataence/encoding/cursor"
)

const (
Expand Down Expand Up @@ -278,23 +279,23 @@ func (this *FastPFOR) decodePage(in []int32, inpos *cursor.Cursor, out []int32,
for run < run_end {
bestb := uint32(grapByte(mybytearray, mybp))
mybp++
cexcept := int32(grapByte(mybytearray, mybp))
cexcept := int32(grapByte(mybytearray, mybp))
mybp++
for k := uint32(0); k < 128; k += 32 {
bitpacking.FastUnpack(in, int(tmpinpos), out, int(tmpoutpos+k), int(bestb))
tmpinpos += bestb
tmpinpos += bestb
}

if cexcept > 0 {
maxbits := uint32(grapByte(mybytearray, mybp))
maxbits := uint32(grapByte(mybytearray, mybp))
mybp++
index := maxbits - bestb
// assuming that the Go compiler is bad, we move everything that is indexed outside the upcoming loop
packedexceptions := this.dataToBePacked[index]
myindex := this.dataPointers[index]

for k := int32(0); k < cexcept; k++ {
pos := uint32(grapByte(mybytearray, mybp))
pos := uint32(grapByte(mybytearray, mybp))
mybp++
exceptvalue := packedexceptions[myindex]
myindex++
Expand Down
7 changes: 4 additions & 3 deletions fastpfor/fastpfor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@
package fastpfor

import (
"github.com/reducedb/encoding/benchtools"
"github.com/reducedb/encoding/cursor"
"github.com/reducedb/encoding/generators"
"log"
"testing"

"github.com/dataence/encoding/benchtools"
"github.com/dataence/encoding/cursor"
"github.com/dataence/encoding/generators"
)

var (
Expand Down
2 changes: 1 addition & 1 deletion integer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
package encoding

import (
"github.com/reducedb/encoding/cursor"
"github.com/dataence/encoding/cursor"
)

type Integer interface {
Expand Down
7 changes: 4 additions & 3 deletions variablebyte/variablebyte.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ package variablebyte

import (
"errors"
"github.com/reducedb/bytebuffer"
"github.com/reducedb/encoding"
"github.com/reducedb/encoding/cursor"

"github.com/dataence/bytebuffer"
"github.com/dataence/encoding"
"github.com/dataence/encoding/cursor"
)

type VariableByte struct {
Expand Down
Loading

0 comments on commit b90e310

Please sign in to comment.