Skip to content

Commit

Permalink
more fixes, updates, optimizations
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhenjl
zhenjl committed Nov 14, 2013
1 parent 98c88d4 commit 4041f8c
Show file tree
Hide file tree
Showing 7 changed files with 547 additions and 55 deletions.
6 changes: 3 additions & 3 deletions benchmark/benchmark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,13 +271,13 @@ func main() {
log.Fatal(err)
}

if err := pprofCodecs(codecs, data, false); err != nil {
if err := testCodecs(codecs, data, false); err != nil {
log.Fatal(err)
}
if err := pprofCodecs(codecs, data, false); err != nil {
if err := testCodecs(codecs, data, false); err != nil {
log.Fatal(err)
}
if err := pprofCodecs(codecs, data, true); err != nil {
if err := testCodecs(codecs, data, true); err != nil {
log.Fatal(err)
}
}
0 ...ark/Benchmarking_Integer_Compression.xlsx → ...lts/Benchmarking_Integer_Compression.xlsx
View file
File renamed without changes.
105 changes: 61 additions & 44 deletions benchtools/benchtools.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,34 +74,77 @@ func PprofCodec(codec encoding.Integer, in []int32, sizes []int) {
}
}

func PprofCompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
f, e := os.Create("cpu.compress.pprof")
if e != nil {
log.Fatal(e)
}
defer f.Close()
func Compress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
return runCompress(codec, in, length, false)
}

pprof.StartCPUProfile(f)
duration, out, err = Compress(codec, in, length)
pprof.StopCPUProfile()
func Uncompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
return runUncompress(codec, in, length, false)
}

return
func PprofCompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
return runCompress(codec, in, length, true)
}

func PprofUncompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
f, e := os.Create("cpu.uncompress.pprof")
if e != nil {
log.Fatal(e)
return runUncompress(codec, in, length, true)
}

func runCompress(codec encoding.Integer, in []int32, length int, prof bool) (duration int64, out []int32, err error) {
out = make([]int32, length*2)
inpos := cursor.New()
outpos := cursor.New()

now := time.Now()
if prof {
f, e := os.Create("cpu.compress.pprof")
if e != nil {
log.Fatal(e)
}
defer f.Close()

pprof.StartCPUProfile(f)
}

if err = codec.Compress(in, inpos, len(in), out, outpos); err != nil {
return 0, nil, err
}
defer f.Close()
since := time.Since(now).Nanoseconds()

pprof.StartCPUProfile(f)
duration, out, err = Uncompress(codec, in, length)
pprof.StopCPUProfile()
if prof {
pprof.StopCPUProfile()
}

return
return since, out[:outpos.Get()], nil
}

func runUncompress(codec encoding.Integer, in []int32, length int, prof bool) (duration int64, out []int32, err error) {
out = make([]int32, length)
inpos := cursor.New()
outpos := cursor.New()

if prof {
f, e := os.Create("cpu.uncompress.pprof")
if e != nil {
log.Fatal(e)
}
defer f.Close()

pprof.StartCPUProfile(f)
}

now := time.Now()
if err = codec.Uncompress(in, inpos, len(in), out, outpos); err != nil {
return 0, nil, err
}
since := time.Since(now).Nanoseconds()

if prof {
pprof.StopCPUProfile()
}

return since, out[:outpos.Get()], nil
}

func RunTestGzip(data []byte) {
log.Printf("encoding/RunTestGzip: Testing comprssion Gzip\n")
Expand Down Expand Up @@ -176,29 +219,3 @@ func RunTestSnappy(data []byte) {
}
}


func Compress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
out = make([]int32, length*2)
inpos := cursor.New()
outpos := cursor.New()

now := time.Now()
if err = codec.Compress(in, inpos, len(in), out, outpos); err != nil {
return 0, nil, err
}

return time.Since(now).Nanoseconds(), out[:outpos.Get()], nil
}

func Uncompress(codec encoding.Integer, in []int32, length int) (duration int64, out []int32, err error) {
out = make([]int32, length)
inpos := cursor.New()
outpos := cursor.New()

now := time.Now()
if err = codec.Uncompress(in, inpos, len(in), out, outpos); err != nil {
return 0, nil, err
}

return time.Since(now).Nanoseconds(), out[:outpos.Get()], nil
}
36 changes: 36 additions & 0 deletions bitpacking/bitpacking.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ func FastPackWithoutMask(in []int32, inpos int, out []int32, outpos int, bit int

return nil
}

func fastpackwithoutmask1(in []int32, inpos int, out []int32, outpos int) {
out[0+outpos] = in[0+inpos] |
((in[1+inpos]) << 1) |
Expand Down Expand Up @@ -6840,6 +6841,41 @@ func fastunpack0(in []int32, inpos int, out []int32, outpos int) {
for i := outpos; i < outpos + 32; i++ {
out[i] = 0
}

/*
out[outpos+0] = 0
out[outpos+1] = 0
out[outpos+2] = 0
out[outpos+3] = 0
out[outpos+4] = 0
out[outpos+5] = 0
out[outpos+6] = 0
out[outpos+7] = 0
out[outpos+8] = 0
out[outpos+9] = 0
out[outpos+10] = 0
out[outpos+11] = 0
out[outpos+12] = 0
out[outpos+13] = 0
out[outpos+14] = 0
out[outpos+15] = 0
out[outpos+16] = 0
out[outpos+17] = 0
out[outpos+18] = 0
out[outpos+19] = 0
out[outpos+20] = 0
out[outpos+21] = 0
out[outpos+22] = 0
out[outpos+23] = 0
out[outpos+24] = 0
out[outpos+25] = 0
out[outpos+26] = 0
out[outpos+27] = 0
out[outpos+28] = 0
out[outpos+29] = 0
out[outpos+30] = 0
out[outpos+31] = 0
*/
}

func fastpack0(in []int32, inpos int, out []int32, outpos int) {
Expand Down
36 changes: 33 additions & 3 deletions bitpacking/delta_bitpacking.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,9 +240,39 @@ func DeltaUnpack(initoffset int32, in []int32, inpos int, out []int32, outpos in
}

func deltaunpack0(initoffset int32, in []int32, inpos int, out []int32, outpos int) {
for i := outpos; i < outpos + 32; i++ {
out[i] = initoffset
}
// Unrolling the loop to get best performance
out[outpos+0] = initoffset
out[outpos+1] = initoffset
out[outpos+2] = initoffset
out[outpos+3] = initoffset
out[outpos+4] = initoffset
out[outpos+5] = initoffset
out[outpos+6] = initoffset
out[outpos+7] = initoffset
out[outpos+8] = initoffset
out[outpos+9] = initoffset
out[outpos+10] = initoffset
out[outpos+11] = initoffset
out[outpos+12] = initoffset
out[outpos+13] = initoffset
out[outpos+14] = initoffset
out[outpos+15] = initoffset
out[outpos+16] = initoffset
out[outpos+17] = initoffset
out[outpos+18] = initoffset
out[outpos+19] = initoffset
out[outpos+20] = initoffset
out[outpos+21] = initoffset
out[outpos+22] = initoffset
out[outpos+23] = initoffset
out[outpos+24] = initoffset
out[outpos+25] = initoffset
out[outpos+26] = initoffset
out[outpos+27] = initoffset
out[outpos+28] = initoffset
out[outpos+29] = initoffset
out[outpos+30] = initoffset
out[outpos+31] = initoffset
}

func deltapack0(initoffset int32, in []int32, inpos int, out []int32, outpos int) {
Expand Down
14 changes: 9 additions & 5 deletions delta/fastpfor/fastpfor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,12 +117,15 @@ func (this *FastPFOR) getBestBFromData(in []int32) (bestb int32, bestc int32, ma
// Get the count of all the leading bit positions for the slice
// Mainly to figure out what's the best (most popular) bit position
for _, v := range in {
l := encoding.LeadingBitPosition(uint32(v))
this.freqs[l] += 1
if l > bestb {
bestb = l
}
this.freqs[encoding.LeadingBitPosition(uint32(v))]++
}
//encoding.FastLeadingBitFrequency128(in, this.freqs)

bestb = 32

for this.freqs[bestb] == 0 {
bestb--
}

maxb = bestb
bestCost := bestb*DefaultBlockSize
Expand Down Expand Up @@ -338,6 +341,7 @@ func (this *FastPFOR) decodePage(in []int32, inpos *cursor.Cursor, out []int32,
out[int(tmpoutpos)+i] = v + offset
offset += v
}

initoffset.Set(int(out[tmpoutpos+DefaultBlockSize-1]))

run += 1
Expand Down
Loading

0 comments on commit 4041f8c

Please sign in to comment.