update lz4 + cleanup dependencies (segmentio#848)

jackjie2016 · Feb 15, 2022 · c28bda1 · c28bda1
1 parent 7888926
commit c28bda1
Show file tree

Hide file tree

Showing 44 changed files with 448 additions and 29 deletions.
diff --git a/compress/lz4/lz4.go b/compress/lz4/lz4.go
@@ -4,7 +4,7 @@ import (
 	"io"
 	"sync"
 
-	"github.com/pierrec/lz4"
+	"github.com/pierrec/lz4/v4"
 )
 
 var (

diff --git a/compress/snappy/go-xerial-snappy/LICENSE b/compress/snappy/go-xerial-snappy/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Evan Huus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/compress/snappy/go-xerial-snappy/README.md b/compress/snappy/go-xerial-snappy/README.md
@@ -0,0 +1,13 @@
+# go-xerial-snappy
+
+[![Build Status](https://travis-ci.org/eapache/go-xerial-snappy.svg?branch=master)](https://travis-ci.org/eapache/go-xerial-snappy)
+
+Xerial-compatible Snappy framing support for golang.
+
+Packages using Xerial for snappy encoding use a framing format incompatible with
+basically everything else in existence. This package wraps Go's built-in snappy
+package to support it.
+
+Apps that use this format include Apache Kafka (see
+https://github.com/dpkp/kafka-python/issues/126#issuecomment-35478921 for
+details).
diff --git a/compress/snappy/go-xerial-snappy/corpus/020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2 b/compress/snappy/go-xerial-snappy/corpus/020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2
diff --git a/compress/snappy/go-xerial-snappy/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3 b/compress/snappy/go-xerial-snappy/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3
@@ -0,0 +1 @@
+����Y
diff --git a/compress/snappy/go-xerial-snappy/corpus/0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9 b/compress/snappy/go-xerial-snappy/corpus/0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9
diff --git a/compress/snappy/go-xerial-snappy/corpus/1 b/compress/snappy/go-xerial-snappy/corpus/1
diff --git a/compress/snappy/go-xerial-snappy/corpus/361a1c6d2a8f80780826c3d83ad391d0475c922f-4 b/compress/snappy/go-xerial-snappy/corpus/361a1c6d2a8f80780826c3d83ad391d0475c922f-4
diff --git a/compress/snappy/go-xerial-snappy/corpus/4117af68228fa64339d362cf980c68ffadff96c8-12 b/compress/snappy/go-xerial-snappy/corpus/4117af68228fa64339d362cf980c68ffadff96c8-12
diff --git a/compress/snappy/go-xerial-snappy/corpus/4142249be82c8a617cf838eef05394ece39becd3-9 b/compress/snappy/go-xerial-snappy/corpus/4142249be82c8a617cf838eef05394ece39becd3-9
diff --git a/compress/snappy/go-xerial-snappy/corpus/41ea8c7d904f1cd913b52e9ead4a96c639d76802-10 b/compress/snappy/go-xerial-snappy/corpus/41ea8c7d904f1cd913b52e9ead4a96c639d76802-10
diff --git a/compress/snappy/go-xerial-snappy/corpus/44083e1447694980c0ee682576e32358c9ee883f-2 b/compress/snappy/go-xerial-snappy/corpus/44083e1447694980c0ee682576e32358c9ee883f-2
diff --git a/compress/snappy/go-xerial-snappy/corpus/4d6b359bd538feaa7d36c89235d07d0a443797ac-1 b/compress/snappy/go-xerial-snappy/corpus/4d6b359bd538feaa7d36c89235d07d0a443797ac-1
diff --git a/compress/snappy/go-xerial-snappy/corpus/521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4 b/compress/snappy/go-xerial-snappy/corpus/521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4
@@ -0,0 +1 @@
+�������Y
diff --git a/compress/snappy/go-xerial-snappy/corpus/526e6f85d1b8777f0d9f70634c9f8b77fbdccdff-7 b/compress/snappy/go-xerial-snappy/corpus/526e6f85d1b8777f0d9f70634c9f8b77fbdccdff-7
diff --git a/compress/snappy/go-xerial-snappy/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e b/compress/snappy/go-xerial-snappy/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e
@@ -0,0 +1 @@
+package 
diff --git a/compress/snappy/go-xerial-snappy/corpus/60cd10738158020f5843b43960158c3d116b3a71-11 b/compress/snappy/go-xerial-snappy/corpus/60cd10738158020f5843b43960158c3d116b3a71-11
diff --git a/compress/snappy/go-xerial-snappy/corpus/652b031b4b9d601235f86ef62523e63d733b8623-3 b/compress/snappy/go-xerial-snappy/corpus/652b031b4b9d601235f86ef62523e63d733b8623-3
diff --git a/compress/snappy/go-xerial-snappy/corpus/684a011f6fdfc7ae9863e12381165e82d2a2e356-9 b/compress/snappy/go-xerial-snappy/corpus/684a011f6fdfc7ae9863e12381165e82d2a2e356-9
diff --git a/compress/snappy/go-xerial-snappy/corpus/72e42fc8e5eaed6a8a077f420fc3bd1f9a7c0919-1 b/compress/snappy/go-xerial-snappy/corpus/72e42fc8e5eaed6a8a077f420fc3bd1f9a7c0919-1
diff --git a/compress/snappy/go-xerial-snappy/corpus/80881d1b911b95e0203b3b0e7dc6360c35f7620f-7 b/compress/snappy/go-xerial-snappy/corpus/80881d1b911b95e0203b3b0e7dc6360c35f7620f-7
@@ -0,0 +1 @@
+墳￩��������������￩�￩��������������������꿽
diff --git a/compress/snappy/go-xerial-snappy/corpus/8484b3082d522e0a1f315db1fa1b2a5118be7cc3-8 b/compress/snappy/go-xerial-snappy/corpus/8484b3082d522e0a1f315db1fa1b2a5118be7cc3-8
diff --git a/compress/snappy/go-xerial-snappy/corpus/9635bb09260f100bc4a2ee4e3b980fecc5b874ce-1 b/compress/snappy/go-xerial-snappy/corpus/9635bb09260f100bc4a2ee4e3b980fecc5b874ce-1
diff --git a/compress/snappy/go-xerial-snappy/corpus/99d36b0b5b1be7151a508dd440ec725a2576c41c-1 b/compress/snappy/go-xerial-snappy/corpus/99d36b0b5b1be7151a508dd440ec725a2576c41c-1
diff --git a/compress/snappy/go-xerial-snappy/corpus/9d339eddb4e2714ea319c3fb571311cb95fdb067-6 b/compress/snappy/go-xerial-snappy/corpus/9d339eddb4e2714ea319c3fb571311cb95fdb067-6
diff --git a/compress/snappy/go-xerial-snappy/corpus/b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4 b/compress/snappy/go-xerial-snappy/corpus/b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4
@@ -0,0 +1 @@
+����Y
diff --git a/compress/snappy/go-xerial-snappy/corpus/c1951b29109ec1017f63535ce3699630f46f54e1-5 b/compress/snappy/go-xerial-snappy/corpus/c1951b29109ec1017f63535ce3699630f46f54e1-5
diff --git a/compress/snappy/go-xerial-snappy/corpus/cb806bc4f67316af02d6ae677332a3b6005a18da-5 b/compress/snappy/go-xerial-snappy/corpus/cb806bc4f67316af02d6ae677332a3b6005a18da-5
@@ -0,0 +1 @@
+墳��������濽
diff --git a/compress/snappy/go-xerial-snappy/corpus/cd7dd228703739e9252c7ea76f1c5f82ab44686a-10 b/compress/snappy/go-xerial-snappy/corpus/cd7dd228703739e9252c7ea76f1c5f82ab44686a-10
diff --git a/compress/snappy/go-xerial-snappy/corpus/ce3671e91907349cea04fc3f2a4b91c65b99461d-3 b/compress/snappy/go-xerial-snappy/corpus/ce3671e91907349cea04fc3f2a4b91c65b99461d-3
diff --git a/compress/snappy/go-xerial-snappy/corpus/ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6 b/compress/snappy/go-xerial-snappy/corpus/ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6
@@ -0,0 +1 @@
+墳￩�����������������꿽
diff --git a/compress/snappy/go-xerial-snappy/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709-1 b/compress/snappy/go-xerial-snappy/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709-1
diff --git a/compress/snappy/go-xerial-snappy/corpus/e2230aa0ecaebb9b890440effa13f501a89247b2-1 b/compress/snappy/go-xerial-snappy/corpus/e2230aa0ecaebb9b890440effa13f501a89247b2-1
diff --git a/compress/snappy/go-xerial-snappy/corpus/efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11 b/compress/snappy/go-xerial-snappy/corpus/efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11
diff --git a/compress/snappy/go-xerial-snappy/corpus/f0445ac39e03978bbc8011316ac8468015ddb72c-1 b/compress/snappy/go-xerial-snappy/corpus/f0445ac39e03978bbc8011316ac8468015ddb72c-1
diff --git a/compress/snappy/go-xerial-snappy/corpus/f241da53c6bc1fe3368c55bf28db86ce15a2c784-2 b/compress/snappy/go-xerial-snappy/corpus/f241da53c6bc1fe3368c55bf28db86ce15a2c784-2
diff --git a/compress/snappy/go-xerial-snappy/fuzz.go b/compress/snappy/go-xerial-snappy/fuzz.go
@@ -0,0 +1,16 @@
+// +build gofuzz
+
+package snappy
+
+func Fuzz(data []byte) int {
+	decode, err := Decode(data)
+	if decode == nil && err == nil {
+		panic("nil error with nil result")
+	}
+
+	if err != nil {
+		return 0
+	}
+
+	return 1
+}
diff --git a/compress/snappy/go-xerial-snappy/snappy.go b/compress/snappy/go-xerial-snappy/snappy.go
@@ -0,0 +1,131 @@
+package snappy
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+
+	master "github.com/klauspost/compress/snappy"
+)
+
+const (
+	sizeOffset = 16
+	sizeBytes  = 4
+)
+
+var (
+	xerialHeader = []byte{130, 83, 78, 65, 80, 80, 89, 0}
+
+	// This is xerial version 1 and minimally compatible with version 1
+	xerialVersionInfo = []byte{0, 0, 0, 1, 0, 0, 0, 1}
+
+	// ErrMalformed is returned by the decoder when the xerial framing
+	// is malformed
+	ErrMalformed = errors.New("malformed xerial framing")
+)
+
+func min(x, y int) int {
+    if x < y {
+        return x
+    }
+    return y
+}
+
+// Encode encodes data as snappy with no framing header.
+func Encode(src []byte) []byte {
+	return master.Encode(nil, src)
+}
+
+// EncodeStream *appends* to the specified 'dst' the compressed
+// 'src' in xerial framing format. If 'dst' does not have enough
+// capacity, then a new slice will be allocated. If 'dst' has
+// non-zero length, then if *must* have been built using this function.
+func EncodeStream(dst, src []byte) []byte {
+	if len(dst) == 0 {
+		dst = append(dst, xerialHeader...)
+		dst = append(dst, xerialVersionInfo...)
+	}
+
+	// Snappy encode in blocks of maximum 32KB
+	var (
+		max = len(src)
+		blockSize = 32 * 1024
+		pos   = 0
+		chunk []byte
+	)
+
+	for pos < max {
+		newPos := min(pos + blockSize, max)
+		chunk = master.Encode(chunk[:cap(chunk)], src[pos:newPos])
+
+		// First encode the compressed size (big-endian)
+		// Put* panics if the buffer is too small, so pad 4 bytes first
+		origLen := len(dst)
+		dst = append(dst, dst[0:4]...)
+		binary.BigEndian.PutUint32(dst[origLen:], uint32(len(chunk)))
+
+		// And now the compressed data
+		dst = append(dst, chunk...)
+		pos = newPos
+	}
+	return dst
+}
+
+// Decode decodes snappy data whether it is traditional unframed
+// or includes the xerial framing format.
+func Decode(src []byte) ([]byte, error) {
+	return DecodeInto(nil, src)
+}
+
+// DecodeInto decodes snappy data whether it is traditional unframed
+// or includes the xerial framing format into the specified `dst`.
+// It is assumed that the entirety of `dst` including all capacity is available
+// for use by this function. If `dst` is nil *or* insufficiently large to hold
+// the decoded `src`, new space will be allocated.
+func DecodeInto(dst, src []byte) ([]byte, error) {
+	var max = len(src)
+	if max < len(xerialHeader) {
+		return nil, ErrMalformed
+	}
+
+	if !bytes.Equal(src[:8], xerialHeader) {
+		return master.Decode(dst[:cap(dst)], src)
+	}
+
+	if max < sizeOffset+sizeBytes {
+		return nil, ErrMalformed
+	}
+
+	if dst == nil {
+		dst = make([]byte, 0, len(src))
+	}
+
+	dst = dst[:0]
+	var (
+		pos   = sizeOffset
+		chunk []byte
+		err       error
+	)
+
+	for pos+sizeBytes <= max {
+		size := int(binary.BigEndian.Uint32(src[pos : pos+sizeBytes]))
+		pos += sizeBytes
+
+		nextPos := pos + size
+		// On architectures where int is 32-bytes wide size + pos could
+		// overflow so we need to check the low bound as well as the
+		// high
+		if nextPos < pos || nextPos > max {
+			return nil, ErrMalformed
+		}
+
+		chunk, err = master.Decode(chunk[:cap(chunk)], src[pos:nextPos])
+
+		if err != nil {
+			return nil, err
+		}
+		pos = nextPos
+		dst = append(dst, chunk...)
+	}
+	return dst, nil
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,7 +4,7 @@ import ( @@
     	"io"
     	"sync"
-    	"github.com/pierrec/lz4"
+    	"github.com/pierrec/lz4/v4"
     )
     var (
@@ Expand Down @@