diff --git a/compress/lz4/lz4.go b/compress/lz4/lz4.go index 2c892ca07..1aa8289b8 100644 --- a/compress/lz4/lz4.go +++ b/compress/lz4/lz4.go @@ -4,7 +4,7 @@ import ( "io" "sync" - "github.com/pierrec/lz4" + "github.com/pierrec/lz4/v4" ) var ( diff --git a/compress/snappy/go-xerial-snappy/LICENSE b/compress/snappy/go-xerial-snappy/LICENSE new file mode 100644 index 000000000..5bf3688d9 --- /dev/null +++ b/compress/snappy/go-xerial-snappy/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Evan Huus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/compress/snappy/go-xerial-snappy/README.md b/compress/snappy/go-xerial-snappy/README.md new file mode 100644 index 000000000..3f2695c72 --- /dev/null +++ b/compress/snappy/go-xerial-snappy/README.md @@ -0,0 +1,13 @@ +# go-xerial-snappy + +[![Build Status](https://travis-ci.org/eapache/go-xerial-snappy.svg?branch=master)](https://travis-ci.org/eapache/go-xerial-snappy) + +Xerial-compatible Snappy framing support for golang. + +Packages using Xerial for snappy encoding use a framing format incompatible with +basically everything else in existence. This package wraps Go's built-in snappy +package to support it. + +Apps that use this format include Apache Kafka (see +https://github.com/dpkp/kafka-python/issues/126#issuecomment-35478921 for +details). diff --git a/compress/snappy/go-xerial-snappy/corpus/020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2 b/compress/snappy/go-xerial-snappy/corpus/020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2 new file mode 100644 index 000000000..1b704ae0e Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3 b/compress/snappy/go-xerial-snappy/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3 new file mode 100644 index 000000000..8c321a617 --- /dev/null +++ b/compress/snappy/go-xerial-snappy/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3 @@ -0,0 +1 @@ +����Y \ No newline at end of file diff --git a/compress/snappy/go-xerial-snappy/corpus/0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9 b/compress/snappy/go-xerial-snappy/corpus/0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9 new file mode 100644 index 000000000..d413114b2 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/1 b/compress/snappy/go-xerial-snappy/corpus/1 new file mode 100644 index 000000000..5e3abaf35 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/1 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/361a1c6d2a8f80780826c3d83ad391d0475c922f-4 b/compress/snappy/go-xerial-snappy/corpus/361a1c6d2a8f80780826c3d83ad391d0475c922f-4 new file mode 100644 index 000000000..d2528bad4 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/361a1c6d2a8f80780826c3d83ad391d0475c922f-4 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/4117af68228fa64339d362cf980c68ffadff96c8-12 b/compress/snappy/go-xerial-snappy/corpus/4117af68228fa64339d362cf980c68ffadff96c8-12 new file mode 100644 index 000000000..38ee90f85 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/4117af68228fa64339d362cf980c68ffadff96c8-12 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/4142249be82c8a617cf838eef05394ece39becd3-9 b/compress/snappy/go-xerial-snappy/corpus/4142249be82c8a617cf838eef05394ece39becd3-9 new file mode 100644 index 000000000..6d5812d8e Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/4142249be82c8a617cf838eef05394ece39becd3-9 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/41ea8c7d904f1cd913b52e9ead4a96c639d76802-10 b/compress/snappy/go-xerial-snappy/corpus/41ea8c7d904f1cd913b52e9ead4a96c639d76802-10 new file mode 100644 index 000000000..6a5f1466b Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/41ea8c7d904f1cd913b52e9ead4a96c639d76802-10 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/44083e1447694980c0ee682576e32358c9ee883f-2 b/compress/snappy/go-xerial-snappy/corpus/44083e1447694980c0ee682576e32358c9ee883f-2 new file mode 100644 index 000000000..6880dda2f Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/44083e1447694980c0ee682576e32358c9ee883f-2 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/4d6b359bd538feaa7d36c89235d07d0a443797ac-1 b/compress/snappy/go-xerial-snappy/corpus/4d6b359bd538feaa7d36c89235d07d0a443797ac-1 new file mode 100644 index 000000000..797a0a319 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/4d6b359bd538feaa7d36c89235d07d0a443797ac-1 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4 b/compress/snappy/go-xerial-snappy/corpus/521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4 new file mode 100644 index 000000000..ad77b2004 --- /dev/null +++ b/compress/snappy/go-xerial-snappy/corpus/521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4 @@ -0,0 +1 @@ +�������Y \ No newline at end of file diff --git a/compress/snappy/go-xerial-snappy/corpus/526e6f85d1b8777f0d9f70634c9f8b77fbdccdff-7 b/compress/snappy/go-xerial-snappy/corpus/526e6f85d1b8777f0d9f70634c9f8b77fbdccdff-7 new file mode 100644 index 000000000..0d3e7cc51 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/526e6f85d1b8777f0d9f70634c9f8b77fbdccdff-7 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e b/compress/snappy/go-xerial-snappy/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e new file mode 100644 index 000000000..59c77e4dc --- /dev/null +++ b/compress/snappy/go-xerial-snappy/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e @@ -0,0 +1 @@ +package \ No newline at end of file diff --git a/compress/snappy/go-xerial-snappy/corpus/60cd10738158020f5843b43960158c3d116b3a71-11 b/compress/snappy/go-xerial-snappy/corpus/60cd10738158020f5843b43960158c3d116b3a71-11 new file mode 100644 index 000000000..801e2cf29 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/60cd10738158020f5843b43960158c3d116b3a71-11 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/652b031b4b9d601235f86ef62523e63d733b8623-3 b/compress/snappy/go-xerial-snappy/corpus/652b031b4b9d601235f86ef62523e63d733b8623-3 new file mode 100644 index 000000000..76bd9d9eb Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/652b031b4b9d601235f86ef62523e63d733b8623-3 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/684a011f6fdfc7ae9863e12381165e82d2a2e356-9 b/compress/snappy/go-xerial-snappy/corpus/684a011f6fdfc7ae9863e12381165e82d2a2e356-9 new file mode 100644 index 000000000..7894c036c Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/684a011f6fdfc7ae9863e12381165e82d2a2e356-9 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/72e42fc8e5eaed6a8a077f420fc3bd1f9a7c0919-1 b/compress/snappy/go-xerial-snappy/corpus/72e42fc8e5eaed6a8a077f420fc3bd1f9a7c0919-1 new file mode 100644 index 000000000..f5a2bb96e Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/72e42fc8e5eaed6a8a077f420fc3bd1f9a7c0919-1 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/80881d1b911b95e0203b3b0e7dc6360c35f7620f-7 b/compress/snappy/go-xerial-snappy/corpus/80881d1b911b95e0203b3b0e7dc6360c35f7620f-7 new file mode 100644 index 000000000..16f802af8 --- /dev/null +++ b/compress/snappy/go-xerial-snappy/corpus/80881d1b911b95e0203b3b0e7dc6360c35f7620f-7 @@ -0,0 +1 @@ +墳￩��������������￩�￩��������������������꿽 \ No newline at end of file diff --git a/compress/snappy/go-xerial-snappy/corpus/8484b3082d522e0a1f315db1fa1b2a5118be7cc3-8 b/compress/snappy/go-xerial-snappy/corpus/8484b3082d522e0a1f315db1fa1b2a5118be7cc3-8 new file mode 100644 index 000000000..a8d5adb3d Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/8484b3082d522e0a1f315db1fa1b2a5118be7cc3-8 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/9635bb09260f100bc4a2ee4e3b980fecc5b874ce-1 b/compress/snappy/go-xerial-snappy/corpus/9635bb09260f100bc4a2ee4e3b980fecc5b874ce-1 new file mode 100644 index 000000000..9292f67b9 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/9635bb09260f100bc4a2ee4e3b980fecc5b874ce-1 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/99d36b0b5b1be7151a508dd440ec725a2576c41c-1 b/compress/snappy/go-xerial-snappy/corpus/99d36b0b5b1be7151a508dd440ec725a2576c41c-1 new file mode 100644 index 000000000..3e250d273 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/99d36b0b5b1be7151a508dd440ec725a2576c41c-1 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/9d339eddb4e2714ea319c3fb571311cb95fdb067-6 b/compress/snappy/go-xerial-snappy/corpus/9d339eddb4e2714ea319c3fb571311cb95fdb067-6 new file mode 100644 index 000000000..b2c05e5cb Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/9d339eddb4e2714ea319c3fb571311cb95fdb067-6 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4 b/compress/snappy/go-xerial-snappy/corpus/b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4 new file mode 100644 index 000000000..fb7157dc4 --- /dev/null +++ b/compress/snappy/go-xerial-snappy/corpus/b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4 @@ -0,0 +1 @@ +����Y \ No newline at end of file diff --git a/compress/snappy/go-xerial-snappy/corpus/c1951b29109ec1017f63535ce3699630f46f54e1-5 b/compress/snappy/go-xerial-snappy/corpus/c1951b29109ec1017f63535ce3699630f46f54e1-5 new file mode 100644 index 000000000..ebe47e8c3 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/c1951b29109ec1017f63535ce3699630f46f54e1-5 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/cb806bc4f67316af02d6ae677332a3b6005a18da-5 b/compress/snappy/go-xerial-snappy/corpus/cb806bc4f67316af02d6ae677332a3b6005a18da-5 new file mode 100644 index 000000000..f926593fd --- /dev/null +++ b/compress/snappy/go-xerial-snappy/corpus/cb806bc4f67316af02d6ae677332a3b6005a18da-5 @@ -0,0 +1 @@ +墳��������濽 \ No newline at end of file diff --git a/compress/snappy/go-xerial-snappy/corpus/cd7dd228703739e9252c7ea76f1c5f82ab44686a-10 b/compress/snappy/go-xerial-snappy/corpus/cd7dd228703739e9252c7ea76f1c5f82ab44686a-10 new file mode 100644 index 000000000..668f70c32 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/cd7dd228703739e9252c7ea76f1c5f82ab44686a-10 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/ce3671e91907349cea04fc3f2a4b91c65b99461d-3 b/compress/snappy/go-xerial-snappy/corpus/ce3671e91907349cea04fc3f2a4b91c65b99461d-3 new file mode 100644 index 000000000..1e1b3b051 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/ce3671e91907349cea04fc3f2a4b91c65b99461d-3 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6 b/compress/snappy/go-xerial-snappy/corpus/ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6 new file mode 100644 index 000000000..db6795b77 --- /dev/null +++ b/compress/snappy/go-xerial-snappy/corpus/ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6 @@ -0,0 +1 @@ +墳￩�����������������꿽 \ No newline at end of file diff --git a/compress/snappy/go-xerial-snappy/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709-1 b/compress/snappy/go-xerial-snappy/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709-1 new file mode 100644 index 000000000..e69de29bb diff --git a/compress/snappy/go-xerial-snappy/corpus/e2230aa0ecaebb9b890440effa13f501a89247b2-1 b/compress/snappy/go-xerial-snappy/corpus/e2230aa0ecaebb9b890440effa13f501a89247b2-1 new file mode 100644 index 000000000..0e3f82ff4 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/e2230aa0ecaebb9b890440effa13f501a89247b2-1 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11 b/compress/snappy/go-xerial-snappy/corpus/efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11 new file mode 100644 index 000000000..203163488 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/f0445ac39e03978bbc8011316ac8468015ddb72c-1 b/compress/snappy/go-xerial-snappy/corpus/f0445ac39e03978bbc8011316ac8468015ddb72c-1 new file mode 100644 index 000000000..b30db18f6 Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/f0445ac39e03978bbc8011316ac8468015ddb72c-1 differ diff --git a/compress/snappy/go-xerial-snappy/corpus/f241da53c6bc1fe3368c55bf28db86ce15a2c784-2 b/compress/snappy/go-xerial-snappy/corpus/f241da53c6bc1fe3368c55bf28db86ce15a2c784-2 new file mode 100644 index 000000000..4909d4e3b Binary files /dev/null and b/compress/snappy/go-xerial-snappy/corpus/f241da53c6bc1fe3368c55bf28db86ce15a2c784-2 differ diff --git a/compress/snappy/go-xerial-snappy/fuzz.go b/compress/snappy/go-xerial-snappy/fuzz.go new file mode 100644 index 000000000..6a46f4784 --- /dev/null +++ b/compress/snappy/go-xerial-snappy/fuzz.go @@ -0,0 +1,16 @@ +// +build gofuzz + +package snappy + +func Fuzz(data []byte) int { + decode, err := Decode(data) + if decode == nil && err == nil { + panic("nil error with nil result") + } + + if err != nil { + return 0 + } + + return 1 +} diff --git a/compress/snappy/go-xerial-snappy/snappy.go b/compress/snappy/go-xerial-snappy/snappy.go new file mode 100644 index 000000000..32f2a3475 --- /dev/null +++ b/compress/snappy/go-xerial-snappy/snappy.go @@ -0,0 +1,131 @@ +package snappy + +import ( + "bytes" + "encoding/binary" + "errors" + + master "github.com/klauspost/compress/snappy" +) + +const ( + sizeOffset = 16 + sizeBytes = 4 +) + +var ( + xerialHeader = []byte{130, 83, 78, 65, 80, 80, 89, 0} + + // This is xerial version 1 and minimally compatible with version 1 + xerialVersionInfo = []byte{0, 0, 0, 1, 0, 0, 0, 1} + + // ErrMalformed is returned by the decoder when the xerial framing + // is malformed + ErrMalformed = errors.New("malformed xerial framing") +) + +func min(x, y int) int { + if x < y { + return x + } + return y +} + +// Encode encodes data as snappy with no framing header. +func Encode(src []byte) []byte { + return master.Encode(nil, src) +} + +// EncodeStream *appends* to the specified 'dst' the compressed +// 'src' in xerial framing format. If 'dst' does not have enough +// capacity, then a new slice will be allocated. If 'dst' has +// non-zero length, then if *must* have been built using this function. +func EncodeStream(dst, src []byte) []byte { + if len(dst) == 0 { + dst = append(dst, xerialHeader...) + dst = append(dst, xerialVersionInfo...) + } + + // Snappy encode in blocks of maximum 32KB + var ( + max = len(src) + blockSize = 32 * 1024 + pos = 0 + chunk []byte + ) + + for pos < max { + newPos := min(pos + blockSize, max) + chunk = master.Encode(chunk[:cap(chunk)], src[pos:newPos]) + + // First encode the compressed size (big-endian) + // Put* panics if the buffer is too small, so pad 4 bytes first + origLen := len(dst) + dst = append(dst, dst[0:4]...) + binary.BigEndian.PutUint32(dst[origLen:], uint32(len(chunk))) + + // And now the compressed data + dst = append(dst, chunk...) + pos = newPos + } + return dst +} + +// Decode decodes snappy data whether it is traditional unframed +// or includes the xerial framing format. +func Decode(src []byte) ([]byte, error) { + return DecodeInto(nil, src) +} + +// DecodeInto decodes snappy data whether it is traditional unframed +// or includes the xerial framing format into the specified `dst`. +// It is assumed that the entirety of `dst` including all capacity is available +// for use by this function. If `dst` is nil *or* insufficiently large to hold +// the decoded `src`, new space will be allocated. +func DecodeInto(dst, src []byte) ([]byte, error) { + var max = len(src) + if max < len(xerialHeader) { + return nil, ErrMalformed + } + + if !bytes.Equal(src[:8], xerialHeader) { + return master.Decode(dst[:cap(dst)], src) + } + + if max < sizeOffset+sizeBytes { + return nil, ErrMalformed + } + + if dst == nil { + dst = make([]byte, 0, len(src)) + } + + dst = dst[:0] + var ( + pos = sizeOffset + chunk []byte + err error + ) + + for pos+sizeBytes <= max { + size := int(binary.BigEndian.Uint32(src[pos : pos+sizeBytes])) + pos += sizeBytes + + nextPos := pos + size + // On architectures where int is 32-bytes wide size + pos could + // overflow so we need to check the low bound as well as the + // high + if nextPos < pos || nextPos > max { + return nil, ErrMalformed + } + + chunk, err = master.Decode(chunk[:cap(chunk)], src[pos:nextPos]) + + if err != nil { + return nil, err + } + pos = nextPos + dst = append(dst, chunk...) + } + return dst, nil +} diff --git a/compress/snappy/go-xerial-snappy/snappy_test.go b/compress/snappy/go-xerial-snappy/snappy_test.go new file mode 100644 index 000000000..bdc41bf4b --- /dev/null +++ b/compress/snappy/go-xerial-snappy/snappy_test.go @@ -0,0 +1,249 @@ +package snappy + +import ( + "bytes" + "testing" +) + +const largeString = `Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur? At vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias except` + +var snappyTestCases = map[string][]byte{ + "REPEATREPEATREPEATREPEATREPEATREPEAT": {36, 20, 82, 69, 80, 69, 65, 84, 118, 6, 0}, + "REALLY SHORT": {12, 44, 82, 69, 65, 76, 76, 89, 32, 83, 72, 79, 82, 84}, + "AXBXCXDXEXFX": {12, 44, 65, 88, 66, 88, 67, 88, 68, 88, 69, 88, 70, 88}, +} + +var snappyStreamTestCases = map[string][]byte{ + "PLAINDATA": {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 11, 9, 32, 80, 76, 65, 73, 78, 68, 65, 84, 65}, + `{"a":"UtaitILHMDAAAAfU","b":"日本"}`: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 39, 37, 144, 123, 34, 97, 34, 58, 34, 85, 116, 97, 105, 116, 73, 76, 72, 77, 68, 65, 65, 65, 65, 102, 85, 34, 44, 34, 98, 34, 58, 34, 230, 151, 165, 230, 156, 172, 34, 125}, + largeString: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3, 89, 128, 8, 240, 90, 83, 101, 100, 32, 117, 116, 32, 112, 101, 114, 115, 112, 105, 99, 105, 97, 116, 105, 115, 32, 117, 110, 100, 101, 32, 111, 109, 110, 105, 115, 32, 105, 115, 116, 101, 32, 110, 97, 116, 117, 115, 32, 101, 114, 114, 111, 114, 32, 115, 105, 116, 32, 118, 111, 108, 117, 112, 116, 97, 116, 101, 109, 32, 97, 99, 99, 117, 115, 97, 110, 116, 105, 117, 109, 32, 100, 111, 108, 111, 114, 101, 109, 113, 117, 101, 32, 108, 97, 117, 100, 97, 5, 22, 240, 60, 44, 32, 116, 111, 116, 97, 109, 32, 114, 101, 109, 32, 97, 112, 101, 114, 105, 97, 109, 44, 32, 101, 97, 113, 117, 101, 32, 105, 112, 115, 97, 32, 113, 117, 97, 101, 32, 97, 98, 32, 105, 108, 108, 111, 32, 105, 110, 118, 101, 110, 116, 111, 114, 101, 32, 118, 101, 114, 105, 116, 97, 1, 141, 4, 101, 116, 1, 36, 88, 115, 105, 32, 97, 114, 99, 104, 105, 116, 101, 99, 116, 111, 32, 98, 101, 97, 116, 97, 101, 32, 118, 105, 1, 6, 120, 100, 105, 99, 116, 97, 32, 115, 117, 110, 116, 32, 101, 120, 112, 108, 105, 99, 97, 98, 111, 46, 32, 78, 101, 109, 111, 32, 101, 110, 105, 109, 5, 103, 0, 109, 46, 180, 0, 12, 113, 117, 105, 97, 17, 16, 0, 115, 5, 209, 72, 97, 115, 112, 101, 114, 110, 97, 116, 117, 114, 32, 97, 117, 116, 32, 111, 100, 105, 116, 5, 9, 36, 102, 117, 103, 105, 116, 44, 32, 115, 101, 100, 9, 53, 32, 99, 111, 110, 115, 101, 113, 117, 117, 110, 1, 42, 20, 109, 97, 103, 110, 105, 32, 9, 245, 16, 115, 32, 101, 111, 115, 1, 36, 28, 32, 114, 97, 116, 105, 111, 110, 101, 17, 96, 33, 36, 1, 51, 36, 105, 32, 110, 101, 115, 99, 105, 117, 110, 116, 1, 155, 1, 254, 16, 112, 111, 114, 114, 111, 1, 51, 36, 115, 113, 117, 97, 109, 32, 101, 115, 116, 44, 1, 14, 13, 81, 5, 183, 4, 117, 109, 1, 18, 0, 97, 9, 19, 4, 32, 115, 1, 149, 12, 109, 101, 116, 44, 9, 135, 76, 99, 116, 101, 116, 117, 114, 44, 32, 97, 100, 105, 112, 105, 115, 99, 105, 32, 118, 101, 108, 50, 173, 0, 24, 110, 111, 110, 32, 110, 117, 109, 9, 94, 84, 105, 117, 115, 32, 109, 111, 100, 105, 32, 116, 101, 109, 112, 111, 114, 97, 32, 105, 110, 99, 105, 100, 33, 52, 20, 117, 116, 32, 108, 97, 98, 33, 116, 4, 101, 116, 9, 106, 0, 101, 5, 219, 20, 97, 109, 32, 97, 108, 105, 5, 62, 33, 164, 8, 114, 97, 116, 29, 212, 12, 46, 32, 85, 116, 41, 94, 52, 97, 100, 32, 109, 105, 110, 105, 109, 97, 32, 118, 101, 110, 105, 33, 221, 72, 113, 117, 105, 115, 32, 110, 111, 115, 116, 114, 117, 109, 32, 101, 120, 101, 114, 99, 105, 33, 202, 104, 111, 110, 101, 109, 32, 117, 108, 108, 97, 109, 32, 99, 111, 114, 112, 111, 114, 105, 115, 32, 115, 117, 115, 99, 105, 112, 105, 13, 130, 8, 105, 111, 115, 1, 64, 12, 110, 105, 115, 105, 1, 150, 5, 126, 44, 105, 100, 32, 101, 120, 32, 101, 97, 32, 99, 111, 109, 5, 192, 0, 99, 41, 131, 33, 172, 8, 63, 32, 81, 1, 107, 4, 97, 117, 33, 101, 96, 118, 101, 108, 32, 101, 117, 109, 32, 105, 117, 114, 101, 32, 114, 101, 112, 114, 101, 104, 101, 110, 100, 101, 114, 105, 65, 63, 12, 105, 32, 105, 110, 1, 69, 16, 118, 111, 108, 117, 112, 65, 185, 1, 47, 24, 105, 116, 32, 101, 115, 115, 101, 1, 222, 64, 109, 32, 110, 105, 104, 105, 108, 32, 109, 111, 108, 101, 115, 116, 105, 97, 101, 46, 103, 0, 0, 44, 1, 45, 16, 32, 105, 108, 108, 117, 37, 143, 45, 36, 0, 109, 5, 110, 65, 33, 20, 97, 116, 32, 113, 117, 111, 17, 92, 44, 115, 32, 110, 117, 108, 108, 97, 32, 112, 97, 114, 105, 9, 165, 24, 65, 116, 32, 118, 101, 114, 111, 69, 34, 44, 101, 116, 32, 97, 99, 99, 117, 115, 97, 109, 117, 115, 1, 13, 104, 105, 117, 115, 116, 111, 32, 111, 100, 105, 111, 32, 100, 105, 103, 110, 105, 115, 115, 105, 109, 111, 115, 32, 100, 117, 99, 105, 1, 34, 80, 113, 117, 105, 32, 98, 108, 97, 110, 100, 105, 116, 105, 105, 115, 32, 112, 114, 97, 101, 115, 101, 101, 87, 17, 111, 56, 116, 117, 109, 32, 100, 101, 108, 101, 110, 105, 116, 105, 32, 97, 116, 65, 89, 28, 99, 111, 114, 114, 117, 112, 116, 105, 1, 150, 0, 115, 13, 174, 5, 109, 8, 113, 117, 97, 65, 5, 52, 108, 101, 115, 116, 105, 97, 115, 32, 101, 120, 99, 101, 112, 116, 0, 0, 0, 1, 0}, +} + +func makeMassive(input string, numCopies int) string { + outBuff := make([]byte, len(input) * numCopies) + + for i := 0; i < numCopies; i++ { + copy(outBuff[len(outBuff):], input) + } + + return string(outBuff) +} + +func TestSnappyEncode(t *testing.T) { + for src, exp := range snappyTestCases { + dst := Encode([]byte(src)) + if !bytes.Equal(dst, exp) { + t.Errorf("Expected %s to generate %v, but was %v", src, exp, dst) + } + } +} + +func TestSnappyEncodeStream(t *testing.T) { + for src, _ := range snappyStreamTestCases { + dst := EncodeStream(nil, []byte(src)) + + // Block size can change the bytes generated, so let's just decode and make sure in matches out + dec, err := Decode(dst) + if err != nil { + t.Error(err) + } + if src != string(dec) { + t.Errorf("Expected decode to match encode orig = %s, decoded = %s", src, string(dec)) + } + } +} + +func TestSnappyLargeStringEncodeStream(t *testing.T) { + massiveString := makeMassive(largeString, 10000) + dst := EncodeStream(nil, []byte(massiveString)) + dec, err := Decode(dst) + if err != nil { + t.Error(err) + } + if massiveString != string(dec) { + t.Errorf("Decoded string didn't match original input (not printing due to size)") + } +} + +func TestSnappyDecode(t *testing.T) { + for exp, src := range snappyTestCases { + dst, err := Decode(src) + if err != nil { + t.Error("Encoding error: ", err) + } else if !bytes.Equal(dst, []byte(exp)) { + t.Errorf("Expected %s to be generated from %v, but was %s", exp, src, string(dst)) + } + } +} + +func TestSnappyDecodeStreams(t *testing.T) { + for exp, src := range snappyStreamTestCases { + dst, err := Decode(src) + if err != nil { + t.Error("Encoding error: ", err) + } else if !bytes.Equal(dst, []byte(exp)) { + t.Errorf("Expected %s to be generated from [%d]byte, but was %s", exp, len(src), string(dst)) + } + } +} + +func TestSnappyDecodeMalformedTruncatedHeader(t *testing.T) { + // Truncated headers should not cause a panic. + for i := 0; i < len(xerialHeader); i++ { + buf := make([]byte, i) + copy(buf, xerialHeader[:i]) + if _, err := Decode(buf); err != ErrMalformed { + t.Errorf("expected ErrMalformed got %v", err) + } + } +} + +func TestSnappyDecodeMalformedTruncatedSize(t *testing.T) { + // Inputs with valid Xerial header but truncated "size" field + sizes := []int{sizeOffset + 1, sizeOffset + 2, sizeOffset + 3} + for _, size := range sizes { + buf := make([]byte, size) + copy(buf, xerialHeader) + if _, err := Decode(buf); err != ErrMalformed { + t.Errorf("expected ErrMalformed got %v", err) + } + } +} + +func TestSnappyDecodeMalformedBNoData(t *testing.T) { + // No data after the size field + buf := make([]byte, 20) + copy(buf, xerialHeader) + // indicate that there's one byte of data to be read + buf[len(buf)-1] = 1 + if _, err := Decode(buf); err != ErrMalformed { + t.Errorf("expected ErrMalformed got %v", err) + } +} + +func TestSnappyMasterDecodeFailed(t *testing.T) { + buf := make([]byte, 21) + copy(buf, xerialHeader) + // indicate that there's one byte of data to be read + buf[len(buf)-2] = 1 + // A payload which will not decode + buf[len(buf)-1] = 1 + if _, err := Decode(buf); err == ErrMalformed || err == nil { + t.Errorf("unexpected err: %v", err) + } +} + +func BenchmarkSnappyDecode(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for n := 0; n < b.N; n++ { + bytes := 0 + for _, test := range snappyTestCases { + dst, err := Decode(test) + if err != nil { + b.Error("Decoding error: ", err) + } + bytes += len(dst) + } + b.SetBytes(int64(bytes)) + } +} + +func BenchmarkSnappyDecodeInto(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + var ( + dst []byte + err error + ) + + for n := 0; n < b.N; n++ { + bytes := 0 + for _, test := range snappyTestCases { + + dst, err = DecodeInto(dst, test) + if err != nil { + b.Error("Decoding error: ", err) + } + bytes += len(dst) + } + b.SetBytes(int64(bytes)) + } +} + +func BenchmarkSnappyStreamDecode(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for n := 0; n < b.N; n++ { + bytes := 0 + for _, test := range snappyStreamTestCases { + dst, err := Decode(test) + if err != nil { + b.Error("Decoding error: ", err) + } + bytes += len(dst) + } + b.SetBytes(int64(bytes)) + } +} + +func BenchmarkSnappyStreamDecodeInto(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + var ( + dst = make([]byte, 1024, 1024) + err error + ) + + for n := 0; n < b.N; n++ { + bytes := 0 + for _, test := range snappyStreamTestCases { + dst, err = DecodeInto(dst, test) + if err != nil { + b.Error("Decoding error: ", err) + } + bytes += len(dst) + } + b.SetBytes(int64(bytes)) + } +} +func BenchmarkSnappyStreamDecodeMassive(b *testing.B) { + massiveString := makeMassive(largeString, 10000) + enc := EncodeStream(nil, []byte(massiveString)) + + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(int64(len(massiveString))) + + for n := 0; n < b.N; n++ { + _, err := Decode(enc) + if err != nil { + b.Error("Decoding error: ", err) + } + } +} + +func BenchmarkSnappyStreamDecodeIntoMassive(b *testing.B) { + massiveString := makeMassive(largeString, 10000) + enc := EncodeStream(nil, []byte(massiveString)) + + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(int64(len(massiveString))) + + var ( + dst = make([]byte, 1024, 1024) + err error + ) + + for n := 0; n < b.N; n++ { + dst, err = DecodeInto(dst, enc) + if err != nil { + b.Error("Decoding error: ", err) + } + } +} diff --git a/compress/snappy/snappy.go b/compress/snappy/snappy.go index fcf72409d..a726ecebc 100644 --- a/compress/snappy/snappy.go +++ b/compress/snappy/snappy.go @@ -4,7 +4,7 @@ import ( "io" "sync" - "github.com/golang/snappy" + "github.com/klauspost/compress/snappy" ) // Framing is an enumeration type used to enable or disable xerial framing of diff --git a/compress/snappy/xerial.go b/compress/snappy/xerial.go index 06b73909e..6445d7486 100644 --- a/compress/snappy/xerial.go +++ b/compress/snappy/xerial.go @@ -5,7 +5,7 @@ import ( "encoding/binary" "io" - "github.com/golang/snappy" + "github.com/klauspost/compress/snappy" ) const defaultBufferSize = 32 * 1024 diff --git a/compress/snappy/xerial_test.go b/compress/snappy/xerial_test.go index 4545bfe2e..fc91a42b2 100644 --- a/compress/snappy/xerial_test.go +++ b/compress/snappy/xerial_test.go @@ -6,8 +6,8 @@ import ( "io" "testing" - goxerialsnappy "github.com/eapache/go-xerial-snappy" - "github.com/golang/snappy" + goxerialsnappy "github.com/segmentio/kafka-go/compress/snappy/go-xerial-snappy" + "github.com/klauspost/compress/snappy" ) // Wrap an io.Reader or io.Writer to disable all copy optimizations like diff --git a/go.mod b/go.mod index cfc9cfdfe..dffdfa9d4 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,8 @@ module github.com/segmentio/kafka-go go 1.15 require ( - github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 - github.com/frankban/quicktest v1.11.3 // indirect - github.com/golang/snappy v0.0.1 - github.com/klauspost/compress v1.9.8 - github.com/pierrec/lz4 v2.6.0+incompatible + github.com/klauspost/compress v1.14.2 + github.com/pierrec/lz4/v4 v4.1.14 github.com/stretchr/testify v1.6.1 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/xdg/stringprep v1.0.0 // indirect diff --git a/go.sum b/go.sum index 5982e2410..25361b443 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,9 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= -github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= -github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw= +github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= +github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -35,8 +22,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=