Support for adding IP Address/Prefix + stream based decoder (rs#49)

* added IPAddr, IPPrefix and stream based cbor decoder * Update README with cbor decoder tool info * Update README in cbor with comparison data
lukaszraczylo · Apr 3, 2018 · 2ccfab3 · 2ccfab3
1 parent 05eafee
commit 2ccfab3
Show file tree

Hide file tree

Showing 22 changed files with 1,188 additions and 682 deletions.
diff --git a/README.md b/README.md
@@ -436,6 +436,9 @@ In addition to the default JSON encoding, `zerolog` can produce binary logs usin
 go build -tags binary_log .
 ```
 
+To Decode binary encoded log files you can use any CBOR decoder. One has been tested to work
+with zerolog library is CSD(https://github.com/toravir/csd/).
+
 ## Benchmarks
 
 All operations are allocation free (those numbers *include* JSON encoding):

diff --git a/array.go b/array.go
@@ -1,6 +1,7 @@
 package zerolog
 
 import (
+	"net"
 	"sync"
 	"time"
 )
@@ -174,3 +175,21 @@ func (a *Array) Interface(i interface{}) *Array {
 	a.buf = appendInterface(appendArrayDelim(a.buf), i)
 	return a
 }
+
+// IPAddr adds IPv4 or IPv6 address to the array
+func (a *Array) IPAddr(ip net.IP) *Array {
+	a.buf = appendIPAddr(appendArrayDelim(a.buf), ip)
+	return a
+}
+
+// IPPrefix adds IPv4 or IPv6 Prefix (IP + mask) to the array
+func (a *Array) IPPrefix(pfx net.IPNet) *Array {
+	a.buf = appendIPPrefix(appendArrayDelim(a.buf), pfx)
+	return a
+}
+
+// MACAddr adds a MAC (Ethernet) address to the array
+func (a *Array) MACAddr(ha net.HardwareAddr) *Array {
+	a.buf = appendMACAddr(appendArrayDelim(a.buf), ha)
+	return a
+}
diff --git a/array_test.go b/array_test.go
@@ -1,6 +1,7 @@
 package zerolog
 
 import (
+	"net"
 	"testing"
 	"time"
 )
@@ -24,8 +25,9 @@ func TestArray(t *testing.T) {
 		Bytes([]byte("b")).
 		Hex([]byte{0x1f}).
 		Time(time.Time{}).
+		IPAddr(net.IP{192, 168, 0, 10}).
 		Dur(0)
-	want := `[true,1,2,3,4,5,6,7,8,9,10,11,12,"a","b","1f","0001-01-01T00:00:00Z",0]`
+	want := `[true,1,2,3,4,5,6,7,8,9,10,11,12,"a","b","1f","0001-01-01T00:00:00Z","192.168.0.10",0]`
 	if got := decodeObjectToStr(a.write([]byte{})); got != want {
 		t.Errorf("Array.write()\ngot:  %s\nwant: %s", got, want)
 	}

diff --git a/context.go b/context.go
@@ -2,6 +2,7 @@ package zerolog
 
 import (
 	"io/ioutil"
+	"net"
 	"time"
 )
 
@@ -330,3 +331,21 @@ func (c Context) Caller() Context {
 	c.l = c.l.Hook(ch)
 	return c
 }
+
+// IPAddr adds IPv4 or IPv6 Address to the context
+func (c Context) IPAddr(key string, ip net.IP) Context {
+	c.l.context = appendIPAddr(appendKey(c.l.context, key), ip)
+	return c
+}
+
+// IPPrefix adds IPv4 or IPv6 Prefix (address and mask) to the context
+func (c Context) IPPrefix(key string, pfx net.IPNet) Context {
+	c.l.context = appendIPPrefix(appendKey(c.l.context, key), pfx)
+	return c
+}
+
+// MACAddr adds MAC address to the context
+func (c Context) MACAddr(key string, ha net.HardwareAddr) Context {
+	c.l.context = appendMACAddr(appendKey(c.l.context, key), ha)
+	return c
+}
diff --git a/encoder_cbor.go b/encoder_cbor.go
@@ -5,6 +5,7 @@ package zerolog
 // This file contains bindings to do binary encoding.
 
 import (
+	"net"
 	"time"
 
 	"github.com/rs/zerolog/internal/cbor"
@@ -211,6 +212,18 @@ func decodeObjectToStr(in []byte) string {
 	return cbor.DecodeObjectToStr(in)
 }
 
+func appendIPAddr(dst []byte, ip net.IP) []byte {
+	return cbor.AppendIPAddr(dst, ip)
+}
+
+func appendIPPrefix(dst []byte, pfx net.IPNet) []byte {
+	return cbor.AppendIPPrefix(dst, pfx)
+}
+
+func appendMACAddr(dst []byte, ha net.HardwareAddr) []byte {
+	return cbor.AppendMACAddr(dst, ha)
+}
+
 // decodeIfBinaryToBytes - converts a binary formatted log msg to a
 // JSON formatted Bytes Log message.
 func decodeIfBinaryToBytes(in []byte) []byte {

diff --git a/encoder_json.go b/encoder_json.go
@@ -6,6 +6,7 @@ package zerolog
 // JSON encoded byte stream.
 
 import (
+	"net"
 	"strconv"
 	"time"
 
@@ -211,6 +212,18 @@ func decodeIfBinaryToBytes(in []byte) []byte {
 	return in
 }
 
+func appendIPAddr(dst []byte, ip net.IP) []byte {
+	return json.AppendIPAddr(dst, ip)
+}
+
+func appendIPPrefix(dst []byte, pfx net.IPNet) []byte {
+	return json.AppendIPPrefix(dst, pfx)
+}
+
+func appendMACAddr(dst []byte, ha net.HardwareAddr) []byte {
+	return json.AppendMACAddr(dst, ha)
+}
+
 func appendHex(in []byte, val []byte) []byte {
-    return json.AppendHex(in, val)
+	return json.AppendHex(in, val)
 }
diff --git a/event.go b/event.go
@@ -2,6 +2,7 @@ package zerolog
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"runtime"
 	"strconv"
@@ -597,3 +598,30 @@ func (e *Event) caller(skip int) *Event {
 	e.buf = appendString(appendKey(e.buf, CallerFieldName), file+":"+strconv.Itoa(line))
 	return e
 }
+
+// IPAddr adds IPv4 or IPv6 Address to the event
+func (e *Event) IPAddr(key string, ip net.IP) *Event {
+	if e == nil {
+		return e
+	}
+	e.buf = appendIPAddr(appendKey(e.buf, key), ip)
+	return e
+}
+
+// IPPrefix adds IPv4 or IPv6 Prefix (address and mask) to the event
+func (e *Event) IPPrefix(key string, pfx net.IPNet) *Event {
+	if e == nil {
+		return e
+	}
+	e.buf = appendIPPrefix(appendKey(e.buf, key), pfx)
+	return e
+}
+
+// MACAddr adds MAC address to the event
+func (e *Event) MACAddr(key string, ha net.HardwareAddr) *Event {
+	if e == nil {
+		return e
+	}
+	e.buf = appendMACAddr(appendKey(e.buf, key), ha)
+	return e
+}
diff --git a/fields.go b/fields.go
@@ -1,6 +1,7 @@
 package zerolog
 
 import (
+	"net"
 	"sort"
 	"time"
 )
@@ -118,6 +119,12 @@ func appendFields(dst []byte, fields map[string]interface{}) []byte {
 			dst = appendDurations(dst, val, DurationFieldUnit, DurationFieldInteger)
 		case nil:
 			dst = appendNil(dst)
+		case net.IP:
+			dst = appendIPAddr(dst, val)
+		case net.IPNet:
+			dst = appendIPPrefix(dst, val)
+		case net.HardwareAddr:
+			dst = appendMACAddr(dst, val)
 		default:
 			dst = appendInterface(dst, val)
 		}

diff --git a/internal/cbor/README.md b/internal/cbor/README.md
@@ -1,74 +1,56 @@
-Reference:
-   CBOR Encoding is described in RFC7049 https://tools.ietf.org/html/rfc7049
+## Reference:
+   CBOR Encoding is described in [RFC7049](https://tools.ietf.org/html/rfc7049)
 
+## Comparison of JSON vs CBOR
 
-Tests and benchmark:
+Two main areas of reduction are:
 
+1. CPU usage to write a log msg 
+2. Size (in bytes) of log messages.
+
+
+CPU Usage savings are below:
 ```
-sprint @ cbor>go test -v -benchmem -bench=.                                                                               
-=== RUN   TestDecodeInteger                                                                                               
---- PASS: TestDecodeInteger (0.00s)                                                                                       
-=== RUN   TestDecodeString                                                                                                
---- PASS: TestDecodeString (0.00s)                                                                                        
-=== RUN   TestDecodeArray                                                                                                 
---- PASS: TestDecodeArray (0.00s)                                                                                         
-=== RUN   TestDecodeMap                                                                                                   
---- PASS: TestDecodeMap (0.00s)                                                                                           
-=== RUN   TestDecodeBool                                                                                                  
---- PASS: TestDecodeBool (0.00s)                                                                                          
-=== RUN   TestDecodeFloat                                                                                                 
---- PASS: TestDecodeFloat (0.00s)                                                                                         
-=== RUN   TestDecodeTimestamp                                                                                             
---- PASS: TestDecodeTimestamp (0.00s)                                                                                     
-=== RUN   TestDecodeCbor2Json                                                                                             
---- PASS: TestDecodeCbor2Json (0.00s)                                                                                     
-=== RUN   TestAppendString                                                                                                
---- PASS: TestAppendString (0.00s)                                                                                        
-=== RUN   TestAppendBytes                                                                                                 
---- PASS: TestAppendBytes (0.00s)                                                                                         
-=== RUN   TestAppendTimeNow                                                                                               
---- PASS: TestAppendTimeNow (0.00s)                                                                                       
-=== RUN   TestAppendTimePastPresentInteger                                                                                
---- PASS: TestAppendTimePastPresentInteger (0.00s)                                                                        
-=== RUN   TestAppendTimePastPresentFloat                                                                                  
---- PASS: TestAppendTimePastPresentFloat (0.00s)                                                                          
-=== RUN   TestAppendNull                                                                                                  
---- PASS: TestAppendNull (0.00s)                                                                                          
-=== RUN   TestAppendBool                                                                                                  
---- PASS: TestAppendBool (0.00s)                                                                                          
-=== RUN   TestAppendBoolArray
---- PASS: TestAppendBoolArray (0.00s)
-=== RUN   TestAppendInt
---- PASS: TestAppendInt (0.00s)
-=== RUN   TestAppendIntArray
---- PASS: TestAppendIntArray (0.00s)
-=== RUN   TestAppendFloat32
---- PASS: TestAppendFloat32 (0.00s)
-goos: linux
-goarch: amd64
-pkg: github.com/toravir/zerolog/internal/cbor
-BenchmarkAppendString/MultiBytesLast-4          30000000                43.3 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendString/NoEncoding-4              30000000                48.2 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendString/EncodingFirst-4           30000000                48.2 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendString/EncodingMiddle-4          30000000                41.7 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendString/EncodingLast-4            30000000                51.8 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendString/MultiBytesFirst-4         50000000                38.0 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendString/MultiBytesMiddle-4        50000000                38.0 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendTime/Integer-4                   50000000                39.6 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendTime/Float-4                     30000000                56.1 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/uint8-4                      50000000                29.1 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/uint16-4                     50000000                30.3 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/uint32-4                     50000000                37.1 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/int8-4                       100000000               21.5 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/int16-4                      50000000                25.8 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/int32-4                      50000000                26.7 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/int-Positive-4               100000000               21.5 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/int-Negative-4               100000000               20.7 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/uint64-4                     50000000                36.7 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendInt/int64-4                      30000000                39.6 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendFloat/Float32-4                  50000000                23.9 ns/op             0 B/op          0 allocs/op
-BenchmarkAppendFloat/Float64-4                  50000000                32.8 ns/op             0 B/op          0 allocs/op
-PASS
-ok      github.com/toravir/zerolog/internal/cbor        34.969s
-sprint @ cbor>
+name                                    JSON time/op    CBOR time/op   delta
+Info-32                                   15.3ns ± 1%    11.7ns ± 3%  -23.78%  (p=0.000 n=9+10)      
+ContextFields-32                          16.2ns ± 2%    12.3ns ± 3%  -23.97%  (p=0.000 n=9+9)       
+ContextAppend-32                          6.70ns ± 0%    6.20ns ± 0%   -7.44%  (p=0.000 n=9+9)       
+LogFields-32                              66.4ns ± 0%    24.6ns ± 2%  -62.89%  (p=0.000 n=10+9)      
+LogArrayObject-32                          911ns ±11%     768ns ± 6%  -15.64%  (p=0.000 n=10+10)     
+LogFieldType/Floats-32                    70.3ns ± 2%    29.5ns ± 1%  -57.98%  (p=0.000 n=10+10)     
+LogFieldType/Err-32                       14.0ns ± 3%    12.1ns ± 8%  -13.20%  (p=0.000 n=8+10)      
+LogFieldType/Dur-32                       17.2ns ± 2%    13.1ns ± 1%  -24.27%  (p=0.000 n=10+9)      
+LogFieldType/Object-32                    54.3ns ±11%    52.3ns ± 7%     ~     (p=0.239 n=10+10)     
+LogFieldType/Ints-32                      20.3ns ± 2%    15.1ns ± 2%  -25.50%  (p=0.000 n=9+10)      
+LogFieldType/Interfaces-32                 642ns ±11%     621ns ± 9%     ~     (p=0.118 n=10+10)     
+LogFieldType/Interface(Objects)-32         635ns ±13%     632ns ± 9%     ~     (p=0.592 n=10+10)     
+LogFieldType/Times-32                      294ns ± 0%      27ns ± 1%  -90.71%  (p=0.000 n=10+9)      
+LogFieldType/Durs-32                       121ns ± 0%      33ns ± 2%  -72.44%  (p=0.000 n=9+9)       
+LogFieldType/Interface(Object)-32         56.6ns ± 8%    52.3ns ± 8%   -7.54%  (p=0.007 n=10+10)     
+LogFieldType/Errs-32                      17.8ns ± 3%    16.1ns ± 2%   -9.71%  (p=0.000 n=10+9)      
+LogFieldType/Time-32                      40.5ns ± 1%    12.7ns ± 6%  -68.66%  (p=0.000 n=8+9)       
+LogFieldType/Bool-32                      12.0ns ± 5%    10.2ns ± 2%  -15.18%  (p=0.000 n=10+8)      
+LogFieldType/Bools-32                     17.2ns ± 2%    12.6ns ± 4%  -26.63%  (p=0.000 n=10+10)     
+LogFieldType/Int-32                       12.3ns ± 2%    11.2ns ± 4%   -9.27%  (p=0.000 n=9+10)      
+LogFieldType/Float-32                     16.7ns ± 1%    12.6ns ± 2%  -24.42%  (p=0.000 n=7+9)       
+LogFieldType/Str-32                       12.7ns ± 7%    11.3ns ± 7%  -10.88%  (p=0.000 n=10+9)      
+LogFieldType/Strs-32                      20.3ns ± 3%    18.2ns ± 3%  -10.25%  (p=0.000 n=9+10)      
+LogFieldType/Interface-32                  183ns ±12%     175ns ± 9%     ~     (p=0.078 n=10+10)     
 ```
+
+Log message size savings is greatly dependent on the number and type of fields in the log message.
+Assuming this log message (with an Integer, timestamp and string, in addition to level).
+
+`{"level":"error","Fault":41650,"time":"2018-04-01T15:18:19-07:00","message":"Some Message"}`
+
+Two measurements were done for the log file sizes - one without any compression, second
+using [compress/zlib](https://golang.org/pkg/compress/zlib/). 
+
+Results for 10,000 log messages:
+
+| Log Format |  Plain File Size (in KB) | Compressed File Size (in KB) |
+| :--- | :---: | :---: |
+| JSON | 920 | 28 |
+| CBOR | 550 | 28 |
+
+The example used to calculate the above data is available in [Examples](examples).
diff --git a/internal/cbor/cbor.go b/internal/cbor/cbor.go
@@ -7,25 +7,34 @@ import "time"
 const (
 	majorOffset   = 5
 	additionalMax = 23
-	//Non Values
+
+	// Non Values.
 	additionalTypeBoolFalse byte = 20
 	additionalTypeBoolTrue  byte = 21
 	additionalTypeNull      byte = 22
-	//Integer (+ve and -ve) Sub-types
+
+	// Integer (+ve and -ve) Sub-types.
 	additionalTypeIntUint8  byte = 24
 	additionalTypeIntUint16 byte = 25
 	additionalTypeIntUint32 byte = 26
 	additionalTypeIntUint64 byte = 27
-	//Float Sub-types
+
+	// Float Sub-types.
 	additionalTypeFloat16 byte = 25
 	additionalTypeFloat32 byte = 26
 	additionalTypeFloat64 byte = 27
 	additionalTypeBreak   byte = 31
-	//Tag Sub-types
-	additionalTypeTimestamp    byte = 01
-	additionalTypeEmbeddedJSON byte = 31
-        additionalTypeTagHexString uint16 = 262
-	//Unspecified number of elements
+
+	// Tag Sub-types.
+	additionalTypeTimestamp byte = 01
+
+	// Extended Tags - from https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml
+	additionalTypeTagNetworkAddr   uint16 = 260
+	additionalTypeTagNetworkPrefix uint16 = 261
+	additionalTypeEmbeddedJSON     uint16 = 262
+	additionalTypeTagHexString     uint16 = 263
+
+	// Unspecified number of elements.
 	additionalTypeInfiniteCount byte = 31
 )
 const (