Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added Unix epoch timestamp support for JSON parser #4633

Merged
merged 5 commits into from
Sep 7, 2018
Merged

Added Unix epoch timestamp support for JSON parser #4633

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
092bd26
Added Unix epoch timestamp support for JSON parser. Corresponding tes…
drenizg Sep 4, 2018
2e365ed
gofmt fixes.
drenizg Sep 4, 2018
bf1eb82
"unix" option now handles epoch in seconds (with decimal part support).
drenizg Sep 4, 2018
76f49cf
Added support to parse unix timestamps coming as strings fields, with…
drenizg Sep 6, 2018
37548b8
Unix timestamp parsing logic now extracted as new function `parseUnix…
drenizg Sep 7, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Unix timestamp parsing logic now extracted as new function `parseUnix… 
…Timestamp`
  • Loading branch information
@drenizg
drenizg committed Sep 7, 2018
commit 37548b8756d9f4c9b62944ec77b91a0d78614dc8
82 changes: 47 additions & 35 deletions plugins/parsers/json/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
"math"
"regexp"
Expand Down Expand Up @@ -49,6 +50,49 @@ func (p *JSONParser) parseArray(buf []byte) ([]telegraf.Metric, error) {
return metrics, nil
}

// format = "unix": epoch is assumed to be in seconds and can come as number or string. Can have a decimal part.
// format = "unix_ms": epoch is assumed to be in milliseconds and can come as number or string. Cannot have a decimal part.
func parseUnixTimestamp(jsonValue interface{}, format string) (time.Time, error) {
timeInt, timeFractional := int64(0), int64(0)
timeEpochStr, ok := jsonValue.(string)
var err error

if !ok {
timeEpochFloat, ok := jsonValue.(float64)
if !ok {
err := fmt.Errorf("time: %v could not be converted to string nor float64", jsonValue)
return time.Time{}, err
}
intPart, frac := math.Modf(timeEpochFloat)
timeInt, timeFractional = int64(intPart), int64(frac*1e9)
} else {
splitted := regexp.MustCompile("[.,]").Split(timeEpochStr, 2)
timeInt, err = strconv.ParseInt(splitted[0], 10, 64)
if err != nil {
return time.Time{}, err
}

if len(splitted) == 2 {
if len(splitted[1]) > 9 {
splitted[1] = splitted[1][:9] //truncates decimal part to nanoseconds precision
}
nanosecStr := splitted[1] + strings.Repeat("0", 9-len(splitted[1])) //adds 0's to the right to obtain a valid number of nanoseconds

timeFractional, err = strconv.ParseInt(nanosecStr, 10, 64)
if err != nil {
return time.Time{}, err
}
}
}
if strings.EqualFold(format, "unix") {
return time.Unix(timeInt, timeFractional).UTC(), nil
} else if strings.EqualFold(format, "unix_ms") {
return time.Unix(timeInt/1000, (timeInt%1000)*1e6).UTC(), nil
} else {
return time.Time{}, errors.New("Invalid unix format")
}
}

func (p *JSONParser) parseObject(metrics []telegraf.Metric, jsonOut map[string]interface{}) ([]telegraf.Metric, error) {

tags := make(map[string]string)
Expand Down Expand Up @@ -80,42 +124,10 @@ func (p *JSONParser) parseObject(metrics []telegraf.Metric, jsonOut map[string]i
return nil, err
}

// unix: epoch is assumed to be in seconds. Can have a decimal part.
// unix_ms: epoch is assumed to be in milliseconds. Cannot have a decimal part.
if strings.EqualFold(p.JSONTimeFormat, "unix") || strings.EqualFold(p.JSONTimeFormat, "unix_ms") {
timeInt, timeFractional := int64(0), int64(0)
timeEpochStr, ok := f.Fields[p.JSONTimeKey].(string)
if !ok {
timeEpochFloat, ok := f.Fields[p.JSONTimeKey].(float64)
if !ok {
err := fmt.Errorf("time: %v could not be converted to string nor float64", f.Fields[p.JSONTimeKey])
return nil, err
}
intPart, frac := math.Modf(timeEpochFloat)
timeInt, timeFractional = int64(intPart), int64(frac*1e9)
} else {
splitted := regexp.MustCompile("[.,]").Split(timeEpochStr, 2)
timeInt, err = strconv.ParseInt(splitted[0], 10, 64)
if err != nil {
return nil, err
}

if len(splitted) == 2 {
if len(splitted[1]) > 9 {
splitted[1] = splitted[1][:9] //truncates decimal part to nanoseconds precision
}
nanosecStr := splitted[1] + strings.Repeat("0", 9-len(splitted[1])) //adds 0's to the right to obtain a valid number of nanoseconds

timeFractional, err = strconv.ParseInt(nanosecStr, 10, 64)
if err != nil {
return nil, err
}
}
}
if strings.EqualFold(p.JSONTimeFormat, "unix") {
nTime = time.Unix(timeInt, timeFractional).UTC()
} else { //unix_ms
nTime = time.Unix(timeInt/1000, (timeInt%1000)*1e6).UTC()
nTime, err = parseUnixTimestamp(f.Fields[p.JSONTimeKey], p.JSONTimeFormat)
if err != nil {
return nil, err
}
} else {
timeStr, ok := f.Fields[p.JSONTimeKey].(string)
Expand Down