diff --git a/Makefile b/Makefile index 2d06f9ddc..e7d810945 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,10 @@ VERSION ?= 0.1.8 prepare: docker login +.PHONY: deps +deps: + go get -v github.com/vitkovskii/insane-doc@v0.0.1 + .PHONY: test test: go test ./fd/ -v -count 1 diff --git a/README.md b/README.md index e8291aa49..92f627b8f 100755 --- a/README.md +++ b/README.md @@ -29,11 +29,11 @@ TBD: throughput on production servers. ## Plugins -**Input**: [dmesg](/plugin/input/dmesg/README.md), [fake](/plugin/input/fake/README.md), [file](/plugin/input/file/README.md), [http](/plugin/input/http/README.md), [k8s](/plugin/input/k8s/README.md), [kafka](/plugin/input/kafka/README.md) +**Input**: [dmesg](plugin/input/dmesg/README.md), [fake](plugin/input/fake/README.md), [file](plugin/input/file/README.md), [http](plugin/input/http/README.md), [journalctl](plugin/input/journalctl/README.md), [k8s](plugin/input/k8s/README.md), [kafka](plugin/input/kafka/README.md) -**Action**: [convert_date](/plugin/action/convert_date/README.md), [debug](/plugin/action/debug/README.md), [discard](/plugin/action/discard/README.md), [flatten](/plugin/action/flatten/README.md), [join](/plugin/action/join/README.md), [json_decode](/plugin/action/json_decode/README.md), [keep_fields](/plugin/action/keep_fields/README.md), [modify](/plugin/action/modify/README.md), [parse_es](/plugin/action/parse_es/README.md), [remove_fields](/plugin/action/remove_fields/README.md), [rename](/plugin/action/rename/README.md), [throttle](/plugin/action/throttle/README.md) +**Action**: [convert_date](plugin/action/convert_date/README.md), [debug](plugin/action/debug/README.md), [discard](plugin/action/discard/README.md), [flatten](plugin/action/flatten/README.md), [join](plugin/action/join/README.md), [json_decode](plugin/action/json_decode/README.md), [keep_fields](plugin/action/keep_fields/README.md), [modify](plugin/action/modify/README.md), [parse_es](plugin/action/parse_es/README.md), [remove_fields](plugin/action/remove_fields/README.md), [rename](plugin/action/rename/README.md), [throttle](plugin/action/throttle/README.md) -**Output**: [devnull](/plugin/output/devnull/README.md), [elasticsearch](/plugin/output/elasticsearch/README.md), [gelf](/plugin/output/gelf/README.md), [kafka](/plugin/output/kafka/README.md), [stdout](/plugin/output/stdout/README.md) +**Output**: [devnull](plugin/output/devnull/README.md), [elasticsearch](plugin/output/elasticsearch/README.md), [gelf](plugin/output/gelf/README.md), [kafka](plugin/output/kafka/README.md), [stdout](plugin/output/stdout/README.md) ## What's next * [Quick start](/docs/quick-start.md) diff --git a/_sidebar.md b/_sidebar.md index 53fd353ea..e424d5130 100644 --- a/_sidebar.md +++ b/_sidebar.md @@ -13,33 +13,34 @@ - **Plugins** - Input - - [dmesg](/plugin/input/dmesg/README.md) - - [fake](/plugin/input/fake/README.md) - - [file](/plugin/input/file/README.md) - - [http](/plugin/input/http/README.md) - - [k8s](/plugin/input/k8s/README.md) - - [kafka](/plugin/input/kafka/README.md) + - [dmesg](plugin/input/dmesg/README.md) + - [fake](plugin/input/fake/README.md) + - [file](plugin/input/file/README.md) + - [http](plugin/input/http/README.md) + - [journalctl](plugin/input/journalctl/README.md) + - [k8s](plugin/input/k8s/README.md) + - [kafka](plugin/input/kafka/README.md) - Action - - [convert_date](/plugin/action/convert_date/README.md) - - [debug](/plugin/action/debug/README.md) - - [discard](/plugin/action/discard/README.md) - - [flatten](/plugin/action/flatten/README.md) - - [join](/plugin/action/join/README.md) - - [json_decode](/plugin/action/json_decode/README.md) - - [keep_fields](/plugin/action/keep_fields/README.md) - - [modify](/plugin/action/modify/README.md) - - [parse_es](/plugin/action/parse_es/README.md) - - [remove_fields](/plugin/action/remove_fields/README.md) - - [rename](/plugin/action/rename/README.md) - - [throttle](/plugin/action/throttle/README.md) + - [convert_date](plugin/action/convert_date/README.md) + - [debug](plugin/action/debug/README.md) + - [discard](plugin/action/discard/README.md) + - [flatten](plugin/action/flatten/README.md) + - [join](plugin/action/join/README.md) + - [json_decode](plugin/action/json_decode/README.md) + - [keep_fields](plugin/action/keep_fields/README.md) + - [modify](plugin/action/modify/README.md) + - [parse_es](plugin/action/parse_es/README.md) + - [remove_fields](plugin/action/remove_fields/README.md) + - [rename](plugin/action/rename/README.md) + - [throttle](plugin/action/throttle/README.md) - Output - - [devnull](/plugin/output/devnull/README.md) - - [elasticsearch](/plugin/output/elasticsearch/README.md) - - [gelf](/plugin/output/gelf/README.md) - - [kafka](/plugin/output/kafka/README.md) - - [stdout](/plugin/output/stdout/README.md) + - [devnull](plugin/output/devnull/README.md) + - [elasticsearch](plugin/output/elasticsearch/README.md) + - [gelf](plugin/output/gelf/README.md) + - [kafka](plugin/output/kafka/README.md) + - [stdout](plugin/output/stdout/README.md) - **Other** diff --git a/docs/installation.md b/docs/installation.md index 9c6e3c191..97b189c82 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,10 +2,10 @@ ## Docker -Images are available on [github packages](https://github.com/ozonru/file.d/packages). +Images are available on [docker hub](https://hub.docker.com/r/ozonru/file.d). Mount config from /my-config.yaml and start `file.d` with this config:
-`docker run -v /my-config.yaml:/my-config.yaml docker.pkg.github.com/ozonru/file.d/file.d-linux-amd64:v0.1.2 /file.d/file.d --config /my-config.yaml` +`docker run -v /my-config.yaml:/my-config.yaml ozonru/file.d:latest-linux-amd64 /file.d/file.d --config /my-config.yaml` ## Downloads To be filled diff --git a/go.mod b/go.mod index 757bd0c1c..49954cbc4 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ require ( github.com/DataDog/zstd v1.4.0 // indirect github.com/Shopify/sarama v1.23.1 github.com/alecthomas/kingpin v2.2.6+incompatible - github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d github.com/bitly/go-simplejson v0.5.0 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/euank/go-kmsg-parser v2.0.0+incompatible @@ -21,13 +21,12 @@ require ( github.com/prometheus/client_golang v1.3.0 github.com/satori/go.uuid v1.2.0 github.com/spf13/pflag v1.0.3 // indirect - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.4.0 + github.com/vitkovskii/insane-doc v0.0.1 // indirect github.com/vitkovskii/insane-json v0.1.0 - go.uber.org/atomic v1.4.0 + go.uber.org/atomic v1.5.0 go.uber.org/automaxprocs v1.2.0 - go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.9.1 - golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284 // indirect + go.uber.org/zap v1.13.0 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect diff --git a/go.sum b/go.sum index 297c27313..59fefe7b4 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.0 h1:vhoV+DUHnRZdKW1i5UMjAk2G4JY8wN4ayRfYDNdEhwo= github.com/DataDog/zstd v1.4.0/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= @@ -14,6 +16,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -36,11 +40,9 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/euank/go-kmsg-parser v1.0.0 h1:rtNgGgSPLxuBSrjPtDNf6oFvT90i/VraFHuX8YBH+SU= github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -70,6 +72,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= @@ -124,7 +127,6 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -137,6 +139,7 @@ github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLk github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -150,30 +153,47 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/vitkovskii/insane-doc v0.0.1 h1:VAo+bhfzbEtJows9f+XcNB/1B1jNA4DCgvuC9ATNmOE= +github.com/vitkovskii/insane-doc v0.0.1/go.mod h1:ebjM3CQC8/330kQYh640VK2sS68k/T1q7TFFgj6jiyY= github.com/vitkovskii/insane-json v0.1.0 h1:EY22tUdAaMpXDxaw4P1Ojs9jEtsJjvLFgWisX8q13m0= github.com/vitkovskii/insane-json v0.1.0/go.mod h1:xQyYcnFJ8ElboaEZG805SrQ7I4QupForGkm0/TnRaZ8= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/automaxprocs v1.2.0 h1:+RUihKM+nmYUoB9w0D0Ov5TJ2PpFO2FgenTxMJiZBZA= go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284 h1:rlLehGeYg6jfoyz/eDqDU1iRXLKfR42nnNh57ytKEWo= golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -181,6 +201,7 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -200,11 +221,19 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZe golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -223,6 +252,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= k8s.io/api v0.0.0-20190620084959-7cf5895f2711 h1:BblVYz/wE5WtBsD/Gvu54KyBUTJMflolzc5I2DTvh50= k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A= k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719/go.mod h1:I4A+glKBHiTgiEjQiCCQfCAIcIMFGt291SmsvcrFzJA= diff --git a/pipeline/processor.go b/pipeline/processor.go index b679a3999..fa5e6b5a3 100644 --- a/pipeline/processor.go +++ b/pipeline/processor.go @@ -9,14 +9,14 @@ import ( type ActionResult int const ( - // pass event to the next action in a pipeline + // ActionPass pass event to the next action in a pipeline ActionPass ActionResult = 0 - // skip further processing of event and request next event from the same stream and source as current + // ActionCollapse skip further processing of event and request next event from the same stream and source as current // plugin may receive event with eventKindTimeout if it takes to long to read next event from same stream ActionCollapse ActionResult = 2 - // skip further processing of event and request next event from any stream and source + // ActionDiscard skip further processing of event and request next event from any stream and source ActionDiscard ActionResult = 1 - // hold event in a plugin and request next event from the same stream and source as current. + // ActionHold hold event in a plugin and request next event from the same stream and source as current. // same as ActionCollapse but held event should be manually committed or returned into pipeline. // check out Commit()/Propagate() functions in InputPluginController. // plugin may receive event with eventKindTimeout if it takes to long to read next event from same stream diff --git a/plugin/README.md b/plugin/README.md index d5f0eda66..997a2201a 100755 --- a/plugin/README.md +++ b/plugin/README.md @@ -4,11 +4,11 @@ ## dmesg It reads kernel events from /dev/kmsg -[More details...](/plugin/input/dmesg/README.md) +[More details...](plugin/input/dmesg/README.md) ## fake It provides an API to test pipelines and other plugins. -[More details...](/plugin/input/fake/README.md) +[More details...](plugin/input/fake/README.md) ## file It watches for files in the provided directory and reads them line by line. @@ -39,7 +39,7 @@ pipelines: persistence_mode: async ``` -[More details...](/plugin/input/file/README.md) +[More details...](plugin/input/file/README.md) ## http Reads events from HTTP requests with the body delimited by a new line. @@ -51,7 +51,11 @@ So you can use Elasticsearch filebeat output plugin to send data to `file.d`. > Plugin answers with HTTP code `OK 200` right after it has read all the request body. > It doesn't wait until events are committed. -[More details...](/plugin/input/http/README.md) +[More details...](plugin/input/http/README.md) +## journalctl +Reads `journalctl` output. + +[More details...](plugin/input/journalctl/README.md) ## k8s It reads Kubernetes logs and also adds pod meta-information. Also, it joins split logs into a single event. @@ -78,22 +82,22 @@ pipelines: read_buffer_size: 2048 ``` -[More details...](/plugin/input/k8s/README.md) +[More details...](plugin/input/k8s/README.md) ## kafka It reads events from multiple Kafka topics using `sarama` library. > It guarantees at "at-least-once delivery" due to the commitment mechanism. -[More details...](/plugin/input/kafka/README.md) +[More details...](plugin/input/kafka/README.md) # Actions ## convert_date It converts field date/time data to different format. -[More details...](/plugin/action/convert_date/README.md) +[More details...](plugin/action/convert_date/README.md) ## debug It logs event to stdout. Useful for debugging. -[More details...](/plugin/action/debug/README.md) +[More details...](plugin/action/debug/README.md) ## discard It drops an event. It is used in a combination with `match_fields`/`match_mode` parameters to filter out the events. @@ -109,7 +113,7 @@ pipelines: ... ``` -[More details...](/plugin/action/discard/README.md) +[More details...](plugin/action/discard/README.md) ## flatten It extracts the object keys and adds them into the root with some prefix. If the provided field isn't an object, an event will be skipped. @@ -126,7 +130,7 @@ pipelines: ``` It transforms `{"animal":{"type":"cat","paws":4}}` into `{"pet_type":"b","pet_paws":"4"}`. -[More details...](/plugin/action/flatten/README.md) +[More details...](plugin/action/flatten/README.md) ## join It makes one big event from the sequence of the events. It is useful for assembling back together "exceptions" or "panics" if they were written line by line. @@ -150,16 +154,16 @@ pipelines: ... ``` -[More details...](/plugin/action/join/README.md) +[More details...](plugin/action/join/README.md) ## json_decode It decodes a JSON string from the event field and merges the result with the event root. If the decoded JSON isn't an object, the event will be skipped. -[More details...](/plugin/action/json_decode/README.md) +[More details...](plugin/action/json_decode/README.md) ## keep_fields It keeps the list of the event fields and removes others. -[More details...](/plugin/action/keep_fields/README.md) +[More details...](plugin/action/keep_fields/README.md) ## modify It modifies the content for a field. It works only with strings. You can provide an unlimited number of config parameters. Each parameter handled as `cfg.FieldSelector`:`cfg.Substitution`. @@ -188,16 +192,16 @@ The resulting event could look like: } ``` -[More details...](/plugin/action/modify/README.md) +[More details...](plugin/action/modify/README.md) ## parse_es It parses HTTP input using Elasticsearch `/_bulk` API format. It converts sources defining create/index actions to the events. Update/delete actions are ignored. > Check out the details in [Elastic Bulk API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). -[More details...](/plugin/action/parse_es/README.md) +[More details...](plugin/action/parse_es/README.md) ## remove_fields It removes the list of the event fields and keeps others. -[More details...](/plugin/action/remove_fields/README.md) +[More details...](plugin/action/remove_fields/README.md) ## rename It renames the fields of the event. You can provide an unlimited number of config parameters. Each parameter handled as `cfg.FieldSelector`:`string`. When `override` is set to `false`, the field won't be renamed in the case of field name collision. @@ -225,22 +229,22 @@ The resulting event could look like: }, ``` -[More details...](/plugin/action/rename/README.md) +[More details...](plugin/action/rename/README.md) ## throttle It discards the events if pipeline throughput gets higher than a configured threshold. -[More details...](/plugin/action/throttle/README.md) +[More details...](plugin/action/throttle/README.md) # Outputs ## devnull It provides an API to test pipelines and other plugins. -[More details...](/plugin/output/devnull/README.md) +[More details...](plugin/output/devnull/README.md) ## elasticsearch It sends events into Elasticsearch. It uses `_bulk` API to send events in batches. If a network error occurs, the batch will infinitely try to be delivered to the random endpoint. -[More details...](/plugin/output/elasticsearch/README.md) +[More details...](plugin/output/elasticsearch/README.md) ## gelf It sends event batches to the GELF endpoint. Transport level protocol TCP or UDP is configurable. > It doesn't support UDP chunking. So don't use UDP if event size may be greater than 8192. @@ -259,15 +263,15 @@ GELF messages are separated by null byte. Each message is a JSON with the follow Every field with an underscore prefix `_` will be treated as an extra field. Allowed characters in field names are letters, numbers, underscores, dashes, and dots. -[More details...](/plugin/output/gelf/README.md) +[More details...](plugin/output/gelf/README.md) ## kafka It sends the event batches to kafka brokers using `sarama` lib. -[More details...](/plugin/output/kafka/README.md) +[More details...](plugin/output/kafka/README.md) ## stdout It writes events to stdout(also known as console). -[More details...](/plugin/output/stdout/README.md) +[More details...](plugin/output/stdout/README.md)
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/action/README.md b/plugin/action/README.md index fb2bbb41d..6881e42c7 100755 --- a/plugin/action/README.md +++ b/plugin/action/README.md @@ -3,11 +3,11 @@ ## convert_date It converts field date/time data to different format. -[More details...](/plugin/action/convert_date/README.md) +[More details...](plugin/action/convert_date/README.md) ## debug It logs event to stdout. Useful for debugging. -[More details...](/plugin/action/debug/README.md) +[More details...](plugin/action/debug/README.md) ## discard It drops an event. It is used in a combination with `match_fields`/`match_mode` parameters to filter out the events. @@ -23,7 +23,7 @@ pipelines: ... ``` -[More details...](/plugin/action/discard/README.md) +[More details...](plugin/action/discard/README.md) ## flatten It extracts the object keys and adds them into the root with some prefix. If the provided field isn't an object, an event will be skipped. @@ -40,7 +40,7 @@ pipelines: ``` It transforms `{"animal":{"type":"cat","paws":4}}` into `{"pet_type":"b","pet_paws":"4"}`. -[More details...](/plugin/action/flatten/README.md) +[More details...](plugin/action/flatten/README.md) ## join It makes one big event from the sequence of the events. It is useful for assembling back together "exceptions" or "panics" if they were written line by line. @@ -64,16 +64,16 @@ pipelines: ... ``` -[More details...](/plugin/action/join/README.md) +[More details...](plugin/action/join/README.md) ## json_decode It decodes a JSON string from the event field and merges the result with the event root. If the decoded JSON isn't an object, the event will be skipped. -[More details...](/plugin/action/json_decode/README.md) +[More details...](plugin/action/json_decode/README.md) ## keep_fields It keeps the list of the event fields and removes others. -[More details...](/plugin/action/keep_fields/README.md) +[More details...](plugin/action/keep_fields/README.md) ## modify It modifies the content for a field. It works only with strings. You can provide an unlimited number of config parameters. Each parameter handled as `cfg.FieldSelector`:`cfg.Substitution`. @@ -102,16 +102,16 @@ The resulting event could look like: } ``` -[More details...](/plugin/action/modify/README.md) +[More details...](plugin/action/modify/README.md) ## parse_es It parses HTTP input using Elasticsearch `/_bulk` API format. It converts sources defining create/index actions to the events. Update/delete actions are ignored. > Check out the details in [Elastic Bulk API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). -[More details...](/plugin/action/parse_es/README.md) +[More details...](plugin/action/parse_es/README.md) ## remove_fields It removes the list of the event fields and keeps others. -[More details...](/plugin/action/remove_fields/README.md) +[More details...](plugin/action/remove_fields/README.md) ## rename It renames the fields of the event. You can provide an unlimited number of config parameters. Each parameter handled as `cfg.FieldSelector`:`string`. When `override` is set to `false`, the field won't be renamed in the case of field name collision. @@ -139,9 +139,9 @@ The resulting event could look like: }, ``` -[More details...](/plugin/action/rename/README.md) +[More details...](plugin/action/rename/README.md) ## throttle It discards the events if pipeline throughput gets higher than a configured threshold. -[More details...](/plugin/action/throttle/README.md) +[More details...](plugin/action/throttle/README.md)
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/action/throttle/README.md b/plugin/action/throttle/README.md index cad44b891..6c9b7ab14 100755 --- a/plugin/action/throttle/README.md +++ b/plugin/action/throttle/README.md @@ -30,6 +30,12 @@ The default events limit that plugin allows per `interval`
+**`limit_kind`** *`string`* *`default=count`* *`options=count|size`* + +What we're limiting: number of messages, or total size of the messages + +
+ **`buckets_count`** *`int`* *`default=60`* How much time buckets to hold in the memory. E.g. if `buckets_count` is `60` and `interval` is `5m`, @@ -48,6 +54,7 @@ Time interval to check event throughput. Rules to override the `default_limit` for different group of event. It's a list of objects. Each object has the `limit` and `conditions` fields. * `limit` – the value which will override the `default_limit`, if `conditions` are met. +* `limit_kind` – the type of a limit: `count` - number of messages, `size` - total size from all messages * `conditions` – the map of `event field name => event field value`. The conditions are checked using `AND` operator.
diff --git a/plugin/input/README.md b/plugin/input/README.md index 1f88d3a0d..425cdc8d9 100755 --- a/plugin/input/README.md +++ b/plugin/input/README.md @@ -3,11 +3,11 @@ ## dmesg It reads kernel events from /dev/kmsg -[More details...](/plugin/input/dmesg/README.md) +[More details...](plugin/input/dmesg/README.md) ## fake It provides an API to test pipelines and other plugins. -[More details...](/plugin/input/fake/README.md) +[More details...](plugin/input/fake/README.md) ## file It watches for files in the provided directory and reads them line by line. @@ -38,7 +38,7 @@ pipelines: persistence_mode: async ``` -[More details...](/plugin/input/file/README.md) +[More details...](plugin/input/file/README.md) ## http Reads events from HTTP requests with the body delimited by a new line. @@ -50,7 +50,11 @@ So you can use Elasticsearch filebeat output plugin to send data to `file.d`. > Plugin answers with HTTP code `OK 200` right after it has read all the request body. > It doesn't wait until events are committed. -[More details...](/plugin/input/http/README.md) +[More details...](plugin/input/http/README.md) +## journalctl +Reads `journalctl` output. + +[More details...](plugin/input/journalctl/README.md) ## k8s It reads Kubernetes logs and also adds pod meta-information. Also, it joins split logs into a single event. @@ -77,10 +81,10 @@ pipelines: read_buffer_size: 2048 ``` -[More details...](/plugin/input/k8s/README.md) +[More details...](plugin/input/k8s/README.md) ## kafka It reads events from multiple Kafka topics using `sarama` library. > It guarantees at "at-least-once delivery" due to the commitment mechanism. -[More details...](/plugin/input/kafka/README.md) +[More details...](plugin/input/kafka/README.md)
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/input/journalctl/README.idoc.md b/plugin/input/journalctl/README.idoc.md new file mode 100644 index 000000000..f234a3636 --- /dev/null +++ b/plugin/input/journalctl/README.idoc.md @@ -0,0 +1,5 @@ +# Journal.d plugin +@introduction + +### Config params +@config-params|description diff --git a/plugin/input/journalctl/README.md b/plugin/input/journalctl/README.md new file mode 100755 index 000000000..eb884a2c7 --- /dev/null +++ b/plugin/input/journalctl/README.md @@ -0,0 +1,21 @@ +# Journal.d plugin +Reads `journalctl` output. + +### Config params +**`offsets_file`** *`string`* *`required`* + +The filename to store offsets of processed messages. + +
+ +**`journal_args`** *`[]string`* *`default=-f -a`* + +Additional args for `journalctl`. +Plugin forces "-o json" and "-c *cursor*" or "-n all", otherwise +you can use any additional args. +> Have a look at https://man7.org/linux/man-pages/man1/journalctl.1.html + +
+ + +
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/input/journalctl/journalctl.go b/plugin/input/journalctl/journalctl.go index 776a2bf19..af803dfa7 100644 --- a/plugin/input/journalctl/journalctl.go +++ b/plugin/input/journalctl/journalctl.go @@ -7,7 +7,7 @@ import ( ) /*{ introduction -Input plugin, that reads journalctl logs +Reads `journalctl` output. }*/ type Plugin struct { params *pipeline.InputPluginParams @@ -27,10 +27,10 @@ type Config struct { //> @3@4@5@6 //> - //> Additional args for journalctl + //> Additional args for `journalctl`. //> Plugin forces "-o json" and "-c *cursor*" or "-n all", otherwise - //> you can use almost any additional args - //> https://man7.org/linux/man-pages/man1/journalctl.1.html + //> you can use any additional args. + //>> Have a look at https://man7.org/linux/man-pages/man1/journalctl.1.html JournalArgs []string `json:"journal_args" default:"-f -a"` //* // for testing mostly diff --git a/plugin/output/README.md b/plugin/output/README.md index 7e1e92e41..51e82dc34 100755 --- a/plugin/output/README.md +++ b/plugin/output/README.md @@ -3,12 +3,12 @@ ## devnull It provides an API to test pipelines and other plugins. -[More details...](/plugin/output/devnull/README.md) +[More details...](plugin/output/devnull/README.md) ## elasticsearch It sends events into Elasticsearch. It uses `_bulk` API to send events in batches. If a network error occurs, the batch will infinitely try to be delivered to the random endpoint. -[More details...](/plugin/output/elasticsearch/README.md) +[More details...](plugin/output/elasticsearch/README.md) ## gelf It sends event batches to the GELF endpoint. Transport level protocol TCP or UDP is configurable. > It doesn't support UDP chunking. So don't use UDP if event size may be greater than 8192. @@ -27,13 +27,13 @@ GELF messages are separated by null byte. Each message is a JSON with the follow Every field with an underscore prefix `_` will be treated as an extra field. Allowed characters in field names are letters, numbers, underscores, dashes, and dots. -[More details...](/plugin/output/gelf/README.md) +[More details...](plugin/output/gelf/README.md) ## kafka It sends the event batches to kafka brokers using `sarama` lib. -[More details...](/plugin/output/kafka/README.md) +[More details...](plugin/output/kafka/README.md) ## stdout It writes events to stdout(also known as console). -[More details...](/plugin/output/stdout/README.md) +[More details...](plugin/output/stdout/README.md)
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file