diff --git a/README.md b/README.md index 9fa1e6760..0a2f484b8 100644 --- a/README.md +++ b/README.md @@ -294,37 +294,37 @@ curl :2020 | jq . The list below shows supported plugins which are based on Fluent Bit v1.7.x+. For more information, please refer to the API docs of each plugin. - [Input](docs/crd.md#input) - - [dummy](docs/plugins/input/dummy.md) - - [tail](docs/plugins/input/tail.md) - - [systemd](docs/plugins/input/systemd.md) + - [dummy](docs/plugins/fluentbit/input/dummy.md) + - [tail](docs/plugins/fluentbit/input/tail.md) + - [systemd](docs/plugins/fluentbit/input/systemd.md) - [Parser](docs/crd.md#parser) - - [json](docs/plugins/parser/json.md) - - [logfmt](docs/plugins/parser/logfmt.md) - - [lstv](docs/plugins/parser/lstv.md) - - [regex](docs/plugins/parser/regex.md) + - [json](docs/plugins/fluentbit/parser/json.md) + - [logfmt](docs/plugins/fluentbit/parser/logfmt.md) + - [lstv](docs/plugins/fluentbit/parser/lstv.md) + - [regex](docs/plugins/fluentbit/parser/regex.md) - [Filter](docs/crd.md#filter) - - [kubernetes](docs/plugins/filter/kubernetes.md) - - [modify](docs/plugins/filter/modify.md) - - [nest](docs/plugins/filter/nest.md) - - [parser](docs/plugins/filter/parser.md) - - [grep](docs/plugins/filter/grep.md) - - [record modifier](docs/plugins/filter/recordmodifier.md) - - [lua](docs/plugins/filter/lua.md) - - [throttle](docs/plugins/filter/throttle.md) - - [aws](docs/plugins/filter/aws.md) - - [multiline](docs/plugins/filter/multiline.md) + - [kubernetes](docs/plugins/fluentbit/filter/kubernetes.md) + - [modify](docs/plugins/fluentbit/filter/modify.md) + - [nest](docs/plugins/fluentbit/filter/nest.md) + - [parser](docs/plugins/fluentbit/filter/parser.md) + - [grep](docs/plugins/fluentbit/filter/grep.md) + - [record modifier](docs/plugins/fluentbit/filter/recordmodifier.md) + - [lua](docs/plugins/fluentbit/filter/lua.md) + - [throttle](docs/plugins/fluentbit/filter/throttle.md) + - [aws](docs/plugins/fluentbit/filter/aws.md) + - [multiline](docs/plugins/fluentbit/filter/multiline.md) - [Output](docs/crd.md#output) - - [elasticsearch](docs/plugins/output/elasticsearch.md) - - [file](docs/plugins/output/file.md) - - [forward](docs/plugins/output/forward.md) - - [http](docs/plugins/output/http.md) - - [kafka](docs/plugins/output/kafka.md) - - [null](docs/plugins/output/null.md) - - [stdout](docs/plugins/output/stdout.md) - - [tcp](docs/plugins/output/tcp.md) - - [loki](docs/plugins/output/loki.md) - - [syslog](docs/plugins/output/syslog.md) - - [datadog](docs/plugins/output/datadog.md) + - [elasticsearch](docs/plugins/fluentbit/output/elasticsearch.md) + - [file](docs/plugins/fluentbit/output/file.md) + - [forward](docs/plugins/fluentbit/output/forward.md) + - [http](docs/plugins/fluentbit/output/http.md) + - [kafka](docs/plugins/fluentbit/output/kafka.md) + - [null](docs/plugins/fluentbit/output/null.md) + - [stdout](docs/plugins/fluentbit/output/stdout.md) + - [tcp](docs/plugins/fluentbit/output/tcp.md) + - [loki](docs/plugins/fluentbit/output/loki.md) + - [syslog](docs/plugins/fluentbit/output/syslog.md) + - [datadog](docs/plugins/fluentbit/output/datadog.md) ## Best Practice diff --git a/apis/fluentbit/v1alpha2/filter_types.go b/apis/fluentbit/v1alpha2/clusterfilter_types.go similarity index 90% rename from apis/fluentbit/v1alpha2/filter_types.go rename to apis/fluentbit/v1alpha2/clusterfilter_types.go index fa3e48464..3501da0fa 100644 --- a/apis/fluentbit/v1alpha2/filter_types.go +++ b/apis/fluentbit/v1alpha2/clusterfilter_types.go @@ -31,7 +31,7 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// FilterSpec defines the desired state of Filter +// FilterSpec defines the desired state of ClusterFilter type FilterSpec struct { // A pattern to match against the tags of incoming records. // It's case-sensitive and support the star (*) character as a wildcard. @@ -69,10 +69,11 @@ type FilterItem struct { } // +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster // +genclient -// Filter defines a Filter configuration. -type Filter struct { +// ClusterFilter defines a cluster-level Filter configuration. +type ClusterFilter struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` // Specification of desired Filter configuration. @@ -81,17 +82,17 @@ type Filter struct { // +kubebuilder:object:root=true -// FilterList contains a list of Filter +// FilterList contains a list of ClusterFilter type FilterList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []Filter `json:"items"` + Items []ClusterFilter `json:"items"` } // +kubebuilder:object:generate:=false -// FilterByName implements sort.Interface for []Filter based on the Name field. -type FilterByName []Filter +// FilterByName implements sort.Interface for []ClusterFilter based on the Name field. +type FilterByName []ClusterFilter func (a FilterByName) Len() int { return len(a) } func (a FilterByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } @@ -138,5 +139,5 @@ func (list FilterList) Load(sl plugins.SecretLoader) (string, error) { } func init() { - SchemeBuilder.Register(&Filter{}, &FilterList{}) + SchemeBuilder.Register(&ClusterFilter{}, &FilterList{}) } diff --git a/apis/fluentbit/v1alpha2/fluentbitconfig_types.go b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go similarity index 82% rename from apis/fluentbit/v1alpha2/fluentbitconfig_types.go rename to apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go index 0d3562e47..0c0da00ff 100644 --- a/apis/fluentbit/v1alpha2/fluentbitconfig_types.go +++ b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go @@ -29,7 +29,7 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// FluentBitConfigSpec defines the desired state of FluentBitConfig +// FluentBitConfigSpec defines the desired state of ClusterFluentBitConfig type FluentBitConfigSpec struct { // Service defines the global behaviour of the Fluent Bit engine. Service *Service `json:"service,omitempty"` @@ -41,6 +41,9 @@ type FluentBitConfigSpec struct { OutputSelector metav1.LabelSelector `json:"outputSelector,omitempty"` // Select parser plugins ParserSelector metav1.LabelSelector `json:"parserSelector,omitempty"` + //If namespace is defined, then the configmap and secret for fluent-bit is in this namespace. + //If it is not defined, it is in the namespace of the fluentd-operator + Namespace *string `json:"namespace,omitempty"` } type Service struct { @@ -69,11 +72,11 @@ type Service struct { } // +kubebuilder:object:root=true -// +kubebuilder:resource:shortName=fbc +// +kubebuilder:resource:shortName=fbc,scope=Cluster // +genclient -// FluentBitConfig is the Schema for the fluentbitconfigs API -type FluentBitConfig struct { +// ClusterFluentBitConfig is the Schema for the cluster-level fluentbitconfigs API +type ClusterFluentBitConfig struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -82,15 +85,15 @@ type FluentBitConfig struct { // +kubebuilder:object:root=true -// FluentBitConfigList contains a list of FluentBitConfig +// FluentBitConfigList contains a list of ClusterFluentBitConfig type FluentBitConfigList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []FluentBitConfig `json:"items"` + Items []ClusterFluentBitConfig `json:"items"` } func init() { - SchemeBuilder.Register(&FluentBitConfig{}, &FluentBitConfigList{}) + SchemeBuilder.Register(&ClusterFluentBitConfig{}, &FluentBitConfigList{}) } func (s *Service) Params() *params.KVs { @@ -125,7 +128,7 @@ func (s *Service) Params() *params.KVs { return m } -func (cfg FluentBitConfig) RenderMainConfig(sl plugins.SecretLoader, inputs InputList, filters FilterList, outputs OutputList) (string, error) { +func (cfg ClusterFluentBitConfig) RenderMainConfig(sl plugins.SecretLoader, inputs InputList, filters FilterList, outputs OutputList) (string, error) { var buf bytes.Buffer // The Service defines the global behaviour of the Fluent Bit engine. @@ -161,7 +164,7 @@ func (cfg FluentBitConfig) RenderMainConfig(sl plugins.SecretLoader, inputs Inpu return buf.String(), nil } -func (cfg FluentBitConfig) RenderParserConfig(sl plugins.SecretLoader, parsers ParserList) (string, error) { +func (cfg ClusterFluentBitConfig) RenderParserConfig(sl plugins.SecretLoader, parsers ParserList) (string, error) { var buf bytes.Buffer parserSections, err := parsers.Load(sl) @@ -188,13 +191,13 @@ func (a ByName) Len() int { return len(a) } func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name } -func (cfg FluentBitConfig) RenderLuaScript(cl plugins.ConfigMapLoader, filters FilterList) ([]Script, error) { +func (cfg ClusterFluentBitConfig) RenderLuaScript(cl plugins.ConfigMapLoader, filters FilterList, namespace string) ([]Script, error) { scripts := make([]Script, 0) for _, f := range filters.Items { for _, p := range f.Spec.FilterItems { if p.Lua != nil { - script, err := cl.LoadConfigMap(p.Lua.Script) + script, err := cl.LoadConfigMap(p.Lua.Script, namespace) if err != nil { return nil, err } diff --git a/apis/fluentbit/v1alpha2/fluentbitconfig_types_test.go b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types_test.go similarity index 96% rename from apis/fluentbit/v1alpha2/fluentbitconfig_types_test.go rename to apis/fluentbit/v1alpha2/clusterfluentbitconfig_types_test.go index 4b65dc05c..3e5f207e2 100644 --- a/apis/fluentbit/v1alpha2/fluentbitconfig_types_test.go +++ b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types_test.go @@ -94,7 +94,7 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) { "lbl1": "lvl1", } - inputObj := &Input{ + inputObj := &ClusterInput{ TypeMeta: metav1.TypeMeta{ APIVersion: "fluentbit.fluent.io/v1alpha2", Kind: "Input", @@ -120,10 +120,10 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) { } inputs := InputList{ - Items: []Input{*inputObj}, + Items: []ClusterInput{*inputObj}, } - filterObj := &Filter{ + filterObj := &ClusterFilter{ TypeMeta: metav1.TypeMeta{ APIVersion: "fluentbit.fluent.io/v1alpha2", Kind: "Filter", @@ -181,10 +181,10 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) { } filters := FilterList{ - Items: []Filter{*filterObj}, + Items: []ClusterFilter{*filterObj}, } - syslogOut := Output{ + syslogOut := ClusterOutput{ TypeMeta: metav1.TypeMeta{ APIVersion: "fluentbit.fluent.io/v1alpha2", Kind: "Output", @@ -218,7 +218,7 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) { headers["X-Log-Header-0"] = "testing" headers["X-Log-Header-App-ID"] = "9780495d9db3" - httpOutput := Output{ + httpOutput := ClusterOutput{ TypeMeta: metav1.TypeMeta{ APIVersion: "fluentbit.fluent.io/v1alpha2", Kind: "Output", @@ -247,10 +247,10 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) { } outputs := OutputList{ - Items: []Output{syslogOut, httpOutput}, + Items: []ClusterOutput{syslogOut, httpOutput}, } - cfg := FluentBitConfig{ + cfg := ClusterFluentBitConfig{ Spec: FluentBitConfigSpec{Service: &Service{ Daemon: ptrBool(false), FlushSeconds: ptrInt64(1), diff --git a/apis/fluentbit/v1alpha2/input_types.go b/apis/fluentbit/v1alpha2/clusterinput_types.go similarity index 89% rename from apis/fluentbit/v1alpha2/input_types.go rename to apis/fluentbit/v1alpha2/clusterinput_types.go index 1e4e21c5c..c3a3be5a5 100644 --- a/apis/fluentbit/v1alpha2/input_types.go +++ b/apis/fluentbit/v1alpha2/clusterinput_types.go @@ -30,7 +30,7 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// InputSpec defines the desired state of Input +// InputSpec defines the desired state of ClusterInput type InputSpec struct { // A user friendly alias name for this input plugin. // Used in metrics for distinction of each configured input. @@ -44,10 +44,11 @@ type InputSpec struct { } // +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster // +genclient -// Input is the Schema for the inputs API -type Input struct { +// ClusterInput is the Schema for the inputs API +type ClusterInput struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -56,7 +57,7 @@ type Input struct { // +kubebuilder:object:generate:=false // InputByName implements sort.Interface for []Input based on the Name field. -type InputByName []Input +type InputByName []ClusterInput func (a InputByName) Len() int { return len(a) } func (a InputByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } @@ -64,11 +65,11 @@ func (a InputByName) Less(i, j int) bool { return a[i].Name < a[j].Name } // +kubebuilder:object:root=true -// InputList contains a list of Input +// ClusterInputList contains a list of Input type InputList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []Input `json:"items"` + Items []ClusterInput `json:"items"` } func (list InputList) Load(sl plugins.SecretLoader) (string, error) { @@ -107,5 +108,5 @@ func (list InputList) Load(sl plugins.SecretLoader) (string, error) { } func init() { - SchemeBuilder.Register(&Input{}, &InputList{}) + SchemeBuilder.Register(&ClusterInput{}, &InputList{}) } diff --git a/apis/fluentbit/v1alpha2/output_types.go b/apis/fluentbit/v1alpha2/clusteroutput_types.go similarity index 92% rename from apis/fluentbit/v1alpha2/output_types.go rename to apis/fluentbit/v1alpha2/clusteroutput_types.go index 87e94dc40..5328648d5 100644 --- a/apis/fluentbit/v1alpha2/output_types.go +++ b/apis/fluentbit/v1alpha2/clusteroutput_types.go @@ -31,7 +31,7 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// OutputSpec defines the desired state of Output +// OutputSpec defines the desired state of ClusterOutput type OutputSpec struct { // A pattern to match against the tags of incoming records. // It's case sensitive and support the star (*) character as a wildcard. @@ -72,10 +72,11 @@ type OutputSpec struct { } // +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster // +genclient -// Output is the Schema for the outputs API -type Output struct { +// ClusterOutput is the Schema for the cluster-level outputs API +type ClusterOutput struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -84,16 +85,16 @@ type Output struct { // +kubebuilder:object:root=true -// OutputList contains a list of Output +// OutputList contains a list of ClusterOutput type OutputList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []Output `json:"items"` + Items []ClusterOutput `json:"items"` } // +kubebuilder:object:generate:=false // OutputByName implements sort.Interface for []Output based on the Name field. -type OutputByName []Output +type OutputByName []ClusterOutput func (a OutputByName) Len() int { return len(a) } func (a OutputByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } @@ -144,5 +145,5 @@ func (list OutputList) Load(sl plugins.SecretLoader) (string, error) { } func init() { - SchemeBuilder.Register(&Output{}, &OutputList{}) + SchemeBuilder.Register(&ClusterOutput{}, &OutputList{}) } diff --git a/apis/fluentbit/v1alpha2/parser_types.go b/apis/fluentbit/v1alpha2/clusterparser_types.go similarity index 91% rename from apis/fluentbit/v1alpha2/parser_types.go rename to apis/fluentbit/v1alpha2/clusterparser_types.go index 1fd3f4773..a64e2a4dc 100644 --- a/apis/fluentbit/v1alpha2/parser_types.go +++ b/apis/fluentbit/v1alpha2/clusterparser_types.go @@ -30,7 +30,7 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// ParserSpec defines the desired state of Parser +// ParserSpec defines the desired state of ClusterParser type ParserSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file @@ -58,10 +58,11 @@ type Decorder struct { } // +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster // +genclient -// Parser is the Schema for the parsers API -type Parser struct { +// ClusterParser is the Schema for the cluster-level parsers API +type ClusterParser struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -70,16 +71,16 @@ type Parser struct { // +kubebuilder:object:root=true -// ParserList contains a list of Parser +// ParserList contains a list of ClusterParser type ParserList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []Parser `json:"items"` + Items []ClusterParser `json:"items"` } // +kubebuilder:object:generate:=false // ParserByName implements sort.Interface for []Parser based on the Name field. -type ParserByName []Parser +type ParserByName []ClusterParser func (a ParserByName) Len() int { return len(a) } func (a ParserByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } @@ -129,5 +130,5 @@ func (list ParserList) Load(sl plugins.SecretLoader) (string, error) { } func init() { - SchemeBuilder.Register(&Parser{}, &ParserList{}) + SchemeBuilder.Register(&ClusterParser{}, &ParserList{}) } diff --git a/apis/fluentbit/v1alpha2/plugins/configmap_types.go b/apis/fluentbit/v1alpha2/plugins/configmap_types.go index 5db0611bd..8745f273f 100644 --- a/apis/fluentbit/v1alpha2/plugins/configmap_types.go +++ b/apis/fluentbit/v1alpha2/plugins/configmap_types.go @@ -21,9 +21,9 @@ func NewConfigMapLoader(c client.Client, ns string) ConfigMapLoader { } } -func (cl ConfigMapLoader) LoadConfigMap(selector v1.ConfigMapKeySelector) (string, error) { +func (cl ConfigMapLoader) LoadConfigMap(selector v1.ConfigMapKeySelector, namespace string) (string, error) { var configMap v1.ConfigMap - if err := cl.client.Get(context.Background(), client.ObjectKey{Name: selector.Name, Namespace: cl.namespace}, &configMap); err != nil { + if err := cl.client.Get(context.Background(), client.ObjectKey{Name: selector.Name, Namespace: namespace}, &configMap); err != nil { return "", err } diff --git a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go index b4b96742d..da59ea6f7 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go index 2b3c0857c..c95dcb6e3 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go index f2ae835ac..ad8f42e8b 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go index cc233b240..4985ba660 100644 --- a/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/apis/fluentbit/v1alpha2/plugins/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/zz_generated.deepcopy.go index e0e34ad3c..7939eb28c 100644 --- a/apis/fluentbit/v1alpha2/plugins/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index 2c5c90925..9bf3edb4b 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* @@ -31,46 +30,150 @@ import ( ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Decorder) DeepCopyInto(out *Decorder) { +func (in *ClusterFilter) DeepCopyInto(out *ClusterFilter) { *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Decorder. -func (in *Decorder) DeepCopy() *Decorder { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterFilter. +func (in *ClusterFilter) DeepCopy() *ClusterFilter { if in == nil { return nil } - out := new(Decorder) + out := new(ClusterFilter) in.DeepCopyInto(out) return out } +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterFilter) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Filter) DeepCopyInto(out *Filter) { +func (in *ClusterFluentBitConfig) DeepCopyInto(out *ClusterFluentBitConfig) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Filter. -func (in *Filter) DeepCopy() *Filter { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterFluentBitConfig. +func (in *ClusterFluentBitConfig) DeepCopy() *ClusterFluentBitConfig { if in == nil { return nil } - out := new(Filter) + out := new(ClusterFluentBitConfig) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Filter) DeepCopyObject() runtime.Object { +func (in *ClusterFluentBitConfig) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterInput) DeepCopyInto(out *ClusterInput) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterInput. +func (in *ClusterInput) DeepCopy() *ClusterInput { + if in == nil { + return nil + } + out := new(ClusterInput) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterInput) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterOutput) DeepCopyInto(out *ClusterOutput) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterOutput. +func (in *ClusterOutput) DeepCopy() *ClusterOutput { + if in == nil { + return nil + } + out := new(ClusterOutput) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterOutput) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterParser) DeepCopyInto(out *ClusterParser) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterParser. +func (in *ClusterParser) DeepCopy() *ClusterParser { + if in == nil { + return nil + } + out := new(ClusterParser) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterParser) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Decorder) DeepCopyInto(out *Decorder) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Decorder. +func (in *Decorder) DeepCopy() *Decorder { + if in == nil { + return nil + } + out := new(Decorder) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FilterItem) DeepCopyInto(out *FilterItem) { *out = *in @@ -148,7 +251,7 @@ func (in *FilterList) DeepCopyInto(out *FilterList) { in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]Filter, len(*in)) + *out = make([]ClusterFilter, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -222,32 +325,6 @@ func (in *FluentBit) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FluentBitConfig) DeepCopyInto(out *FluentBitConfig) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluentBitConfig. -func (in *FluentBitConfig) DeepCopy() *FluentBitConfig { - if in == nil { - return nil - } - out := new(FluentBitConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *FluentBitConfig) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FluentBitConfigList) DeepCopyInto(out *FluentBitConfigList) { *out = *in @@ -255,7 +332,7 @@ func (in *FluentBitConfigList) DeepCopyInto(out *FluentBitConfigList) { in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]FluentBitConfig, len(*in)) + *out = make([]ClusterFluentBitConfig, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -292,6 +369,11 @@ func (in *FluentBitConfigSpec) DeepCopyInto(out *FluentBitConfigSpec) { in.FilterSelector.DeepCopyInto(&out.FilterSelector) in.OutputSelector.DeepCopyInto(&out.OutputSelector) in.ParserSelector.DeepCopyInto(&out.ParserSelector) + if in.Namespace != nil { + in, out := &in.Namespace, &out.Namespace + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluentBitConfigSpec. @@ -416,32 +498,6 @@ func (in *FluentBitStatus) DeepCopy() *FluentBitStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Input) DeepCopyInto(out *Input) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Input. -func (in *Input) DeepCopy() *Input { - if in == nil { - return nil - } - out := new(Input) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Input) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InputList) DeepCopyInto(out *InputList) { *out = *in @@ -449,7 +505,7 @@ func (in *InputList) DeepCopyInto(out *InputList) { in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]Input, len(*in)) + *out = make([]ClusterInput, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -504,32 +560,6 @@ func (in *InputSpec) DeepCopy() *InputSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Output) DeepCopyInto(out *Output) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Output. -func (in *Output) DeepCopy() *Output { - if in == nil { - return nil - } - out := new(Output) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Output) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OutputList) DeepCopyInto(out *OutputList) { *out = *in @@ -537,7 +567,7 @@ func (in *OutputList) DeepCopyInto(out *OutputList) { in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]Output, len(*in)) + *out = make([]ClusterOutput, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -637,32 +667,6 @@ func (in *OutputSpec) DeepCopy() *OutputSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Parser) DeepCopyInto(out *Parser) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Parser. -func (in *Parser) DeepCopy() *Parser { - if in == nil { - return nil - } - out := new(Parser) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Parser) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ParserList) DeepCopyInto(out *ParserList) { *out = *in @@ -670,7 +674,7 @@ func (in *ParserList) DeepCopyInto(out *ParserList) { in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]Parser, len(*in)) + *out = make([]ClusterParser, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/apis/fluentd/v1alpha1/fluentd_types.go b/apis/fluentd/v1alpha1/fluentd_types.go index 3b7ca8c52..d80cb37e5 100644 --- a/apis/fluentd/v1alpha1/fluentd_types.go +++ b/apis/fluentd/v1alpha1/fluentd_types.go @@ -35,7 +35,7 @@ type FluentdSpec struct { // Numbers of the Fluentd instance Replicas *int32 `json:"replicas,omitempty"` // Numbers of the workers in Fluentd instance - Workers *int32 `json:"workers,omitempty"` + Workers *int32 `json:"workers,omitempty"` // Fluentd image. Image string `json:"image,omitempty"` // Fluentd Watcher command line arguments. diff --git a/apis/fluentd/v1alpha1/plugins/common/zz_generated.deepcopy.go b/apis/fluentd/v1alpha1/plugins/common/zz_generated.deepcopy.go index 94c06b7ae..78a8bbe1b 100644 --- a/apis/fluentd/v1alpha1/plugins/common/zz_generated.deepcopy.go +++ b/apis/fluentd/v1alpha1/plugins/common/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/apis/fluentd/v1alpha1/zz_generated.deepcopy.go b/apis/fluentd/v1alpha1/zz_generated.deepcopy.go index 0ea9eb074..a5ab786e4 100644 --- a/apis/fluentd/v1alpha1/zz_generated.deepcopy.go +++ b/apis/fluentd/v1alpha1/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml new file mode 100644 index 000000000..0d95cc63b --- /dev/null +++ b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml @@ -0,0 +1,518 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: clusterfilters.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: ClusterFilter + listKind: ClusterFilterList + plural: clusterfilters + singular: clusterfilter + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: ClusterFilter defines a cluster-level Filter configuration. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of desired Filter configuration. + properties: + filters: + description: A set of filter plugins in order. + items: + properties: + aws: + description: Aws defines a Aws configuration. + properties: + accountID: + description: The account ID for current EC2 instance.Default + is false. + type: boolean + amiID: + description: The EC2 instance image id.Default is false. + type: boolean + az: + description: The availability zone; for example, "us-east-1a". + Default is true. + type: boolean + ec2InstanceID: + description: The EC2 instance ID.Default is true. + type: boolean + ec2InstanceType: + description: The EC2 instance type.Default is false. + type: boolean + hostName: + description: The hostname for current EC2 instance.Default + is false. + type: boolean + imdsVersion: + description: Specify which version of the instance metadata + service to use. Valid values are 'v1' or 'v2'. + enum: + - v1 + - v2 + type: string + privateIP: + description: The EC2 instance private ip.Default is false. + type: boolean + vpcID: + description: The VPC ID for current EC2 instance.Default + is false. + type: boolean + type: object + grep: + description: Grep defines Grep Filter configuration. + properties: + exclude: + description: 'Exclude records which field matches the regular + expression. Value Format: FIELD REGEX' + type: string + regex: + description: 'Keep records which field matches the regular + expression. Value Format: FIELD REGEX' + type: string + type: object + kubernetes: + description: Kubernetes defines Kubernetes Filter configuration. + properties: + annotations: + description: Include Kubernetes resource annotations in + the extra metadata. + type: boolean + bufferSize: + description: Set the buffer size for HTTP client when reading + responses from Kubernetes API server. + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + dummyMeta: + description: If set, use dummy-meta data (for test/dev purposes) + type: boolean + k8sLoggingExclude: + description: Allow Kubernetes Pods to exclude their logs + from the log processor (read more about it in Kubernetes + Annotations section). + type: boolean + k8sLoggingParser: + description: Allow Kubernetes Pods to suggest a pre-defined + Parser (read more about it in Kubernetes Annotations section) + type: boolean + keepLog: + description: When Keep_Log is disabled, the log field is + removed from the incoming message once it has been successfully + merged (Merge_Log must be enabled as well). + type: boolean + kubeCAFile: + description: CA certificate file + type: string + kubeCAPath: + description: Absolute path to scan for certificate files + type: string + kubeMetaPreloadCacheDir: + description: If set, Kubernetes meta-data can be cached/pre-loaded + from files in JSON format in this directory, named as + namespace-pod.meta + type: string + kubeTagPrefix: + description: When the source records comes from Tail input + plugin, this option allows to specify what's the prefix + used in Tail configuration. + type: string + kubeTokenFile: + description: Token file + type: string + kubeURL: + description: API Server end-point + type: string + labels: + description: Include Kubernetes resource labels in the extra + metadata. + type: boolean + mergeLog: + description: When enabled, it checks if the log field content + is a JSON string map, if so, it append the map fields + as part of the log structure. + type: boolean + mergeLogKey: + description: When Merge_Log is enabled, the filter tries + to assume the log field from the incoming message is a + JSON string message and make a structured representation + of it at the same level of the log field in the map. Now + if Merge_Log_Key is set (a string name), all the new structured + fields taken from the original log content are inserted + under the new key. + type: string + mergeLogTrim: + description: When Merge_Log is enabled, trim (remove possible + \n or \r) field values. + type: boolean + mergeParser: + description: Optional parser name to specify how to parse + the data contained in the log key. Recommended use is + for developers or testing only. + type: string + regexParser: + description: Set an alternative Parser to process record + Tag and extract pod_name, namespace_name, container_name + and docker_id. The parser must be registered in a parsers + file (refer to parser filter-kube-test as an example). + type: string + tlsDebug: + description: Debug level between 0 (nothing) and 4 (every + detail). + format: int32 + type: integer + tlsVerify: + description: When enabled, turns on certificate validation + when connecting to the Kubernetes API server. + type: boolean + useJournal: + description: When enabled, the filter reads logs coming + in Journald format. + type: boolean + type: object + lua: + description: Lua defines Lua Filter configuration. + properties: + call: + description: Lua function name that will be triggered to + do filtering. It's assumed that the function is declared + inside the Script defined above. + type: string + protectedMode: + description: If enabled, Lua script will be executed in + protected mode. It prevents to crash when invalid Lua + script is executed. Default is true. + type: boolean + script: + description: Path to the Lua script that will be used. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + timeAsTable: + description: By default when the Lua script is invoked, + the record timestamp is passed as a Floating number which + might lead to loss precision when the data is converted + back. If you desire timestamp precision enabling this + option will pass the timestamp as a Lua table with keys + sec for seconds since epoch and nsec for nanoseconds. + type: boolean + typeIntKey: + description: If these keys are matched, the fields are converted + to integer. If more than one key, delimit by space. Note + that starting from Fluent Bit v1.6 integer data types + are preserved and not converted to double as in previous + versions. + items: + type: string + type: array + required: + - call + - script + type: object + modify: + description: Modify defines Modify Filter configuration. + properties: + conditions: + description: All conditions have to be true for the rules + to be applied. + items: + description: The plugin supports the following conditions + properties: + aKeyMatches: + description: Is true if a key matches regex KEY + type: string + keyDoesNotExist: + additionalProperties: + type: string + description: Is true if KEY does not exist + type: object + keyExists: + description: Is true if KEY exists + type: string + keyValueDoesNotEqual: + additionalProperties: + type: string + description: Is true if KEY exists and its value is + not VALUE + type: object + keyValueDoesNotMatch: + additionalProperties: + type: string + description: Is true if key KEY exists and its value + does not match VALUE + type: object + keyValueEquals: + additionalProperties: + type: string + description: Is true if KEY exists and its value is + VALUE + type: object + keyValueMatches: + additionalProperties: + type: string + description: Is true if key KEY exists and its value + matches VALUE + type: object + matchingKeysDoNotHaveMatchingValues: + additionalProperties: + type: string + description: Is true if all keys matching KEY have + values that do not match VALUE + type: object + matchingKeysHaveMatchingValues: + additionalProperties: + type: string + description: Is true if all keys matching KEY have + values that match VALUE + type: object + noKeyMatches: + description: Is true if no key matches regex KEY + type: string + type: object + type: array + rules: + description: Rules are applied in the order they appear, + with each rule operating on the result of the previous + rule. + items: + description: The plugin supports the following rules + properties: + add: + additionalProperties: + type: string + description: Add a key/value pair with key KEY and + value VALUE if KEY does not exist + type: object + copy: + additionalProperties: + type: string + description: Copy a key/value pair with key KEY to + COPIED_KEY if KEY exists AND COPIED_KEY does not + exist + type: object + hardCopy: + additionalProperties: + type: string + description: Copy a key/value pair with key KEY to + COPIED_KEY if KEY exists. If COPIED_KEY already + exists, this field is overwritten + type: object + hardRename: + additionalProperties: + type: string + description: Rename a key/value pair with key KEY + to RENAMED_KEY if KEY exists. If RENAMED_KEY already + exists, this field is overwritten + type: object + remove: + description: Remove a key/value pair with key KEY + if it exists + type: string + removeRegex: + description: Remove all key/value pairs with key matching + regexp KEY + type: string + removeWildcard: + description: Remove all key/value pairs with key matching + wildcard KEY + type: string + rename: + additionalProperties: + type: string + description: Rename a key/value pair with key KEY + to RENAMED_KEY if KEY exists AND RENAMED_KEY does + not exist + type: object + set: + additionalProperties: + type: string + description: Add a key/value pair with key KEY and + value VALUE. If KEY already exists, this field is + overwritten + type: object + type: object + type: array + type: object + multiline: + description: Multiline defines a Multiline configuration. + properties: + keyContent: + description: Key name that holds the content to process. + Note that a Multiline Parser definition can already specify + the key_content to use, but this option allows to overwrite + that value for the purpose of the filter. + type: string + parser: + description: Specify one or multiple Multiline Parsing definitions + to apply to the content. You can specify multiple multiline + parsers to detect different formats by separating them + with a comma. + type: string + required: + - parser + type: object + nest: + description: Nest defines Nest Filter configuration. + properties: + addPrefix: + description: Prefix affected keys with this string + type: string + nestUnder: + description: Nest records matching the Wildcard under this + key + type: string + nestedUnder: + description: Lift records nested under the Nested_under + key + type: string + operation: + description: Select the operation nest or lift + enum: + - nest + - lift + type: string + removePrefix: + description: Remove prefix from affected keys if it matches + this string + type: string + wildcard: + description: Nest records which field matches the wildcard + items: + type: string + type: array + type: object + parser: + description: Parser defines Parser Filter configuration. + properties: + keyName: + description: Specify field name in record to parse. + type: string + parser: + description: Specify the parser name to interpret the field. + Multiple Parser entries are allowed (split by comma). + type: string + preserveKey: + description: Keep original Key_Name field in the parsed + result. If false, the field will be removed. + type: boolean + reserveData: + description: Keep all other original fields in the parsed + result. If false, all other original fields will be removed. + type: boolean + unescapeKey: + description: 'If the key is a escaped string (e.g: stringify + JSON), unescape the string before to apply the parser.' + type: boolean + type: object + recordModifier: + description: RecordModifier defines Record Modifier Filter configuration. + properties: + records: + description: Append fields. This parameter needs key and + value pair. + items: + type: string + type: array + removeKeys: + description: If the key is matched, that field is removed. + items: + type: string + type: array + whitelistKeys: + description: If the key is not matched, that field is removed. + items: + type: string + type: array + type: object + rewriteTag: + description: RewriteTag defines a RewriteTag configuration. + properties: + emitterName: + description: When the filter emits a record under the new + Tag, there is an internal emitter plugin that takes care + of the job. Since this emitter expose metrics as any other + component of the pipeline, you can use this property to + configure an optional name for it. + type: string + rules: + description: 'Defines the matching criteria and the format + of the Tag for the matching record. The Rule format have + four components: KEY REGEX NEW_TAG KEEP.' + items: + type: string + type: array + type: object + throttle: + description: Throttle defines a Throttle configuration. + properties: + interval: + description: Interval is the time interval expressed in + "sleep" format. e.g. 3s, 1.5m, 0.5h, etc. + pattern: ^\d+(\.[0-9]{0,2})?(s|m|h|d)?$ + type: string + printStatus: + description: PrintStatus represents whether to print status + messages with current rate and the limits to information + logs. + type: boolean + rate: + description: Rate is the amount of messages for the time. + format: int64 + type: integer + window: + description: Window is the amount of intervals to calculate + average over. + format: int64 + type: integer + type: object + type: object + type: array + match: + description: A pattern to match against the tags of incoming records. + It's case-sensitive and support the star (*) character as a wildcard. + type: string + matchRegex: + description: A regular expression to match against the tags of incoming + records. Use this option if you want to use the full regex syntax. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml new file mode 100644 index 000000000..892c93732 --- /dev/null +++ b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -0,0 +1,276 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: clusterfluentbitconfigs.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: ClusterFluentBitConfig + listKind: ClusterFluentBitConfigList + plural: clusterfluentbitconfigs + shortNames: + - fbc + singular: clusterfluentbitconfig + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: ClusterFluentBitConfig is the Schema for the cluster-level fluentbitconfigs + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FluentBitConfigSpec defines the desired state of ClusterFluentBitConfig + properties: + filterSelector: + description: Select filter plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + inputSelector: + description: Select input plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + namespace: + description: If namespace is defined, then the configmap and secret + for fluent-bit is in this namespace. If it is not defined, it is + in the namespace of the fluentd-operator + type: string + outputSelector: + description: Select output plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + parserSelector: + description: Select parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + service: + description: Service defines the global behaviour of the Fluent Bit + engine. + properties: + daemon: + description: If true go to background on start + type: boolean + flushSeconds: + description: Interval to flush output + format: int64 + type: integer + graceSeconds: + description: Wait time on exit + format: int64 + type: integer + httpListen: + description: Address to listen + pattern: ^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}$ + type: string + httpPort: + description: Port to listen + format: int32 + maximum: 65535 + minimum: 1 + type: integer + httpServer: + description: If true enable statistics HTTP server + type: boolean + logFile: + description: File to log diagnostic output + type: string + logLevel: + description: Diagnostic level (error/warning/info/debug/trace) + enum: + - error + - warning + - info + - debug + - trace + type: string + parsersFile: + description: Optional 'parsers' config file (can be multiple) + type: string + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml new file mode 100644 index 000000000..ec31c77ec --- /dev/null +++ b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml @@ -0,0 +1,284 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: clusterinputs.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: ClusterInput + listKind: ClusterInputList + plural: clusterinputs + singular: clusterinput + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: ClusterInput is the Schema for the inputs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InputSpec defines the desired state of ClusterInput + properties: + alias: + description: A user friendly alias name for this input plugin. Used + in metrics for distinction of each configured input. + type: string + dummy: + description: Dummy defines Dummy Input configuration. + properties: + dummy: + description: Dummy JSON record. + type: string + rate: + description: Events number generated per second. + format: int32 + type: integer + samples: + description: Sample events to generate. + format: int32 + type: integer + tag: + description: Tag name associated to all records comming from this + plugin. + type: string + type: object + systemd: + description: Systemd defines Systemd Input configuration. + properties: + db: + description: Specify the database file to keep track of monitored + files and offsets. + type: string + dbSync: + description: 'Set a default synchronization (I/O) method. values: + Extra, Full, Normal, Off. This flag affects how the internal + SQLite engine do synchronization to disk, for more details about + each option please refer to this section. note: this option + was introduced on Fluent Bit v1.4.6.' + enum: + - Extra + - Full + - Normal + - "Off" + type: string + maxEntries: + description: When Fluent Bit starts, the Journal might have a + high number of logs in the queue. In order to avoid delays and + reduce memory usage, this option allows to specify the maximum + number of log entries that can be processed per round. Once + the limit is reached, Fluent Bit will continue processing the + remaining log entries once Journald performs the notification. + type: integer + maxFields: + description: Set a maximum number of fields (keys) allowed per + record. + type: integer + path: + description: Optional path to the Systemd journal directory, if + not set, the plugin will use default paths to read local-only + logs. + type: string + readFromTail: + description: Start reading new entries. Skip entries already stored + in Journald. + enum: + - "on" + - "off" + type: string + stripUnderscores: + description: Remove the leading underscore of the Journald field + (key). For example the Journald field _PID becomes the key PID. + enum: + - "on" + - "off" + type: string + systemdFilter: + description: 'Allows to perform a query over logs that contains + a specific Journald key/value pairs, e.g: _SYSTEMD_UNIT=UNIT. + The Systemd_Filter option can be specified multiple times in + the input section to apply multiple filters as required.' + items: + type: string + type: array + systemdFilterType: + description: Define the filter type when Systemd_Filter is specified + multiple times. Allowed values are And and Or. With And a record + is matched only when all of the Systemd_Filter have a match. + With Or a record is matched when any of the Systemd_Filter has + a match. + enum: + - And + - Or + type: string + tag: + description: 'The tag is used to route messages but on Systemd + plugin there is an extra functionality: if the tag includes + a star/wildcard, it will be expanded with the Systemd Unit file + (e.g: host.* => host.UNIT_NAME).' + type: string + type: object + tail: + description: Tail defines Tail Input configuration. + properties: + bufferChunkSize: + description: Set the initial buffer size to read files data. This + value is used too to increase buffer size. The value must be + according to the Unit Size specification. + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + bufferMaxSize: + description: 'Set the limit of the buffer size per monitored file. + When a buffer needs to be increased (e.g: very long lines), + this value is used to restrict how much the memory buffer can + grow. If reading a file exceed this limit, the file is removed + from the monitored file list The value must be according to + the Unit Size specification.' + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + db: + description: Specify the database file to keep track of monitored + files and offsets. + type: string + dbSync: + description: 'Set a default synchronization (I/O) method. Values: + Extra, Full, Normal, Off.' + enum: + - Extra + - Full + - Normal + - "Off" + type: string + disableInotifyWatcher: + description: DisableInotifyWatcher will disable inotify and use + the file stat watcher instead. + type: boolean + dockerMode: + description: If enabled, the plugin will recombine split Docker + log lines before passing them to any parser as configured above. + This mode cannot be used at the same time as Multiline. + type: boolean + dockerModeFlushSeconds: + description: Wait period time in seconds to flush queued unfinished + split lines. + format: int64 + type: integer + excludePath: + description: 'Set one or multiple shell patterns separated by + commas to exclude files matching a certain criteria, e.g: exclude_path=*.gz,*.zip' + type: string + ignoredOlder: + description: Ignores records which are older than this time in + seconds. Supports m,h,d (minutes, hours, days) syntax. Default + behavior is to read all records from specified files. Only available + when a Parser is specificied and it can parse the time of a + record. + pattern: ^\d+(m|h|d)?$ + type: string + key: + description: When a message is unstructured (no parser applied), + it's appended as a string under the key name log. This option + allows to define an alternative name for that key. + type: string + memBufLimit: + description: Set a limit of memory that Tail plugin can use when + appending data to the Engine. If the limit is reach, it will + be paused; when the data is flushed it resumes. + type: string + multiline: + description: If enabled, the plugin will try to discover multiline + messages and use the proper parsers to compose the outgoing + messages. Note that when this option is enabled the Parser option + is not used. + type: boolean + multilineFlushSeconds: + description: Wait period time in seconds to process queued multiline + messages + format: int64 + type: integer + multilineParser: + description: This will help to reassembly multiline messages originally + split by Docker or CRI Specify one or Multiline Parser definition + to apply to the content. + type: string + parser: + description: Specify the name of a parser to interpret the entry + as a structured message. + type: string + parserFirstline: + description: Name of the parser that matchs the beginning of a + multiline message. Note that the regular expression defined + in the parser must include a group name (named capture) + type: string + parserN: + description: Optional-extra parser to interpret and structure + multiline entries. This option can be used to define multiple + parsers. + items: + type: string + type: array + path: + description: Pattern specifying a specific log files or multiple + ones through the use of common wildcards. + type: string + pathKey: + description: If enabled, it appends the name of the monitored + file as part of the record. The value assigned becomes the key + in the map. + type: string + readFromHead: + description: For new discovered files on start (without a database + offset/position), read the content from the head of the file, + not tail. + type: boolean + refreshIntervalSeconds: + description: The interval of refreshing the list of watched files + in seconds. + format: int64 + type: integer + rotateWaitSeconds: + description: Specify the number of extra time in seconds to monitor + a file once is rotated in case some pending data is flushed. + format: int64 + type: integer + skipLongLines: + description: When a monitored file reach it buffer capacity due + to a very long line (Buffer_Max_Size), the default behavior + is to stop monitoring that file. Skip_Long_Lines alter that + behavior and instruct Fluent Bit to skip long lines and continue + processing other lines that fits into the buffer size. + type: boolean + tag: + description: Set a tag (with regex-extract fields) that will be + placed on lines read. E.g. kube... + type: string + tagRegex: + description: Set a regex to exctract fields from the file + type: string + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml new file mode 100644 index 000000000..a51b92c08 --- /dev/null +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -0,0 +1,1336 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: clusteroutputs.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: ClusterOutput + listKind: ClusterOutputList + plural: clusteroutputs + singular: clusteroutput + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: ClusterOutput is the Schema for the cluster-level outputs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OutputSpec defines the desired state of ClusterOutput + properties: + alias: + description: A user friendly alias name for this output plugin. Used + in metrics for distinction of each configured output. + type: string + datadog: + description: DataDog defines DataDog Output configuration. + properties: + apikey: + description: Your Datadog API key. + type: string + compress: + description: Compress the payload in GZIP format. Datadog supports + and recommends setting this to gzip. + type: string + dd_message_key: + description: By default, the plugin searches for the key 'log' + and remap the value to the key 'message'. If the property is + set, the plugin will search the property name key. + type: string + dd_service: + description: The human readable name for your service generating + the logs. + type: string + dd_source: + description: A human readable name for the underlying technology + of your service. + type: string + dd_tags: + description: The tags you want to assign to your logs in Datadog. + type: string + host: + description: Host is the Datadog server where you are sending + your logs. + type: string + include_tag_key: + description: If enabled, a tag is appended to output. The key + name is used tag_key property. + type: boolean + json_date_key: + description: Date key name for output. + type: string + provider: + description: To activate the remapping, specify configuration + flag provider. + type: string + proxy: + description: Specify an HTTP Proxy. + type: string + tag_key: + description: The key name of tag. If include_tag_key is false, + This property is ignored. + type: string + tls: + description: TLS controls whether to use end-to-end security communications + security protocol. Datadog recommends setting this to on. + type: boolean + type: object + es: + description: Elasticsearch defines Elasticsearch Output configuration. + properties: + awsAuth: + description: Enable AWS Sigv4 Authentication for Amazon ElasticSearch + Service. + type: string + awsExternalID: + description: External ID for the AWS IAM Role specified with aws_role_arn. + type: string + awsRegion: + description: Specify the AWS region for Amazon ElasticSearch Service. + type: string + awsRoleARN: + description: AWS IAM Role to assume to put records to your Amazon + ES cluster. + type: string + awsSTSEndpoint: + description: Specify the custom sts endpoint to be used with STS + API for Amazon ElasticSearch Service. + type: string + bufferSize: + description: Specify the buffer size used to read the response + from the Elasticsearch HTTP service. This option is useful for + debugging purposes where is required to read full responses, + note that response size grows depending of the number of records + inserted. To set an unlimited amount of memory set this value + to False, otherwise the value must be according to the Unit + Size specification. + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + cloudAuth: + description: Specify the credentials to use to connect to Elastic's + Elasticsearch Service running on Elastic Cloud. + type: string + cloudID: + description: If you are using Elastic's Elasticsearch Service + you can specify the cloud_id of the cluster running. + type: string + currentTimeIndex: + description: Use current time for index generation instead of + message record + type: boolean + generateID: + description: When enabled, generate _id for outgoing records. + This prevents duplicate records when retrying ES. + type: boolean + host: + description: IP address or hostname of the target Elasticsearch + instance + type: string + httpPassword: + description: Password for user defined in HTTP_User + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + type: object + httpUser: + description: Optional username credential for Elastic X-Pack access + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + type: object + idKey: + description: If set, _id will be the value of the key from incoming + record and Generate_ID option is ignored. + type: string + includeTagKey: + description: When enabled, it append the Tag name to the record. + type: boolean + index: + description: Index name + type: string + logstashDateFormat: + description: Time format (based on strftime) to generate the second + part of the Index name. + type: string + logstashFormat: + description: 'Enable Logstash format compatibility. This option + takes a boolean value: True/False, On/Off' + type: boolean + logstashPrefix: + description: 'When Logstash_Format is enabled, the Index name + is composed using a prefix and the date, e.g: If Logstash_Prefix + is equals to ''mydata'' your index will become ''mydata-YYYY.MM.DD''. + The last string appended belongs to the date when the data is + being generated.' + type: string + logstashPrefixKey: + description: Prefix keys with this string + type: string + path: + description: Elasticsearch accepts new data on HTTP query path + "/_bulk". But it is also possible to serve Elasticsearch behind + a reverse proxy on a subpath. This option defines such path + on the fluent-bit side. It simply adds a path prefix in the + indexing HTTP POST URI. + type: string + pipeline: + description: Newer versions of Elasticsearch allows setting up + filters called pipelines. This option allows defining which + pipeline the database should use. For performance reasons is + strongly suggested parsing and filtering on Fluent Bit side, + avoid pipelines. + type: string + port: + description: TCP port of the target Elasticsearch instance + format: int32 + maximum: 65535 + minimum: 1 + type: integer + replaceDots: + description: When enabled, replace field name dots with underscore, + required by Elasticsearch 2.0-2.3. + type: boolean + suppressTypeName: + description: When enabled, mapping types is removed and Type option + is ignored. Types are deprecated in APIs in v7.0. This options + is for v7.0 or later. + type: string + tagKey: + description: When Include_Tag_Key is enabled, this property defines + the key name for the tag. + type: string + timeKey: + description: When Logstash_Format is enabled, each record will + get a new timestamp field. The Time_Key property defines the + name of that field. + type: string + timeKeyFormat: + description: When Logstash_Format is enabled, this property defines + the format of the timestamp. + type: string + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + traceError: + description: When enabled print the elasticsearch API calls to + stdout when elasticsearch returns an error + type: boolean + traceOutput: + description: When enabled print the elasticsearch API calls to + stdout (for diag only) + type: boolean + type: + description: Type name + type: string + type: object + file: + description: File defines File Output configuration. + properties: + delimiter: + description: The character to separate each pair. Applicable only + if format is csv or ltsv. + type: string + file: + description: Set file name to store the records. If not set, the + file name will be the tag associated with the records. + type: string + format: + description: 'The format of the file content. See also Format + section. Default: out_file.' + enum: + - out_file + - plain + - csv + - ltsv + - template + type: string + labelDelimiter: + description: The character to separate each pair. Applicable only + if format is ltsv. + type: string + path: + description: Absolute directory path to store files. If not set, + Fluent Bit will write the files on it's own positioned directory. + type: string + template: + description: The format string. Applicable only if format is template. + type: string + type: object + firehose: + description: Firehose defines Firehose Output configuration. + properties: + autoRetryRequests: + description: Immediately retry failed requests to AWS services + once. This option does not affect the normal Fluent Bit retry + mechanism with backoff. Instead, it enables an immediate retry + with no delay for networking errors, which may help improve + throughput when there are transient/random networking issues. + type: boolean + dataKeys: + description: By default, the whole log record will be sent to + Kinesis. If you specify a key name(s) with this option, then + only those keys and values will be sent to Kinesis. For example, + if you are using the Fluentd Docker log driver, you can specify + data_keys log and only the log message will be sent to Kinesis. + If you specify multiple keys, they should be comma delimited. + type: string + deliveryStream: + description: The name of the Kinesis Firehose Delivery stream + that you want log records sent to. + type: string + endpoint: + description: Specify a custom endpoint for the Kinesis Firehose + API. + type: string + logKey: + description: By default, the whole log record will be sent to + Firehose. If you specify a key name with this option, then only + the value of that key will be sent to Firehose. For example, + if you are using the Fluentd Docker log driver, you can specify + log_key log and only the log message will be sent to Firehose. + type: string + region: + description: The AWS region. + type: string + roleARN: + description: ARN of an IAM role to assume (for cross account access). + type: string + stsEndpoint: + description: Specify a custom endpoint for the STS API; used to + assume your custom role provided with role_arn. + type: string + timeKey: + description: Add the timestamp to the record under this key. By + default, the timestamp from Fluent Bit will not be added to + records sent to Kinesis. + type: string + timeKeyFormat: + description: strftime compliant format string for the timestamp; + for example, %Y-%m-%dT%H *string This option is used with time_key. + You can also use %L for milliseconds and %f for microseconds. + If you are using ECS FireLens, make sure you are running Amazon + ECS Container Agent v1.42.0 or later, otherwise the timestamps + associated with your container logs will only have second precision. + type: string + required: + - deliveryStream + - region + type: object + forward: + description: Forward defines Forward Output configuration. + properties: + emptySharedKey: + description: Use this option to connect to Fluentd with a zero-length + secret. + type: boolean + host: + description: Target host where Fluent-Bit or Fluentd are listening + for Forward messages. + type: string + password: + description: Specify the password corresponding to the username. + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + type: object + port: + description: TCP Port of the target service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + requireAckResponse: + description: Send "chunk"-option and wait for "ack" response from + server. Enables at-least-once and receiving server can control + rate of traffic. (Requires Fluentd v0.14.0+ server) + type: boolean + selfHostname: + description: Default value of the auto-generated certificate common + name (CN). + type: string + sendOptions: + description: Always send options (with "size"=count of messages) + type: boolean + sharedKey: + description: A key string known by the remote Fluentd used for + authorization. + type: string + timeAsInteger: + description: Set timestamps in integer format, it enable compatibility + mode for Fluentd v0.12 series. + type: boolean + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + username: + description: Specify the username to present to a Fluentd server + that enables user_auth. + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + type: object + type: object + http: + description: HTTP defines HTTP Output configuration. + properties: + allowDuplicatedHeaders: + description: Specify if duplicated headers are allowed. If a duplicated + header is found, the latest key/value set is preserved. + type: boolean + compress: + description: Set payload compression mechanism. Option available + is 'gzip' + type: string + format: + description: Specify the data format to be used in the HTTP request + body, by default it uses msgpack. Other supported formats are + json, json_stream and json_lines and gelf. + enum: + - msgpack + - json + - json_stream + - json_lines + - gelf + type: string + gelfFullMessageKey: + description: Specify the key to use for the full message in gelf + format + type: string + gelfHostKey: + description: Specify the key to use for the host in gelf format + type: string + gelfLevelKey: + description: Specify the key to use for the level in gelf format + type: string + gelfShortMessgeKey: + description: Specify the key to use as the short message in gelf + format + type: string + gelfTimestampKey: + description: Specify the key to use for timestamp in gelf format + type: string + headerTag: + description: Specify an optional HTTP header field for the original + message tag. + type: string + headers: + additionalProperties: + type: string + description: Add a HTTP header key/value pair. Multiple headers + can be set. + type: object + host: + description: IP address or hostname of the target HTTP Server + type: string + httpPassword: + description: Basic Auth Password. Requires HTTP_User to be set + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + type: object + httpUser: + description: Basic Auth Username + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + type: object + jsonDateFormat: + description: 'Specify the format of the date. Supported formats + are double, epoch and iso8601 (eg: 2018-05-30T09:39:52.000681Z)' + type: string + jsonDateKey: + description: Specify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + port: + description: TCP port of the target HTTP Server + format: int32 + maximum: 65535 + minimum: 1 + type: integer + proxy: + description: Specify an HTTP Proxy. The expected format of this + value is http://host:port. Note that https is not supported + yet. + type: string + tls: + description: HTTP output plugin supports TTL/SSL, for more details + about the properties available and general configuration, please + refer to the TLS/SSL section. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + uri: + description: 'Specify an optional HTTP URI for the target web + server, e.g: /something' + type: string + type: object + kafka: + description: Kafka defines Kafka Output configuration. + properties: + brokers: + description: 'Single of multiple list of Kafka Brokers, e.g: 192.168.1.3:9092, + 192.168.1.4:9092.' + type: string + dynamicTopic: + description: adds unknown topics (found in Topic_Key) to Topics. + So in Topics only a default topic needs to be configured + type: boolean + format: + description: 'Specify data format, options available: json, msgpack.' + type: string + messageKey: + description: Optional key to store the message + type: string + messageKeyField: + description: If set, the value of Message_Key_Field in the record + will indicate the message key. If not set nor found in the record, + Message_Key will be used (if set). + type: string + queueFullRetries: + description: Fluent Bit queues data into rdkafka library, if for + some reason the underlying library cannot flush the records + the queue might fills up blocking new addition of records. The + queue_full_retries option set the number of local retries to + enqueue the data. The default value is 10 times, the interval + between each retry is 1 second. Setting the queue_full_retries + value to 0 set's an unlimited number of retries. + format: int64 + type: integer + rdkafka: + additionalProperties: + type: string + description: '{property} can be any librdkafka properties' + type: object + timestampFormat: + description: iso8601 or double + type: string + timestampKey: + description: Set the key to store the record timestamp + type: string + topicKey: + description: 'If multiple Topics exists, the value of Topic_Key + in the record will indicate the topic to use. E.g: if Topic_Key + is router and the record is {"key1": 123, "router": "route_2"}, + Fluent Bit will use topic route_2. Note that if the value of + Topic_Key is not present in Topics, then by default the first + topic in the Topics list will indicate the topic to be used.' + type: string + topics: + description: Single entry or list of topics separated by comma + (,) that Fluent Bit will use to send messages to Kafka. If only + one topic is set, that one will be used for all records. Instead + if multiple topics exists, the one set in the record by Topic_Key + will be used. + type: string + type: object + loki: + description: Loki defines Loki Output configuration. + properties: + autoKubernetesLabels: + description: If set to true, it will add all Kubernetes labels + to the Stream labels. + enum: + - "on" + - "off" + type: string + host: + description: Loki hostname or IP address. + type: string + httpPassword: + description: Password for user defined in HTTP_User Set HTTP basic + authentication password + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + type: object + httpUser: + description: Set HTTP basic authentication user name. + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + type: object + labelKeys: + description: Optional list of record keys that will be placed + as stream labels. This configuration property is for records + key only. + items: + type: string + type: array + labels: + description: Stream labels for API request. It can be multiple + comma separated of strings specifying key=value pairs. In addition + to fixed parameters, it also allows to add custom record keys + (similar to label_keys property). + items: + type: string + type: array + lineFormat: + description: Format to use when flattening the record to a log + line. Valid values are json or key_value. If set to json, the + log line sent to Loki will be the Fluent Bit record dumped as + JSON. If set to key_value, the log line will be each item in + the record concatenated together (separated by a single space) + in the format. + enum: + - json + - key_value + type: string + port: + description: Loki TCP port + format: int32 + maximum: 65535 + minimum: 1 + type: integer + tenantID: + description: Tenant ID used by default to push logs to Loki. If + omitted or empty it assumes Loki is running in single-tenant + mode and no X-Scope-OrgID header is sent. + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + type: object + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + required: + - host + type: object + match: + description: A pattern to match against the tags of incoming records. + It's case sensitive and support the star (*) character as a wildcard. + type: string + matchRegex: + description: A regular expression to match against the tags of incoming + records. Use this option if you want to use the full regex syntax. + type: string + "null": + description: Null defines Null Output configuration. + type: object + retry_limit: + description: RetryLimit represents configuration for the scheduler + which can be set independently on each output section. This option + allows to disable retries or impose a limit to try N times and then + discard the data after reaching that limit. + type: string + stdout: + description: Stdout defines Stdout Output configuration. + properties: + format: + description: Specify the data format to be printed. Supported + formats are msgpack json, json_lines and json_stream. + enum: + - msgpack + - json + - json_lines + - json_stream + type: string + jsonDateFormat: + description: 'Specify the format of the date. Supported formats + are double, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and epoch.' + enum: + - double + - iso8601 + - epoch + type: string + jsonDateKey: + description: Specify the name of the date field in output. + type: string + type: object + syslog: + description: Syslog defines Syslog Output configuration. + properties: + host: + description: Host domain or IP address of the remote Syslog server. + type: string + mode: + description: Mode of the desired transport type, the available + options are tcp, tls and udp. + type: string + port: + description: TCP or UDP port of the remote Syslog server. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + syslogAppnameKey: + description: Key name from the original record that contains the + application name that generated the message. + type: string + syslogFacilityKey: + description: Key from the original record that contains the Syslog + facility number. + type: string + syslogFormat: + description: Syslog protocol format to use, the available options + are rfc3164 and rfc5424. + type: string + syslogHostnameKey: + description: Key name from the original record that contains the + hostname that generated the message. + type: string + syslogMaxSize: + description: Maximum size allowed per message, in bytes. + format: int32 + type: integer + syslogMessageIDKey: + description: Key name from the original record that contains the + Message ID associated to the message. + type: string + syslogMessageKey: + description: Key key name that contains the message to deliver. + type: string + syslogProcessIDKey: + description: Key name from the original record that contains the + Process ID that generated the message. + type: string + syslogSDKey: + description: Key name from the original record that contains the + Structured Data (SD) content. + type: string + syslogSeverityKey: + description: Key from the original record that contains the Syslog + severity number. + type: string + tls: + description: Syslog output plugin supports TTL/SSL, for more details + about the properties available and general configuration, please + refer to the TLS/SSL section. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object + tcp: + description: TCP defines TCP Output configuration. + properties: + format: + description: Specify the data format to be printed. Supported + formats are msgpack json, json_lines and json_stream. + enum: + - msgpack + - json + - json_lines + - json_stream + type: string + host: + description: Target host where Fluent-Bit or Fluentd are listening + for Forward messages. + type: string + jsonDateFormat: + description: 'Specify the format of the date. Supported formats + are double, epoch and iso8601 (eg: 2018-05-30T09:39:52.000681Z)' + enum: + - double + - epoch + - iso8601 + type: string + jsonDateKey: + description: TSpecify the name of the time key in the output record. + To disable the time key just set the value to false. + type: string + port: + description: TCP Port of the target service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource represents a source for the value + of a secret. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/fluentbit.fluent.io_clusterparsers.yaml b/config/crd/bases/fluentbit.fluent.io_clusterparsers.yaml new file mode 100644 index 000000000..aa9355892 --- /dev/null +++ b/config/crd/bases/fluentbit.fluent.io_clusterparsers.yaml @@ -0,0 +1,115 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: clusterparsers.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: ClusterParser + listKind: ClusterParserList + plural: clusterparsers + singular: clusterparser + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: ClusterParser is the Schema for the cluster-level parsers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ParserSpec defines the desired state of ClusterParser + properties: + decoders: + description: 'Decoders are a built-in feature available through the + Parsers file, each Parser definition can optionally set one or multiple + decoders. There are two type of decoders type: Decode_Field and + Decode_Field_As.' + items: + properties: + decodeField: + description: If the content can be decoded in a structured message, + append that structure message (keys and values) to the original + log message. + type: string + decodeFieldAs: + description: Any content decoded (unstructured or structured) + will be replaced in the same key/value, no extra keys are + added. + type: string + type: object + type: array + json: + description: JSON defines json parser configuration. + properties: + timeFormat: + description: Time_Format, eg. %Y-%m-%dT%H:%M:%S %z + type: string + timeKeep: + description: Time_Keep + type: boolean + timeKey: + description: Time_Key + type: string + type: object + logfmt: + description: Logfmt defines logfmt parser configuration. + type: object + ltsv: + description: LTSV defines ltsv parser configuration. + properties: + timeFormat: + description: Time_Format, eg. %Y-%m-%dT%H:%M:%S %z + type: string + timeKeep: + description: Time_Keep + type: boolean + timeKey: + description: Time_Key + type: string + types: + type: string + type: object + regex: + description: Regex defines regex parser configuration. + properties: + regex: + type: string + timeFormat: + description: Time_Format, eg. %Y-%m-%dT%H:%M:%S %z + type: string + timeKeep: + description: Time_Keep + type: boolean + timeKey: + description: Time_Key + type: string + types: + type: string + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml index 2fdb47071..0be0c1a16 100644 --- a/config/crd/bases/fluentbit.fluent.io_filters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_filters.yaml @@ -14,7 +14,7 @@ spec: listKind: FilterList plural: filters singular: filter - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: diff --git a/config/crd/bases/fluentbit.fluent.io_fluentbitclusterconfigs.yaml b/config/crd/bases/fluentbit.fluent.io_fluentbitclusterconfigs.yaml new file mode 100644 index 000000000..84fae9b8e --- /dev/null +++ b/config/crd/bases/fluentbit.fluent.io_fluentbitclusterconfigs.yaml @@ -0,0 +1,276 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: fluentbitclusterconfigs.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: FluentBitClusterConfig + listKind: FluentBitClusterConfigList + plural: fluentbitclusterconfigs + shortNames: + - fbc + singular: fluentbitclusterconfig + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: FluentBitClusterConfig is the Schema for the cluster-level fluentbitconfigs + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterFluentBitConfigSpec defines the desired state of FluentBitConfig + properties: + clusterFilterSelector: + description: Select filter plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + clusterInputSelector: + description: Select input plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + clusterOutputSelector: + description: Select output plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + clusterParserSelector: + description: Select parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + namespace: + description: If namespace is defined, then the configmap and secret + for fluent-bit is in this namespace. If it is not defined, it is + in the namespace of the fluentd-operator + type: string + service: + description: Service defines the global behaviour of the Fluent Bit + engine. + properties: + daemon: + description: If true go to background on start + type: boolean + flushSeconds: + description: Interval to flush output + format: int64 + type: integer + graceSeconds: + description: Wait time on exit + format: int64 + type: integer + httpListen: + description: Address to listen + pattern: ^\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}$ + type: string + httpPort: + description: Port to listen + format: int32 + maximum: 65535 + minimum: 1 + type: integer + httpServer: + description: If true enable statistics HTTP server + type: boolean + logFile: + description: File to log diagnostic output + type: string + logLevel: + description: Diagnostic level (error/warning/info/debug/trace) + enum: + - error + - warning + - info + - debug + - trace + type: string + parsersFile: + description: Optional 'parsers' config file (can be multiple) + type: string + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/fluentbit.fluent.io_fluentbitconfigs.yaml b/config/crd/bases/fluentbit.fluent.io_fluentbitconfigs.yaml index 9563853bf..1951ecf0d 100644 --- a/config/crd/bases/fluentbit.fluent.io_fluentbitconfigs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_fluentbitconfigs.yaml @@ -16,7 +16,7 @@ spec: shortNames: - fbc singular: fluentbitconfig - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -126,6 +126,11 @@ spec: are ANDed. type: object type: object + namespace: + description: If namespace is defined, then the configmap and secret + for fluent-bit is in this namespace. If it is not defined, it is + in the namespace of the fluentd-operator + type: string outputSelector: description: Select output plugins properties: diff --git a/config/crd/bases/fluentbit.fluent.io_inputs.yaml b/config/crd/bases/fluentbit.fluent.io_inputs.yaml index 082124cbb..bb9cd872e 100644 --- a/config/crd/bases/fluentbit.fluent.io_inputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_inputs.yaml @@ -14,7 +14,7 @@ spec: listKind: InputList plural: inputs singular: input - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index 562435909..a5e53806b 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -14,7 +14,7 @@ spec: listKind: OutputList plural: outputs singular: output - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: diff --git a/config/crd/bases/fluentbit.fluent.io_parsers.yaml b/config/crd/bases/fluentbit.fluent.io_parsers.yaml index 24d16340f..7a3516cc4 100644 --- a/config/crd/bases/fluentbit.fluent.io_parsers.yaml +++ b/config/crd/bases/fluentbit.fluent.io_parsers.yaml @@ -14,7 +14,7 @@ spec: listKind: ParserList plural: parsers singular: parser - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: diff --git a/controllers/fluentbitconfig_controller.go b/controllers/fluentbitconfig_controller.go index 9a44cde25..0a3d86d72 100644 --- a/controllers/fluentbitconfig_controller.go +++ b/controllers/fluentbitconfig_controller.go @@ -18,6 +18,8 @@ package controllers import ( "context" + "fmt" + "os" "fluent.io/fluent-operator/apis/fluentbit/v1alpha2/plugins" "github.com/go-logr/logr" @@ -58,7 +60,7 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ _ = r.Log.WithValues("fluentbitconfig", req.NamespacedName) var cfgs fluentbitv1alpha2.FluentBitConfigList - if err := r.List(ctx, &cfgs, client.InNamespace(req.Namespace)); err != nil { + if err := r.List(ctx, &cfgs); err != nil { if errors.IsNotFound(err) { return ctrl.Result{}, nil } @@ -72,7 +74,7 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ if err != nil { return ctrl.Result{}, err } - if err = r.List(ctx, &inputs, client.InNamespace(req.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + if err = r.List(ctx, &inputs, client.MatchingLabelsSelector{Selector: selector}); err != nil { return ctrl.Result{}, err } @@ -82,7 +84,7 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ if err != nil { return ctrl.Result{}, err } - if err = r.List(ctx, &filters, client.InNamespace(req.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + if err = r.List(ctx, &filters, client.MatchingLabelsSelector{Selector: selector}); err != nil { return ctrl.Result{}, err } @@ -92,7 +94,7 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ if err != nil { return ctrl.Result{}, err } - if err = r.List(ctx, &outputs, client.InNamespace(req.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + if err = r.List(ctx, &outputs, client.MatchingLabelsSelector{Selector: selector}); err != nil { return ctrl.Result{}, err } @@ -102,12 +104,17 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ if err != nil { return ctrl.Result{}, err } - if err = r.List(ctx, &parsers, client.InNamespace(req.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + if err = r.List(ctx, &parsers, client.MatchingLabelsSelector{Selector: selector}); err != nil { return ctrl.Result{}, err } - + var ns string + if cfg.Spec.Namespace != nil { + ns = fmt.Sprintf(*cfg.Spec.Namespace) + } else { + ns = os.Getenv("NAMESPACE") + } // Inject config data into Secret - sl := plugins.NewSecretLoader(r.Client, cfg.Namespace, r.Log) + sl := plugins.NewSecretLoader(r.Client, ns, r.Log) mainAppCfg, err := cfg.RenderMainConfig(sl, inputs, filters, outputs) if err != nil { return ctrl.Result{}, err @@ -117,8 +124,8 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } - cl := plugins.NewConfigMapLoader(r.Client, cfg.Namespace) - scripts, err := cfg.RenderLuaScript(cl, filters) + cl := plugins.NewConfigMapLoader(r.Client, ns) + scripts, err := cfg.RenderLuaScript(cl, filters, ns) if err != nil { return ctrl.Result{}, err } @@ -127,7 +134,7 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ sec := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: cfg.Name, - Namespace: cfg.Namespace, + Namespace: ns, }, } @@ -174,11 +181,11 @@ func (r *FluentBitConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { } return ctrl.NewControllerManagedBy(mgr). - For(&fluentbitv1alpha2.FluentBitConfig{}). + For(&fluentbitv1alpha2.ClusterFluentBitConfig{}). Owns(&corev1.Secret{}). - Watches(&source.Kind{Type: &fluentbitv1alpha2.Input{}}, &handler.EnqueueRequestForObject{}). - Watches(&source.Kind{Type: &fluentbitv1alpha2.Filter{}}, &handler.EnqueueRequestForObject{}). - Watches(&source.Kind{Type: &fluentbitv1alpha2.Output{}}, &handler.EnqueueRequestForObject{}). - Watches(&source.Kind{Type: &fluentbitv1alpha2.Parser{}}, &handler.EnqueueRequestForObject{}). + Watches(&source.Kind{Type: &fluentbitv1alpha2.ClusterInput{}}, &handler.EnqueueRequestForObject{}). + Watches(&source.Kind{Type: &fluentbitv1alpha2.ClusterFilter{}}, &handler.EnqueueRequestForObject{}). + Watches(&source.Kind{Type: &fluentbitv1alpha2.ClusterOutput{}}, &handler.EnqueueRequestForObject{}). + Watches(&source.Kind{Type: &fluentbitv1alpha2.ClusterParser{}}, &handler.EnqueueRequestForObject{}). Complete(r) } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 5fd971704..8c35ab0fc 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -63,10 +63,10 @@ var _ = BeforeSuite(func() { err = fluentbitv1alpha2.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - + err = fluentdv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - + //+kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/docs/crd.md b/docs/crd.md index 57201fd44..128f8c306 100644 --- a/docs/crd.md +++ b/docs/crd.md @@ -52,13 +52,13 @@ Filter defines a Filter configuration. | Field | Description | Scheme | | ----- | ----------- | ------ | -| grep | Grep defines Grep Filter configuration. | *[filter.Grep](plugins/filter/grep.md) | -| recordModifier | RecordModifier defines Record Modifier Filter configuration. | *[filter.RecordModifier](plugins/filter/recordmodifier.md) | -| kubernetes | Kubernetes defines Kubernetes Filter configuration. | *[filter.Kubernetes](plugins/filter/kubernetes.md) | -| modify | Modify defines Modify Filter configuration. | *[filter.Modify](plugins/filter/modify.md) | -| nest | Nest defines Nest Filter configuration. | *[filter.Nest](plugins/filter/nest.md) | -| parser | Parser defines Parser Filter configuration. | *[filter.Parser](plugins/filter/parser.md) | -| lua | Lua defines Lua Filter configuration. | *[filter.Lua](plugins/filter/lua.md) | +| grep | Grep defines Grep Filter configuration. | *[filter.Grep](plugins/fluentbit/filter/grep.md) | +| recordModifier | RecordModifier defines Record Modifier Filter configuration. | *[filter.RecordModifier](plugins/fluentbit/filter/recordmodifier.md) | +| kubernetes | Kubernetes defines Kubernetes Filter configuration. | *[filter.Kubernetes](plugins/fluentbit/filter/kubernetes.md) | +| modify | Modify defines Modify Filter configuration. | *[filter.Modify](plugins/fluentbit/filter/modify.md) | +| nest | Nest defines Nest Filter configuration. | *[filter.Nest](plugins/fluentbit/filter/nest.md) | +| parser | Parser defines Parser Filter configuration. | *[filter.Parser](plugins/fluentbit/filter/parser.md) | +| lua | Lua defines Lua Filter configuration. | *[filter.Lua](plugins/fluentbit/filter/lua.md) | [Back to TOC](#table-of-contents) ## FilterList @@ -192,9 +192,9 @@ InputSpec defines the desired state of Input | Field | Description | Scheme | | ----- | ----------- | ------ | | alias | A user friendly alias name for this input plugin. | string | -| dummy | Dummy defines Dummy Input configuration. | *[input.Dummy](plugins/input/dummy.md) | -| tail | Tail defines Tail Input configuration. | *[input.Tail](plugins/input/tail.md) | -| systemd | Systemd defines Systemd Input configuration. | *[input.Systemd](plugins/input/systemd.md) | +| dummy | Dummy defines Dummy Input configuration. | *[input.Dummy](plugins/fluentbit/input/dummy.md) | +| tail | Tail defines Tail Input configuration. | *[input.Tail](plugins/fluentbit/input/tail.md) | +| systemd | Systemd defines Systemd Input configuration. | *[input.Systemd](plugins/fluentbit/input/systemd.md) | [Back to TOC](#table-of-contents) ## Output @@ -230,17 +230,17 @@ OutputSpec defines the desired state of Output | matchRegex | A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. | string | | alias | A user friendly alias name for this output plugin. | string | | retry_limit | This option allows to disable retries or impose a limit to try N times and then discard the data after reaching that limit. | string | -| es | Elasticsearch defines Elasticsearch Output configuration. | *[output.Elasticsearch](plugins/output/elasticsearch.md) | -| file | File defines File Output configuration. | *[output.File](plugins/output/file.md) | -| forward | Forward defines Forward Output configuration. | *[output.Forward](plugins/output/forward.md) | -| http | HTTP defines HTTP Output configuration. | *[output.HTTP](plugins/output/http.md) | -| kafka | Kafka defines Kafka Output configuration. | *[output.Kafka](plugins/output/kafka.md) | -| null | Null defines Null Output configuration. | *[output.Null](plugins/output/null.md) | -| stdout | Stdout defines Stdout Output configuration. | *[output.Stdout](plugins/output/stdout.md) | -| tcp | TCP defines TCP Output configuration. | *[output.TCP](plugins/output/tcp.md) | -| loki | Loki defines Loki Output configuration. | *[output.Loki](plugins/output/loki.md) | -| syslog | Syslog defines Syslog Output configuration. | *[output.Syslog](plugins/output/syslog.md) | -| datadog | DataDog defines DataDog Output configuration. | *[output.DataDog](plugins/output/datadog.md) | +| es | Elasticsearch defines Elasticsearch Output configuration. | *[output.Elasticsearch](plugins/fluentbit/output/elasticsearch.md) | +| file | File defines File Output configuration. | *[output.File](plugins/fluentbit/output/file.md) | +| forward | Forward defines Forward Output configuration. | *[output.Forward](plugins/fluentbit/output/forward.md) | +| http | HTTP defines HTTP Output configuration. | *[output.HTTP](plugins/fluentbit/output/http.md) | +| kafka | Kafka defines Kafka Output configuration. | *[output.Kafka](plugins/fluentbit/output/kafka.md) | +| null | Null defines Null Output configuration. | *[output.Null](plugins/fluentbit/output/null.md) | +| stdout | Stdout defines Stdout Output configuration. | *[output.Stdout](plugins/fluentbit/output/stdout.md) | +| tcp | TCP defines TCP Output configuration. | *[output.TCP](plugins/fluentbit/output/tcp.md) | +| loki | Loki defines Loki Output configuration. | *[output.Loki](plugins/fluentbit/output/loki.md) | +| syslog | Syslog defines Syslog Output configuration. | *[output.Syslog](plugins/fluentbit/output/syslog.md) | +| datadog | DataDog defines DataDog Output configuration. | *[output.DataDog](plugins/fluentbit/output/datadog.md) | [Back to TOC](#table-of-contents) ## Parser @@ -272,10 +272,10 @@ ParserSpec defines the desired state of Parser | Field | Description | Scheme | | ----- | ----------- | ------ | -| json | JSON defines json parser configuration. | *[parser.JSON](plugins/parser/json.md) | -| regex | Regex defines regex parser configuration. | *[parser.Regex](plugins/parser/regex.md) | -| ltsv | LTSV defines ltsv parser configuration. | *[parser.LSTV](plugins/parser/lstv.md) | -| logfmt | Logfmt defines logfmt parser configuration. | *[parser.Logfmt](plugins/parser/logfmt.md) | +| json | JSON defines json parser configuration. | *[parser.JSON](plugins/fluentbit/parser/json.md) | +| regex | Regex defines regex parser configuration. | *[parser.Regex](plugins/fluentbit/parser/regex.md) | +| ltsv | LTSV defines ltsv parser configuration. | *[parser.LSTV](plugins/fluentbit/parser/lstv.md) | +| logfmt | Logfmt defines logfmt parser configuration. | *[parser.Logfmt](plugins/fluentbit/parser/logfmt.md) | | decoders | Decoders are a built-in feature available through the Parsers file, each Parser definition can optionally set one or multiple decoders. There are two type of decoders type: Decode_Field and Decode_Field_As. | []Decorder | [Back to TOC](#table-of-contents) diff --git a/docs/plugins/filter/aws.md b/docs/plugins/fluentbit/filter/aws.md similarity index 100% rename from docs/plugins/filter/aws.md rename to docs/plugins/fluentbit/filter/aws.md diff --git a/docs/plugins/filter/grep.md b/docs/plugins/fluentbit/filter/grep.md similarity index 100% rename from docs/plugins/filter/grep.md rename to docs/plugins/fluentbit/filter/grep.md diff --git a/docs/plugins/filter/kubernetes.md b/docs/plugins/fluentbit/filter/kubernetes.md similarity index 100% rename from docs/plugins/filter/kubernetes.md rename to docs/plugins/fluentbit/filter/kubernetes.md diff --git a/docs/plugins/filter/lua.md b/docs/plugins/fluentbit/filter/lua.md similarity index 100% rename from docs/plugins/filter/lua.md rename to docs/plugins/fluentbit/filter/lua.md diff --git a/docs/plugins/filter/modify.md b/docs/plugins/fluentbit/filter/modify.md similarity index 100% rename from docs/plugins/filter/modify.md rename to docs/plugins/fluentbit/filter/modify.md diff --git a/docs/plugins/filter/multiline.md b/docs/plugins/fluentbit/filter/multiline.md similarity index 100% rename from docs/plugins/filter/multiline.md rename to docs/plugins/fluentbit/filter/multiline.md diff --git a/docs/plugins/filter/nest.md b/docs/plugins/fluentbit/filter/nest.md similarity index 100% rename from docs/plugins/filter/nest.md rename to docs/plugins/fluentbit/filter/nest.md diff --git a/docs/plugins/filter/parser.md b/docs/plugins/fluentbit/filter/parser.md similarity index 100% rename from docs/plugins/filter/parser.md rename to docs/plugins/fluentbit/filter/parser.md diff --git a/docs/plugins/filter/recordmodifier.md b/docs/plugins/fluentbit/filter/recordmodifier.md similarity index 100% rename from docs/plugins/filter/recordmodifier.md rename to docs/plugins/fluentbit/filter/recordmodifier.md diff --git a/docs/plugins/filter/throttle.md b/docs/plugins/fluentbit/filter/throttle.md similarity index 100% rename from docs/plugins/filter/throttle.md rename to docs/plugins/fluentbit/filter/throttle.md diff --git a/docs/plugins/input/dummy.md b/docs/plugins/fluentbit/input/dummy.md similarity index 100% rename from docs/plugins/input/dummy.md rename to docs/plugins/fluentbit/input/dummy.md diff --git a/docs/plugins/input/systemd.md b/docs/plugins/fluentbit/input/systemd.md similarity index 100% rename from docs/plugins/input/systemd.md rename to docs/plugins/fluentbit/input/systemd.md diff --git a/docs/plugins/input/tail.md b/docs/plugins/fluentbit/input/tail.md similarity index 100% rename from docs/plugins/input/tail.md rename to docs/plugins/fluentbit/input/tail.md diff --git a/docs/plugins/output/datadog.md b/docs/plugins/fluentbit/output/datadog.md similarity index 100% rename from docs/plugins/output/datadog.md rename to docs/plugins/fluentbit/output/datadog.md diff --git a/docs/plugins/output/elasticsearch.md b/docs/plugins/fluentbit/output/elasticsearch.md similarity index 100% rename from docs/plugins/output/elasticsearch.md rename to docs/plugins/fluentbit/output/elasticsearch.md diff --git a/docs/plugins/output/file.md b/docs/plugins/fluentbit/output/file.md similarity index 100% rename from docs/plugins/output/file.md rename to docs/plugins/fluentbit/output/file.md diff --git a/docs/plugins/output/forward.md b/docs/plugins/fluentbit/output/forward.md similarity index 100% rename from docs/plugins/output/forward.md rename to docs/plugins/fluentbit/output/forward.md diff --git a/docs/plugins/output/http.md b/docs/plugins/fluentbit/output/http.md similarity index 100% rename from docs/plugins/output/http.md rename to docs/plugins/fluentbit/output/http.md diff --git a/docs/plugins/output/kafka.md b/docs/plugins/fluentbit/output/kafka.md similarity index 100% rename from docs/plugins/output/kafka.md rename to docs/plugins/fluentbit/output/kafka.md diff --git a/docs/plugins/output/loki.md b/docs/plugins/fluentbit/output/loki.md similarity index 100% rename from docs/plugins/output/loki.md rename to docs/plugins/fluentbit/output/loki.md diff --git a/docs/plugins/output/null.md b/docs/plugins/fluentbit/output/null.md similarity index 100% rename from docs/plugins/output/null.md rename to docs/plugins/fluentbit/output/null.md diff --git a/docs/plugins/output/stdout.md b/docs/plugins/fluentbit/output/stdout.md similarity index 100% rename from docs/plugins/output/stdout.md rename to docs/plugins/fluentbit/output/stdout.md diff --git a/docs/plugins/output/syslog.md b/docs/plugins/fluentbit/output/syslog.md similarity index 100% rename from docs/plugins/output/syslog.md rename to docs/plugins/fluentbit/output/syslog.md diff --git a/docs/plugins/output/tcp.md b/docs/plugins/fluentbit/output/tcp.md similarity index 100% rename from docs/plugins/output/tcp.md rename to docs/plugins/fluentbit/output/tcp.md diff --git a/docs/plugins/parser/json.md b/docs/plugins/fluentbit/parser/json.md similarity index 100% rename from docs/plugins/parser/json.md rename to docs/plugins/fluentbit/parser/json.md diff --git a/docs/plugins/parser/logfmt.md b/docs/plugins/fluentbit/parser/logfmt.md similarity index 100% rename from docs/plugins/parser/logfmt.md rename to docs/plugins/fluentbit/parser/logfmt.md diff --git a/docs/plugins/parser/lstv.md b/docs/plugins/fluentbit/parser/lstv.md similarity index 100% rename from docs/plugins/parser/lstv.md rename to docs/plugins/fluentbit/parser/lstv.md diff --git a/docs/plugins/parser/regex.md b/docs/plugins/fluentbit/parser/regex.md similarity index 100% rename from docs/plugins/parser/regex.md rename to docs/plugins/fluentbit/parser/regex.md diff --git a/docs/plugins/secret.md b/docs/plugins/fluentbit/secret.md similarity index 100% rename from docs/plugins/secret.md rename to docs/plugins/fluentbit/secret.md diff --git a/docs/plugins/tls.md b/docs/plugins/fluentbit/tls.md similarity index 100% rename from docs/plugins/tls.md rename to docs/plugins/fluentbit/tls.md diff --git a/manifests/forward/fluentbit-output-forward.yaml b/manifests/forward/fluentbit-output-forward.yaml index 87d64d62d..21b3bc5a0 100644 --- a/manifests/forward/fluentbit-output-forward.yaml +++ b/manifests/forward/fluentbit-output-forward.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Output +kind: ClusterOutput metadata: name: fluentd - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/auditd/filter-auditd.yaml b/manifests/logging-stack/auditd/filter-auditd.yaml index 18b69f64d..85b3763ba 100644 --- a/manifests/logging-stack/auditd/filter-auditd.yaml +++ b/manifests/logging-stack/auditd/filter-auditd.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io -kind: Filter +kind: ClusterFilter metadata: name: filter-audit-logs - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/auditd/input-auditd.yaml b/manifests/logging-stack/auditd/input-auditd.yaml index d55d38310..2dba8f6f5 100644 --- a/manifests/logging-stack/auditd/input-auditd.yaml +++ b/manifests/logging-stack/auditd/input-auditd.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io -kind: Input +kind: ClusterInput metadata: name: auditd-input - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/auditd/output-auditd-elasticsearch.yaml b/manifests/logging-stack/auditd/output-auditd-elasticsearch.yaml index 0b4efb155..95352e441 100644 --- a/manifests/logging-stack/auditd/output-auditd-elasticsearch.yaml +++ b/manifests/logging-stack/auditd/output-auditd-elasticsearch.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io -kind: Output +kind: ClusterOutput metadata: name: auditd-to-es - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/filter-containerd.yaml b/manifests/logging-stack/filter-containerd.yaml index 403fec00e..30f444746 100644 --- a/manifests/logging-stack/filter-containerd.yaml +++ b/manifests/logging-stack/filter-containerd.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Filter +kind: ClusterFilter metadata: name: containerd - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/filter-kubernetes.yaml b/manifests/logging-stack/filter-kubernetes.yaml index 89e58b524..a2e1eaa70 100644 --- a/manifests/logging-stack/filter-kubernetes.yaml +++ b/manifests/logging-stack/filter-kubernetes.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Filter +kind: ClusterFilter metadata: name: kubernetes - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/filter-systemd.yaml b/manifests/logging-stack/filter-systemd.yaml index 49c3e96b4..2a6297926 100644 --- a/manifests/logging-stack/filter-systemd.yaml +++ b/manifests/logging-stack/filter-systemd.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Filter +kind: ClusterFilter metadata: name: systemd - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/fluentbitconfig-fluentBitConfig.yaml b/manifests/logging-stack/fluentbitconfig-fluentBitConfig.yaml index b97b83f15..25e409173 100644 --- a/manifests/logging-stack/fluentbitconfig-fluentBitConfig.yaml +++ b/manifests/logging-stack/fluentbitconfig-fluentBitConfig.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: FluentBitConfig +kind: ClusterFluentBitConfig metadata: name: fluent-bit-config - namespace: kubesphere-logging-system labels: app.kubernetes.io/name: fluent-bit spec: diff --git a/manifests/logging-stack/input-systemd-docker.yaml b/manifests/logging-stack/input-systemd-docker.yaml index 553627446..5bd5cbe73 100644 --- a/manifests/logging-stack/input-systemd-docker.yaml +++ b/manifests/logging-stack/input-systemd-docker.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Input +kind: ClusterInput metadata: name: docker - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/input-systemd-kubelet.yaml b/manifests/logging-stack/input-systemd-kubelet.yaml index 1d7575984..fd3c18b82 100644 --- a/manifests/logging-stack/input-systemd-kubelet.yaml +++ b/manifests/logging-stack/input-systemd-kubelet.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Input +kind: ClusterInput metadata: name: kubelet - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/input-tail.yaml b/manifests/logging-stack/input-tail.yaml index a65f3cf1b..95840b162 100644 --- a/manifests/logging-stack/input-tail.yaml +++ b/manifests/logging-stack/input-tail.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Input +kind: ClusterInput metadata: name: tail - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/output-elasticsearch.yaml b/manifests/logging-stack/output-elasticsearch.yaml index 26a7bf241..fc85bea19 100644 --- a/manifests/logging-stack/output-elasticsearch.yaml +++ b/manifests/logging-stack/output-elasticsearch.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Output +kind: ClusterOutput metadata: name: es - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/output-forward.yaml b/manifests/logging-stack/output-forward.yaml index 0e7b3da65..572e75175 100644 --- a/manifests/logging-stack/output-forward.yaml +++ b/manifests/logging-stack/output-forward.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Output +kind: ClusterOutput metadata: name: fluentd - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "false" fluentbit.fluent.io/component: logging diff --git a/manifests/logging-stack/output-kafka.yaml b/manifests/logging-stack/output-kafka.yaml index 2aa35b035..ea9e59abf 100644 --- a/manifests/logging-stack/output-kafka.yaml +++ b/manifests/logging-stack/output-kafka.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Output +kind: ClusterOutput metadata: name: kafka - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "false" fluentbit.fluent.io/component: logging diff --git a/manifests/quick-start/fluentbit.yaml b/manifests/quick-start/fluentbit.yaml index 1307821e4..7c59be910 100644 --- a/manifests/quick-start/fluentbit.yaml +++ b/manifests/quick-start/fluentbit.yaml @@ -11,10 +11,9 @@ spec: --- apiVersion: fluentbit.fluent.io -kind: FluentBitConfig +kind: ClusterFluentBitConfig metadata: name: fluent-bit-config - namespace: kubesphere-logging-system labels: app.kubernetes.io/name: fluent-bit spec: @@ -27,10 +26,9 @@ spec: --- apiVersion: fluentbit.fluent.io -kind: Input +kind: ClusterInput metadata: name: dummy - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" spec: @@ -39,10 +37,9 @@ spec: --- apiVersion: fluentbit.fluent.io -kind: Output +kind: ClusterOutput metadata: name: stdout - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" spec: diff --git a/manifests/regex-parser/filter-parser.yaml b/manifests/regex-parser/filter-parser.yaml index 32f8db188..c31956d12 100644 --- a/manifests/regex-parser/filter-parser.yaml +++ b/manifests/regex-parser/filter-parser.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Filter +kind: ClusterFilter metadata: name: parser - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" spec: diff --git a/manifests/regex-parser/fluentbitconfig-fluentBitConfig.yaml b/manifests/regex-parser/fluentbitconfig-fluentBitConfig.yaml index 9efa84619..7c1b6474e 100644 --- a/manifests/regex-parser/fluentbitconfig-fluentBitConfig.yaml +++ b/manifests/regex-parser/fluentbitconfig-fluentBitConfig.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: FluentBitConfig +kind: ClusterFluentBitConfig metadata: name: fluent-bit-config - namespace: kubesphere-logging-system labels: app.kubernetes.io/name: fluent-bit spec: diff --git a/manifests/regex-parser/input-tail.yaml b/manifests/regex-parser/input-tail.yaml index 87362b142..fcef9b46c 100644 --- a/manifests/regex-parser/input-tail.yaml +++ b/manifests/regex-parser/input-tail.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Input +kind: ClusterInput metadata: name: tail - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" fluentbit.fluent.io/component: logging diff --git a/manifests/regex-parser/output-stdout.yaml b/manifests/regex-parser/output-stdout.yaml index ad7a73f77..ac7ae2b8f 100644 --- a/manifests/regex-parser/output-stdout.yaml +++ b/manifests/regex-parser/output-stdout.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Output +kind: ClusterOutput metadata: name: stdout - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" spec: diff --git a/manifests/regex-parser/parser-regex.yaml b/manifests/regex-parser/parser-regex.yaml index 2dcfae265..09b6ffee0 100644 --- a/manifests/regex-parser/parser-regex.yaml +++ b/manifests/regex-parser/parser-regex.yaml @@ -1,8 +1,7 @@ apiVersion: fluentbit.fluent.io/v1alpha2 -kind: Parser +kind: ClusterParser metadata: name: my-regex - namespace: kubesphere-logging-system labels: fluentbit.fluent.io/enabled: "true" spec: diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 2b909d94c..c5f059b6e 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -1518,7 +1518,7 @@ spec: listKind: FilterList plural: filters singular: filter - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -2452,7 +2452,7 @@ spec: shortNames: - fbc singular: fluentbitconfig - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -2562,6 +2562,11 @@ spec: are ANDed. type: object type: object + namespace: + description: If namespace is defined, then the configmap and secret + for fluent-bit is in this namespace. If it is not defined, it is + in the namespace of the fluentd-operator + type: string outputSelector: description: Select output plugins properties: @@ -8352,7 +8357,7 @@ spec: listKind: InputList plural: inputs singular: input - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -8634,7 +8639,7 @@ spec: listKind: OutputList plural: outputs singular: output - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -10876,7 +10881,7 @@ spec: listKind: ParserList plural: parsers singular: parser - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: diff --git a/manifests/setup/fluent-operator-deployment.yaml b/manifests/setup/fluent-operator-deployment.yaml index e9ef97076..45a2cbf24 100644 --- a/manifests/setup/fluent-operator-deployment.yaml +++ b/manifests/setup/fluent-operator-deployment.yaml @@ -40,6 +40,12 @@ spec: containers: - name: fluent-operator image: kubesphere/fluent-operator:v1.0.0 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace resources: limits: cpu: 200m diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index a38e4ec6f..957f6d69a 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -1518,7 +1518,7 @@ spec: listKind: FilterList plural: filters singular: filter - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -2452,7 +2452,7 @@ spec: shortNames: - fbc singular: fluentbitconfig - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -2562,6 +2562,11 @@ spec: are ANDed. type: object type: object + namespace: + description: If namespace is defined, then the configmap and secret + for fluent-bit is in this namespace. If it is not defined, it is + in the namespace of the fluentd-operator + type: string outputSelector: description: Select output plugins properties: @@ -8352,7 +8357,7 @@ spec: listKind: InputList plural: inputs singular: input - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -8634,7 +8639,7 @@ spec: listKind: OutputList plural: outputs singular: output - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -10876,7 +10881,7 @@ spec: listKind: ParserList plural: parsers singular: parser - scope: Namespaced + scope: Cluster versions: - name: v1alpha2 schema: @@ -11113,7 +11118,13 @@ spec: app.kubernetes.io/name: fluent-operator spec: containers: - - image: kubesphere/fluent-operator:v1.0.0 + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: kubesphere/fluent-operator:v1.0.0 name: fluent-operator resources: limits: