Skip to content

Commit

Permalink
Change fluentbit's crd from namesapce level to cluster level
Browse files Browse the repository at this point in the history 
Signed-off-by: chengdehao <dehaocheng@kubesphere.io>
  • Loading branch information
@wenchajun
wenchajun committed Feb 14, 2022
1 parent fa5c99c commit dee5769
Show file tree
Hide file tree
Showing 84 changed files with 3,128 additions and 307 deletions.
56 changes: 28 additions & 28 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,37 +294,37 @@ curl <podIP>:2020 | jq .
The list below shows supported plugins which are based on Fluent Bit v1.7.x+. For more information, please refer to the API docs of each plugin.

- [Input](docs/crd.md#input)
- [dummy](docs/plugins/input/dummy.md)
- [tail](docs/plugins/input/tail.md)
- [systemd](docs/plugins/input/systemd.md)
- [dummy](docs/plugins/fluentbit/input/dummy.md)
- [tail](docs/plugins/fluentbit/input/tail.md)
- [systemd](docs/plugins/fluentbit/input/systemd.md)
- [Parser](docs/crd.md#parser)
- [json](docs/plugins/parser/json.md)
- [logfmt](docs/plugins/parser/logfmt.md)
- [lstv](docs/plugins/parser/lstv.md)
- [regex](docs/plugins/parser/regex.md)
- [json](docs/plugins/fluentbit/parser/json.md)
- [logfmt](docs/plugins/fluentbit/parser/logfmt.md)
- [lstv](docs/plugins/fluentbit/parser/lstv.md)
- [regex](docs/plugins/fluentbit/parser/regex.md)
- [Filter](docs/crd.md#filter)
- [kubernetes](docs/plugins/filter/kubernetes.md)
- [modify](docs/plugins/filter/modify.md)
- [nest](docs/plugins/filter/nest.md)
- [parser](docs/plugins/filter/parser.md)
- [grep](docs/plugins/filter/grep.md)
- [record modifier](docs/plugins/filter/recordmodifier.md)
- [lua](docs/plugins/filter/lua.md)
- [throttle](docs/plugins/filter/throttle.md)
- [aws](docs/plugins/filter/aws.md)
- [multiline](docs/plugins/filter/multiline.md)
- [kubernetes](docs/plugins/fluentbit/filter/kubernetes.md)
- [modify](docs/plugins/fluentbit/filter/modify.md)
- [nest](docs/plugins/fluentbit/filter/nest.md)
- [parser](docs/plugins/fluentbit/filter/parser.md)
- [grep](docs/plugins/fluentbit/filter/grep.md)
- [record modifier](docs/plugins/fluentbit/filter/recordmodifier.md)
- [lua](docs/plugins/fluentbit/filter/lua.md)
- [throttle](docs/plugins/fluentbit/filter/throttle.md)
- [aws](docs/plugins/fluentbit/filter/aws.md)
- [multiline](docs/plugins/fluentbit/filter/multiline.md)
- [Output](docs/crd.md#output)
- [elasticsearch](docs/plugins/output/elasticsearch.md)
- [file](docs/plugins/output/file.md)
- [forward](docs/plugins/output/forward.md)
- [http](docs/plugins/output/http.md)
- [kafka](docs/plugins/output/kafka.md)
- [null](docs/plugins/output/null.md)
- [stdout](docs/plugins/output/stdout.md)
- [tcp](docs/plugins/output/tcp.md)
- [loki](docs/plugins/output/loki.md)
- [syslog](docs/plugins/output/syslog.md)
- [datadog](docs/plugins/output/datadog.md)
- [elasticsearch](docs/plugins/fluentbit/output/elasticsearch.md)
- [file](docs/plugins/fluentbit/output/file.md)
- [forward](docs/plugins/fluentbit/output/forward.md)
- [http](docs/plugins/fluentbit/output/http.md)
- [kafka](docs/plugins/fluentbit/output/kafka.md)
- [null](docs/plugins/fluentbit/output/null.md)
- [stdout](docs/plugins/fluentbit/output/stdout.md)
- [tcp](docs/plugins/fluentbit/output/tcp.md)
- [loki](docs/plugins/fluentbit/output/loki.md)
- [syslog](docs/plugins/fluentbit/output/syslog.md)
- [datadog](docs/plugins/fluentbit/output/datadog.md)

## Best Practice

Expand Down
17 changes: 9 additions & 8 deletions apis/fluentbit/v1alpha2/filter_types.go → ...fluentbit/v1alpha2/clusterfilter_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.

// FilterSpec defines the desired state of Filter
// FilterSpec defines the desired state of ClusterFilter
type FilterSpec struct {
// A pattern to match against the tags of incoming records.
// It's case-sensitive and support the star (*) character as a wildcard.
Expand Down Expand Up @@ -69,10 +69,11 @@ type FilterItem struct {
}

// +kubebuilder:object:root=true
// +kubebuilder:resource:scope=Cluster
// +genclient

// Filter defines a Filter configuration.
type Filter struct {
// ClusterFilter defines a cluster-level Filter configuration.
type ClusterFilter struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
// Specification of desired Filter configuration.
Expand All @@ -81,17 +82,17 @@ type Filter struct {

// +kubebuilder:object:root=true

// FilterList contains a list of Filter
// FilterList contains a list of ClusterFilter
type FilterList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []Filter `json:"items"`
Items []ClusterFilter `json:"items"`
}

// +kubebuilder:object:generate:=false

// FilterByName implements sort.Interface for []Filter based on the Name field.
type FilterByName []Filter
// FilterByName implements sort.Interface for []ClusterFilter based on the Name field.
type FilterByName []ClusterFilter

func (a FilterByName) Len() int { return len(a) }
func (a FilterByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
Expand Down Expand Up @@ -138,5 +139,5 @@ func (list FilterList) Load(sl plugins.SecretLoader) (string, error) {
}

func init() {
SchemeBuilder.Register(&Filter{}, &FilterList{})
SchemeBuilder.Register(&ClusterFilter{}, &FilterList{})
}
25 changes: 14 additions & 11 deletions ...uentbit/v1alpha2/fluentbitconfig_types.go → .../v1alpha2/clusterfluentbitconfig_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.

// FluentBitConfigSpec defines the desired state of FluentBitConfig
// FluentBitConfigSpec defines the desired state of ClusterFluentBitConfig
type FluentBitConfigSpec struct {
// Service defines the global behaviour of the Fluent Bit engine.
Service *Service `json:"service,omitempty"`
Expand All @@ -41,6 +41,9 @@ type FluentBitConfigSpec struct {
OutputSelector metav1.LabelSelector `json:"outputSelector,omitempty"`
// Select parser plugins
ParserSelector metav1.LabelSelector `json:"parserSelector,omitempty"`
//If namespace is defined, then the configmap and secret for fluent-bit is in this namespace.
//If it is not defined, it is in the namespace of the fluentd-operator
Namespace *string `json:"namespace,omitempty"`
}

type Service struct {
Expand Down Expand Up @@ -69,11 +72,11 @@ type Service struct {
}

// +kubebuilder:object:root=true
// +kubebuilder:resource:shortName=fbc
// +kubebuilder:resource:shortName=fbc,scope=Cluster
// +genclient

// FluentBitConfig is the Schema for the fluentbitconfigs API
type FluentBitConfig struct {
// ClusterFluentBitConfig is the Schema for the cluster-level fluentbitconfigs API
type ClusterFluentBitConfig struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Expand All @@ -82,15 +85,15 @@ type FluentBitConfig struct {

// +kubebuilder:object:root=true

// FluentBitConfigList contains a list of FluentBitConfig
// FluentBitConfigList contains a list of ClusterFluentBitConfig
type FluentBitConfigList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []FluentBitConfig `json:"items"`
Items []ClusterFluentBitConfig `json:"items"`
}

func init() {
SchemeBuilder.Register(&FluentBitConfig{}, &FluentBitConfigList{})
SchemeBuilder.Register(&ClusterFluentBitConfig{}, &FluentBitConfigList{})
}

func (s *Service) Params() *params.KVs {
Expand Down Expand Up @@ -125,7 +128,7 @@ func (s *Service) Params() *params.KVs {
return m
}

func (cfg FluentBitConfig) RenderMainConfig(sl plugins.SecretLoader, inputs InputList, filters FilterList, outputs OutputList) (string, error) {
func (cfg ClusterFluentBitConfig) RenderMainConfig(sl plugins.SecretLoader, inputs InputList, filters FilterList, outputs OutputList) (string, error) {
var buf bytes.Buffer

// The Service defines the global behaviour of the Fluent Bit engine.
Expand Down Expand Up @@ -161,7 +164,7 @@ func (cfg FluentBitConfig) RenderMainConfig(sl plugins.SecretLoader, inputs Inpu
return buf.String(), nil
}

func (cfg FluentBitConfig) RenderParserConfig(sl plugins.SecretLoader, parsers ParserList) (string, error) {
func (cfg ClusterFluentBitConfig) RenderParserConfig(sl plugins.SecretLoader, parsers ParserList) (string, error) {
var buf bytes.Buffer

parserSections, err := parsers.Load(sl)
Expand All @@ -188,13 +191,13 @@ func (a ByName) Len() int { return len(a) }
func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

func (cfg FluentBitConfig) RenderLuaScript(cl plugins.ConfigMapLoader, filters FilterList) ([]Script, error) {
func (cfg ClusterFluentBitConfig) RenderLuaScript(cl plugins.ConfigMapLoader, filters FilterList, namespace string) ([]Script, error) {

scripts := make([]Script, 0)
for _, f := range filters.Items {
for _, p := range f.Spec.FilterItems {
if p.Lua != nil {
script, err := cl.LoadConfigMap(p.Lua.Script)
script, err := cl.LoadConfigMap(p.Lua.Script, namespace)
if err != nil {
return nil, err
}
Expand Down
16 changes: 8 additions & 8 deletions ...it/v1alpha2/fluentbitconfig_types_test.go → ...pha2/clusterfluentbitconfig_types_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) {
"lbl1": "lvl1",
}

inputObj := &Input{
inputObj := &ClusterInput{
TypeMeta: metav1.TypeMeta{
APIVersion: "fluentbit.fluent.io/v1alpha2",
Kind: "Input",
Expand All @@ -120,10 +120,10 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) {
}

inputs := InputList{
Items: []Input{*inputObj},
Items: []ClusterInput{*inputObj},
}

filterObj := &Filter{
filterObj := &ClusterFilter{
TypeMeta: metav1.TypeMeta{
APIVersion: "fluentbit.fluent.io/v1alpha2",
Kind: "Filter",
Expand Down Expand Up @@ -181,10 +181,10 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) {
}

filters := FilterList{
Items: []Filter{*filterObj},
Items: []ClusterFilter{*filterObj},
}

syslogOut := Output{
syslogOut := ClusterOutput{
TypeMeta: metav1.TypeMeta{
APIVersion: "fluentbit.fluent.io/v1alpha2",
Kind: "Output",
Expand Down Expand Up @@ -218,7 +218,7 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) {
headers["X-Log-Header-0"] = "testing"
headers["X-Log-Header-App-ID"] = "9780495d9db3"

httpOutput := Output{
httpOutput := ClusterOutput{
TypeMeta: metav1.TypeMeta{
APIVersion: "fluentbit.fluent.io/v1alpha2",
Kind: "Output",
Expand Down Expand Up @@ -247,10 +247,10 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) {
}

outputs := OutputList{
Items: []Output{syslogOut, httpOutput},
Items: []ClusterOutput{syslogOut, httpOutput},
}

cfg := FluentBitConfig{
cfg := ClusterFluentBitConfig{
Spec: FluentBitConfigSpec{Service: &Service{
Daemon: ptrBool(false),
FlushSeconds: ptrInt64(1),
Expand Down
15 changes: 8 additions & 7 deletions apis/fluentbit/v1alpha2/input_types.go → .../fluentbit/v1alpha2/clusterinput_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.

// InputSpec defines the desired state of Input
// InputSpec defines the desired state of ClusterInput
type InputSpec struct {
// A user friendly alias name for this input plugin.
// Used in metrics for distinction of each configured input.
Expand All @@ -44,10 +44,11 @@ type InputSpec struct {
}

// +kubebuilder:object:root=true
// +kubebuilder:resource:scope=Cluster
// +genclient

// Input is the Schema for the inputs API
type Input struct {
// ClusterInput is the Schema for the inputs API
type ClusterInput struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Expand All @@ -56,19 +57,19 @@ type Input struct {

// +kubebuilder:object:generate:=false
// InputByName implements sort.Interface for []Input based on the Name field.
type InputByName []Input
type InputByName []ClusterInput

func (a InputByName) Len() int { return len(a) }
func (a InputByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a InputByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

// +kubebuilder:object:root=true

// InputList contains a list of Input
// ClusterInputList contains a list of Input
type InputList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []Input `json:"items"`
Items []ClusterInput `json:"items"`
}

func (list InputList) Load(sl plugins.SecretLoader) (string, error) {
Expand Down Expand Up @@ -107,5 +108,5 @@ func (list InputList) Load(sl plugins.SecretLoader) (string, error) {
}

func init() {
SchemeBuilder.Register(&Input{}, &InputList{})
SchemeBuilder.Register(&ClusterInput{}, &InputList{})
}
15 changes: 8 additions & 7 deletions apis/fluentbit/v1alpha2/output_types.go → ...fluentbit/v1alpha2/clusteroutput_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.

// OutputSpec defines the desired state of Output
// OutputSpec defines the desired state of ClusterOutput
type OutputSpec struct {
// A pattern to match against the tags of incoming records.
// It's case sensitive and support the star (*) character as a wildcard.
Expand Down Expand Up @@ -72,10 +72,11 @@ type OutputSpec struct {
}

// +kubebuilder:object:root=true
// +kubebuilder:resource:scope=Cluster
// +genclient

// Output is the Schema for the outputs API
type Output struct {
// ClusterOutput is the Schema for the cluster-level outputs API
type ClusterOutput struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Expand All @@ -84,16 +85,16 @@ type Output struct {

// +kubebuilder:object:root=true

// OutputList contains a list of Output
// OutputList contains a list of ClusterOutput
type OutputList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []Output `json:"items"`
Items []ClusterOutput `json:"items"`
}

// +kubebuilder:object:generate:=false
// OutputByName implements sort.Interface for []Output based on the Name field.
type OutputByName []Output
type OutputByName []ClusterOutput

func (a OutputByName) Len() int { return len(a) }
func (a OutputByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
Expand Down Expand Up @@ -144,5 +145,5 @@ func (list OutputList) Load(sl plugins.SecretLoader) (string, error) {
}

func init() {
SchemeBuilder.Register(&Output{}, &OutputList{})
SchemeBuilder.Register(&ClusterOutput{}, &OutputList{})
}
Loading

0 comments on commit dee5769

Please sign in to comment.