Skip to content

Commit

Permalink
Merge pull request fluent#1220 from bakervos/feature/additional-elast…
Browse files Browse the repository at this point in the history
…icsearch-options

Feature/additional elasticsearch options
  • Loading branch information
benjaminhuo authored Jun 27, 2024
2 parents d8e2060 + 7985b5e commit 6989ede
Show file tree
Hide file tree
Showing 60 changed files with 35,712 additions and 33,287 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi

CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
controller-gen: go-deps ## Download controller-gen locally if necessary.
$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.3)
$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0)

GINKGO = $(shell pwd)/bin/ginkgo
ginkgo: go-deps ## Download controller-gen locally if necessary.
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion apis/fluentbit/v1alpha2/plugins/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion apis/fluentbit/v1alpha2/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

37 changes: 37 additions & 0 deletions apis/fluentd/v1alpha1/plugins/output/es.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,33 @@ type ElasticsearchCommon struct {
ClientKey *string `json:"clientKey,omitempty"`
// Optional, password for ClientKey file
ClientKeyPassword *plugins.Secret `json:"clientKeyPassword,omitempty"`
// Optional, Always update the template, even if it already exists (default: false)
TemplateOverwrite *bool `json:"templateOverwrite,omitempty"`
// Optional, You can specify times of retry putting template (default: 10)
MaxRetryPuttingTemplate *uint32 `json:"maxRetryPuttingTemplate,omitempty"`
// Optional, Indicates whether to fail when max_retry_putting_template is exceeded. If you have multiple output plugin, you could use this property to do not fail on fluentd statup (default: false)
FailOnPuttingTemplateRetryExceeded *bool `json:"failOnPuttingTemplateRetryExceeded,omitempty"`
// Optional, Indicates that the plugin should reset connection on any error (reconnect on next send) (default: false)
ReconnectOnError *bool `json:"reconnectOnError,omitempty"`
// Optional, Automatically reload connection after 10000 documents (default: true)
ReloadConnections *bool `json:"reloadConnections,omitempty"`
// Optional, Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the request, this can be useful to quickly remove a dead node from the list of addresses (default: false)
ReloadOnFailure *bool `json:"reloadOnFailure,omitempty"`
// Optional, HTTP Timeout (default: 5)
// +kubebuilder:validation:Pattern:="^\\d+(s|m|h|d)$"
RequestTimeout *string `json:"requestTimeout,omitempty"`
// Optional, Suppress '[types removal]' warnings on elasticsearch 7.x
SuppressTypeName *bool `json:"suppressTypeName,omitempty"`
// Optional, Enable Index Lifecycle Management (ILM)
EnableIlm *bool `json:"enableIlm,omitempty"`
// Optional, Specify ILM policy id
IlmPolicyId *string `json:"ilmPolicyId,omitempty"`
// Optional, Specify ILM policy contents as Hash
IlmPolicy *string `json:"ilmPolicy,omitempty"`
// Optional, Specify whether overwriting ilm policy or not
IlmPolicyOverwrite *bool `json:"ilmPolicyOverride,omitempty"`
// Optional, Enable logging of 400 reason without enabling debug log level
LogEs400Reason *bool `json:"logEs400Reason,omitempty"`
}

type Elasticsearch struct {
Expand All @@ -53,4 +80,14 @@ type ElasticsearchDataStream struct {

// You can specify Elasticsearch data stream name by this parameter. This parameter is mandatory for elasticsearch_data_stream
DataStreamName *string `json:"dataStreamName"`
// Optional, You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template
DataStreamTemplateName *string `json:"dataStreamTemplateName,omitempty"`
// Optional, Specify whether index patterns should include a wildcard (*) when creating an index template. This is particularly useful to prevent errors in scenarios where index templates are generated automatically, and multiple services with distinct suffixes are in use
DataStreamTemplateUseIndexPatternsWildcard *bool `json:"dataStreamTemplateUseIndexPatternsWildcard,omitempty"`
// Optional, You can specify the name of an existing ILM policy, which will be applied to the data stream. If not present, it creates a new ILM default policy (unless data_stream_template_name is defined, in that case the ILM will be set to the one specified in the matching index template)
DataStreamIlmName *string `json:"dataStreamIlmName,omitempty"`
// Optional, You can specify the ILM policy contents as hash. If not present, it will apply the ILM default policy
DataStreamIlmPolicy *string `json:"dataStreamIlmPolicy,omitempty"`
// Optional, Specify whether the data stream ILM policy should be overwritten
DataStreamIlmPolicyOverwrite *bool `json:"dataStreamIlmPolicyOverwrite,omitempty"`
}
72 changes: 72 additions & 0 deletions apis/fluentd/v1alpha1/plugins/output/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,58 @@ func (o *Output) elasticsearchPluginCommon(common *ElasticsearchCommon, parent *
parent.InsertPairs("path", fmt.Sprint(*common.Path))
}

if common.TemplateOverwrite != nil {
parent.InsertPairs("template_overwrite", fmt.Sprint(*common.TemplateOverwrite))
}

if common.MaxRetryPuttingTemplate != nil {
parent.InsertPairs("max_retry_putting_template", fmt.Sprint(*common.MaxRetryPuttingTemplate))
}

if common.FailOnPuttingTemplateRetryExceeded != nil {
parent.InsertPairs("fail_on_putting_template_retry_exceed", fmt.Sprint(*common.FailOnPuttingTemplateRetryExceeded))
}

if common.ReconnectOnError != nil {
parent.InsertPairs("reconnect_on_error", fmt.Sprint(*common.ReconnectOnError))
}

if common.ReloadConnections != nil {
parent.InsertPairs("reload_connections", fmt.Sprint(*common.ReloadConnections))
}

if common.ReloadOnFailure != nil {
parent.InsertPairs("reload_on_failure", fmt.Sprint(*common.ReloadOnFailure))
}

if common.RequestTimeout != nil {
parent.InsertPairs("request_timeout", fmt.Sprint(*common.RequestTimeout))
}

if common.SuppressTypeName != nil {
parent.InsertPairs("suppress_type_name", fmt.Sprint(*common.SuppressTypeName))
}

if common.EnableIlm != nil {
parent.InsertPairs("enable_ilm", fmt.Sprint(*common.EnableIlm))
}

if common.IlmPolicyId != nil {
parent.InsertPairs("ilm_policy_id", fmt.Sprint(*common.IlmPolicyId))
}

if common.IlmPolicy != nil {
parent.InsertPairs("ilm_policy", fmt.Sprint(*common.IlmPolicy))
}

if common.IlmPolicyOverwrite != nil {
parent.InsertPairs("ilm_policy_overwrite", fmt.Sprint(*common.IlmPolicyOverwrite))
}

if common.LogEs400Reason != nil {
parent.InsertPairs("log_es_400_reason", fmt.Sprint(*common.LogEs400Reason))
}

return parent, nil
}

Expand Down Expand Up @@ -487,6 +539,26 @@ func (o *Output) elasticsearchDataStreamPlugin(parent *params.PluginStore, loade
parent.InsertPairs("data_stream_name", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamName))
}

if o.ElasticsearchDataStream.DataStreamTemplateName != nil {
parent.InsertPairs("data_stream_template_name", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamTemplateName))
}

if o.ElasticsearchDataStream.DataStreamTemplateUseIndexPatternsWildcard != nil {
parent.InsertPairs("data_stream_template_use_index_patterns_wildcard", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamTemplateUseIndexPatternsWildcard))
}

if o.ElasticsearchDataStream.DataStreamIlmName != nil {
parent.InsertPairs("data_stream_ilm_name", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamIlmName))
}

if o.ElasticsearchDataStream.DataStreamIlmPolicy != nil {
parent.InsertPairs("data_stream_ilm_policy", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamIlmPolicy))
}

if o.ElasticsearchDataStream.DataStreamIlmPolicyOverwrite != nil {
parent.InsertPairs("data_stream_ilm_policy_overwrite", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamIlmPolicyOverwrite))
}

return parent, nil
}

Expand Down
1 change: 0 additions & 1 deletion apis/fluentd/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 6989ede

Please sign in to comment.