High CPU usage (when idle) when there are multiple out_foward block with multiple downstream nodes #1852
Open
Description
Fluentd version 0.14.20
OS: centos7
AWS EC2 m4.large
Problem: CPU usage is very high. The fluentd only setup with 1 worker and the usage is consistently >90% on 1 CPU core even when there's no traffic at all.
I notice the CPU usage increase as the number of block increase or the number of dowstream nodes increase.
My fluentd config is as below:
<system>
root_dir /var/log/fluentd
</system>
<source>
@type forward
port 24226
chunk_size_limit 128M
<security>
self_hostname "receiver-1"
shared_key "xxxxxx"
</security>
<transport tls>
cert_path "<path-to-public-key>"
private_key_path "<path-to-private-key>"
</transport>
</source>
<match data_1>
@type copy
<store>
@id data_1_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_2>
@type copy
<store>
@id data_2_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_3>
@type copy
<store>
@id data_3_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_4>
@type copy
<store>
@id data_4_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_5>
@type copy
<store>
@id data_5_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
<store>
@id data_5_downstream_2
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_2-1
port 24224
</server>
<server>
host downstream_2-2
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_6>
@type copy
<store>
@id data_6_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
<store>
@id data_6_downstream_2
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_2-1
port 24224
</server>
<server>
host downstream_2-2
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_7>
@type copy
<store>
@id data_7_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
<store>
@id data_7_downstream_2
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_2-1
port 24224
</server>
<server>
host downstream_2-2
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_8>
@type copy
<store>
@id data_8_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_9>
@type copy
<store>
@id data_9_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
<store>
@id data_9_downstream_2
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_2-1
port 24224
</server>
<server>
host downstream_2-2
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_10>
@type copy
<store>
@id data_10_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
<store>
@id data_10_downstream_2
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_2-1
port 24224
</server>
<server>
host downstream_2-2
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_11>
@type copy
<store>
@id data_11_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
<store>
@id data_11_downstream_2
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_2-1
port 24224
</server>
<server>
host downstream_2-2
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match data_12>
@type copy
<store>
@id data_12_downstream_1
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_1-1
port 24224
</server>
<server>
host downstream_1-2
port 24224
</server>
<server>
host downstream_1-3
port 24224
</server>
<server>
host downstream_1-4
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
<store>
@id data_12_downstream_2
@type forward
require_ack_response true
transport tls
tls_cert_path "<path-to-CA-cert>"
<security>
self_hostname "receiver-1"
shared_key "yyyyyy"
</security>
<server>
host downstream_2-1
port 24224
</server>
<server>
host downstream_2-2
port 24224
</server>
<buffer tag,t>
@type file
flush_mode interval
flush_interval 2s
retry_forever true
retry_type periodic
retry_wait 60s
chunk_limit_size 67108864
total_limit_size 68719476736
</buffer>
</store>
</match>
<match **>
@type null
</match>