Day 81

Nongzhsh · Mar 28, 2022 · ac9ddf3 · ac9ddf3
1 parent 719d23e
commit ac9ddf3
Show file tree

Hide file tree

Showing 18 changed files with 553 additions and 5 deletions.
diff --git a/Days/Images/Day81_Monitoring1.png b/Days/Images/Day81_Monitoring1.png
diff --git a/Days/Images/Day81_Monitoring2.png b/Days/Images/Day81_Monitoring2.png
diff --git a/Days/Monitoring/Fluentd/introduction/configurations/containers-fluent.conf b/Days/Monitoring/Fluentd/introduction/configurations/containers-fluent.conf
@@ -0,0 +1,13 @@
+<source>
+  @type tail
+  format json
+  read_from_head true
+  tag docker.log
+  path /fluentd/log/containers/*/*-json.log
+  pos_file /tmp/container-logs.pos
+</source>
+
+# <match docker.log>
+#   @type file
+#   path /output/docker.log
+# </match>
diff --git a/Days/Monitoring/Fluentd/introduction/configurations/elastic-fluent.conf b/Days/Monitoring/Fluentd/introduction/configurations/elastic-fluent.conf
@@ -0,0 +1,26 @@
+# where to send http logs
+<match http-*.log>
+  @type elasticsearch
+  host elasticsearch
+  port 9200
+  index_name fluentd-http
+  type_name fluentd
+</match>
+
+#where to send file logs
+<match file-myapp.log>
+  @type elasticsearch
+  host elasticsearch
+  port 9200
+  index_name fluentd-file
+  type_name fluentd
+</match>
+
+#where to send docker logs
+<match docker.log>
+  @type elasticsearch
+  host elasticsearch
+  port 9200
+  index_name fluentd-docker
+  type_name fluentd
+</match>
diff --git a/Days/Monitoring/Fluentd/introduction/configurations/file-fluent.conf b/Days/Monitoring/Fluentd/introduction/configurations/file-fluent.conf
@@ -0,0 +1,20 @@
+<source>
+  @type tail
+  format json
+  read_from_head true
+  tag file-myapp.log
+  path /fluentd/log/files/example-log.log
+  pos_file /tmp/example-log.log.pos
+</source>
+
+<filter file-myapp.log>
+  @type record_transformer
+  <record>
+    host_param "#{Socket.gethostname}"
+  </record>
+</filter>
+
+# <match file-myapp.log>
+#   @type file
+#   path /output/file-myapp.log
+# </match>
diff --git a/Days/Monitoring/Fluentd/introduction/configurations/fluent.conf b/Days/Monitoring/Fluentd/introduction/configurations/fluent.conf
@@ -0,0 +1,16 @@
+################################################################
+# This source reads tail of a file
+@include file-fluent.conf
+
+################################################################
+
+# This source gets incoming logs over HTTP
+@include http-fluent.conf
+
+################################################################
+# This source gets all logs from local docker host
+@include containers-fluent.conf
+
+################################################################
+# Send all logs to elastic search
+@include elastic-fluent.conf
diff --git a/Days/Monitoring/Fluentd/introduction/configurations/http-fluent.conf b/Days/Monitoring/Fluentd/introduction/configurations/http-fluent.conf
@@ -0,0 +1,19 @@
+<source>
+  @type http
+  port 9880
+  bind 0.0.0.0
+  body_size_limit 32m
+  keepalive_timeout 10s
+</source>
+
+<filter http-*.log>
+  @type record_transformer
+  <record>
+    host_param "#{Socket.gethostname}"
+  </record>
+</filter>
+
+# <match http-*.log>
+#   @type file
+#   path /output/http.log
+# </match>
diff --git a/Days/Monitoring/Fluentd/introduction/docker-compose.yaml b/Days/Monitoring/Fluentd/introduction/docker-compose.yaml
@@ -0,0 +1,49 @@
+version: "3"
+services:
+  fluentd:
+    container_name: fluentd
+    user: root
+    build:
+      context: .
+    image: fluentd
+    volumes:
+    - /var/lib/docker/containers:/fluentd/log/containers # Example: Reading docker logs
+    - ./file:/fluentd/log/files/ #Example: Reading logs from a file
+    - ./configurations:/fluentd/etc/
+    - ./logs:/output/ # Example: Fluentd will collect logs and store it here for demo
+    logging:
+      driver: "local"
+  # This app sends logs to Fluentd via HTTP
+  http-myapp:
+    container_name: http-myapp
+    image: alpine
+    volumes:
+    - ./http:/app
+    command: [ /bin/sh , -c , "apk add --no-cache curl && chmod +x /app/app.sh && ./app/app.sh"]
+  # This app writes logs to a local file
+  file-myapp:
+    container_name: file-myapp
+    image: alpine
+    volumes:
+    - ./file:/app
+    command: [ /bin/sh , -c , "chmod +x /app/app.sh && ./app/app.sh"]
+  elasticsearch: # port 9200
+    image: elasticsearch:7.9.1
+    container_name: elasticsearch
+    environment:
+    - node.name=elasticsearch
+    - cluster.initial_master_nodes=elasticsearch
+    - bootstrap.memory_lock=true
+    - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+  kibana:
+    image: kibana:7.9.1
+    container_name: kibana
+    ports:
+      - "5601:5601"
+    environment:
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
diff --git a/Days/Monitoring/Fluentd/introduction/dockerfile b/Days/Monitoring/Fluentd/introduction/dockerfile
@@ -0,0 +1,4 @@
+FROM fluent/fluentd:v1.11-debian
+
+USER root
+RUN gem install fluent-plugin-elasticsearch
diff --git a/Days/Monitoring/Fluentd/introduction/file/app.sh b/Days/Monitoring/Fluentd/introduction/file/app.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+while true
+do
+	echo "Writing log to a file"
+  echo '{"app":"file-myapp"}' >> /app/example-log.log
+	sleep 5
+done