chore: wip configuring minio, fluent, and elasticsearch

johnny-rice · Dec 10, 2023 · 6cd8128 · 6cd8128
1 parent c18e154
commit 6cd8128
Show file tree

Hide file tree

Showing 8 changed files with 75 additions and 150 deletions.
diff --git a/.env.example b/.env.example
@@ -7,13 +7,13 @@ REDIS_HOST=redis
 REDIS_PORT=6379
 REDIS_SECRET=changeme
 LOGS_LEVEL=verbose
-LOGS_FORMAT=text
-LOGS_TRANSPORT=none
-LOGS_DRIVER_HOST=logsdriver
+LOGS_FORMAT=json
+LOGS_TRANSPORT=fluent
+LOGS_DRIVER_HOST=fluent
 LOGS_DRIVER_PORT=24224
 LOGS_AGGREGRATOR_PORT=9200
-LOGS_AGGREGRATOR_HOST=logsaggregator
-LOGS_OPT_TAG_PREFIX=fonoster
+LOGS_AGGREGRATOR_HOST=elasticsearch
+LOGS_OPT_TAG_PREFIX=fonoster-logs
 
 ##
 # Web UI Configuration

diff --git a/.gitpod.yml b/.gitpod.yml
@@ -11,8 +11,8 @@ tasks:
   - name: Run dependencies
     command: |
       cp .env.example .env
-      docker-compose -f docker-compose.yaml -f docker-compose.dev.yaml \
-        up redis rtpengine voice mediaserver routr
+      docker-compose -f compose.yaml -f compose.dev.yaml \
+        up minio redis rtpengine voice mediaserver routr fluent elasticsearch
   - name: Build and start application
     init: npm run make
     command: npm start

diff --git a/compose.dev.yaml b/compose.dev.yaml
@@ -17,3 +17,15 @@ services:
   vault:
     ports:
       - 8200:8200
+
+  fluent:
+    ports:
+      - 24224:24224
+
+  elasticsearch:
+    ports:
+      - 9200:9200
+
+  minio:
+    ports:
+      - 9000:9000
diff --git a/compose.yaml b/compose.yaml
@@ -11,12 +11,17 @@ services:
       WEBUI_APISERVER_ENDPOINT: ${WEBUI_APISERVER_ENDPOINT}
       WEBUI_GITHUB_CLIENT_ID: ${WEBUI_GITHUB_CLIENT_ID}
       WEBUI_GITHUB_CLIENT_SECRET: ${WEBUI_GITHUB_CLIENT_SECRET} 
+    logging:
+      options:
+        tag: fonoster-logs
 
   apiserver:
     image: fonoster/apiserver:latest
     restart: unless-stopped
     environment:
       LOGS_LEVEL: ${LOGS_LEVEL}
+      LOGS_TRANSPORT: ${LOGS_TRANSPORT}
+      LOGS_FORMAT: ${LOGS_FORMAT}
       APISERVER_GLOBAL_SIP_DOMAIN: ${APISERVER_GLOBAL_SIP_DOMAIN}
       APISERVER_ROUTR_API_HOST: ${APISERVER_ROUTR_API_HOST}
       APISERVER_ROUTR_API_PORT: ${APISERVER_ROUTR_API_PORT}
@@ -44,6 +49,9 @@ services:
       LOGS_AGGREGRATOR_PORT: ${LOGS_AGGREGRATOR_PORT}
     ports:
       - 50052:50052
+    logging:
+      options:
+        tag: fonoster-logs
 
   routr:
     image: fonoster/routr:1.2.8
@@ -71,6 +79,9 @@ services:
     ports:
       - 5060-5063:5060-5063
       - 5060:5060/udp
+    logging:
+      options:
+        tag: fonoster-logs
 
   # RTPEngine uses a range of ports to handle RTP traffic. Because exposing a large range of ports 
   # is not possible in Docker, we need to use network_mode: host.
@@ -101,6 +112,8 @@ services:
       EXTERN_ADDR: ${DOCKER_HOST_ADDRESS}
       GRPC_ALLOW_INSECURE: ${GRPC_ALLOW_INSECURE}
       LOGS_LEVEL: ${LOGS_LEVEL}
+      LOGS_TRANSPORT: ${LOGS_TRANSPORT}
+      LOGS_FORMAT: ${LOGS_FORMAT}
       EXTERN_PORT: ${MEDIASERVER_EXTERN_PORT}
       APISERVER_ENDPOINT: ${MEDIASERVER_APISERVER_ENDPOINT}
       ARI_EXTERNAL_URL: ${MEDIASERVER_ARI_EXTERNAL_URL}
@@ -117,6 +130,9 @@ services:
       - shared:/home/fonoster
     expose:
       - 8080
+    logging:
+      options:
+        tag: fonoster-logs
 
   voice:
     image: fonoster/rox:latest
@@ -126,6 +142,11 @@ services:
       - 3001
     environment:
       LOGS_LEVEL: ${LOGS_LEVEL}
+      LOGS_TRANSPORT: ${LOGS_TRANSPORT}
+      LOGS_FORMAT: ${LOGS_FORMAT}
+    logging:
+      options:
+        tag: fonoster-logs
 
   redis:
     image: redis:7.0.8-alpine
@@ -145,5 +166,36 @@ services:
     cap_add:
       - IPC_LOCK
 
+  fluent:
+    image: fonoster/fluent:latest
+    restart: unless-stopped
+    expose:
+      - 24224
+    volumes:
+      - ./etc/fluent.conf:/fluentd/etc/fluent.conf:ro
+
+  elasticsearch:
+    image: elasticsearch:7.17.15
+    platform: linux/x86_64
+    restart: unless-stopped
+    environment:
+      - "discovery.type=single-node"
+    expose:
+      - 9200
+    volumes:
+      - shared:/usr/share/elasticsearch/data:rw
+
+  minio:
+    image: minio/minio:RELEASE.2023-12-09T18-17-51Z
+    restart: unless-stopped
+    command: minio server /fonoster
+    environment:
+      MINIO_ACCESS_KEY: ${APISERVER_S3_SERVER_USERNAME}
+      MINIO_SECRET_KEY: ${APISERVER_S3_SERVER_SECRET}
+    expose:
+      - 9000
+    volumes:
+      - shared:/fonoster  
+
 volumes:
   shared:
diff --git a/etc/fluent.conf b/etc/fluent.conf
@@ -7,7 +7,7 @@
   @type copy
   <store>
     @type elasticsearch
-    host logsaggregator.fonoster
+    host elasticsearch
     port 9200
     logstash_format true
     logstash_prefix fluentd

diff --git a/mods/webui/package.json b/mods/webui/package.json
@@ -17,15 +17,12 @@
     "@fonoster/sdk": "^0.4.8",
     "@headlessui/react": "^1.4.2",
     "@heroicons/react": "^1.0.5",
-    "@sentry/browser": "^6.19.1",
-    "@sentry/tracing": "^6.19.1",
     "@supabase/supabase-js": "^1.29.1",
     "@supabase/ui": "^0.36.2",
     "@tailwindcss/forms": "^0.4.0",
     "autoprefixer": "^10.4.1",
     "axios": "^0.24.0",
     "dotenv": "^16.3.1",
-    "mixpanel-browser": "^2.45.0",
     "next": "12.3.4",
     "next-auth": "4.15.2",
     "next-query-params": "^2.1.2",
@@ -50,7 +47,6 @@
     "zustand": "^3.6.9"
   },
   "devDependencies": {
-    "@types/mixpanel-browser": "^2.38.1",
     "@types/node": "^17.0.8",
     "@types/react": "^17.0.38",
     "typescript": "^4.5.4"

diff --git a/mods/webui/src/pages/api/sdk/monitor.ts b/mods/webui/src/pages/api/sdk/monitor.ts
@@ -1,4 +1,4 @@
-const Monitor = require('@fonoster/monitor')
+const Monitor = require('@fonoster/monitor/dist/client/monitor')
 import { NextApiRequest, NextApiResponse } from 'next'
 
 import { getServerCurrentProject } from '@/mods/auth/lib/getUserLogged'
@@ -9,7 +9,7 @@ export default async function handle(
   req: NextApiRequest,
   res: NextApiResponse
 ) {
-  const manager = new Monitor(getServerCurrentProject(req))
+  const monitor = new Monitor(getServerCurrentProject(req))
 
   const get = async () => {
     const { time = TIMES[0].value, eventType, level } = req.query
@@ -34,7 +34,7 @@ export default async function handle(
       })
     }
 
-    return manager.searchEvents({
+    return monitor.searchEvents({
       query: {
         bool: {
           must,

diff --git a/package-lock.json b/package-lock.json