feat: (server) using filebeat to send logs to elasticsearch (foyzulk…

…arim#86)
aninda052 · Sep 27, 2023 · 601d837 · 601d837
1 parent fd83c66
commit 601d837
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 38 deletions.
diff --git a/server/EKA-docker/docker-compose.yml b/server/EKA-docker/docker-compose.yml
@@ -82,13 +82,11 @@ services:
       timeout: 10s
       retries: 120
 
-  # Kibana to display monitoring data
+  # APM for mesuring performance
   apm-server:
     depends_on:
       elasticsearch:
         condition: service_healthy
-      kibana:
-        condition: service_healthy
     image: docker.elastic.co/apm/apm-server:8.7.1
     container_name: apm-server
     ports:
@@ -108,18 +106,32 @@ services:
       interval: 10s
       retries: 120
       test: curl -I --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null https://localhost:8200/
+
+  # Filebeat for sending logs to elasticsearch
+  filebeat:
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+    image: docker.elastic.co/beats/filebeat:8.7.1
+    container_name: filebeat
+    # command: >
+    #   bash -c '
+    #     chmod go-w /usr/share/filebeat/filebeat.yml;
+    #   '
+    volumes:
+      - ../logs:/var/log/server
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
+    restart: unless-stopped
+    environment:
+      - ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST:-elasticsearch}
+      - KIBANA_HOST=${KIBANA_HOST:-kibana}
+      - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME:-elastic}
+      - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD:-change_me}
+    labels:
+      co.elastic.logs/json.overwrite_keys: true 
+      co.elastic.logs/json.add_error_key: true 
+      co.elastic.logs/json.expand_keys: true 
 
 volumes:
   elasticsearch-data:
     driver: local
-
-# networks:
-#   es-net:
-#     driver: bridge
-
-
-        #  -E apm-server.kibana.username=elastic
-        #  -E apm-server.kibana.password=change_me
-        #  -E apm-server.kibana.enabled=true
-        #  -E apm-server.kibana.host=kibana:5601
-        #  -E apm-server.kibana.protocol=http
diff --git a/server/EKA-docker/filebeat.yml b/server/EKA-docker/filebeat.yml
@@ -0,0 +1,15 @@
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+  - "/var/log/server/*.log"
+processors: 
+  - add_host_metadata: ~
+  # - add_cloud_metadata: ~
+  # - add_docker_metadata: ~
+  # - add_kubernetes_metadata: ~
+
+output.elasticsearch:
+  hosts: ["${ELASTICSEARCH_HOST}:9200"]
+  username: ${ELASTICSEARCH_USERNAME}
+  password: ${ELASTICSEARCH_PASSWORD}
diff --git a/server/package.json b/server/package.json
@@ -26,6 +26,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "@elastic/ecs-winston-format": "^1.3.1",
     "bcrypt": "^5.1.0",
     "bullmq": "^3.5.11",
     "compression": "^1.7.4",

diff --git a/server/src/logger.js b/server/src/logger.js
@@ -1,30 +1,33 @@
 const os = require('os');
 const { createLogger, format, transports } = require('winston');
 const { Loggly } = require('winston-loggly-bulk');
+const ecsFormat = require('@elastic/ecs-winston-format')
 require('winston-daily-rotate-file');
 
+const LOG_DIR = 'logs'
 class LogManager {
   static instance;
   constructor() {
     this.logger = createLogger({
       level: 'info',
-      format: format.combine(
-        format.timestamp({
-          format: 'YYYY-MM-DD HH:mm:ss',
-        }),
-        format.errors({ stack: true }),
-        format.splat(),
-        format.json()
-      ),
-      exceptionHandlers: [new transports.File({ filename: 'exception.log' })],
-      rejectionHandlers: [new transports.File({ filename: 'rejections.log' })],
+      // format:format.combine(
+      //   format.timestamp({
+      //     format: 'YYYY-MM-DD HH:mm:ss',
+      //   }),
+      //   format.errors({ stack: true }),
+      //   format.splat(),
+      //   format.json()
+      // ),
+      format: ecsFormat(),
+      exceptionHandlers: [new transports.File({ filename: `${LOG_DIR}/exception.log` })],
+      rejectionHandlers: [new transports.File({ filename: `${LOG_DIR}/rejections.log` })],
       defaultMeta: { service: process.env.npm_lifecycle_event },
       transports: [
-        new transports.File({ filename: 'error.log', level: 'error' }),
-        new transports.File({ filename: 'combined.log' }),
+        new transports.File({ filename: `${LOG_DIR}/error.log`, level: 'error' }),
+        new transports.File({ filename: `${LOG_DIR}/combined.log` }),
         new transports.DailyRotateFile({
           level: 'info',
-          filename: 'application-%DATE%.log',
+          filename: `${LOG_DIR}/application-%DATE%.log`,
           datePattern: 'YYYY-MM-DD-HH',
           zippedArchive: true,
           maxSize: '20m',

diff --git a/server/src/modules/models/video/controller.js b/server/src/modules/models/video/controller.js
@@ -17,11 +17,11 @@ const {
 const BASE_URL = `/api/videos`;
 
 const setupRoutes = (app) => {
-  console.log(`Setting up routes for ${BASE_URL}`);
+  // console.log(`Setting up routes for ${BASE_URL}`);
 
   // return empty response with success message for the base route
   app.get(`${BASE_URL}/`, async (req, res) => {
-    console.log(`GET`, req.params);
+    // console.log(`GET`, req.params);
     const data = await search({});
     res.send({
       status: 'success',
@@ -32,7 +32,7 @@ const setupRoutes = (app) => {
   });
 
   app.get(`${BASE_URL}/detail/:id`, async (req, res) => {
-    console.log(`GET`, req.params);
+    // console.log(`GET`, req.params);
     const video = await updateViewCount(req.params.id);
     if (video instanceof Error) {
       return res.status(400).json(JSON.parse(video.message));
@@ -42,13 +42,13 @@ const setupRoutes = (app) => {
 
   // TODO: Proper searching with paging and ordering
   app.post(`${BASE_URL}/search`, async (req, res) => {
-    console.log('POST search', req.body);
+    // console.log('POST search', req.body);
     const result = await search(req.body);
     res.send(result);
   });
 
   app.post(`${BASE_URL}/count`, async (req, res) => {
-    console.log('POST count', req.body);
+    // console.log('POST count', req.body);
     const result = await count(req.body);
     res.send({count: result});
   });
@@ -84,7 +84,7 @@ const setupRoutes = (app) => {
   });
 
   app.delete(`${BASE_URL}/delete/:id`, async (req, res) => {
-    console.log('DELETE', req.params.id);
+    // console.log('DELETE', req.params.id);
     if (req.params.id) {
       const result = await deleteById(req.params.id);
       if (result instanceof Error) {
@@ -110,10 +110,10 @@ const setupRoutes = (app) => {
 
   const fileFilter = (req, file, cb) => {
     if (file.mimetype === 'video/mp4' || file.mimetype === 'video/webm') {
-      console.log('file type supported', file);
+      // console.log('file type supported', file);
       cb(null, true);
     } else {
-      console.log('file type not supported', file);
+      // console.log('file type not supported', file);
       cb(new multer.MulterError('File type not supported'), false);
     }
   };
@@ -132,7 +132,7 @@ const setupRoutes = (app) => {
         res.status(400).json({ status: 'error', error: err });
         return;
       } else {
-        console.log('upload success', req.file);
+        // console.log('upload success', req.file);
         // res.status(200).json({ status: "success", message: "upload success" });
         next();
       }
@@ -155,10 +155,10 @@ const setupRoutes = (app) => {
         viewCount: 0,
         duration: 0,
       };
-      console.log('dbPayload', dbPayload);
+      // console.log('dbPayload', dbPayload);
       // TODO: save the file info and get the id from the database
       const result = await insert(dbPayload);
-      console.log('result', result);
+      // console.log('result', result);
       await addQueueItem(QUEUE_EVENTS.VIDEO_UPLOADED, {
         id: result.insertedId.toString(),
         ...req.body,