- Added promtail and loki

- Added loki dashboard and update plg resources - Added backend file logger - UPdated nginx to write logs to file
raft-tech · Sep 5, 2024 · 8cbb557 · 8cbb557
1 parent c7f0010
commit 8cbb557
Show file tree

Hide file tree

Showing 7 changed files with 257 additions and 7 deletions.
diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml
@@ -94,6 +94,25 @@ services:
       - web
       - celery-exporter
       - postgres-exporter
+
+  promtail:
+    restart: always
+    image: grafana/promtail:3.0.1
+    ports:
+      - 9080:9080
+    volumes:
+      - ./plg/promtail/config.yml:/etc/promtail/config.yml
+      - ~/tdp-logs/nginx:/var/log/nginx
+      - logs:/logs
+    command: -config.file=/etc/promtail/config.yml
+
+  loki:
+    restart: always
+    image: grafana/loki:3.0.1
+    ports:
+      - 3100:3100
+    volumes:
+      - loki_data:/loki
 
   celery-exporter:
     restart: always
@@ -157,6 +176,7 @@ services:
       - ELASTICSEARCH_LOG_INDEX_SLOW_LEVEL
     volumes:
       - .:/tdpapp
+      - logs:/logs
     image: tdp
     build: .
     command: >
@@ -189,6 +209,8 @@ volumes:
   elastic_data:
   prometheus_data:
   grafana_data:
+  loki_data:
+  logs:
 
 
 networks:

diff --git a/tdrs-backend/plg/grafana/dashboards/loki_test_dashboard.json b/tdrs-backend/plg/grafana/dashboards/loki_test_dashboard.json
@@ -0,0 +1,182 @@
+{
+    "__inputs": [
+      {
+        "name": "DS_LOKI",
+        "label": "Loki",
+        "description": "",
+        "type": "datasource",
+        "pluginId": "loki",
+        "pluginName": "Loki"
+      }
+    ],
+    "__elements": {},
+    "__requires": [
+      {
+        "type": "grafana",
+        "id": "grafana",
+        "name": "Grafana",
+        "version": "11.2.0"
+      },
+      {
+        "type": "panel",
+        "id": "logs",
+        "name": "Logs",
+        "version": ""
+      },
+      {
+        "type": "datasource",
+        "id": "loki",
+        "name": "Loki",
+        "version": "1.0.0"
+      }
+    ],
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": null,
+    "links": [],
+    "panels": [
+      {
+        "datasource": {
+          "type": "loki",
+          "uid": "${DS_LOKI}"
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "options": {
+          "dedupStrategy": "none",
+          "enableLogDetails": true,
+          "prettifyLogMessage": false,
+          "showCommonLabels": false,
+          "showLabels": false,
+          "showTime": false,
+          "sortOrder": "Descending",
+          "wrapLogMessage": false
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "loki",
+              "uid": "${DS_LOKI}"
+            },
+            "editorMode": "code",
+            "expr": "{job=\"django\"}",
+            "queryType": "range",
+            "refId": "A"
+          }
+        ],
+        "title": "Django Logs",
+        "type": "logs"
+      },
+      {
+        "datasource": {
+          "type": "loki",
+          "uid": "${DS_LOKI}"
+        },
+        "gridPos": {
+          "h": 10,
+          "w": 24,
+          "x": 0,
+          "y": 8
+        },
+        "id": 2,
+        "options": {
+          "dedupStrategy": "none",
+          "enableLogDetails": true,
+          "prettifyLogMessage": false,
+          "showCommonLabels": false,
+          "showLabels": false,
+          "showTime": false,
+          "sortOrder": "Descending",
+          "wrapLogMessage": false
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "loki",
+              "uid": "${DS_LOKI}"
+            },
+            "editorMode": "code",
+            "expr": "{job=\"nginx\"}",
+            "queryType": "range",
+            "refId": "A"
+          }
+        ],
+        "title": "Nginx Logs",
+        "type": "logs"
+      },
+      {
+        "datasource": {
+          "type": "loki",
+          "uid": "${DS_LOKI}"
+        },
+        "gridPos": {
+          "h": 9,
+          "w": 24,
+          "x": 0,
+          "y": 18
+        },
+        "id": 3,
+        "options": {
+          "dedupStrategy": "none",
+          "enableLogDetails": true,
+          "prettifyLogMessage": false,
+          "showCommonLabels": false,
+          "showLabels": false,
+          "showTime": false,
+          "sortOrder": "Descending",
+          "wrapLogMessage": false
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "loki",
+              "uid": "${DS_LOKI}"
+            },
+            "editorMode": "code",
+            "expr": "{job=\"varlogs\"}",
+            "queryType": "range",
+            "refId": "A"
+          }
+        ],
+        "title": "Promtail varlogs",
+        "type": "logs"
+      }
+    ],
+    "schemaVersion": 39,
+    "tags": [],
+    "templating": {
+      "list": []
+    },
+    "time": {
+      "from": "now-30m",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "browser",
+    "title": "Loki Test",
+    "uid": "edx0f84e7zwu8a",
+    "version": 6,
+    "weekStart": ""
+  }
diff --git a/tdrs-backend/plg/grafana/datasources.yml b/tdrs-backend/plg/grafana/datasources.yml
@@ -16,3 +16,10 @@ datasources:
     withCredentials: false
     isDefault: true
     typeLogoUrl: "public/app/plugins/datasource/prometheus/img/prometheus_logo.svg"
+  - name: Loki
+    type: loki
+    access: proxy
+    url: "http://loki:3100"
+    editable: false
+    jsonData:
+      maxLines: 1000
diff --git a/tdrs-backend/plg/promtail/config.yml b/tdrs-backend/plg/promtail/config.yml
@@ -0,0 +1,32 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: varlogs
+        __path__: /var/log/*log
+  - job_name: django
+    static_configs:
+      - targets:
+        - localhost
+        labels:
+          job: django
+          __path__: /logs/django.log
+  - job_name: nginx
+    static_configs:
+      - targets:
+        - localhost
+        labels:
+          job: nginx
+          __path__: /var/log/nginx/*log
diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py
@@ -233,30 +233,35 @@ class Common(Configuration):
                 "class": "logging.StreamHandler",
                 "formatter": "color",
             },
+            "file": {
+                "class": "logging.FileHandler",
+                "formatter": "verbose",
+                "filename": '/logs/django.log',
+            }
         },
         "loggers": {
             "tdpservice": {
-               "handlers": ["application"],
+               "handlers": ["application", "file"],
                "propagate": True,
                "level": LOGGING_LEVEL
             },
             "tdpservice.parsers": {
-               "handlers": ["application"],
+               "handlers": ["application", "file"],
                "propagate": False,
                "level": LOGGING_LEVEL
             },
-            "django": {"handlers": ["console"], "propagate": True},
+            "django": {"handlers": ["console", "file"], "propagate": True},
             "django.server": {
-                "handlers": ["django.server"],
+                "handlers": ["django.server", "file"],
                 "propagate": False,
                 "level": LOGGING_LEVEL
             },
             "django.request": {
-                "handlers": ["console"],
+                "handlers": ["console", "file"],
                 "propagate": False,
                 "level": LOGGING_LEVEL
             },
-            "django.db.backends": {"handlers": ["console"], "level": "INFO"},
+            "django.db.backends": {"handlers": ["console", "file"], "level": "INFO"},
         },
     }
     es_logger = logging.getLogger('elasticsearch')

diff --git a/tdrs-frontend/docker-compose.yml b/tdrs-frontend/docker-compose.yml
@@ -24,6 +24,7 @@ services:
       - default
     volumes:
       - ./:/home/node/app
+      - ~/tdp-logs/nginx:/var/log/nginx
     environment:
       - NGINX_FRONTEND=tdp-frontend
       - BACK_END=web

diff --git a/tdrs-frontend/nginx/local/default.conf.template b/tdrs-frontend/nginx/local/default.conf.template
@@ -105,7 +105,8 @@ http {
                                ' Proxy Pass: "$proxy_host" "$upstream_addr"';
 
     access_log /dev/stdout compression;
-    #access_log stderr compression;
+    access_log /var/log/nginx/access_log.log compression;
+    error_log /var/log/nginx/error_log.log warn;
 
     # Content caching
     # saves cached fies in /tmp