maj dashboard OK

2024-02-07 10:56:49 +01:00 · 2024-02-07 10:56:49 +01:00 · dd026a876d
commit dd026a876d
parent 28a2c97512
10 changed files with 1710 additions and 3275 deletions
--- a/README.md
+++ b/README.md
@ -2,64 +2,124 @@

 Simple stack pour monitorer les logs des conteneurs ou applications d'un serveur via LOKI / PROMTAIL

-## CONFIGURATION
+![ARCHI](docs/promtail-loki.png)

-### LOKI AGENT HOST
+## UTILISATION LOCALE

-Pour que LOKI récupère les logs des conteneurs il faut ajouter les labels au docker-compose.yml:
+```bash
+docker-compose up -d
+```
+
+Grafana est disponible à l'adresse: http://localhost:3000
+> user: admin / password: admin
+
+## CONFIGURATION PROD
+
+De base promtail est configuré pour faire remonter les logs systèmes (/var/log)
+
+### PROMTAIL AGENT HOST
+
+Promtail est l'agent qui va pusher les logs vers Loki:
+```yml
+  promtail:
+    image:  grafana/promtail:2.9.4
+    container_name: promtail
+    volumes:
+      - ./promtail/config.yml:/etc/promtail/config.yml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/log:/var/log
+    command: -config.file=/etc/promtail/config.yml
+    depends_on:
+      - loki
+```
+
+Pour que LOKI récupère les logs des conteneurs il faut ajouter les labels aux conteneurs dont on veux monitorer les logs:
 ```yml
    labels:
      logging: "promtail"
      logging_jobname: "containerlogs"
 ```

+### GRAFANA
+
+C'est sur le serveur de Grafana que l'on déploie Loki:
+
+```yml
+version: "3"
+
+services:
+  loki:
+    image: grafana/loki:2.9.4
+    container_name: loki
+    ports:
+      - 3100:3100
+    volumes:
+      - ./loki/config:/etc/loki
+      - ./loki/cert:/etc/loki/cert
+    command: -config.file=/etc/loki/config.yml
+```
+
+Ajouter une datasource en entrant l'URI du serveur Loki ainsi que le certificat (/etc/loki/cert/ca.crt)
+
+![AJOUT DATASOURCE](docs/datasource.png)
+
 #### TLS 

 - Create certs:

+> Renseigner les nom du serveur LOKI ainsi que son DNS, idem pour l'agent Promtail dans le .env
+
 ```bash
-openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -keyout gn-prod.key -out gn-prod.crt -subj "/C=FR/ST=PARIS/L=GarageNum/O=prom/CN=legaragenumerique.fr" -addext "subjectAltName = DNS:gnprod"
+./certificates.sh
 ```
-> remplacer Les infos (C=FR, ST=PARIS, L=GarageNum, ...)

 - TLS config:
+
+Décommenter les lignes concernant le TLS dans promtail/config/yml comme suit:
+
+```yaml
+clients:
+  # LOCAL
+  # - url: http://loki:3100/loki/api/v1/push
+  
+  # DISTANT TLS
+  - url: https://loki-dns-serveur:3100/loki/api/v1/push
+    tls_config:
+      ca_file: /usr/allen/loki/cert/ca.crt
+      cert_file: /usr/allen/loki/cert/promtail.client.crt
+      key_file: /usr/allen/loki/cert/client.key
+      server_name: loki-dns-serveur
+      insecure_skip_verify: false
+```
+
+Idem pour loki/config/config.yml:
+
 ```yaml
 server:
  http_listen_port: 3100
-  grpc_listen_port: 9443
  
+# DISTANT TLS
+  grpc_listen_port: 9096
  http_tls_config:
+    cert_file: /etc/loki/cert/loki.server.crt
+    key_file: /etc/loki/cert/server.key
    client_auth_type: RequireAndVerifyClientCert
-    client_ca_file: /opt/loki/certs/loki_CA_.cer 
-    cert_file: /etc/loki/server.crt
-    key_file: /etc/loki/server.key
-  grpc_tls_config:
-    client_auth_type: RequireAndVerifyClientCert
-    client_ca_file: /opt/loki/certs/loki_CA_.cer   
-    cert_file: /etc/loki/server.crt
-    key_file: /etc/loki/server.key
+    client_ca_file: /etc/loki/cert/ca.crt
 ```

-### GRAFANA HOST
-
-
-
-## UTILISATION
-
-```bash
-docker-compose up -d
-```
+> Modifier loki-dns-serveur avec le vrai dns du serveur

 ## DASHBOARD

 > import dashboard ID: 17514
 > Faire la dashboard standard

- [ ] schema type
- [ ] provisionner dashboard
- [ ] Pormtail config
+- [X] schema type
+- [X] provisionner dashboard
+- [X] Promtail config
 - [ ] TLS config (https)
- [ ] SSH logs
+- [X] SSH logs


 > le scraping des metrics du serveur distant se font désormais via TLS
--- a/certificates.sh
+++ b/certificates.sh
@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Load .env
+if [ -f .env ]; then
+    export $(grep -v '^#' .env | xargs -0)
+else
+    echo "Error: .env file not found."
+    exit 1
+fi
+
+CERT_DIR="loki/cert"
+mkdir -p "$CERT_DIR"
+
+# Root CA certificate
+openssl req -newkey rsa:4096 -nodes -keyout ca.key -subj "$CA_SUBJECT" -out ca.csr
+openssl x509 -req -days 3650 -in ca.csr -signkey ca.key -out "$CERT_DIR/ca.crt"
+
+# Server certificate
+openssl req -newkey rsa:4096 -nodes -keyout "$CERT_DIR/server.key" -subj "$SERVER_SUBJECT" -out "$CERT_DIR/server.csr"
+openssl x509 -req -extfile <(printf "subjectAltName=$SERVER_DNS") -days 1365 -in "$CERT_DIR/server.csr" -CA "$CERT_DIR/ca.crt" -CAkey ca.key -CAcreateserial -out "$CERT_DIR/server.crt"
+
+# Client certificate
+openssl req -newkey rsa:4096 -nodes -keyout "$CERT_DIR/client.key" -subj "$CLIENT_SUBJECT" -out "$CERT_DIR/client.csr"
+openssl x509 -req -extfile <(printf "subjectAltName=$CLIENT_DNS") -days 1365 -in "$CERT_DIR/client.csr" -CA "$CERT_DIR/ca.crt" -CAkey ca.key -CAcreateserial -out "$CERT_DIR/client.crt"
+
+# Clean up!
+rm -f ca.csr "$CERT_DIR/server.csr" "$CERT_DIR/client.csr" ca.srl
+
+echo "Certificate generation completed successfully. Certificates are stored in the '$CERT_DIR' directory."
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -2,37 +2,37 @@ version: '3.8'

 services:
 # APP EXEMPLE
-  # nginx-app:
-  #   container_name: nginx-app
-  #   image: nginx
-  #   # NECESSARY FOR LOKI
-  #   labels:
-  #     logging: "promtail"
-  #     logging_jobname: "containerlogs"
-  #   ports:
-  #     - 8080:80
+  nginx-app:
+    container_name: nginx-app
+    image: nginx
+    # NECESSARY FOR LOKI
+    labels:
+      logging: "promtail"
+      logging_jobname: "containerlogs"
+    ports:
+      - 8080:80

-  # grafana:
-  #   image: grafana/grafana:latest
-  #   container_name: grafana
-  #   ports:
-  #     - 3000:3000
-  #   volumes:
-  #     - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - 3000:3000
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning
      # - ./grafana/dashboards:/var/lib/grafana/dashboards

  loki:
-    image: grafana/loki:latest
+    image: grafana/loki:2.9.4
    container_name: loki
    ports:
      - 3100:3100
    volumes:
      - ./loki/config:/etc/loki
-      - ./loki/certs:/etc/loki/certs
+      - ./loki/cert:/etc/loki/cert
    command: -config.file=/etc/loki/config.yml

  promtail:
-    image:  grafana/promtail:latest
+    image:  grafana/promtail:2.9.4
    container_name: promtail
    volumes:
      - ./promtail/config.yml:/etc/promtail/config.yml
--- a/docs/promtail-loki.png
+++ b/docs/promtail-loki.png
--- a/grafana/dashboards/dashboard-exemple.json
+++ b/grafana/dashboards/dashboard-exemple.json
@ -1,81 +0,0 @@
-{
-    "annotations": {
-      "list": [
-        {
-          "builtIn": 1,
-          "datasource": {
-            "type": "grafana",
-            "uid": "-- Grafana --"
-          },
-          "enable": true,
-          "hide": true,
-          "iconColor": "rgba(0, 211, 255, 1)",
-          "name": "Annotations & Alerts",
-          "type": "dashboard"
-        }
-      ]
-    },
-    "editable": true,
-    "fiscalYearStartMonth": 0,
-    "graphTooltip": 0,
-    "id": 1,
-    "links": [],
-    "liveNow": false,
-    "panels": [
-      {
-        "datasource": {
-          "type": "loki",
-          "uid": "P8E80F9AEF21F6940"
-        },
-        "gridPos": {
-          "h": 9,
-          "w": 24,
-          "x": 0,
-          "y": 0
-        },
-        "id": 1,
-        "options": {
-          "dedupStrategy": "none",
-          "enableLogDetails": true,
-          "prettifyLogMessage": false,
-          "showCommonLabels": false,
-          "showLabels": false,
-          "showTime": false,
-          "sortOrder": "Descending",
-          "wrapLogMessage": false
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "loki",
-              "uid": "P8E80F9AEF21F6940"
-            },
-            "editorMode": "builder",
-            "expr": "{container=\"nginx-app\"} |= ``",
-            "key": "Q-d83b192b-23c0-4458-9a28-0d178f451096-0",
-            "queryType": "range",
-            "refId": "A"
-          }
-        ],
-        "title": "nginx-app logs",
-        "transformations": [],
-        "type": "logs"
-      }
-    ],
-    "refresh": "5s",
-    "schemaVersion": 39,
-    "tags": [],
-    "templating": {
-      "list": []
-    },
-    "time": {
-      "from": "now-6h",
-      "to": "now"
-    },
-    "timepicker": {},
-    "timezone": "",
-    "title": "dashboard-exemple",
-    "uid": "f5bb84b2-b3f1-4776-9ab5-5d2389adfaec",
-    "version": 1,
-    "weekStart": ""
-  }
--- a/grafana/dashboards/ssh-dashboard.json
+++ b/grafana/dashboards/ssh-dashboard.json
--- a/grafana/provisioning/dashboards.yaml
+++ b/grafana/provisioning/dashboards.yaml
@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: 'default'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    options:
+      path: /etc/grafana/provisioning/dashboards
--- a/grafana/provisioning/dashboards/ssh-dashboard.json
+++ b/grafana/provisioning/dashboards/ssh-dashboard.json
@ -26,7 +26,7 @@
  "fiscalYearStartMonth": 0,
  "gnetId": 17514,
  "graphTooltip": 0,
-    "id": 1,
+  "id": 2,
  "links": [],
  "liveNow": false,
  "panels": [
@ -1173,12 +1173,10 @@
            "mode": "absolute",
            "steps": [
              {
-                  "color": "green",
-                  "value": null
+                "color": "green"
              }
            ]
-            },
-            "unitScale": true
+          }
        },
        "overrides": []
      },
@ -1190,7 +1188,6 @@
      },
      "id": 13,
      "options": {
-          "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
@ -1201,7 +1198,7 @@
        },
        "showHeader": true
      },
-        "pluginVersion": "10.3.1",
+      "pluginVersion": "9.2.5",
      "targets": [
        {
          "datasource": {
@ -1300,12 +1297,10 @@
            "mode": "absolute",
            "steps": [
              {
-                  "color": "green",
-                  "value": null
+                "color": "green"
              }
            ]
-            },
-            "unitScale": true
+          }
        },
        "overrides": []
      },
@ -1317,7 +1312,6 @@
      },
      "id": 14,
      "options": {
-          "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
@ -1328,7 +1322,7 @@
        },
        "showHeader": true
      },
-        "pluginVersion": "10.3.1",
+      "pluginVersion": "9.2.5",
      "targets": [
        {
          "datasource": {
@ -1405,7 +1399,7 @@
      "type": "table"
    }
  ],
-    "refresh": "30s",
+  "refresh": "1m",
  "revision": 2,
  "schemaVersion": 39,
  "tags": [
@ -1491,10 +1485,10 @@
        "current": {
          "selected": true,
          "text": [
-              "ssh-logs"
+            "All"
          ],
          "value": [
-              "ssh-logs"
+            "$__all"
          ]
        },
        "datasource": {
@ -1523,9 +1517,13 @@
      {
        "allValue": ".*",
        "current": {
-            "selected": false,
-            "text": "All",
-            "value": "$__all"
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
        },
        "datasource": {
          "type": "loki",
@ -1553,25 +1551,13 @@
    ]
  },
  "time": {
-      "from": "now-24h",
+    "from": "now-6h",
    "to": "now"
  },
-    "timepicker": {
-      "hidden": true,
-      "refresh_intervals": [
-        "30s",
-        "1m",
-        "5m",
-        "15m",
-        "30m",
-        "1h",
-        "2h",
-        "1d"
-      ]
-    },
-    "timezone": "browser",
+  "timepicker": {},
+  "timezone": "",
  "title": "SSH Logs",
-    "uid": "OMEuTfqVk",
-    "version": 5,
+  "uid": "OMEuTfqVk5",
+  "version": 2,
  "weekStart": ""
-  }
+}
--- a/loki/config/config.yml
+++ b/loki/config/config.yml
@ -2,10 +2,14 @@ auth_enabled: false

 server:
  http_listen_port: 3100
-  # HTTPS /TLS
-  # http_tls_config: &tls_server_config
-  #   cert_file: /etc/loki/cert.pem
-  #   key_file: /etc/loki/key.pem
+  
+# DISTANT TLS
+  # grpc_listen_port: 9096
+  # http_tls_config:
+  #   cert_file: /etc/loki/cert/loki.server.crt
+  #   key_file: /etc/loki/cert/server.key
+  #   client_auth_type: RequireAndVerifyClientCert
+  #   client_ca_file: /etc/loki/cert/ca.crt

 common:
  path_prefix: /loki
--- a/promtail/config.yml
+++ b/promtail/config.yml
@ -32,12 +32,15 @@ positions:
 clients:
  # LOCAL
  - url: http://loki:3100/loki/api/v1/push
+  
  # DISTANT WITH TLS
-  # - url: http://<Redacted>/loki/api/v1/push
+  # - url: https://loki-dns-serveur:3100/loki/api/v1/push
    # tls_config:
-  #     ca_file: /etc/loki/certs/ca.crt
-  #     cert_file: /etc/loki/certs/cert.pem
-  #     key_file: /etc/loki/certs/key.pem
+    #   ca_file: /etc/loki/cert/ca.crt
+    #   cert_file: /etc/loki/cert/promtail.client.crt
+    #   key_file: /etc/loki/cert/client.key
+    #   server_name: lokiserver.com
+    #   insecure_skip_verify: false