update promtail version

README.md

@@ -2,28 +2,174 @@
 
 Simple stack pour monitorer les logs des conteneurs ou applications d'un serveur via LOKI / PROMTAIL
 
-## CONFIGURATION
+![ARCHI](docs/promtail-loki.png)
 
-Pour que LOKI récupère les logs des conteneurs il faut ajouter les labels au docker-compose.yml:
+## UTILISATION LOCALE
+
+```bash
+docker-compose up -d
+```
+
+Grafana est disponible à l'adresse: http://localhost:3000
+> user: admin / password: admin
+
+## CONFIGURATION PROD
+
+De base promtail est configuré pour faire remonter les logs systèmes (/var/log)
+
+### PROMTAIL AGENT HOST
+
+Promtail est l'agent qui va pusher les logs vers Loki:
+```yml
+  promtail:
+    image:  grafana/promtail:2.9.4
+    container_name: promtail
+    volumes:
+      - ./promtail/config.yml:/etc/promtail/config.yml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/log:/var/log
+    command: -config.file=/etc/promtail/config.yml
+```
+
+Pour que LOKI récupère les logs des conteneurs il faut ajouter les labels aux conteneurs dont on veux monitorer les logs (optionnel):
 ```yml
     labels:
       logging: "promtail"
       logging_jobname: "containerlogs"
 ```
 
-## UTILISATION
+### GRAFANA
+
+C'est sur le serveur de Grafana que l'on déploie Loki (de préférence):
+
+```yml
+version: "3"
+
+services:
+  loki:
+    image: grafana/loki:2.9.4
+    container_name: loki
+    ports:
+      - 3100:3100
+    volumes:
+      - ./loki/config:/etc/loki
+      - ./loki/cert:/etc/loki/cert
+    command: -config.file=/etc/loki/config.yml
+```
+
+### TLS 
+
+Les metrics des agents promtail transitent par le WAN et nécessitent d'être encryptées.
+
+- Creation des certificats:
+
+> Renseigner les nom du serveur LOKI ainsi que son DNS, idem pour l'agent Promtail et lançer le script:
 
 ```bash
-docker-compose up -d
+sudo ./certificates.sh
 ```
 
+#### LOKI
+
+Décommenter les lignes concernant le TLS dans promtail/config/yml comme suit:
+
+```yaml
+clients:
+  # LOCAL
+  # - url: http://loki:3100/loki/api/v1/push
+  
+  # DISTANT TLS
+  - url: https://loki-dns-serveur:3100/loki/api/v1/push
+    tls_config:
+      ca_file: /etc/promtail/cert/ca.crt
+      cert_file: /etc/promtail/cert/promtail.client.crt
+      key_file: /etc/promtail/cert/client.key
+      server_name: loki-dns-serveur
+      insecure_skip_verify: false
+```
+
+#### PROMTAIL
+
+Idem pour loki/config/config.yml:
+
+```yaml
+server:
+  http_listen_port: 3100
+  
+# DISTANT TLS
+  grpc_listen_port: 9096
+  http_tls_config:
+    cert_file: /etc/loki/cert/loki.server.crt
+    key_file: /etc/loki/cert/server.key
+    client_auth_type: RequireAndVerifyClientCert
+    client_ca_file: /etc/loki/cert/ca.crt
+```
+
+#### NGINX
+
+Nginx reverse proxy configuration:
+
+```
+upstream loki {
+        server 127.0.0.1:3100;
+}
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name loki.mon-domaine.tld;
+    rewrite ^(.*) https://loki.mon-domaine.tld$1 permanent;
+}
+
+server {
+    listen [::]:443 ssl;
+    listen 443 ssl;
+    server_name loki.mon-domaine.tld;
+
+    client_max_body_size 200M;
+
+    location / {
+            proxy_buffering off;
+            proxy_pass http://loki;
+            proxy_pass_request_headers on;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+    }
+
+    error_log /var/log/nginx/loki.mon-domaine.tld-proxy-error.log;
+    access_log /var/log/nginx/loki.mon-domaine.tld-proxy-access.log;
+
+    ssl_certificate           /etc/letsencrypt/live/loki.mon-domaine.tld/fullchain.pem;
+    ssl_certificate_key       /etc/letsencrypt/live/loki.mon-domaine.tld/privkey.pem;
+
+    ssl_protocols       TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
+    ssl_ciphers         HIGH:!aNULL:!MD5;
+}
+```
+
+> Modifier loki-dns-serveur avec le vrai dns du serveur
+
+#### GRAFANA
+
+Configurer:
+- l'URI du serveur Loki 
+- Le certificat CA (/etc/loki/cert/ca.crt)
+- Le certificat client (/etc/loki/cert/server.crt)
+- La clé client (/etc/loki/cert/server.key)
+
+
+![AJOUT DATASOURCE](docs/datasource.png)
+
 ## DASHBOARD
 
 > import dashboard ID: 17514
 > Faire la dashboard standard
 
-- [ ] schema type
-- [ ] provisionner dashboard
-- [ ] Pormtail config
-- [ ] TLS config (https)
-- [ ] SSH logs
+## TO DO
+
+- [X] schema type
+- [X] provisionner dashboard
+- [X] Promtail config
+- [X] TLS config (https)
+- [X] SSH logs

certificates.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+if [ "$(id -u)" -ne 0 ]
+then
+    echo "Ce script doit être exécuté en tant qu'utilisateur root"
+    exit 1
+fi
+
+generate_certificates() {
+    domain=$1
+    key_file="${domain}.key"
+    csr_file="${domain}.csr"
+    crt_file="${domain}.crt"
+
+    openssl req -newkey rsa:4096 -nodes -keyout "${key_file}" -subj "/C=CN/ST=GD/L=SZ/O=Acme, Inc./CN=${domain}" -out "${csr_file}"
+    openssl x509 -req -extfile <(printf "subjectAltName=DNS:${domain},DNS:www.${domain}") -days 1365 -in "${csr_file}" -CA ca.crt -CAkey ca.key -CAcreateserial -out "${crt_file}"
+
+    mv "${crt_file}" "${key_file}" "${2}/cert/"
+}
+
+openssl genrsa -out ca.key 4096
+openssl req -new -x509 -days 365 -key ca.key -subj "/C=CN/ST=GD/L=SZ/O=Acme, Inc./CN=Acme Root CA" -out ca.crt
+
+mkdir -p loki/cert
+mkdir -p promtail/cert
+
+generate_certificates "lokiserver.com" "loki"
+generate_certificates "promtailclient.com" "promtail"
+
+cp ca.crt loki/cert/
+mv ca.crt promtail/cert/
+
+rm -rf ca.key ca.srl *.csr
+
+echo "Done!"

compose.yml

@@ -5,12 +5,12 @@ services:
   nginx-app:
     container_name: nginx-app
     image: nginx
+    ports:
+      - 8080:80
     # NECESSARY FOR LOKI
     labels:
       logging: "promtail"
       logging_jobname: "containerlogs"
-    ports:
-      - 8080:80
 
   grafana:
     image: grafana/grafana:latest
@@ -18,27 +18,28 @@ services:
     ports:
       - 3000:3000
     volumes:
-      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
-      - ./grafana/dashboards:/var/lib/grafana/dashboards
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      # - ./grafana/dashboards:/var/lib/grafana/dashboards
 
   loki:
-    image: grafana/loki:latest
+    image: grafana/loki:2.9.4
     container_name: loki
     ports:
       - 3100:3100
     volumes:
       - ./loki/config:/etc/loki
-      - ./loki/certs:/etc/loki/certs
+      - ./loki/cert:/etc/loki/cert:ro
     command: -config.file=/etc/loki/config.yml
 
   promtail:
-    image:  grafana/promtail:latest
+    image:  grafana/promtail:3.6.10
     container_name: promtail
     volumes:
       - ./promtail/config.yml:/etc/promtail/config.yml
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
       - /var/run/docker.sock:/var/run/docker.sock
-      - /var/log:/var/log
+      - /var/log:/var/log:ro
+      - ./promtail/cert:/etc/promtail/cert
     command: -config.file=/etc/promtail/config.yml
     depends_on:
       - loki

grafana/dashboards/dashboard-exemple.json

@@ -1,81 +0,0 @@
-{
-    "annotations": {
-      "list": [
-        {
-          "builtIn": 1,
-          "datasource": {
-            "type": "grafana",
-            "uid": "-- Grafana --"
-          },
-          "enable": true,
-          "hide": true,
-          "iconColor": "rgba(0, 211, 255, 1)",
-          "name": "Annotations & Alerts",
-          "type": "dashboard"
-        }
-      ]
-    },
-    "editable": true,
-    "fiscalYearStartMonth": 0,
-    "graphTooltip": 0,
-    "id": 1,
-    "links": [],
-    "liveNow": false,
-    "panels": [
-      {
-        "datasource": {
-          "type": "loki",
-          "uid": "P8E80F9AEF21F6940"
-        },
-        "gridPos": {
-          "h": 9,
-          "w": 24,
-          "x": 0,
-          "y": 0
-        },
-        "id": 1,
-        "options": {
-          "dedupStrategy": "none",
-          "enableLogDetails": true,
-          "prettifyLogMessage": false,
-          "showCommonLabels": false,
-          "showLabels": false,
-          "showTime": false,
-          "sortOrder": "Descending",
-          "wrapLogMessage": false
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "loki",
-              "uid": "P8E80F9AEF21F6940"
-            },
-            "editorMode": "builder",
-            "expr": "{container=\"nginx-app\"} |= ``",
-            "key": "Q-d83b192b-23c0-4458-9a28-0d178f451096-0",
-            "queryType": "range",
-            "refId": "A"
-          }
-        ],
-        "title": "nginx-app logs",
-        "transformations": [],
-        "type": "logs"
-      }
-    ],
-    "refresh": "5s",
-    "schemaVersion": 39,
-    "tags": [],
-    "templating": {
-      "list": []
-    },
-    "time": {
-      "from": "now-6h",
-      "to": "now"
-    },
-    "timepicker": {},
-    "timezone": "",
-    "title": "dashboard-exemple",
-    "uid": "f5bb84b2-b3f1-4776-9ab5-5d2389adfaec",
-    "version": 1,
-    "weekStart": ""
-  }

grafana/provisioning/dashboards.yaml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: 'default'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    options:
+      path: /etc/grafana/provisioning/dashboards

loki/config/config.yml

@@ -2,10 +2,14 @@ auth_enabled: false
 
 server:
   http_listen_port: 3100
-  # HTTPS /TLS
-  # http_tls_config: &tls_server_config
-  #   cert_file: /etc/loki/cert.pem
-  #   key_file: /etc/loki/key.pem
+  
+# DISTANT TLS
+  # grpc_listen_port: 9096
+  # http_tls_config:
+  #   cert_file: /etc/loki/cert/loki.server.crt
+  #   key_file: /etc/loki/cert/server.key
+  #   client_auth_type: RequireAndVerifyClientCert
+  #   client_ca_file: /etc/loki/cert/ca.crt
 
 common:
   path_prefix: /loki

promtail/config.yml

@@ -32,12 +32,15 @@ positions:
 clients:
   # LOCAL
   - url: http://loki:3100/loki/api/v1/push
+  
   # DISTANT WITH TLS
-  # - url: http://<Redacted>/loki/api/v1/push
-  #   tls_config:
-  #     ca_file: /etc/loki/certs/ca.crt
-  #     cert_file: /etc/loki/certs/cert.pem
-  #     key_file: /etc/loki/certs/key.pem
+  # - url: https://loki-dns-serveur:3100/loki/api/v1/push
+    # tls_config:
+    #   ca_file: /etc/loki/cert/ca.crt
+    #   cert_file: /etc/loki/cert/promtail.client.crt
+    #   key_file: /etc/loki/cert/client.key
+    #   server_name: lokiserver.com
+    #   insecure_skip_verify: false