From f3097a269ae340170fe22301e25ecbca9f065a1f Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 6 Feb 2024 11:33:12 +0100 Subject: [PATCH] stack demo OK, missing dashboard provisionning --- grafana/dashboards/ssh-dashboard.json | 1577 +++++++++++++++++ .../provisioning/dashboards/providers.yaml | 2 +- .../dashboards/ssh-dashboard.json | 1577 +++++++++++++++++ loki/config/config.yml | 45 +- promtail/config.yml | 4 +- 5 files changed, 3198 insertions(+), 7 deletions(-) create mode 100644 grafana/dashboards/ssh-dashboard.json create mode 100644 grafana/provisioning/dashboards/ssh-dashboard.json diff --git a/grafana/dashboards/ssh-dashboard.json b/grafana/dashboards/ssh-dashboard.json new file mode 100644 index 0000000..bd19f8c --- /dev/null +++ b/grafana/dashboards/ssh-dashboard.json @@ -0,0 +1,1577 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Loki v2 SSH Logs", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 17514, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "panels": [], + "title": "SSH - Total Stats", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by(instance) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | __error__=\"\" [$__interval]))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Total Opened Connection", + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by(instance) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Failed|: Invalid|: Connection closed by authenticating user\" | __error__=\"\" [$__interval]))", + "hide": false, + "queryType": "range", + "refId": "A" + } + ], + "title": "Total Failed Connection", + "transformations": [ + { + "id": "merge", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "/^IP$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed\" |~\".* from .*\" | pattern `<_> from port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "A", + "resolution": 1 + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed\" !~\".* from .*\" | pattern `<_> user <_> port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "B" + } + ], + "title": "Total Failed - Unique IP", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "rows", + "valueLabel": "ip" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "178.40.119.51": false, + "194.154.240.221": false, + "label": true + }, + "indexByName": {}, + "renameByName": { + "value": "IP" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" | __error__=\"\" [$__interval])", + "queryType": "range", + "refId": "A" + } + ], + "title": "SSH Log Lines", + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "decbytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "bytes_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" | __error__=\"\" [$__interval])", + "queryType": "range", + "refId": "A" + } + ], + "title": "SSH Log in bytes", + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 15, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by (username) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | pattern `<_> session opened for user (` | username !~\".* by \" | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ username }}", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by (username) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | pattern `<_> session opened for user <_>` | username !~\".*(uid=.*)\" | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ username }}", + "queryType": "range", + "refId": "B" + } + ], + "title": "Session Opened by User", + "transformations": [], + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 16, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by (username) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed .* user\" | pattern `<_> user <_> port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ username }}", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by (username) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Failed\" !~\"invalid user\" | pattern `<_> for from <_> port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ username }}", + "queryType": "range", + "refId": "B" + } + ], + "title": "Failed Attempt by User", + "transformations": [ + { + "id": "joinByLabels", + "options": { + "value": "username" + } + } + ], + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 9, + "options": { + "dedupStrategy": "signature", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" ", + "queryType": "range", + "refId": "A" + } + ], + "title": "SSH Recent Log", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 22, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Accepted\" | pattern `<_> Accepted <_> for <_> from port <_>` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "A", + "resolution": 1 + } + ], + "title": "Session Opened by Unique IP", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "rows" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "label": true + }, + "indexByName": {}, + "renameByName": { + "value": "IP" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 14 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed\" |~\".* from .*\" | pattern `<_> from port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "A", + "resolution": 1 + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed\" !~\".* from .*\" | pattern `<_> user <_> port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "B" + } + ], + "title": "Failed by Unique IP", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "rows" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "label": true + }, + "indexByName": {}, + "renameByName": { + "value": "IP" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [], + "title": "Detailed Stats", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 20, + "maxDataPoints": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Accepted\" | pattern `<_> Accepted <_> for from port <_>` | __error__=\"\"", + "hide": false, + "legendFormat": "{{ ip }} {{ username }}", + "queryType": "range", + "refId": "A", + "resolution": 1 + } + ], + "title": "Session Opened by User and IP", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "Time": false, + "env": true, + "filename": true, + "id": true, + "job": true, + "label": true, + "labels": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": { + "label": "", + "value": "" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 23, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Failed .* user\" | pattern `<_> user from <_> port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Failed\" !~\"invalid user\" | pattern `<_> for from port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "B" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Connection closed by authenticating user\" | pattern `<_> user port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "C" + } + ], + "title": "SSH Failure by User and IP", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "env": true, + "filename": true, + "id": true, + "job": true, + "labels": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": { + "Time": "", + "env": "", + "instance": "", + "job": "", + "tsNs": "" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | pattern `<_> session opened for user (` | username !~\".* by \" | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | pattern `<_> session opened for user <_>` | username !~\".*(uid=.*)\" | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "B" + } + ], + "title": "SSH Session Opened by User", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "env": true, + "filename": true, + "id": true, + "job": true, + "labels": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": { + "Time": "", + "env": "", + "instance": "", + "job": "", + "tsNs": "" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed .* user\" | pattern `<_> user <_> port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Failed\" !~\"invalid user\" | pattern `<_> for from <_> port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "B" + } + ], + "title": "SSH Failure by User", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "env": true, + "filename": true, + "id": true, + "job": true, + "labels": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": { + "Time": "", + "env": "", + "instance": "", + "job": "", + "tsNs": "" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "revision": 2, + "schemaVersion": 39, + "tags": [ + "loki", + "linux", + "ssh" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Loki", + "value": "P8E80F9AEF21F6940" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "filename", + "value": "filename" + }, + "datasource": { + "type": "loki", + "uid": "$datasource" + }, + "definition": "label_names()", + "hide": 0, + "includeAll": false, + "label": "Label Name", + "multi": false, + "name": "label_name", + "options": [], + "query": "label_names()", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "loki", + "uid": "$datasource" + }, + "definition": "label_values($label_value)", + "hide": 0, + "includeAll": true, + "label": "Label Value", + "multi": true, + "name": "label_value", + "options": [], + "query": "label_values($label_name)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "ssh-logs" + ], + "value": [ + "ssh-logs" + ] + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "label": "job", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "{$label_name=~\"$label_value\"}", + "type": 1 + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "label": "instance", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "{$label_name=~\"$label_value\"}", + "type": 1 + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "SSH Logs", + "uid": "OMEuTfqVk", + "version": 5, + "weekStart": "" + } \ No newline at end of file diff --git a/grafana/provisioning/dashboards/providers.yaml b/grafana/provisioning/dashboards/providers.yaml index 1176eb9..1716011 100644 --- a/grafana/provisioning/dashboards/providers.yaml +++ b/grafana/provisioning/dashboards/providers.yaml @@ -17,4 +17,4 @@ providers: disableDeletion: false updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards options: - path: /var/lib/grafana/dashboards \ No newline at end of file + path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/grafana/provisioning/dashboards/ssh-dashboard.json b/grafana/provisioning/dashboards/ssh-dashboard.json new file mode 100644 index 0000000..bd19f8c --- /dev/null +++ b/grafana/provisioning/dashboards/ssh-dashboard.json @@ -0,0 +1,1577 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Loki v2 SSH Logs", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 17514, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "panels": [], + "title": "SSH - Total Stats", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by(instance) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | __error__=\"\" [$__interval]))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Total Opened Connection", + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by(instance) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Failed|: Invalid|: Connection closed by authenticating user\" | __error__=\"\" [$__interval]))", + "hide": false, + "queryType": "range", + "refId": "A" + } + ], + "title": "Total Failed Connection", + "transformations": [ + { + "id": "merge", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "count" + ], + "fields": "/^IP$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed\" |~\".* from .*\" | pattern `<_> from port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "A", + "resolution": 1 + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed\" !~\".* from .*\" | pattern `<_> user <_> port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "B" + } + ], + "title": "Total Failed - Unique IP", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "rows", + "valueLabel": "ip" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "178.40.119.51": false, + "194.154.240.221": false, + "label": true + }, + "indexByName": {}, + "renameByName": { + "value": "IP" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" | __error__=\"\" [$__interval])", + "queryType": "range", + "refId": "A" + } + ], + "title": "SSH Log Lines", + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + }, + "unit": "decbytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "bytes_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" | __error__=\"\" [$__interval])", + "queryType": "range", + "refId": "A" + } + ], + "title": "SSH Log in bytes", + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 15, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by (username) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | pattern `<_> session opened for user (` | username !~\".* by \" | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ username }}", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by (username) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | pattern `<_> session opened for user <_>` | username !~\".*(uid=.*)\" | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ username }}", + "queryType": "range", + "refId": "B" + } + ], + "title": "Session Opened by User", + "transformations": [], + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 16, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by (username) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed .* user\" | pattern `<_> user <_> port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ username }}", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "sum by (username) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Failed\" !~\"invalid user\" | pattern `<_> for from <_> port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ username }}", + "queryType": "range", + "refId": "B" + } + ], + "title": "Failed Attempt by User", + "transformations": [ + { + "id": "joinByLabels", + "options": { + "value": "username" + } + } + ], + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 9, + "options": { + "dedupStrategy": "signature", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" ", + "queryType": "range", + "refId": "A" + } + ], + "title": "SSH Recent Log", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 22, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Accepted\" | pattern `<_> Accepted <_> for <_> from port <_>` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "A", + "resolution": 1 + } + ], + "title": "Session Opened by Unique IP", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "rows" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "label": true + }, + "indexByName": {}, + "renameByName": { + "value": "IP" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 14 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed\" |~\".* from .*\" | pattern `<_> from port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "A", + "resolution": 1 + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count by (ip) (count_over_time({$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed\" !~\".* from .*\" | pattern `<_> user <_> port` | __error__=\"\" [$__interval]))", + "hide": false, + "legendFormat": "{{ ip }}", + "queryType": "range", + "refId": "B" + } + ], + "title": "Failed by Unique IP", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "rows" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "label": true + }, + "indexByName": {}, + "renameByName": { + "value": "IP" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [], + "title": "Detailed Stats", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 20, + "maxDataPoints": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Accepted\" | pattern `<_> Accepted <_> for from port <_>` | __error__=\"\"", + "hide": false, + "legendFormat": "{{ ip }} {{ username }}", + "queryType": "range", + "refId": "A", + "resolution": 1 + } + ], + "title": "Session Opened by User and IP", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "Time": false, + "env": true, + "filename": true, + "id": true, + "job": true, + "label": true, + "labels": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": { + "label": "", + "value": "" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 23, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Failed .* user\" | pattern `<_> user from <_> port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Failed\" !~\"invalid user\" | pattern `<_> for from port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "B" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Connection closed by authenticating user\" | pattern `<_> user port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "C" + } + ], + "title": "SSH Failure by User and IP", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "env": true, + "filename": true, + "id": true, + "job": true, + "labels": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": { + "Time": "", + "env": "", + "instance": "", + "job": "", + "tsNs": "" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | pattern `<_> session opened for user (` | username !~\".* by \" | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": session opened for\" | pattern `<_> session opened for user <_>` | username !~\".*(uid=.*)\" | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "B" + } + ], + "title": "SSH Session Opened by User", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "env": true, + "filename": true, + "id": true, + "job": true, + "labels": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": { + "Time": "", + "env": "", + "instance": "", + "job": "", + "tsNs": "" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |~\": Invalid|: Connection closed by authenticating user|: Failed .* user\" | pattern `<_> user <_> port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{$label_name=~\"$label_value\", job=~\"$job\", instance=~\"$instance\"} |=\"sshd[\" |=\": Failed\" !~\"invalid user\" | pattern `<_> for from <_> port` | __error__=\"\"", + "hide": false, + "queryType": "range", + "refId": "B" + } + ], + "title": "SSH Failure by User", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "extractFields", + "options": { + "format": "auto", + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "env": true, + "filename": true, + "id": true, + "job": true, + "labels": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": { + "Time": "", + "env": "", + "instance": "", + "job": "", + "tsNs": "" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "revision": 2, + "schemaVersion": 39, + "tags": [ + "loki", + "linux", + "ssh" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Loki", + "value": "P8E80F9AEF21F6940" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "loki", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "filename", + "value": "filename" + }, + "datasource": { + "type": "loki", + "uid": "$datasource" + }, + "definition": "label_names()", + "hide": 0, + "includeAll": false, + "label": "Label Name", + "multi": false, + "name": "label_name", + "options": [], + "query": "label_names()", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "loki", + "uid": "$datasource" + }, + "definition": "label_values($label_value)", + "hide": 0, + "includeAll": true, + "label": "Label Value", + "multi": true, + "name": "label_value", + "options": [], + "query": "label_values($label_name)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "ssh-logs" + ], + "value": [ + "ssh-logs" + ] + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "label": "job", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "{$label_name=~\"$label_value\"}", + "type": 1 + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "label": "instance", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "{$label_name=~\"$label_value\"}", + "type": 1 + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "SSH Logs", + "uid": "OMEuTfqVk", + "version": 5, + "weekStart": "" + } \ No newline at end of file diff --git a/loki/config/config.yml b/loki/config/config.yml index 9b33ee4..5b2117f 100644 --- a/loki/config/config.yml +++ b/loki/config/config.yml @@ -28,8 +28,46 @@ schema_config: prefix: index_ period: 24h -ruler: - alertmanager_url: http://localhost:9093 + +frontend: + # Maximum number of outstanding requests per tenant per frontend; requests + # beyond this error with HTTP 429. + # CLI flag: -querier.max-outstanding-requests-per-tenant + max_outstanding_per_tenant: 2048 # default = 100] + +# query_range: +# parallelise_shardable_queries: false + # max_retries: 5 + # split_queries_by_interval: 15m + # cache_results: true + # results_cache: + # cache: + # enable_fifocache: true + # fifocache: + # max_size_items: 1024 + # validity: 24h + +query_range: + parallelise_shardable_queries: false + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + +chunk_store_config: + max_look_back_period: 0s + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 + ttl: 24h + +# analytics: +# reporting_enabled: false + +# ruler: +# alertmanager_url: http://localhost:9093 # By default, Loki will send anonymous, but uniquely-identifiable usage and configuration # analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ @@ -41,5 +79,4 @@ ruler: # Refer to the buildReport method to see what goes into a report. # # If you would like to disable reporting, uncomment the following lines: -#analytics: -# reporting_enabled: false + diff --git a/promtail/config.yml b/promtail/config.yml index b8eef86..ecc57b7 100644 --- a/promtail/config.yml +++ b/promtail/config.yml @@ -65,7 +65,7 @@ scrape_configs: - targets: - localhost labels: - host: lenovo + # host: lenovo job: ssh-logs - __path__: /var/log/auth.log + __path__: /var/log/auth.log*