summaryrefslogtreecommitdiff
path: root/machines/xnet/monitoring
diff options
context:
space:
mode:
Diffstat (limited to 'machines/xnet/monitoring')
-rw-r--r--machines/xnet/monitoring/22604_rev2.json1565
-rw-r--r--machines/xnet/monitoring/default.nix76
-rw-r--r--machines/xnet/monitoring/grafana.nix105
-rw-r--r--machines/xnet/monitoring/prometheus.nix15
4 files changed, 1679 insertions, 82 deletions
diff --git a/machines/xnet/monitoring/22604_rev2.json b/machines/xnet/monitoring/22604_rev2.json
new file mode 100644
index 0000000..0adeb22
--- /dev/null
+++ b/machines/xnet/monitoring/22604_rev2.json
@@ -0,0 +1,1565 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_PROMETHEUS",
+ "label": "Prometheus",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "panel",
+ "id": "gauge",
+ "name": "Gauge",
+ "version": ""
+ },
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "11.4.0"
+ },
+ {
+ "type": "panel",
+ "id": "piechart",
+ "name": "Pie chart",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "stat",
+ "name": "Stat",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "table",
+ "name": "Table",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Extended smartctl-exporter Dashboard",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 0,
+ "links": [],
+ "mappings": [],
+ "min": 20,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "celsius"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 13,
+ "x": 0,
+ "y": 0
+ },
+ "id": 1,
+ "options": {
+ "legend": {
+ "calcs": ["mean", "lastNotNull", "max"],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "expr": "avg(smartctl_device_temperature{instance=~\"$node\", device=~\"$disk\"} * on(instance, device) group_left(interface,serial_number,model_name) smartctl_device{interface=~\"$type\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"}) by (instance,device,model_name)",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "1m",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}} {{device}} {{model_name}}",
+ "refId": "B"
+ }
+ ],
+ "title": "Disk Temperature",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 2,
+ "x": 13,
+ "y": 0
+ },
+ "id": 13,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": ["lastNotNull"],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(smartctl_device{instance=~\"$node\", device=~\"$disk\", interface=~\"$type\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"})",
+ "instant": false,
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Devices",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "min": 0,
+ "noValue": "0",
+ "unit": "none"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": ["Value"],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 2,
+ "x": 15,
+ "y": 0
+ },
+ "id": 16,
+ "options": {
+ "displayLabels": ["percent"],
+ "legend": {
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "values": ["value"]
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": ["lastNotNull"],
+ "fields": "/^Value$/",
+ "values": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(smartctl_device) by (interface)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "__auto",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducers": []
+ }
+ }
+ ],
+ "type": "piechart"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 7,
+ "x": 17,
+ "y": 0
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": ["lastNotNull"],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "smartctl_device_media_errors{instance=~\"$node\", device=~\"$disk\"} * on(instance, device) group_left(interface,serial_number,model_family,model_name) smartctl_device{interface=~\"$type\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"} > 0",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}} {{device}} {{model_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Media Errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "min": 0,
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 2,
+ "x": 13,
+ "y": 4
+ },
+ "id": 12,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": ["lastNotNull"],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "expr": "count((smartctl_device_smart_status{instance=~\"$node\",device=~\"$disk\"} * on(instance, device) group_left(interface,serial_number,model_family,model_name) smartctl_device) == 0)",
+ "format": "table",
+ "instant": true,
+ "intervalFactor": 1,
+ "legendFormat": "__auto",
+ "refId": "A"
+ }
+ ],
+ "title": "Unhealthy",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducers": []
+ }
+ }
+ ],
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 0,
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byValue",
+ "options": {
+ "op": "gte",
+ "reducer": "allIsZero",
+ "value": 0
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": true,
+ "tooltip": true,
+ "viz": false
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 7,
+ "x": 17,
+ "y": 4
+ },
+ "id": 5,
+ "options": {
+ "legend": {
+ "calcs": ["lastNotNull"],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_critical_warning{instance=~\"$node\", device=~\"$disk\"} * on(instance, device) group_left(interface,serial_number,model_family,model_name) smartctl_device{interface=~\"$type\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"} > 0",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Critical Warnings {{device}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Critical Warnings",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 0
+ },
+ {
+ "color": "green",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 2,
+ "x": 15,
+ "y": 6
+ },
+ "id": 15,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "center",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": ["lastNotNull"],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "count(count(smartctl_device) by (instance))",
+ "instant": true,
+ "legendFormat": "nodes",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "filterable": false,
+ "inspect": false,
+ "minWidth": 150
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "dateTimeFromNow"
+ },
+ {
+ "id": "decimals",
+ "value": 2
+ },
+ {
+ "id": "custom.align"
+ },
+ {
+ "id": "displayName",
+ "value": "Last Checked"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Value #A"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Power On Time"
+ },
+ {
+ "id": "unit",
+ "value": "s"
+ },
+ {
+ "id": "decimals",
+ "value": 2
+ },
+ {
+ "id": "custom.align"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "model_name"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Device Model"
+ },
+ {
+ "id": "custom.align"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "device"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Device"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Value #B"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Power Cycles"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "instance"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Node"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Node"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 80
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Device"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 107
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Last Checked"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 142
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Device Model"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 291
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Power On Time"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 162
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Power Cycles"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 117
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 13,
+ "x": 0,
+ "y": 8
+ },
+ "id": 10,
+ "interval": "1h",
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "enablePagination": true,
+ "fields": "",
+ "reducer": ["sum"],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Power On Time"
+ }
+ ]
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max(smartctl_device_power_on_seconds{instance=~\"$node\", device=~\"$disk\"} * on(instance, device) group_left(interface,serial_number,model_family,model_name) smartctl_device{interface=~\"$type\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"}) by (instance, device, model_name)",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "__auto",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max(smartctl_device_power_cycle_count{instance=~\"$node\", device=~\"$disk\"} * on(instance, device) group_left(interface,serial_number,model_family,model_name) smartctl_device{interface=~\"$type\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"}) by (instance, device, model_name)",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Power Cycles",
+ "refId": "B"
+ }
+ ],
+ "title": "Disk Lifetime",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducers": []
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total Written"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Total Written"
+ },
+ {
+ "id": "unit",
+ "value": "bytes"
+ },
+ {
+ "id": "decimals",
+ "value": 2
+ },
+ {
+ "id": "custom.align"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Time"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "dateTimeFromNow"
+ },
+ {
+ "id": "custom.align"
+ },
+ {
+ "id": "displayName",
+ "value": "Last Checked"
+ },
+ {
+ "id": "custom.hidden",
+ "value": true
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "model_name"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Device Model"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "device"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Device"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "instance"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Node"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "serial_number"
+ },
+ "properties": [
+ {
+ "id": "displayName",
+ "value": "Serial Number"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Device"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 115
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "instance"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 77
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Serial Number"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 177
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "model_name"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 277
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Total Written"
+ },
+ "properties": [
+ {
+ "id": "custom.width",
+ "value": 123
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 11,
+ "x": 13,
+ "y": 8
+ },
+ "id": 3,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "enablePagination": true,
+ "fields": "",
+ "reducer": ["sum"],
+ "show": false
+ },
+ "showHeader": true,
+ "sortBy": [
+ {
+ "desc": true,
+ "displayName": "Total Written"
+ }
+ ]
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max(smartctl_device_bytes_written{instance=~\"$node\", device=~\"$disk\"} * on(instance, device) group_left(interface,serial_number,model_family,model_name) smartctl_device{interface=~\"$type\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"}) by (device, instance, model_name, serial_number)",
+ "format": "table",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "__auto",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "max((smartctl_device_attribute{instance=~\"$node\", device=~\"$disk\", attribute_name=\"Total_LBAs_Written\", attribute_value_type=\"raw\"} * on (instance,device) smartctl_device_block_size{instance=~\"$node\", device=~\"$disk\",blocks_type=\"physical\"}) * on(instance, device) group_left(interface,serial_number,model_family,model_name) smartctl_device{interface=~\"$type\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"}) by (device, instance, model_name, serial_number)\n",
+ "format": "table",
+ "hide": false,
+ "instant": true,
+ "legendFormat": "__auto",
+ "range": false,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Data Written on SSDs",
+ "transformations": [
+ {
+ "id": "merge",
+ "options": {
+ "reducers": []
+ }
+ },
+ {
+ "id": "calculateField",
+ "options": {
+ "alias": "Total Written",
+ "binary": {
+ "left": {
+ "fixed": ""
+ },
+ "right": {
+ "fixed": ""
+ }
+ },
+ "mode": "reduceRow",
+ "reduce": {
+ "include": ["Value #B", "Value #A"],
+ "reducer": "sum"
+ },
+ "replaceFields": false,
+ "window": {
+ "reducer": "mean",
+ "windowAlignment": "trailing",
+ "windowSize": 0.1,
+ "windowSizeMode": "percentage"
+ }
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "names": [
+ "Time",
+ "device",
+ "instance",
+ "model_name",
+ "serial_number",
+ "Total Written"
+ ]
+ }
+ }
+ }
+ ],
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "displayName": "",
+ "mappings": [],
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 75
+ },
+ {
+ "color": "red",
+ "value": 90
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 13,
+ "x": 0,
+ "y": 18
+ },
+ "id": 8,
+ "options": {
+ "minVizHeight": 75,
+ "minVizWidth": 75,
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": ["last"],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "sizing": "auto"
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sort_desc(100 - smartctl_device_attribute{instance=~\"$node\", device=~\"$disk\", attribute_name=\"Wear_Leveling_Count\",attribute_value_type=\"value\"} * on(instance, device) group_left(interface,serial_number,model_family,model_name) smartctl_device{interface=~\"sat\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"})",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}:{{device}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Wear Level on SATA Devices",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "displayName": "",
+ "mappings": [],
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "#EAB839",
+ "value": 75
+ },
+ {
+ "color": "red",
+ "value": 90
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 11,
+ "x": 13,
+ "y": 18
+ },
+ "id": 14,
+ "options": {
+ "minVizHeight": 75,
+ "minVizWidth": 75,
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": ["last"],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "sizing": "auto"
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sort_desc(smartctl_device_percentage_used{instance=~\"$node\", device=~\"$disk\"} * on(instance, device) group_left(interface,serial_number,model_family,model_name) smartctl_device{interface=~\"nvme\",serial_number=~\"$serial_number\",model_name=~\"$device_model\"})\n",
+ "format": "time_series",
+ "instant": true,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}:{{device}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Wear Level on NVMe Devices",
+ "type": "gauge"
+ }
+ ],
+ "refresh": "1m",
+ "schemaVersion": 40,
+ "tags": ["prometheus", "node_exporter", "smartmon"],
+ "templating": {
+ "list": [
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "definition": "label_values(smartctl_version,instance)",
+ "includeAll": true,
+ "multi": true,
+ "name": "node",
+ "options": [],
+ "query": {
+ "qryType": 1,
+ "query": "label_values(smartctl_version,instance)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "type": "query"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "definition": "label_values(smartctl_device,device)",
+ "includeAll": true,
+ "multi": true,
+ "name": "disk",
+ "options": [],
+ "query": {
+ "qryType": 1,
+ "query": "label_values(smartctl_device,device)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "type": "query"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "definition": "label_values(smartctl_device,interface)",
+ "includeAll": true,
+ "multi": true,
+ "name": "type",
+ "options": [],
+ "query": {
+ "qryType": 1,
+ "query": "label_values(smartctl_device,interface)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "type": "query"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "definition": "label_values(smartctl_device,model_name)",
+ "includeAll": true,
+ "multi": true,
+ "name": "device_model",
+ "options": [],
+ "query": {
+ "qryType": 1,
+ "query": "label_values(smartctl_device,model_name)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "type": "query"
+ },
+ {
+ "current": {},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "definition": "label_values(smartctl_device,serial_number)",
+ "includeAll": true,
+ "multi": true,
+ "name": "serial_number",
+ "options": [],
+ "query": {
+ "qryType": 1,
+ "query": "label_values(smartctl_device,serial_number)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "sort": 1,
+ "type": "query"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": ["1m", "5m", "15m", "30m", "1h", "2h", "1d"]
+ },
+ "timezone": "browser",
+ "title": "SMARTctl Exporter Dashboard",
+ "uid": "ce8j0dmrrej9cc",
+ "version": 47,
+ "weekStart": "",
+ "gnetId": 22604
+}
diff --git a/machines/xnet/monitoring/default.nix b/machines/xnet/monitoring/default.nix
index 87ae724..eed4963 100644
--- a/machines/xnet/monitoring/default.nix
+++ b/machines/xnet/monitoring/default.nix
@@ -1,38 +1,58 @@
{ config, ... }:
let
- inherit (config.networking) hostName;
+ inherit (builtins)
+ map
+ ;
+
inherit (config.services.prometheus) exporters;
in
{
- imports = [ ./grafana.nix ./prometheus.nix ];
+ imports = [ ./grafana.nix ];
- exporters = {
- node = {
- enable = true;
- enabledCollectors = [ "processes" "systemd" ];
- };
- systemd = {
- enable = true;
- extraFlags = [
- "--systemd.collector.enable-ip-accounting"
- "--systemd.collector.enable-restart-count"
- ];
+ config = {
+ services.prometheus = {
+ enable = config.xnet.monitoring.hostGrafana;
+ globalConfig.scrape_interval = "1m";
+
+ exporters = {
+ node = {
+ enable = true;
+ openFirewall = true;
+ enabledCollectors = [ "processes" "systemd" ];
+ };
+
+ # systemd = {
+ # enable = true;
+ # openFirewall = true;
+ # extraFlags = [
+ # "--systemd.collector.enable-ip-accounting"
+ # "--systemd.collector.enable-restart-count"
+ # ];
+ # };
+
+ smartctl = {
+ enable = true;
+ openFirewall = true;
+ };
+ };
+
+ scrapeConfigs =
+ let
+ hosts = [ "iridium" "radon" ];
+ in
+ [{
+ job_name = "node";
+ static_configs = [{
+ targets = map (h: "${h}:${toString exporters.node.port}") hosts;
+ }];
+ }
+ {
+ job_name = "smartctl";
+ static_configs = [{
+ targets = map (h: "${h}:${toString exporters.smartctl.port}") hosts;
+ }];
+ }];
};
};
-
- services.prometheus.scrapeConfigs = [
- {
- job_name = "node";
- static_configs = [{
- targets = [ "${hostName}:${toString exporters.node.port}" ];
- }];
- }
- {
- job_name = "systemd";
- static_configs = [{
- targets = [ "${hostName}:${toString exporters.systemd.port}" ];
- }];
- }
- ];
}
diff --git a/machines/xnet/monitoring/grafana.nix b/machines/xnet/monitoring/grafana.nix
index 6a37ce1..e95318d 100644
--- a/machines/xnet/monitoring/grafana.nix
+++ b/machines/xnet/monitoring/grafana.nix
@@ -1,51 +1,78 @@
-{ config, pkgs, ... }: {
- services.grafana.provision = {
- enable = true;
- datasources.settings.datasources = [{
- name = "Prometheus";
- type = "prometheus";
- url = "http://localhost:9090";
- access = "proxy";
- editable = false;
- }];
+{ config, pkgs, lib, ... }:
+let
+ inherit (lib) mkOption mkIf types;
- dashboards.settings.providers = [{
- name = "Fetched Dashboards";
- options.path = "/etc/grafana/dashboards";
- }];
+ inherit (config.xnet.monitoring)
+ hostGrafana
+ ;
+in
+{
+ options.xnet.monitoring = {
+ hostGrafana = mkOption {
+ type = types.bool;
+ default = false;
+ description = "Scrape hosts and host the data.";
+ };
};
- environment.etc = {
- "grafana/dashboards/node-exporter.json" = {
- user = "grafana";
- group = "grafana";
- source = pkgs.fetchurl {
- url = "https://grafana.com/api/dashboards/1860/revisions/37/download";
- hash = "sha256-1DE1aaanRHHeCOMWDGdOS1wBXxOF84UXAjJzT5Ek6mM=";
- };
+ config = mkIf hostGrafana {
+ services.grafana.provision = {
+ enable = true;
+ datasources.settings.datasources = [{
+ name = "Prometheus";
+ type = "prometheus";
+ url = "http://localhost:9090";
+ access = "proxy";
+ editable = false;
+ }];
+
+ dashboards.settings.providers = [{
+ name = "Fetched Dashboards";
+ options.path = "/etc/grafana/dashboards";
+ }];
};
- };
- services.grafana = {
- enable = true;
- settings.server = {
- domain = "grafana.web.4kb.net";
- protocol = "socket";
+ environment.etc = {
+ "grafana/dashboards/node-exporter.json" = {
+ user = "grafana";
+ group = "grafana";
+ source = pkgs.fetchurl {
+ url = "https://grafana.com/api/dashboards/1860/revisions/37/download";
+ hash = "sha256-1DE1aaanRHHeCOMWDGdOS1wBXxOF84UXAjJzT5Ek6mM=";
+ };
+ };
+ "grafana/dashboards/smartctl-exporter.json" = {
+ user = "grafana";
+ group = "grafana";
+ source = ./grafana.nix;
+ # source = pkgs.fetchurl {
+ # url = "https://grafana.com/api/dashboards/22604/revisions/2/download";
+ # hash = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
+ # };
+ };
};
- settings."auth.anonymous" = {
- enabled = true;
- org_role = "Admin";
+
+ services.grafana = {
+ enable = true;
+ settings.server = {
+ domain = "grafana.plaza.4kb.net";
+ protocol = "socket";
+ };
+ settings."auth.anonymous" = {
+ enabled = true;
+ org_role = "Admin";
+ };
};
- };
- users.groups.grafana.members = [ "nginx" ];
- systemd.services.nginx.serviceConfig.ProtectHome = false;
+ users.groups.grafana.members = [ "nginx" ];
+ systemd.services.nginx.serviceConfig.ProtectHome = false;
- services.nginx.virtualHosts."${config.services.grafana.settings.server.domain}" = {
- useACMEHost = "4kb.net";
- addSSL = true;
- locations."/" = {
- proxyPass = "http://unix:/${toString config.services.grafana.settings.server.socket}";
+ services.nginx.virtualHosts."${config.services.grafana.settings.server.domain}" = {
+ # useACMEHost = "4kb.net";
+ # addSSL = true;
+ locations."/" = {
+ proxyPass = "http://unix:/${toString config.services.grafana.settings.server.socket}";
+ };
};
};
}
diff --git a/machines/xnet/monitoring/prometheus.nix b/machines/xnet/monitoring/prometheus.nix
deleted file mode 100644
index e32e078..0000000
--- a/machines/xnet/monitoring/prometheus.nix
+++ /dev/null
@@ -1,15 +0,0 @@
-{ config, ... }: {
- services.prometheus = {
- enable = true;
- globalConfig.scrape_interval = "1m";
- # scrapeConfigs = [{
- # job_name = "node";
- # relabel_configs = [{
- # source_labels = [ "__address__" ];
- # regex = "(.*):[0-9]+";
- # target_label = "instance";
- # replacement = "$1";
- # }];
- # }];
- };
-}
generated by cgit v1.3.1 (git 2.54.0) at 2026-09-04 09:37:09 +0000