diff --git a/hosts/ildkule/services/monitoring/dashboards/mysql.json b/hosts/ildkule/services/monitoring/dashboards/mysql.json index 21b0e0b..f3dd8d7 100644 --- a/hosts/ildkule/services/monitoring/dashboards/mysql.json +++ b/hosts/ildkule/services/monitoring/dashboards/mysql.json @@ -13,7 +13,7 @@ ] }, "description": "", - "editable": true, + "editable": false, "gnetId": 11323, "graphTooltip": 1, "id": 31, @@ -1899,7 +1899,7 @@ "dashes": false, "datasource": "$datasource", "decimals": 0, - "description": "***System Memory***: Total Memory for the system.\\\n***InnoDB Buffer Pool Data***: InnoDB maintains a storage area called the buffer pool for caching data and indexes in memory.\\\n***TokuDB Cache Size***: Similar in function to the InnoDB Buffer Pool, TokuDB will allocate 50% of the installed RAM for its own cache.\\\n***Key Buffer Size***: Index blocks for MYISAM tables are buffered and are shared by all threads. key_buffer_size is the size of the buffer used for index blocks.\\\n***Adaptive Hash Index Size***: When InnoDB notices that some index values are being accessed very frequently, it builds a hash index for them in memory on top of B-Tree indexes.\\\n ***Query Cache Size***: The query cache stores the text of a SELECT statement together with the corresponding result that was sent to the client. The query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time.\\\n***InnoDB Dictionary Size***: The data dictionary is InnoDB 's internal catalog of tables. InnoDB stores the data dictionary on disk, and loads entries into memory while the server is running.\\\n***InnoDB Log Buffer Size***: The MySQL InnoDB log buffer allows transactions to run without having to write the log to disk before the transactions commit.", + "description": "***System Memory***: Total Memory for the system.\\\n***InnoDB Buffer Pool Data***: InnoDB maintains a storage area called the buffer pool for caching data and indexes in memory.\\\n***TokuDB Cache Size***: Similar in function to the InnoDB Buffer Pool, TokuDB will allocate 50% of the installed RAM for its own cache.\\\n***Key Buffer Size***: Index blocks for MYISAM tables are buffered and are shared by all threads. key_buffer_size is the size of the buffer used for index blocks.\\\n***Adaptive Hash Index Size***: When InnoDB notices that some index values are being accessed very frequently, it builds a hash index for them in memory on top of B-Tree indexes.\\\n ***Query Cache Size***: The query cache stores the text of a SELECT statement together with the corresponding result that was sent to the client. The query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time.\\\n***InnoDB Dictionary Size***: The data dictionary is InnoDB ‘s internal catalog of tables. InnoDB stores the data dictionary on disk, and loads entries into memory while the server is running.\\\n***InnoDB Log Buffer Size***: The MySQL InnoDB log buffer allows transactions to run without having to write the log to disk before the transactions commit.", "editable": true, "error": false, "fieldConfig": { @@ -3690,7 +3690,7 @@ }, "hide": 0, "includeAll": false, - "label": "Data Source", + "label": "Data source", "multi": false, "name": "datasource", "options": [], @@ -3713,12 +3713,12 @@ "definition": "label_values(mysql_up, job)", "hide": 0, "includeAll": true, - "label": "job", + "label": "Job", "multi": true, "name": "job", "options": [], "query": "label_values(mysql_up, job)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, @@ -3742,12 +3742,12 @@ "definition": "label_values(mysql_up, instance)", "hide": 0, "includeAll": true, - "label": "instance", + "label": "Instance", "multi": true, "name": "instance", "options": [], "query": "label_values(mysql_up, instance)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, diff --git a/hosts/ildkule/services/monitoring/dashboards/node-exporter-full.json b/hosts/ildkule/services/monitoring/dashboards/node-exporter-full.json index 967d1f5..fdc3a00 100644 --- a/hosts/ildkule/services/monitoring/dashboards/node-exporter-full.json +++ b/hosts/ildkule/services/monitoring/dashboards/node-exporter-full.json @@ -1,16 +1,11 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, { "type": "panel", "id": "gauge", @@ -21,7 +16,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.2.3" + "version": "11.6.1" }, { "type": "datasource", @@ -45,7 +40,6 @@ "annotations": { "list": [ { - "$$hashKey": "object:1058", "builtIn": 1, "datasource": { "type": "datasource", @@ -67,8 +61,7 @@ }, "editable": true, "fiscalYearStartMonth": 0, - "gnetId": 1860, - "graphTooltip": 0, + "graphTooltip": 1, "id": null, "links": [ { @@ -88,14 +81,9 @@ "url": "https://grafana.com/grafana/dashboards/1860" } ], - "liveNow": false, "panels": [ { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -104,60 +92,42 @@ }, "id": 261, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], "title": "Quick CPU / Mem / Disk", "type": "row" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Busy state of all CPU cores together", + "description": "Resource pressure via PSI", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, + "decimals": 1, + "links": [], + "mappings": [], + "max": 1, "min": 0, "thresholds": { - "mode": "absolute", + "mode": "percentage", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "green" }, { - "color": "rgba(237, 129, 40, 0.89)", - "value": 85 + "color": "dark-yellow", + "value": 70 }, { - "color": "rgba(245, 54, 54, 0.9)", - "value": 95 + "color": "dark-red", + "value": 90 } ] }, - "unit": "percent" + "unit": "percentunit" }, "overrides": [] }, @@ -167,9 +137,19 @@ "x": 0, "y": 1 }, - "id": 20, - "links": [], + "id": 323, "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ @@ -178,40 +158,73 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "showUnfilled": true, + "sizing": "auto", + "text": {}, + "valueMode": "color" }, - "pluginVersion": "9.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "range": true, + "exemplar": false, + "expr": "irate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "legendFormat": "CPU", + "range": false, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "legendFormat": "Mem", + "range": false, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "legendFormat": "I/O", + "range": false, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "legendFormat": "Irq", + "range": false, + "refId": "D", + "step": 240 } ], - "title": "CPU Busy", - "type": "gauge" + "title": "Pressure", + "type": "bargauge" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Busy state of all CPU cores together (5 min average)", + "description": "Overall CPU busy percentage (averaged across all cores)", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, + "decimals": 1, "mappings": [ { "options": { @@ -229,8 +242,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -252,10 +264,11 @@ "x": 3, "y": 1 }, - "id": 155, - "links": [], + "id": 20, "options": { - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -264,37 +277,37 @@ "values": false }, "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "9.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "avg(node_load5{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100", - "format": "time_series", - "hide": false, - "intervalFactor": 1, + "editorMode": "code", + "exemplar": false, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", instance=\"$node\"}[$__rate_interval])))", + "instant": true, + "legendFormat": "", + "range": false, "refId": "A", "step": 240 } ], - "title": "Sys Load (5m avg)", + "title": "CPU Busy", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Busy state of all CPU cores together (15 min average)", + "description": "System load over all CPU cores together", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, + "decimals": 1, "mappings": [ { "options": { @@ -312,8 +325,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -335,10 +347,11 @@ "x": 6, "y": 1 }, - "id": 19, - "links": [], + "id": 155, "options": { - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -347,37 +360,37 @@ "values": false }, "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "9.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "avg(node_load15{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100", - "hide": false, - "intervalFactor": 1, + "editorMode": "code", + "exemplar": false, + "expr": "scalar(node_load1{instance=\"$node\",job=\"$job\"}) * 100 / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "format": "time_series", + "instant": true, + "range": false, "refId": "A", "step": 240 } ], - "title": "Sys Load (15m avg)", + "title": "Sys Load", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Non available RAM memory", + "description": "Real RAM usage excluding cache and reclaimable memory", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, - "decimals": 0, + "decimals": 1, "mappings": [], "max": 100, "min": 0, @@ -385,8 +398,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -408,11 +420,11 @@ "x": 9, "y": 1 }, - "hideTimeOverride": false, "id": 16, - "links": [], "options": { - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -421,31 +433,18 @@ "values": false }, "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "9.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "((node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100", + "editorMode": "code", + "exemplar": false, + "expr": "clamp_min((1 - (node_memory_MemAvailable_bytes{instance=\"$node\", job=\"$job\"} / node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"})) * 100, 0)", "format": "time_series", - "hide": true, - "intervalFactor": 1, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 - ((node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"} * 100) / node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, + "instant": true, + "range": false, "refId": "B", "step": 240 } @@ -456,14 +455,15 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Used Swap", + "description": "Percentage of swap space currently used by the system", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, + "decimals": 1, "mappings": [ { "options": { @@ -481,8 +481,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -505,9 +504,10 @@ "y": 1 }, "id": 21, - "links": [], "options": { - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -516,17 +516,17 @@ "values": false }, "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "9.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100", - "intervalFactor": 1, + "editorMode": "code", + "exemplar": false, + "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"})) * 100", + "instant": true, + "range": false, "refId": "A", "step": 240 } @@ -537,7 +537,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, "description": "Used Root FS", "fieldConfig": { @@ -545,6 +545,7 @@ "color": { "mode": "thresholds" }, + "decimals": 1, "mappings": [ { "options": { @@ -562,8 +563,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -586,9 +586,10 @@ "y": 1 }, "id": 154, - "links": [], "options": { - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -597,18 +598,18 @@ "values": false }, "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "9.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"})", + "editorMode": "code", + "exemplar": false, + "expr": "(\n (node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"}\n - node_filesystem_avail_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"})\n / node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"}\n) * 100\n", "format": "time_series", - "intervalFactor": 1, + "instant": true, + "range": false, "refId": "A", "step": 240 } @@ -619,9 +620,9 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Total number of CPU cores", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -642,12 +643,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -662,13 +658,13 @@ "y": 1 }, "id": 14, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -676,21 +672,20 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", + "exemplar": false, "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 240 + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" } ], "title": "CPU Cores", @@ -699,89 +694,9 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "System uptime", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 20, - "y": 1 - }, - "hideTimeOverride": true, - "id": 15, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total RootFS", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -803,8 +718,165 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 1 + }, + "id": 75, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 1 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -827,13 +899,13 @@ "y": 3 }, "id": 23, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -841,19 +913,19 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", + "exemplar": false, "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, + "instant": true, + "range": false, "refId": "A", "step": 240 } @@ -864,15 +936,15 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Total RAM", + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, - "decimals": 0, + "decimals": 1, "mappings": [ { "options": { @@ -888,8 +960,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -897,24 +968,24 @@ } ] }, - "unit": "bytes" + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 2, - "w": 2, + "w": 4, "x": 20, "y": 3 }, - "id": 75, - "links": [], + "id": 15, "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -922,109 +993,27 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", - "intervalFactor": 1, + "editorMode": "code", + "exemplar": false, + "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", + "instant": true, + "range": false, "refId": "A", "step": 240 } ], - "title": "RAM Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total SWAP", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 2, - "x": 22, - "y": 3 - }, - "id": 18, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", - "intervalFactor": 1, - "refId": "A", - "step": 240 - } - ], - "title": "SWAP Total", + "title": "Uptime", "type": "stat" }, { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -1033,35 +1022,28 @@ }, "id": 263, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], "title": "Basic CPU / Mem / Net / Disk", "type": "row" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Basic CPU info", + "description": "CPU time spent busy vs idle, split by activity type", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", @@ -1070,6 +1052,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, @@ -1093,12 +1076,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -1135,36 +1113,6 @@ } ] }, - { - "matcher": { - "id": "byName", - "options": "Busy Iowait" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Idle" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byName", @@ -1219,7 +1167,6 @@ "y": 6 }, "id": 77, - "links": [], "options": { "legend": { "calcs": [], @@ -1229,93 +1176,64 @@ "width": 250 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "exemplar": false, + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "hide": false, - "intervalFactor": 1, + "instant": false, "legendFormat": "Busy System", "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "Busy User", "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Busy Iowait", "range": true, "refId": "C", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Busy IRQs", "range": true, "refId": "D", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Busy Other", "range": true, "refId": "E", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Idle", "range": true, "refId": "F", @@ -1328,20 +1246,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Basic memory usage", + "description": "RAM and swap usage overview, including caches", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", @@ -1350,6 +1270,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1373,12 +1294,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -1388,172 +1304,7 @@ { "matcher": { "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "SWAP Used" + "options": "Swap used" }, "properties": [ { @@ -1568,112 +1319,7 @@ { "matcher": { "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Total" + "options": "Total" }, "properties": [ { @@ -1699,7 +1345,7 @@ { "matcher": { "id": "byName", - "options": "RAM Cache + Buffer" + "options": "Cache + Buffer" }, "properties": [ { @@ -1714,7 +1360,7 @@ { "matcher": { "id": "byName", - "options": "RAM Free" + "options": "Free" }, "properties": [ { @@ -1725,32 +1371,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Avaliable" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#DEDAF7", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] } ] }, @@ -1761,7 +1381,6 @@ "y": 6 }, "id": 78, - "links": [], "options": { "legend": { "calcs": [], @@ -1771,71 +1390,55 @@ "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "RAM Total", + "legendFormat": "Total", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "RAM Used", + "legendFormat": "Used", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "RAM Cache + Buffer", + "legendFormat": "Cache + Buffer", + "range": true, "refId": "C", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "RAM Free", + "legendFormat": "Free", + "range": true, "refId": "D", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SWAP Used", + "legendFormat": "Swap used", + "range": true, "refId": "E", "step": 240 } @@ -1846,20 +1449,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Basic network info per interface", + "description": "Per-interface network traffic (receive and transmit) in bits per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", @@ -1868,6 +1473,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1890,367 +1496,17 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, "unit": "bps" }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Recv_bytes_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_drop_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_errs_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_bytes_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_drop_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_errs_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_drop_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#967302", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_errs_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_bytes_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_drop_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#967302", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_errs_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byRegexp", - "options": "/.*trans.*/" + "options": "/.*Tx.*/" }, "properties": [ { @@ -2268,7 +1524,6 @@ "y": 13 }, "id": 74, - "links": [], "options": { "legend": { "calcs": [], @@ -2277,33 +1532,28 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "recv {{device}}", + "legendFormat": "Rx {{device}}", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "editorMode": "code", + "expr": "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "trans {{device}} ", + "legendFormat": "Tx {{device}} ", + "range": true, "refId": "B", "step": 240 } @@ -2314,20 +1564,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Disk space used of all filesystems mounted", + "description": "Percentage of filesystem space used for each mounted device", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", @@ -2336,6 +1588,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2360,12 +1613,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -2380,7 +1628,6 @@ "y": 13 }, "id": 152, - "links": [], "options": { "legend": { "calcs": [], @@ -2389,21 +1636,19 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})", + "editorMode": "code", + "expr": "((node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", device!~\"rootfs\"} - node_filesystem_avail_bytes{instance=\"$node\", job=\"$job\", device!~\"rootfs\"}) / node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", device!~\"rootfs\"}) * 100", "format": "time_series", - "intervalFactor": 1, "legendFormat": "{{mountpoint}}", + "range": true, "refId": "A", "step": 240 } @@ -2413,10 +1658,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -2428,20 +1669,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "CPU time usage split by state, normalized across all CPU cores", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "percentage", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 70, "gradientMode": "none", @@ -2450,6 +1693,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, @@ -2473,12 +1717,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -2604,6 +1843,35 @@ } } ] + }, + { + "matcher": { + "id": "byName", + "options": "Guest CPU usage" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] } ] }, @@ -2611,17 +1879,15 @@ "h": 12, "w": 12, "x": 0, - "y": 7 + "y": 21 }, "id": 3, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -2629,124 +1895,93 @@ "width": 250 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "System - Processes executing in kernel mode", "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"user\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, "legendFormat": "User - Normal processes executing in user mode", "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"nice\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Nice - Niced processes executing in user mode", "range": true, "refId": "C", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"iowait\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Iowait - Waiting for I/O to complete", "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"irq\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "legendFormat": "Irq - Servicing interrupts", + "range": true, "refId": "E", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"softirq\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Irq - Servicing interrupts", + "legendFormat": "Softirq - Servicing softirqs", "range": true, "refId": "F", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"steal\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Softirq - Servicing softirqs", + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", "range": true, "refId": "G", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "legendFormat": "Idle - Waiting for something to happen", "range": true, "refId": "H", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))) > 0", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Idle - Waiting for something to happen", + "legendFormat": "Guest CPU usage", "range": true, - "refId": "J", + "refId": "I", "step": 240 } ], @@ -2756,20 +1991,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Breakdown of physical memory and swap usage. Hardware-detected memory errors are also displayed", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", @@ -2778,6 +2015,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2801,12 +2039,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -3119,17 +2352,15 @@ "h": 12, "w": 12, "x": 12, - "y": 7 + "y": 21 }, "id": 24, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -3137,148 +2368,117 @@ "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "Apps - Memory used by user-space applications", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "range": true, "refId": "C", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "range": true, "refId": "D", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "Cache - Parked file data (file content) cache", + "range": true, "refId": "E", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "range": true, "refId": "F", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "Unused - Free memory unassigned", + "range": true, "refId": "G", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "Swap - Swap space used", + "range": true, "refId": "H", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "range": true, "refId": "I", "step": 240 } ], - "title": "Memory Stack", + "title": "Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Incoming and outgoing network traffic per interface", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bits out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", @@ -3287,6 +2487,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3309,82 +2510,17 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, "unit": "bps" }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "receive_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "receive_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byRegexp", - "options": "/.*Trans.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -3399,50 +2535,43 @@ "h": 12, "w": 12, "x": 0, - "y": 19 + "y": 433 }, "id": 84, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "editorMode": "code", + "expr": "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit", + "legendFormat": "{{device}} - Tx out", + "range": true, "refId": "B", "step": 240 } @@ -3453,20 +2582,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Network interface utilization as a percentage of its maximum capacity", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", @@ -3475,6 +2606,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3493,84 +2625,98 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, - "unit": "bytes" + "unit": "percentunit" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] }, "gridPos": { "h": 12, "w": 12, "x": 12, - "y": 19 + "y": 433 }, - "id": 156, - "links": [], + "id": 338, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])\n / ignoring(speed) node_network_speed_bytes{instance=\"$node\",job=\"$job\", speed!=\"-1\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}}", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "(rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])\n / ignoring(speed) node_network_speed_bytes{instance=\"$node\",job=\"$job\", speed!=\"-1\"})", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Disk Space Used", + "title": "Network Saturation", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Disk I/O operations per second for each device", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "IO read (-) / write (+)", + "axisLabel": "read (-) / write (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -3579,6 +2725,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3601,12 +2748,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -3624,321 +2766,6 @@ "value": "negative-Y" } ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] } ] }, @@ -3946,48 +2773,41 @@ "h": 12, "w": 12, "x": 0, - "y": 31 + "y": 445 }, "id": 229, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "intervalFactor": 4, - "legendFormat": "{{device}} - Reads completed", + "editorMode": "code", + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"}[$__rate_interval])", + "legendFormat": "{{device}} - Read", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Writes completed", + "editorMode": "code", + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"}[$__rate_interval])", + "legendFormat": "{{device}} - Write", + "range": true, "refId": "B", "step": 240 } @@ -3998,20 +2818,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Disk I/O throughput per device", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes read (-) / write (+)", + "axisLabel": "read (-) / write (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", @@ -4020,6 +2842,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4042,37 +2865,17 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, "unit": "Bps" }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "io time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byRegexp", - "options": "/.*read*./" + "options": "/.*Read*./" }, "properties": [ { @@ -4080,93 +2883,6 @@ "value": "negative-Y" } ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byType", - "options": "time" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] } ] }, @@ -4174,84 +2890,78 @@ "h": 12, "w": 12, "x": 12, - "y": 31 + "y": 445 }, "id": 42, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"}[$__rate_interval])", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}} - Successfully read bytes", + "legendFormat": "{{device}} - Read", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"}[$__rate_interval])", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}} - Successfully written bytes", + "legendFormat": "{{device}} - Write", + "range": true, "refId": "B", "step": 240 } ], - "title": "I/O Usage Read / Write", + "title": "Disk Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Amount of available disk space per mounted filesystem, excluding rootfs. Based on block availability to non-root users", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "%util", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 40, + "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4275,119 +2985,105 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, - "unit": "percentunit" + "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "io time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byType", - "options": "time" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 12, "w": 12, "x": 0, - "y": 43 + "y": 457 }, - "id": 127, - "links": [], + "id": 43, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])", + "editorMode": "code", + "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}}", + "legendFormat": "{{mountpoint}}", + "metric": "", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "legendFormat": "{{mountpoint}} - Free", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "legendFormat": "{{mountpoint}} - Size", + "range": true, + "refId": "C", + "step": 240 } ], - "title": "I/O Utilization", + "title": "Filesystem Space Available", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Disk usage (used = total - available) per mountpoint", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "percentage", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 70, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 3, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -4401,18 +3097,210 @@ "mode": "off" } }, + "links": [], "mappings": [], - "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 457 + }, + "id": 156, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Filesystem Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Percentage of time the disk was actively processing I/O operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "color": "red", - "value": 80 + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 469 + }, + "id": 127, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"} [$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk I/O Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "How often tasks experience CPU, memory, or I/O delays. “Some” indicates partial slowdown; “Full” indicates all tasks are stalled. Based on Linux PSI metrics:\nhttps://docs.kernel.org/accounting/psi.html", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "some (-) / full (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } ] }, @@ -4422,30 +3310,24 @@ { "matcher": { "id": "byRegexp", - "options": "/^Guest - /" + "options": "/.*Some.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#5195ce", - "mode": "fixed" - } + "id": "custom.fillOpacity", + "value": 0 } ] }, { "matcher": { "id": "byRegexp", - "options": "/^GuestNice - /" + "options": "/.*Some.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#c15c17", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] } @@ -4455,74 +3337,92 @@ "h": 12, "w": 12, "x": 12, - "y": 43 + "y": 469 }, - "id": 319, + "id": 322, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", - "sort": "desc" + "sort": "none" } }, + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", - "hide": false, - "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", + "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "CPU - Some", "range": true, - "refId": "A" + "refId": "CPU some", + "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", - "hide": false, - "legendFormat": "GuestNice - Time spent running a niced guest (virtual CPU for guest operating system)", + "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Memory - Some", "range": true, - "refId": "B" + "refId": "Memory some", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_memory_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Memory - Full", + "range": true, + "refId": "Memory full", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "I/O - Some", + "range": true, + "refId": "I/O some", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_io_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "I/O - Full", + "range": true, + "refId": "I/O full", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "IRQ - Full", + "range": true, + "refId": "A", + "step": 240 } ], - "title": "CPU spent seconds in guests (VMs)", + "title": "Pressure Stall Information", "type": "timeseries" } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], "title": "CPU / Memory / Net / Disk", "type": "row" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -4534,390 +3434,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Displays committed memory usage versus the system's commit limit. Exceeding the limit is allowed under Linux overcommit policies but may increase OOM risks under high load", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 136, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Active / Inactive", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -4926,6 +3458,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4950,271 +3483,12 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "bytes" }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byRegexp", @@ -5239,18 +3513,16 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 38 + "x": 0, + "y": 732 }, "id": 135, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -5258,455 +3530,54 @@ "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", + "legendFormat": "Committed_AS – Memory promised to processes (not necessarily used)", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", + "legendFormat": "CommitLimit - Max allowable committed memory", + "range": true, "refId": "B", "step": 240 } ], - "title": "Memory Commited", + "title": "Memory Committed", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Memory currently dirty (modified but not yet written to disk), being actively written back, or held by writeback buffers. High dirty or writeback memory may indicate disk I/O pressure or delayed flushing", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 191, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Active_file - File-backed memory on active LRU list", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", - "refId": "D", - "step": 240 - } - ], - "title": "Memory Active / Inactive Detail", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -5715,6 +3586,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -5739,365 +3611,74 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 48 + "y": 732 }, "id": 130, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Writeback - Memory which is actively being written back to disk", + "legendFormat": "Writeback – Memory currently being flushed to disk", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", + "legendFormat": "WritebackTmp – FUSE temporary writeback buffers", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", + "legendFormat": "Dirty – Memory marked dirty (pending write to disk)", + "range": true, "refId": "C", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "NFS Unstable – Pages sent to NFS server, awaiting storage commit", + "range": true, + "refId": "D", + "step": 240 } ], "title": "Memory Writeback and Dirty", @@ -6106,440 +3687,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Kernel slab memory usage, separated into reclaimable and non-reclaimable categories. Reclaimable memory can be freed under memory pressure (e.g., caches), while unreclaimable memory is locked by the kernel for core functions", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 138, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Mapped - Used memory in mapped pages files which have been mmaped, such as libraries", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ShmemPmdMapped - Ammount of shared (shmem/tmpfs) memory backed by huge pages", - "refId": "D", - "step": 240 - } - ], - "title": "Memory Shared and Mapped", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -6548,6 +3711,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6572,351 +3736,54 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 58 + "x": 0, + "y": 932 }, "id": 131, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", + "legendFormat": "SUnreclaim – Non-reclaimable slab memory (kernel objects)", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", + "legendFormat": "SReclaimable – Potentially reclaimable slab memory (e.g., inode cache)", + "range": true, "refId": "B", "step": 240 } @@ -6927,19 +3794,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Memory used for mapped files (such as libraries) and shared memory (shmem and tmpfs), including variants backed by huge pages", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -6948,6 +3818,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -6972,26 +3843,150 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "bytes" }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 932 + }, + "id": 138, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Mapped – Memory mapped from files (e.g., libraries, mmap)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Shmem – Shared memory used by processes and tmpfs", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "ShmemHugePages – Shared memory (shmem/tmpfs) allocated with HugePages", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PMD Mapped – Shmem/tmpfs backed by Transparent HugePages (PMD)", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory Shared and Mapped", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Proportion of memory pages in the kernel's active and inactive LRU lists relative to total RAM. Active pages have been recently used, while inactive pages are less recently accessed but still resident in memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, "overrides": [ { "matcher": { - "id": "byName", - "options": "Active" + "id": "byRegexp", + "options": "/.*Active.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#99440A", + "fixedColor": "green", "mode": "fixed" } } @@ -6999,254 +3994,14 @@ }, { "matcher": { - "id": "byName", - "options": "Buffers" + "id": "byRegexp", + "options": "/.*Inactive.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", + "fixedColor": "dark-blue", "mode": "fixed" } } @@ -7258,65 +4013,435 @@ "h": 10, "w": 12, "x": 0, - "y": 68 + "y": 942 }, - "id": 70, - "links": [], + "id": 136, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "(node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}) \n/ \n(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "legendFormat": "Inactive – Less recently used memory, more likely to be reclaimed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "(node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}) \n/ \n(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})\n", + "format": "time_series", + "legendFormat": "Active – Recently used memory, retained unless under pressure", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Memory LRU Active / Inactive (%)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Breakdown of memory pages in the kernel's active and inactive LRU lists, separated by anonymous (heap, tmpfs) and file-backed (caches, mmap) pages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 942 + }, + "id": 191, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Inactive_anon – Anonymous memory on inactive LRU (incl. tmpfs & swap cache)", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Active_file - File-backed memory on active LRU list", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Active_anon – Anonymous memory on active LRU (incl. tmpfs & swap cache)", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory LRU Active / Inactive Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Tracks kernel memory used for CPU-local structures, per-thread stacks, and bounce buffers used for I/O on DMA-limited devices. These areas are typically small but critical for low-level operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 952 + }, + "id": 160, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "KernelStack – Kernel stack memory (per-thread, non-reclaimable)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PerCPU – Dynamically allocated per-CPU memory (used by kernel modules)", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Bounce Memory – I/O buffer for DMA-limited devices", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Memory Kernel / CPU / IO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Usage of the kernel's vmalloc area, which provides virtual memory allocations for kernel modules and drivers. Includes total, used, and largest free block sizes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Total.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 952 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocChunk - Largest contigious block of vmalloc area which is free", + "legendFormat": "Vmalloc Free Chunk – Largest available block in vmalloc area", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocTotal - Total size of vmalloc memory area", + "legendFormat": "Vmalloc Total – Total size of the vmalloc memory area", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", + "legendFormat": "Vmalloc Used – Portion of vmalloc area currently in use", + "range": true, "refId": "C", "step": 240 } @@ -7327,19 +4452,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Memory used by anonymous pages (not backed by files), including standard and huge page allocations. Includes heap, stack, and memory-mapped anonymous regions", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -7348,6 +4476,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -7372,707 +4501,54 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 68 - }, - "id": 159, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Bounce - Memory used for block device bounce buffers", - "refId": "A", - "step": 240 - } - ], - "title": "Memory Bounce", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Inactive *./" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 78 + "y": 962 }, "id": 129, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "AnonHugePages - Memory in anonymous huge pages", + "legendFormat": "AnonHugePages – Anonymous memory using HugePages", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "AnonPages - Memory in user pages not backed by files", + "legendFormat": "AnonPages – Anonymous memory (non-file-backed)", + "range": true, "refId": "B", "step": 240 } @@ -8083,19 +4559,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Memory that is locked in RAM and cannot be swapped out. Includes both kernel-unevictable memory and user-level memory locked with mlock()", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -8104,6 +4583,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8128,106 +4608,12 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "bytes" }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byName", @@ -8242,156 +4628,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] } ] }, @@ -8399,1567 +4635,15 @@ "h": 10, "w": 12, "x": 12, - "y": 78 - }, - "id": 160, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Kernel / CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "pages", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 88 - }, - "id": 140, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", - "refId": "C", - "step": 240 - } - ], - "title": "Memory HugePages Counter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 88 - }, - "id": 71, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages - Total size of the pool of huge pages", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Hugepagesize - Huge Page size", - "refId": "B", - "step": 240 - } - ], - "title": "Memory HugePages Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 98 - }, - "id": 128, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "DirectMap1G - Amount of pages mapped as this size", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "DirectMap2M - Amount of pages mapped as this size", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "DirectMap4K - Amount of pages mapped as this size", - "refId": "C", - "step": 240 - } - ], - "title": "Memory DirectMap", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 98 + "y": 962 }, "id": 137, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -9967,33 +4651,28 @@ "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", + "legendFormat": "Unevictable – Kernel-pinned memory (not swappable)", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", + "legendFormat": "Mlocked – Application-locked memory via mlock()", + "range": true, "refId": "B", "step": 240 } @@ -10004,19 +4683,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "How much memory is directly mapped in the kernel using different page sizes (4K, 2M, 1G). Helps monitor large page utilization in the direct map region", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -10025,6 +4707,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10314,21 +4997,6 @@ } ] }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byName", @@ -10350,53 +5018,185 @@ "h": 10, "w": 12, "x": 0, - "y": 108 + "y": 972 }, - "id": 132, - "links": [], + "id": 128, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet commited to the storage", + "legendFormat": "DirectMap 1G – Memory mapped with 1GB pages", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "DirectMap 2M – Memory mapped with 2MB pages", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "DirectMap 4K – Memory mapped with 4KB pages", + "range": true, + "refId": "C", + "step": 240 } ], - "title": "Memory NFS", + "title": "Memory DirectMap", "type": "timeseries" - } - ], - "targets": [ + }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "refId": "A" + "description": "Displays HugePages memory usage in bytes, including allocated, free, reserved, and surplus memory. All values are calculated based on the number of huge pages multiplied by their configured size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 972 + }, + "id": 140, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "HugePages Used – Currently allocated", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "HugePages Reserved – Promised but unused", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "HugePages Surplus – Dynamic pool extension", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "HugePages Total – Reserved memory", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory HugePages", + "type": "timeseries" } ], "title": "Memory Meminfo", @@ -10404,10 +5204,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -10419,19 +5215,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Rate of memory pages being read from or written to disk (page-in and page-out operations). High page-out may indicate memory pressure or swapping activity", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "pages out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -10440,6 +5239,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10463,20 +5263,16 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*out/" + "options": "/.*out.*/" }, "properties": [ { @@ -10491,50 +5287,43 @@ "h": 10, "w": 12, "x": 0, - "y": 25 + "y": 733 }, "id": 176, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pagesin - Page in operations", + "legendFormat": "Pagesin - Page in ops", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pagesout - Page out operations", + "legendFormat": "Pagesout - Page out ops", + "range": true, "refId": "B", "step": 240 } @@ -10545,19 +5334,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Rate at which memory pages are being swapped in from or out to disk. High swap-out activity may indicate memory pressure", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "pages out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -10566,6 +5358,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10589,20 +5382,16 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*out/" + "options": "/.*out.*/" }, "properties": [ { @@ -10617,50 +5406,43 @@ "h": 10, "w": 12, "x": 12, - "y": 25 + "y": 733 }, "id": 22, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Pswpin - Pages swapped in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Pswpout - Pages swapped out", + "range": true, "refId": "B", "step": 240 } @@ -10671,19 +5453,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Rate of memory page faults, split into total, major (disk-backed), and derived minor (non-disk) faults. High major fault rates may indicate memory pressure or insufficient RAM", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "faults", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -10692,6 +5477,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10716,275 +5502,16 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pgfault - Page major and minor fault operations" + "options": "Pgfault - Page major and minor fault ops" }, "properties": [ { @@ -10995,7 +5522,24 @@ "id": "custom.stacking", "value": { "group": false, - "mode": "normal" + "mode": "none" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" } } ] @@ -11006,17 +5550,15 @@ "h": 10, "w": 12, "x": 0, - "y": 35 + "y": 913 }, "id": 175, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -11024,45 +5566,37 @@ "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pgfault - Page major and minor fault operations", + "legendFormat": "Pgfault - Page major and minor fault ops", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pgmajfault - Major page fault operations", + "legendFormat": "Pgmajfault - Major page fault ops", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pgminfault - Minor page fault operations", + "legendFormat": "Pgminfault - Minor page fault ops", + "range": true, "refId": "C", "step": 240 } @@ -11073,19 +5607,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Rate of Out-of-Memory (OOM) kill events. A non-zero value indicates the kernel has terminated one or more processes due to memory exhaustion", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -11094,6 +5631,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11118,296 +5656,22 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", - "options": "Active" + "options": "OOM Kills" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", + "fixedColor": "dark-red", "mode": "fixed" } } @@ -11419,39 +5683,35 @@ "h": 10, "w": 12, "x": 12, - "y": 35 + "y": 913 }, "id": 307, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "oom killer invocations ", + "legendFormat": "OOM Kills", + "range": true, "refId": "A", "step": 240 } @@ -11460,24 +5720,11 @@ "type": "timeseries" } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], "title": "Memory Vmstat", "type": "row" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -11489,20 +5736,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Tracks the system clock's estimated and maximum error, as well as its offset from the reference clock (e.g., via NTP). Useful for detecting synchronization drift", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "seconds", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -11511,6 +5760,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11534,123 +5784,92 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "s" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Variation*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 40 + "y": 734 }, "id": 260, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "Estimated error in seconds", + "legendFormat": "Estimated error", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "Time offset in between local system and reference clock", + "legendFormat": "Offset local vs reference", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum error in seconds", + "legendFormat": "Maximum error", + "range": true, "refId": "C", "step": 240 } ], - "title": "Time Syncronized Drift", + "title": "Time Synchronized Drift", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "NTP phase-locked loop (PLL) time constant used by the kernel to control time adjustments. Lower values mean faster correction but less stability", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -11659,6 +5878,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11682,10 +5902,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -11697,39 +5913,35 @@ "h": 10, "w": 12, "x": 12, - "y": 40 + "y": 734 }, "id": 291, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "Phase-locked loop time adjust", + "legendFormat": "PLL Time Constant", + "range": true, "refId": "A", "step": 240 } @@ -11740,20 +5952,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Shows whether the system clock is synchronized to a reliable time source, and the current frequency correction ratio applied by the kernel to maintain synchronization", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -11762,6 +5976,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11785,107 +6000,104 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Variation*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 50 + "y": 884 }, "id": 168, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", + "legendFormat": "Sync status (1 = ok)", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "Local clock frequency adjustment", + "legendFormat": "Frequency Adjustment", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "legendFormat": "Tick Interval", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "legendFormat": "TAI Offset", + "range": true, + "refId": "D", + "step": 240 } ], - "title": "Time Syncronized Status", + "title": "Time Synchronized Status", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Displays the PPS signal's frequency offset and stability (jitter) in hertz. Useful for monitoring high-precision time sources like GPS or atomic clocks", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "seconds", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -11894,6 +6106,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11917,10 +6130,114 @@ "steps": [ { "color": "green" - }, + } + ] + }, + "unit": "rothz" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 884 + }, + "id": 333, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_timex_pps_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Frequency Offset", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_pps_stability_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Frequency Stability", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "PPS Frequency / Stability", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Tracks PPS signal timing jitter and shift compared to system clock", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -11931,68 +6248,184 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 50 + "x": 0, + "y": 894 }, - "id": 294, - "links": [], + "id": 334, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_timex_pps_jitter_seconds{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "Seconds between clock ticks", + "legendFormat": "PPS Jitter", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_timex_pps_shift_seconds{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "International Atomic Time (TAI) offset", + "legendFormat": "PPS Shift", + "range": true, "refId": "B", "step": 240 } ], - "title": "Time Misc", + "title": "PPS Time Accuracy", "type": "timeseries" - } - ], - "targets": [ + }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "refId": "A" + "description": "Rate of PPS synchronization diagnostics including calibration events, jitter violations, errors, and frequency stability exceedances", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 894 + }, + "id": 335, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_timex_pps_calibration_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Calibrations/sec", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_timex_pps_error_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Errors/sec", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_timex_pps_stability_exceeded_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Stability Exceeded/sec", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_timex_pps_jitter_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Jitter Events/sec", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "PPS Sync Events", + "type": "timeseries" } ], "title": "System Timesync", @@ -12000,10 +6433,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -12015,19 +6444,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Processes currently in runnable or blocked states. Helps identify CPU contention or I/O wait bottlenecks.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12036,6 +6468,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12075,50 +6508,43 @@ "h": 10, "w": 12, "x": 0, - "y": 27 + "y": 735 }, "id": 62, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Processes blocked waiting for I/O to complete", + "legendFormat": "Blocked (I/O Wait)", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Processes in runnable state", + "legendFormat": "Runnable (Ready for CPU)", + "range": true, "refId": "B", "step": 240 } @@ -12129,19 +6555,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Current number of processes in each state (e.g., running, sleeping, zombie). Requires --collector.processes to be enabled in node_exporter", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12150,6 +6579,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12174,231 +6604,93 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 315, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ state }}", - "refId": "A", - "step": 240 - } - ], - "title": "Processes State", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "forks / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 148, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Processes forks second", - "refId": "A", - "step": 240 - } - ], - "title": "Processes Forks", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*Max.*/" + "id": "byName", + "options": "D" }, "properties": [ { - "id": "custom.fillOpacity", - "value": 0 + "id": "displayName", + "value": "Uninterruptible Sleeping" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "I" + }, + "properties": [ + { + "id": "displayName", + "value": "Idle Kernel Thread" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "R" + }, + "properties": [ + { + "id": "displayName", + "value": "Running" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S" + }, + "properties": [ + { + "id": "displayName", + "value": "Interruptible Sleeping" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "T" + }, + "properties": [ + { + "id": "displayName", + "value": "Stopped" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "X" + }, + "properties": [ + { + "id": "displayName", + "value": "Dead" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Z" + }, + "properties": [ + { + "id": "displayName", + "value": "Zombie" } ] } @@ -12408,101 +6700,61 @@ "h": 10, "w": 12, "x": 12, - "y": 37 + "y": 735 }, - "id": 149, - "links": [], + "id": 315, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, + "editorMode": "code", + "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "Processes virtual memory size in bytes", + "legendFormat": "{{ state }}", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum amount of virtual memory available in bytes", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Processes virtual memory size in bytes", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum amount of virtual memory available in bytes", - "refId": "D", - "step": 240 } ], - "title": "Processes Memory", + "title": "Processes Detailed States", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Rate of new processes being created on the system (forks/sec).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12511,6 +6763,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12530,6 +6783,103 @@ "links": [], "mappings": [], "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 765 + }, + "id": 148, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Process Forks per second", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Processes Forks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Shows CPU saturation per core, calculated as the proportion of time spent waiting to run relative to total time demanded (running + waiting).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -12542,6 +6892,140 @@ } ] }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*waiting.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 765 + }, + "id": 305, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "interval": "", + "legendFormat": "CPU {{ cpu }} - Running", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "interval": "", + "legendFormat": "CPU {{cpu}} - Waiting Queue", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])\n/\n(irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) + irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))\n", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{cpu}}", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "CPU Saturation per Core", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of active PIDs on the system and the configured maximum allowed. Useful for detecting PID exhaustion risk. Requires --collector.processes in node_exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, "unit": "short" }, "overrides": [ @@ -12561,6 +7045,16 @@ { "id": "custom.fillOpacity", "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } } ] } @@ -12570,52 +7064,45 @@ "h": 10, "w": 12, "x": 0, - "y": 47 + "y": 775 }, "id": 313, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "Number of PIDs", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "PIDs limit", + "range": true, "refId": "B", "step": 240 } @@ -12626,147 +7113,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Number of active threads on the system and the configured thread limit. Useful for monitoring thread pressure. Requires --collector.processes in node_exporter", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*waiting.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 305, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU {{ cpu }} - seconds spent running a process", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU", - "refId": "B", - "step": 240 - } - ], - "title": "Process schedule stats Running / Waiting", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12775,6 +7137,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12799,10 +7162,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -12825,6 +7184,16 @@ { "id": "custom.fillOpacity", "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } } ] } @@ -12834,52 +7203,45 @@ "h": 10, "w": 12, "x": 12, - "y": 57 + "y": 775 }, "id": 314, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "Allocated threads", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "Threads limit", + "range": true, "refId": "B", "step": 240 } @@ -12888,24 +7250,11 @@ "type": "timeseries" } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], "title": "System Processes", "type": "row" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -12917,19 +7266,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Per-second rate of context switches and hardware interrupts. High values may indicate intense CPU or I/O activity", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12938,6 +7290,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12962,14 +7315,10 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [] }, @@ -12977,51 +7326,43 @@ "h": 10, "w": 12, "x": 0, - "y": 42 + "y": 816 }, "id": 8, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Context switches", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "hide": false, - "intervalFactor": 1, "legendFormat": "Interrupts", + "range": true, "refId": "B", "step": 240 } @@ -13032,19 +7373,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "System load average over 1, 5, and 15 minutes. Reflects the number of active or waiting processes. Values above CPU core count may indicate overload", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13053,6 +7397,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13077,79 +7422,104 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CPU Core Count" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 42 + "y": 816 }, "id": 7, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_load1{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 4, "legendFormat": "Load 1m", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_load5{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 4, "legendFormat": "Load 5m", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_load15{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 4, "legendFormat": "Load 15m", + "range": true, "refId": "C", "step": 240 + }, + { + "editorMode": "code", + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "format": "time_series", + "legendFormat": "CPU Core Count", + "range": true, + "refId": "D", + "step": 240 } ], "title": "System Load", @@ -13158,27 +7528,31 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Real-time CPU frequency scaling per core, including average minimum and maximum allowed scaling frequencies", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 20, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13197,57 +7571,80 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "hertz" }, "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*Critical*./" + "id": "byName", + "options": "Max" }, "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, { "id": "color", "value": { - "fixedColor": "#E24D42", + "fixedColor": "dark-red", "mode": "fixed" } }, { - "id": "custom.fillOpacity", - "value": 0 + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } } ] }, { "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" + "id": "byName", + "options": "Min" }, "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, { "id": "color", "value": { - "fixedColor": "#EF843C", + "fixedColor": "blue", "mode": "fixed" } }, { - "id": "custom.fillOpacity", - "value": 0 + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } } ] } @@ -13257,62 +7654,81 @@ "h": 10, "w": 12, "x": 0, - "y": 52 + "y": 826 }, - "id": 259, - "links": [], + "id": 321, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_cpu_scaling_frequency_hertz{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ type }} - {{ info }}", + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_max_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "interval": "", + "legendFormat": "Max", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_min_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "interval": "", + "legendFormat": "Min", + "range": true, + "refId": "C", + "step": 240 } ], - "title": "Interrupts Detail", + "title": "CPU Frequency Scaling", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Rate of scheduling timeslices executed per CPU. Reflects how frequently the scheduler switches tasks on each core", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13321,6 +7737,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13344,14 +7761,10 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [] }, @@ -13359,62 +7772,61 @@ "h": 10, "w": 12, "x": 12, - "y": 52 + "y": 826 }, "id": 306, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "CPU {{ cpu }}", + "range": true, "refId": "A", "step": 240 } ], - "title": "Schedule timeslices executed by each cpu", + "title": "CPU Schedule Timeslices", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Breaks down hardware interrupts by type and device. Useful for diagnosing IRQ load on network, disk, or CPU interfaces. Requires --collector.interrupts to be enabled in node_exporter", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13423,6 +7835,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13447,14 +7860,10 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [] }, @@ -13462,289 +7871,178 @@ "h": 10, "w": 12, "x": 0, - "y": 62 + "y": 836 }, - "id": 151, - "links": [], + "id": 259, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "editorMode": "code", + "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{ type }} - {{ info }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "IRQ Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of bits of entropy currently available to the system's random number generators (e.g., /dev/random). Low values may indicate that random number generation could block or degrade performance of cryptographic operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbits" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Entropy pool max" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 836 + }, + "id": 151, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Entropy available to random number generators", + "legendFormat": "Entropy available", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_entropy_pool_size_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Entropy pool max", + "range": true, + "refId": "B", + "step": 240 } ], "title": "Entropy", "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 62 - }, - "id": 308, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Time spent", - "refId": "A", - "step": 240 - } - ], - "title": "CPU time spent in user and system contexts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 72 - }, - "id": 64, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum open file descriptors", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Open file descriptors", - "refId": "B", - "step": 240 - } - ], - "title": "File Descriptors", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" } ], "title": "System Misc", @@ -13752,10 +8050,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -13767,19 +8061,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Monitors hardware sensor temperatures and critical thresholds as exposed by Linux hwmon. Includes CPU, GPU, and motherboard sensors where available", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "temperature", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13788,6 +8085,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13812,10 +8110,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -13840,25 +8134,6 @@ "value": 0 } ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] } ] }, @@ -13866,117 +8141,98 @@ "h": 10, "w": 12, "x": 0, - "y": 43 + "y": 737 }, "id": 158, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} temp", + "legendFormat": "{{ chip_name }} {{ sensor }}", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}", + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", "format": "time_series", "hide": true, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} Critical Alarm", + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Alarm", "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} Critical", + "legendFormat": "{{ chip_name }} {{ sensor }} Critical", + "range": true, "refId": "C", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}", + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", "format": "time_series", "hide": true, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} Critical Historical", + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Historical", "refId": "D", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}", + "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", "format": "time_series", "hide": true, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip }} {{ sensor }} Max", + "legendFormat": "{{ chip_name }} {{ sensor }} Max", "refId": "E", "step": 240 } ], - "title": "Hardware temperature monitor", + "title": "Hardware Temperature Monitor", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Shows how hard each cooling device (fan/throttle) is working relative to its maximum capacity", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13985,6 +8241,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14008,14 +8265,10 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "percent" }, "overrides": [ { @@ -14043,76 +8296,61 @@ "h": 10, "w": 12, "x": 12, - "y": 43 + "y": 737 }, "id": 300, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "100 * node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"} / node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "Current {{ name }} in {{ type }}", + "legendFormat": "{{ name }} - {{ type }} ", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Max {{ name }} in {{ type }}", - "refId": "B", - "step": 240 } ], - "title": "Throttle cooling device", + "title": "Cooling Device Utilization", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Shows the online status of power supplies (e.g., AC, battery). A value of 1-Yes indicates the power supply is active/online", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14121,6 +8359,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14151,7 +8390,7 @@ } ] }, - "unit": "short" + "unit": "bool_yes_no" }, "overrides": [] }, @@ -14159,90 +8398,61 @@ "h": 10, "w": 12, "x": 0, - "y": 53 + "y": 747 }, "id": 302, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, "legendFormat": "{{ power_supply }} online", + "range": true, "refId": "A", "step": 240 } ], - "title": "Power supply", + "title": "Power Supply", "type": "timeseries" - } - ], - "targets": [ + }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Hardware Misc", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 296, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Displays the current fan speeds (RPM) from hardware sensors via the hwmon interface", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14251,6 +8461,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14275,77 +8486,97 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "rotrpm" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 30 + "x": 12, + "y": 747 }, - "id": 297, - "links": [], + "id": 325, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_hwmon_fan_rpm{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ name }} Connections", + "legendFormat": "{{ chip_name }} {{ sensor }}", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_hwmon_fan_min_rpm{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "legendFormat": "{{ chip_name }} {{ sensor }} rpm min", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Systemd Sockets", + "title": "Hardware Fan Speed", "type": "timeseries" - }, + } + ], + "title": "Hardware Misc", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 296, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Current number of systemd units in each operational state, such as active, failed, inactive, or transitioning", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14354,6 +8585,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14377,10 +8609,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -14402,21 +8630,6 @@ } ] }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF9830", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byName", @@ -14432,21 +8645,6 @@ } ] }, - { - "matcher": { - "id": "byName", - "options": "Deactivating" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFCB7D", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byName", @@ -14461,113 +8659,415 @@ } } ] + }, + { + "matcher": { + "id": "byName", + "options": "Deactivating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] } ] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 30 + "x": 0, + "y": 4228 }, "id": 298, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "Activating", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "Active", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "Deactivating", + "range": true, "refId": "C", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "Failed", + "range": true, "refId": "D", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "Inactive", + "range": true, "refId": "E", "step": 240 } ], "title": "Systemd Units State", "type": "timeseries" - } - ], - "targets": [ + }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "refId": "A" + "description": "Current number of active connections per systemd socket, as reported by the Node Exporter systemd collector", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 4228 + }, + "id": 331, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_systemd_socket_current_connections{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ name }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets Current", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of accepted connections per second for each systemd socket", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 4238 + }, + "id": 297, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{ name }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets Accepted", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of systemd socket connection refusals per second, typically due to service unavailability or backlog overflow", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 4238 + }, + "id": 332, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_systemd_socket_refused_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{ name }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets Refused", + "type": "timeseries" } ], "title": "Systemd", @@ -14575,10 +9075,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -14590,20 +9086,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "The number (after merges) of I/O requests completed per second for the device", + "description": "Number of I/O operations completed per second for the device (after merges), including both reads and writes", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "IO read (-) / write (+)", + "axisLabel": "read (–) / write (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14612,6 +9110,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14635,10 +9134,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -14660,298 +9155,13 @@ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -14963,72 +9173,67 @@ "h": 10, "w": 12, "x": 0, - "y": 31 + "y": 29 }, "id": 9, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 4, - "legendFormat": "{{device}} - Reads completed", + "legendFormat": "{{device}} - Read", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Writes completed", + "legendFormat": "{{device}} - Write", + "range": true, "refId": "B", "step": 240 } ], - "title": "Disk IOps Completed", + "title": "Disk Read/Write IOps", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "The number of bytes read from or written to the device per second", + "description": "Number of bytes read from or written to the device per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes read (-) / write (+)", + "axisLabel": "read (–) / write (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -15037,6 +9242,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15060,10 +9266,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -15085,298 +9287,13 @@ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -15388,82 +9305,80 @@ "h": 10, "w": 12, "x": 12, - "y": 31 + "y": 29 }, "id": 33, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 4, - "legendFormat": "{{device}} - Read bytes", + "legendFormat": "{{device}} - Read", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", + "exemplar": false, "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Written bytes", + "instant": false, + "legendFormat": "{{device}} - Write", + "range": true, "refId": "B", "step": 240 } ], - "title": "Disk R/W Data", + "title": "Disk Read/Write Data", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "description": "Average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "time. read (-) / write (+)", + "axisLabel": "read (–) / write (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 30, + "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15487,10 +9402,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -15512,298 +9423,13 @@ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -15815,52 +9441,43 @@ "h": 10, "w": 12, "x": 0, - "y": 41 + "y": 389 }, "id": 37, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - Read wait time avg", + "legendFormat": "{{device}} - Read", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}} - Write wait time avg", + "legendFormat": "{{device}} - Write", + "range": true, "refId": "B", "step": 240 } @@ -15871,20 +9488,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "The average queue length of the requests that were issued to the device", + "description": "Average queue length of the requests that were issued to the device", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "aqu-sz", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -15893,6 +9512,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15917,10 +9537,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -15930,7 +9546,7 @@ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" + "options": "/sda_*/" }, "properties": [ { @@ -15941,291 +9557,6 @@ } } ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] } ] }, @@ -16233,38 +9564,34 @@ "h": 10, "w": 12, "x": 12, - "y": 41 + "y": 389 }, "id": 35, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "interval": "", - "intervalFactor": 4, "legendFormat": "{{device}}", + "range": true, "refId": "A", "step": 240 } @@ -16275,20 +9602,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "The number of read and write requests merged per second that were queued to the device", + "description": "Number of read and write requests merged per second that were queued to the device", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "I/Os", + "axisLabel": "read (–) / write (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -16297,6 +9626,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16320,10 +9650,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -16345,298 +9671,13 @@ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -16648,48 +9689,41 @@ "h": 10, "w": 12, "x": 0, - "y": 51 + "y": 399 }, "id": 133, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Read merged", + "legendFormat": "{{device}} - Read", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Write merged", + "legendFormat": "{{device}} - Write", + "range": true, "refId": "B", "step": 240 } @@ -16700,28 +9734,31 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", + "description": "Percentage of time the disk spent actively processing I/O operations, including general I/O, discards (TRIM), and write cache flushes", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "%util", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 30, + "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16746,10 +9783,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -16759,298 +9792,13 @@ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -17062,52 +9810,54 @@ "h": 10, "w": 12, "x": 12, - "y": 51 + "y": 399 }, "id": 36, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - IO", + "legendFormat": "{{device}} - General IO", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - discard", + "legendFormat": "{{device}} - Discard/TRIM", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_flush_requests_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}} - Flush (write cache)", + "range": true, + "refId": "C", + "step": 240 } ], "title": "Time Spent Doing I/Os", @@ -17116,20 +9866,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", + "description": "Per-second rate of discard (TRIM) and flush (write cache) operations. Useful for monitoring low-level disk activity on SSDs and advanced storage", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "Outstanding req.", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -17138,6 +9890,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -17156,317 +9909,27 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "ops" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -17478,62 +9941,78 @@ "h": 10, "w": 12, "x": 0, - "y": 61 + "y": 409 }, - "id": 34, - "links": [], + "id": 301, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - IO now", + "legendFormat": "{{device}} - Discards completed", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}} - Discards merged", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_flush_requests_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}} - Flush", + "range": true, + "refId": "C", + "step": 240 } ], - "title": "Instantaneous Queue Size", + "title": "Disk Ops Discards / Flush", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Shows how many disk sectors are discarded (TRIMed) per second. Useful for monitoring SSD behavior and storage efficiency", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "IOs", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -17542,6 +10021,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -17565,311 +10045,22 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "iops" + "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -17881,65 +10072,154 @@ "h": 10, "w": 12, "x": 12, - "y": 61 + "y": 409 }, - "id": 301, - "links": [], + "id": 326, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_disk_discarded_sectors_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - Discards completed", + "legendFormat": "{{device}}", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}} - Discards merged", - "refId": "B", - "step": 240 } ], - "title": "Disk IOps Discards completed / merged", + "title": "Disk Sectors Discarded Successfully", "type": "timeseries" - } - ], - "targets": [ + }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "refId": "A" + "description": "Number of in-progress I/O requests at the time of sampling (active requests in the disk queue)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 419 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Instantaneous Queue Size", + "type": "timeseries" } ], "title": "Storage Disk", @@ -17947,10 +10227,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -17962,20 +10238,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Number of file descriptors currently allocated system-wide versus the system limit. Important for detecting descriptor exhaustion risks", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -17984,6 +10262,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18008,300 +10287,84 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 43, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Available", - "metric": "", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Free", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Size", - "refId": "C", - "step": 240 - } - ], - "title": "Filesystem space available", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "file nodes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 41, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Free file nodes", - "refId": "A", - "step": 240 - } - ], - "title": "File Nodes Free", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "files", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ + "properties": [ { - "color": "green" + "id": "custom.fillOpacity", + "value": 0 }, { - "color": "red", - "value": 80 + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } } ] - }, - "unit": "short" - }, - "overrides": [] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 56 + "y": 30 }, "id": 28, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 4, "legendFormat": "Max open files", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, "legendFormat": "Open files", + "range": true, "refId": "B", "step": 240 } @@ -18312,20 +10375,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Number of free file nodes (inodes) available per mounted filesystem. A low count may prevent file creation even if disk space is available", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "file Nodes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -18334,6 +10399,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18358,10 +10424,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -18373,63 +10435,60 @@ "h": 10, "w": 12, "x": 12, - "y": 56 + "y": 30 }, - "id": 219, - "links": [], + "id": 41, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "editorMode": "code", + "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - File nodes total", + "legendFormat": "{{mountpoint}}", + "range": true, "refId": "A", "step": 240 } ], - "title": "File Nodes Size", + "title": "File Nodes Free", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Indicates filesystems mounted in read-only mode or reporting device-level I/O errors.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -18438,6 +10497,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18463,1277 +10523,81 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "/ ReadOnly" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 66 - }, - "id": 44, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - ReadOnly", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Device error", - "refId": "B", - "step": 240 - } - ], - "title": "Filesystem in ReadOnly / Error", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Storage Filesystem", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 272, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "receive_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "receive_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 60, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic by Packets", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 142, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive errors", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Rransmit errors", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 143, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive drop", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit drop", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Drop", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 43 - }, - "id": 141, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive compressed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit compressed", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Compressed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 146, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive multicast", - "refId": "A", - "step": 240 - } - ], - "title": "Network Traffic Multicast", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 144, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive fifo", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit fifo", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Fifo", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "pps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 145, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive frame", - "refId": "A", - "step": 240 - } - ], - "title": "Network Traffic Frame", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "unit": "bool_yes_no" }, "overrides": [] }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 63 - }, - "id": 231, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Statistic transmit_carrier", - "refId": "A", - "step": 240 - } - ], - "title": "Network Traffic Carrier", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 73 + "y": 370 }, - "id": 232, - "links": [], + "id": 44, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true, - "width": 300 + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit colls", + "legendFormat": "{{mountpoint}} - ReadOnly", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", + "format": "time_series", + "interval": "", + "legendFormat": "{{mountpoint}} - Device error", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Network Traffic Colls", + "title": "Filesystem in ReadOnly / Error", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Number of file nodes (inodes) available per mounted filesystem. Reflects maximum file capacity regardless of disk size", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "entries", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19742,6 +10606,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -19766,10 +10631,1360 @@ "steps": [ { "color": "green" - }, + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 370 + }, + "id": 219, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "File Nodes Size", + "type": "timeseries" + } + ], + "title": "Storage Filesystem", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 272, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of network packets received and transmitted per second, by interface.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "color": "red", - "value": 80 + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic by Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of packet-level errors for each network interface. Receive errors may indicate physical or driver issues; transmit errors may reflect collisions or hardware faults", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 142, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of dropped packets per network interface. Receive drops can indicate buffer overflow or driver issues; transmit drops may result from outbound congestion or queuing limits", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 251 + }, + "id": 143, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Drop", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of compressed network packets received and transmitted per interface. These are common in low-bandwidth or special interfaces like PPP or SLIP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 251 + }, + "id": 141, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Compressed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of incoming multicast packets received per network interface. Multicast is used by protocols such as mDNS, SSDP, and some streaming or cluster services", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 261 + }, + "id": 146, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Multicast", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of received packets that could not be processed due to missing protocol or handler in the kernel. May indicate unsupported traffic or misconfiguration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 261 + }, + "id": 327, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_nohandler_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic NoHandler", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of frame errors on received packets, typically caused by physical layer issues such as bad cables, duplex mismatches, or hardware problems", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 271 + }, + "id": 145, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Frame", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Tracks FIFO buffer overrun errors on network interfaces. These occur when incoming or outgoing packets are dropped due to queue or buffer overflows, often indicating congestion or hardware limits", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 271 + }, + "id": 144, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Fifo", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of packet collisions detected during transmission. Mostly relevant on half-duplex or legacy Ethernet networks", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 281 + }, + "id": 232, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Collision", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of carrier errors during transmission. These typically indicate physical layer issues like faulty cabling or duplex mismatches", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 281 + }, + "id": 231, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Carrier Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of ARP entries per interface. Useful for detecting excessive ARP traffic or table growth due to scanning or misconfiguration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 291 + }, + "id": 230, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "{{ device }} ARP Table", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "ARP Entries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Current and maximum connection tracking entries used by Netfilter (nf_conntrack). High usage approaching the limit may cause packet drops or connection issues", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } ] }, @@ -19785,13 +12000,23 @@ { "id": "color", "value": { - "fixedColor": "#890F02", + "fixedColor": "dark-red", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } } ] } @@ -19801,73 +12026,69 @@ "h": 10, "w": 12, "x": 12, - "y": 73 + "y": 291 }, "id": 61, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, "legendFormat": "NF conntrack entries", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, "legendFormat": "NF conntrack limit", + "range": true, "refId": "B", "step": 240 } ], - "title": "NF Contrack", + "title": "NF Conntrack", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Operational and physical link status of each network interface. Values are Yes for 'up' or link present, and No for 'down' or no carrier.\"", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "Entries", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19876,6 +12097,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -19894,20 +12116,15 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bool_yes_no" }, "overrides": [] }, @@ -19915,86 +12132,64 @@ "h": 10, "w": 12, "x": 0, - "y": 83 + "y": 301 }, - "id": 230, - "links": [], + "id": 309, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ device }} - ARP entries", + "hide": true, + "legendFormat": "{{interface}} - Operational state UP", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link", + "refId": "B" } ], - "title": "ARP Entries", + "title": "Network Operational Status", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Maximum speed of each network interface as reported by the operating system. This is a static hardware capability, not current throughput", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "decimals": 0, + "fieldMinMax": false, "links": [], "mappings": [], "min": 0, @@ -20003,202 +12198,69 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "bytes" + "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, - "w": 12, + "w": 6, "x": 12, - "y": 83 - }, - "id": 288, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ device }} - Bytes", - "refId": "A", - "step": 240 - } - ], - "title": "MTU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 93 + "y": 301 }, "id": 280, - "links": [], "options": { + "displayMode": "basic", "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "maxVizHeight": 30, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"} * 8", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ device }} - Speed", + "legendFormat": "{{ device }}", + "range": true, "refId": "A", "step": 240 } ], "title": "Speed", - "type": "timeseries" + "type": "bargauge" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "MTU (Maximum Transmission Unit) in bytes for each network interface. Affects packet size and transmission efficiency", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "packets", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "decimals": 0, "links": [], @@ -20209,10 +12271,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -20222,63 +12280,83 @@ }, "gridPos": { "h": 10, - "w": 12, - "x": 12, - "y": 93 + "w": 6, + "x": 18, + "y": 301 }, - "id": 289, - "links": [], + "id": 288, "options": { + "displayMode": "basic", "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "maxVizHeight": 30, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ device }} - Interface transmit queue length", + "legendFormat": "{{ device }}", + "range": true, "refId": "A", "step": 240 } ], - "title": "Queue Length", - "type": "timeseries" - }, + "title": "MTU", + "type": "bargauge" + } + ], + "title": "Network Traffic", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 273, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Tracks TCP socket usage and memory per node", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packetes drop (-) / process (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20287,6 +12365,795 @@ "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 63, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Allocated Sockets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "In-Use Sockets", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Orphaned Sockets", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TIME_WAIT Sockets", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Sockstat TCP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of UDP and UDPLite sockets currently in use", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 124, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDPLite - In-Use Sockets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDP - In-Use Sockets", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Sockstat UDP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Total number of sockets currently in use across all protocols (TCP, UDP, UNIX, etc.), as reported by /proc/net/sockstat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 126, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Total sockets", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Sockstat Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of FRAG and RAW sockets currently in use. RAW sockets are used for custom protocols or tools like ping; FRAG sockets are used internally for IP packet defragmentation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 125, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "FRAG - In-Use Sockets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "RAW - In-Use Sockets", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat FRAG / RAW", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Kernel memory used by TCP, UDP, and IP fragmentation buffers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 220, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TCP", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDP", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "Fragmentation", + "range": true, + "refId": "C" + } + ], + "title": "Sockstat Memory Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Average memory used per socket (TCP/UDP). Helps tune net.ipv4.tcp_rmem / tcp_wmem", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 339, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"} / node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TCP", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"} / node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDP", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Sockstat Average Socket Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "TCP/UDP socket memory usage in kernel (in pages)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 336, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TCP", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDP", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP/UDP Kernel Buffer Memory Pages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Packets processed and dropped by the softnet network stack per CPU. Drops may indicate CPU saturation or network driver limitations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "drop (-) / process (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20310,14 +13177,10 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, "overrides": [ { @@ -20337,18 +13200,16 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 103 + "x": 12, + "y": 62 }, "id": 290, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -20356,35 +13217,30 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "CPU {{cpu}} - Processed", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", - "intervalFactor": 1, "legendFormat": "CPU {{cpu}} - Dropped", + "range": true, "refId": "B", "step": 240 } @@ -20395,19 +13251,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "How often the kernel was unable to process all packets in the softnet queue before time ran out. Frequent squeezes may indicate CPU contention or driver inefficiency", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20416,6 +13275,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20446,25 +13306,23 @@ } ] }, - "unit": "short" + "unit": "eps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 103 + "x": 0, + "y": 72 }, "id": 310, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -20472,22 +13330,20 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU {{cpu}} - Squeezed", + "legendFormat": "CPU {{cpu}} - Times Squeezed", + "range": true, "refId": "A", "step": 240 } @@ -20498,19 +13354,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Tracks the number of packets processed or dropped by Receive Packet Steering (RPS), a mechanism to distribute packet processing across CPUs", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20519,6 +13378,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20542,330 +13402,46 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 113 - }, - "id": 309, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{interface}} - Operational state UP", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "instant": false, - "legendFormat": "{{device}} - Physical link state", - "refId": "B" - } - ], - "title": "Network Operational Status", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], - "title": "Network Traffic", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 273, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ + "properties": [ { - "color": "green" + "id": "custom.transform", + "value": "negative-Y" }, { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } } ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 63, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_alloc - Allocated sockets", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_inuse - Tcp sockets currently in use", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_mem - Used memory for tcp", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_orphan - Orphan sockets", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_tw - Sockets wating close", - "refId": "E", - "step": 240 - } - ], - "title": "Sockstat TCP", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 48 + "y": 72 }, - "id": 124, - "links": [], + "id": 330, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -20873,412 +13449,36 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_softnet_received_rps_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "legendFormat": "CPU {{cpu}} - Processed", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_softnet_flow_limit_count_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "UDP_inuse - Udp sockets currently in use", + "legendFormat": "CPU {{cpu}} - Dropped", + "range": true, "refId": "B", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "UDP_mem - Used memory for udp", - "refId": "C", - "step": 240 } ], - "title": "Sockstat UDP", + "title": "Softnet RPS", "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "counter", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 125, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "FRAG_inuse - Frag sockets currently in use", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RAW_inuse - Raw sockets currently in use", - "refId": "C", - "step": 240 - } - ], - "title": "Sockstat FRAG / RAW", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 220, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "mem_bytes - TCP sockets in that state", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "mem_bytes - UDP sockets in that state", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 1, - "legendFormat": "FRAG_memory - Used memory for frag", - "refId": "C" - } - ], - "title": "Sockstat Memory Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "sockets", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 68 - }, - "id": 126, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Sockets_used - Sockets currently in use", - "refId": "A", - "step": 240 - } - ], - "title": "Sockstat Used", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" } ], "title": "Network Sockstat", @@ -21286,10 +13486,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -21301,19 +13497,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Rate of octets sent and received at the IP layer, as reported by /proc/net/netstat", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "octects out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -21322,6 +13521,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21345,20 +13545,16 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*Out.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -21373,17 +13569,15 @@ "h": 10, "w": 12, "x": 0, - "y": 49 + "y": 163 }, "id": 221, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -21391,34 +13585,29 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "InOctets - Received octets", + "legendFormat": "IP Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "OutOctets - Sent octets", + "legendFormat": "IP Tx out", + "range": true, "refId": "B", "step": 240 } @@ -21429,123 +13618,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Rate of TCP segments sent and received per second, including data and control segments", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "datagrams", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 81, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 300 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Forwarding - IP forwarding", - "refId": "A", - "step": 240 - } - ], - "title": "Netstat IP Forwarding", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "messages out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -21554,6 +13642,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21584,256 +13673,13 @@ } ] }, - "unit": "short" + "unit": "pps" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 59 - }, - "id": 115, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", - "refId": "B", - "step": 240 - } - ], - "title": "ICMP In / Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "messages out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 59 - }, - "id": 50, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", - "refId": "A", - "step": 240 - } - ], - "title": "ICMP Errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "datagrams out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -21847,357 +13693,53 @@ "id": "byRegexp", "options": "/.*Snd.*/" }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] + "properties": [] } ] }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 69 - }, - "id": 55, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "InDatagrams - Datagrams received", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "OutDatagrams - Datagrams sent", - "refId": "B", - "step": 240 - } - ], - "title": "UDP In / Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "datagrams", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 69 - }, - "id": 109, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RcvbufErrors - UDP buffer errors received", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "SndbufErrors - UDP buffer errors send", - "refId": "E", - "step": 240 - } - ], - "title": "UDP Errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "datagrams out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Snd.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 79 + "y": 163 }, "id": 299, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "instant": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "legendFormat": "TCP Rx in", "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "legendFormat": "TCP Tx out", + "range": true, "refId": "B", "step": 240 } @@ -22208,20 +13750,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Rate of UDP datagrams sent and received per second, based on /proc/net/netstat", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22230,6 +13774,128 @@ "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 193 + }, + "id": 55, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "UDP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of ICMP messages sent and received per second, including error and control messages", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22248,7 +13914,6 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -22261,128 +13926,92 @@ } ] }, - "unit": "short" + "unit": "pps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 79 + "y": 193 }, - "id": 104, - "links": [], + "id": 115, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "legendFormat": "ICMP Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "legendFormat": "ICMP Tx out", + "range": true, "refId": "B", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "OutRsts - Segments sent with RST flag", - "refId": "F" } ], - "title": "TCP Errors", + "title": "ICMP In / Out", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Tracks various TCP error and congestion-related events, including retransmissions, timeouts, dropped connections, and buffer issues", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "connections", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22391,6 +14020,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22410,6 +14040,172 @@ "links": [], "mappings": [], "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 203 + }, + "id": 104, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Listen Overflows", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Listen Drops", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "SYN Retransmits", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Segment Retransmits", + "range": true, + "refId": "D" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Receive Errors", + "range": true, + "refId": "E" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "RST Sent", + "range": true, + "refId": "F" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPRcvQDrop{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Receive Queue Drops", + "range": true, + "refId": "G" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPOFOQueue{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Out-of-order Queued", + "range": true, + "refId": "H" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPTimeouts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "TCP Timeouts", + "range": true, + "refId": "I" + } + ], + "title": "TCP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of UDP and UDPLite datagram delivery errors, including missing listeners, buffer overflows, and protocol-specific issues", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -22422,13 +14218,395 @@ } ] }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 203 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Rx in Errors", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP No Listener", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "UDPLite Rx in Errors", + "range": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Rx in Buffer Errors", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Tx out Buffer Errors", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "UDP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of incoming ICMP messages that contained protocol-specific errors, such as bad checksums or invalid lengths", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 213 + }, + "id": 50, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "ICMP Rx In", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "ICMP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Rate of TCP SYN cookies sent, validated, and failed. These are used to protect against SYN flood attacks and manage TCP handshake resources under load", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "eps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Failed.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 213 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "SYN Cookies Failed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "SYN Cookies Validated", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "SYN Cookies Sent", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "TCP SynCookie", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of currently established TCP connections and the system's max supported limit. On Linux, MaxConn may return -1 to indicate a dynamic/unlimited configuration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*MaxConn *./" + "options": "/.*Max*./" }, "properties": [ { @@ -22441,6 +14619,16 @@ { "id": "custom.fillOpacity", "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } } ] } @@ -22450,54 +14638,45 @@ "h": 10, "w": 12, "x": 0, - "y": 89 + "y": 223 }, "id": 85, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "legendFormat": "Current Connections", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dinamic is \"-1\")", + "legendFormat": "Max Connections", + "range": true, "refId": "B", "step": 240 } @@ -22508,20 +14687,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Number of UDP packets currently queued in the receive (RX) and transmit (TX) buffers. A growing queue may indicate a bottleneck", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter out (-) / in (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22530,6 +14711,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22553,119 +14735,82 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Sent.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 89 + "y": 223 }, - "id": 91, - "links": [], + "id": 337, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"rx\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "legendFormat": "UDP Rx in Queue", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"tx\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "SyncookiesRecv - SYN cookies received", + "legendFormat": "UDP Tx out Queue", + "range": true, "refId": "B", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "SyncookiesSent - SYN cookies sent", - "refId": "C", - "step": 240 } ], - "title": "TCP SynCookie", + "title": "UDP Queue", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, + "description": "Rate of TCP connection initiations per second. 'Active' opens are initiated by this host. 'Passive' opens are accepted from incoming connections", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "connections", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22674,6 +14819,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22698,10 +14844,116 @@ "steps": [ { "color": "green" - }, + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 233 + }, + "id": 82, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Active Opens", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Passive Opens", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP Direct Transition", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of TCP sockets in key connection states. Requires the --collector.tcpstat flag on node_exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -22712,79 +14964,89 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 99 + "x": 12, + "y": 233 }, - "id": 82, - "links": [], + "id": 320, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "legendFormat": "Established", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", + "legendFormat": "FIN_WAIT2", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Listen", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TIME_WAIT", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"close_wait\", instance=\"$node\", job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "CLOSE_WAIT", + "range": true, + "refId": "E", + "step": 240 } ], - "title": "TCP Direct Transition", + "title": "TCP Stat", "type": "timeseries" } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" - } - ], "title": "Network Netstat", "type": "row" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "gridPos": { "h": 1, "w": 24, @@ -22796,20 +15058,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Duration of each individual collector executed during a Node Exporter scrape. Useful for identifying slow or failing collectors", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "seconds", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22818,6 +15082,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22856,40 +15121,35 @@ "h": 10, "w": 12, "x": 0, - "y": 50 + "y": 164 }, "id": 40, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "editorMode": "code", "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{collector}} - Scrape duration", + "legendFormat": "{{collector}}", + "range": true, "refId": "A", "step": 240 } @@ -22900,20 +15160,22 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "description": "", + "description": "Rate of CPU time used by the process exposing this metric (user + system mode)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22922,10 +15184,8 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -22948,10 +15208,266 @@ "steps": [ { "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 164 + }, + "id": 308, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Process CPU Usage", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Exporter Process CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Tracks the memory usage of the process exposing this metric (e.g., node_exporter), including current virtual memory and maximum virtual memory limit", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Virtual Memory Limit" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 }, { - "color": "red", - "value": 80 + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Virtual Memory" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 10, + "x": 0, + "y": 174 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "Virtual Memory", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "Virtual Memory Limit", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Exporter Processes Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Number of file descriptors used by the exporter process versus its configured limit", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" } ] }, @@ -22961,19 +15477,53 @@ { "matcher": { "id": "byRegexp", - "options": "/.*error.*/" + "options": "/.*Max*./" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#F2495C", + "fixedColor": "#890F02", "mode": "fixed" } }, { - "id": "custom.transform", - "value": "negative-Y" + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Open file descriptors" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } } ] } @@ -22981,114 +15531,174 @@ }, "gridPos": { "h": 10, - "w": 12, - "x": 12, - "y": 50 + "w": 10, + "x": 10, + "y": 174 }, - "id": 157, - "links": [], + "id": 64, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, + "editorMode": "code", + "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{collector}} - Scrape success", + "legendFormat": "Maximum open file descriptors", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, + "editorMode": "code", + "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{collector}} - Scrape textfile error (1 = true)", + "legendFormat": "Open file descriptors", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Exporter File Descriptor Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "description": "Shows whether each Node Exporter collector scraped successfully (1 = success, 0 = failure), and whether the textfile collector returned an error.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 4, + "x": 20, + "y": 174 + }, + "id": 157, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{collector}}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "1 - node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "textfile", + "range": true, "refId": "B", "step": 240 } ], "title": "Node Exporter Scrape", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refId": "A" + "type": "bargauge" } ], "title": "Node Exporter", "type": "row" } ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", + "refresh": "1m", + "schemaVersion": 41, "tags": [ "linux" ], "templating": { "list": [ { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, + "current": {}, "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", + "label": "Datasource", + "name": "ds_prometheus", "options": [], "query": "prometheus", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { "current": {}, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, "definition": "", - "hide": 0, "includeAll": false, "label": "Job", - "multi": false, "name": "job", "options": [], "query": { @@ -23097,59 +15707,48 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${ds_prometheus}" }, - "definition": "label_values(node_uname_info{job=\"$job\"}, instance)", - "hide": 0, + "definition": "label_values(node_uname_info{job=\"$job\"}, nodename)", "includeAll": false, - "label": "Host:", - "multi": false, + "label": "Nodename", + "name": "nodename", + "options": [], + "query": { + "query": "label_values(node_uname_info{job=\"$job\"}, nodename)", + "refId": "Prometheus-nodename-Variable-Query" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${ds_prometheus}" + }, + "definition": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", + "includeAll": false, + "label": "Instance", "name": "node", "options": [], "query": { - "query": "label_values(node_uname_info{job=\"$job\"}, instance)", + "query": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", "refId": "Prometheus-node-Variable-Query" }, "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" - }, - "hide": 2, - "includeAll": false, - "multi": false, - "name": "diskdevices", - "options": [ - { - "selected": true, - "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" - } - ], - "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "skipUrlSync": false, - "type": "custom" + "type": "query" } ] }, @@ -23157,34 +15756,11 @@ "from": "now-24h", "to": "now" }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, + "timepicker": {}, "timezone": "browser", "title": "Node Exporter Full", "uid": "rYdddlPWk", - "version": 9, - "weekStart": "" -} + "version": 98, + "weekStart": "", + "gnetId": 1860 +} \ No newline at end of file diff --git a/hosts/ildkule/services/monitoring/dashboards/postgres.json b/hosts/ildkule/services/monitoring/dashboards/postgres.json index 232582e..6a07073 100644 --- a/hosts/ildkule/services/monitoring/dashboards/postgres.json +++ b/hosts/ildkule/services/monitoring/dashboards/postgres.json @@ -328,7 +328,7 @@ "rgba(50, 172, 45, 0.97)" ], "datasource": "${DS_PROMETHEUS}", - "format": "decbytes", + "format": "short", "gauge": { "maxValue": 100, "minValue": 0, @@ -411,7 +411,7 @@ "rgba(50, 172, 45, 0.97)" ], "datasource": "${DS_PROMETHEUS}", - "format": "decbytes", + "format": "short", "gauge": { "maxValue": 100, "minValue": 0, @@ -1410,7 +1410,7 @@ "tableColumn": "", "targets": [ { - "expr": "pg_settings_seq_page_cost", + "expr": "pg_settings_seq_page_cost{instance=\"$instance\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" @@ -1872,7 +1872,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -1966,7 +1966,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2060,7 +2060,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2251,7 +2251,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2439,7 +2439,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2589,35 +2589,35 @@ "steppedLine": false, "targets": [ { - "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=\"$instance\"}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_backend_total{instance=\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "buffers_backend", "refId": "A" }, { - "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=\"$instance\"}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_alloc_total{instance=\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "buffers_alloc", "refId": "B" }, { - "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=\"$instance\"}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync_total{instance=\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "backend_fsync", "refId": "C" }, { - "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=\"$instance\"}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint_total{instance=\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "buffers_checkpoint", "refId": "D" }, { - "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=\"$instance\"}[5m])", + "expr": "irate(pg_stat_bgwriter_buffers_clean_total{instance=\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "buffers_clean", @@ -2886,14 +2886,14 @@ "steppedLine": false, "targets": [ { - "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{instance=\"$instance\"}[5m])", + "expr": "irate(pg_stat_bgwriter_checkpoint_write_time_total{instance=\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.", "refId": "B" }, { - "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{instance=\"$instance\"}[5m])", + "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time_total{instance=\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.", @@ -3164,4 +3164,4 @@ "title": "PostgreSQL Database", "uid": "000000039", "version": 1 -} +} \ No newline at end of file diff --git a/hosts/ildkule/services/monitoring/dashboards/synapse.json b/hosts/ildkule/services/monitoring/dashboards/synapse.json index 0c61a16..ceacc10 100644 --- a/hosts/ildkule/services/monitoring/dashboards/synapse.json +++ b/hosts/ildkule/services/monitoring/dashboards/synapse.json @@ -2,7 +2,7 @@ "__inputs": [ { "name": "DS_PROMETHEUS", - "label": "Prometheus", + "label": "prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", @@ -15,13 +15,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.2.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph (old)", - "version": "" + "version": "12.3.1" }, { "type": "panel", @@ -47,7 +41,7 @@ { "builtIn": 1, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "enable": false, "hide": true, @@ -60,10 +54,11 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "enable": true, - "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"master\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"master\"}", + "expr": "(\n changes(process_start_time_seconds{job=\"synapse\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) * on (instance, job, index) group_left(version) synapse_build_info{job=\"synapse\"}", + "hide": false, "iconColor": "purple", "name": "deploys", "titleFormat": "Deployed {{version}}" @@ -87,14 +82,9 @@ "type": "dashboards" } ], - "liveNow": false, "panels": [ { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -103,33 +93,12 @@ }, "id": 73, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], "title": "Overview", "type": "row" }, { - "cards": { - "cardPadding": -1, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -152,14 +121,7 @@ "x": 0, "y": 1 }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, "id": 189, - "legend": { - "show": false - }, - "links": [], "options": { "calculate": false, "calculation": {}, @@ -189,7 +151,8 @@ }, "showValue": "never", "tooltip": { - "show": true, + "mode": "single", + "showColorScale": false, "yHistogram": true }, "yAxis": { @@ -198,14 +161,13 @@ "unit": "s" } }, - "pluginVersion": "9.2.2", - "reverseYBuckets": false, + "pluginVersion": "12.3.1", "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)", + "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le)", "format": "heatmap", "interval": "", "intervalFactor": 1, @@ -214,128 +176,314 @@ } ], "title": "Event Send Time (excluding errors, all workers)", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "s", - "logBase": 2, - "show": true - }, - "yBucketBound": "auto" + "type": "heatmap" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 35, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": 0 + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "s" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Avg" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 3 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "99%" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "90%" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "90%" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF7383", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "75%" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "75%" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "50%" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "50%" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "25%" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "25%" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#1F60C4", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "5%" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5%" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgb(255, 255, 255)", + "mode": "fixed" + } + }, + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.lineWidth", + "value": 3 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Local events being persisted" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96d98D", + "mode": "fixed" + } + }, + { + "id": "custom.showPoints", + "value": "always" + }, + { + "id": "unit", + "value": "hertz" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "All events being persisted" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B877D9", + "mode": "fixed" + } + }, + { + "id": "custom.showPoints", + "value": "always" + }, + { + "id": "unit", + "value": "hertz" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 }, - "hiddenSeries": false, "id": 152, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "connected", "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "Avg", - "fill": 0, - "linewidth": 3 + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - { - "alias": "99%", - "color": "#C4162A", - "fillBelowTo": "90%" - }, - { - "alias": "90%", - "color": "#FF7383", - "fillBelowTo": "75%" - }, - { - "alias": "75%", - "color": "#FFEE52", - "fillBelowTo": "50%" - }, - { - "alias": "50%", - "color": "#73BF69", - "fillBelowTo": "25%" - }, - { - "alias": "25%", - "color": "#1F60C4", - "fillBelowTo": "5%" - }, - { - "alias": "5%", - "lines": false - }, - { - "alias": "Average", - "color": "rgb(255, 255, 255)", - "lines": true, - "linewidth": 3 - }, - { - "alias": "Events", - "color": "#B877D9", - "hideTooltip": true, - "points": true, - "yaxis": 2, - "zindex": -3 + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "12.3.1", "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99%", @@ -343,9 +491,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -354,9 +502,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "75%", @@ -364,9 +512,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "50%", @@ -374,150 +522,148 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))", "legendFormat": "25%", "refId": "F" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) by (le))", "legendFormat": "5%", "refId": "G" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))", + "editorMode": "code", + "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size]))", "legendFormat": "Average", + "range": true, "refId": "H" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))", + "editorMode": "code", + "expr": "sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",server_name=\"$server_name\",code=~\"2..\"}[$bucket_size]))", "hide": false, "instant": false, - "legendFormat": "Events", + "legendFormat": "Local events being persisted", "refId": "E" - } - ], - "thresholds": [ - { - "$$hashKey": "object:283", - "colorMode": "warning", - "fill": false, - "line": true, - "op": "gt", - "value": 1, - "yaxis": "left" }, { - "$$hashKey": "object:284", - "colorMode": "critical", - "fill": false, - "line": true, - "op": "gt", - "value": 2, - "yaxis": "left" + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(synapse_storage_events_persisted_events_total{server_name=\"$server_name\"}[$bucket_size]))", + "hide": false, + "instant": false, + "legendFormat": "All events being persisted", + "refId": "I" } ], - "timeRegions": [], "title": "Event Send Time Quantiles (excluding errors, all workers)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:255", - "format": "s", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:256", - "format": "hertz", - "label": "", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "links": [], + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 10 }, - "hiddenSeries": false, "id": 75, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 3, - "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "12.3.1", "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(process_cpu_seconds_total{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -525,109 +671,100 @@ "refId": "A" } ], - "thresholds": [ - { - "$$hashKey": "object:566", - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1, - "yaxis": "left" - } - ], - "timeRegions": [], "title": "CPU usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:538", - "format": "percentunit", - "logBase": 1, - "max": "1.5", - "min": "0", - "show": true - }, - { - "$$hashKey": "object:539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 10 }, - "hiddenSeries": false, "id": 198, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 3, - "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "12.3.1", "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "process_resident_memory_bytes{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -638,53 +775,22 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})", + "expr": "sum(process_resident_memory_bytes{job=~\"$job\",index=~\"$index\"}) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "hide": true, "interval": "", "legendFormat": "total", "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Memory", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "transformations": [], - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1560", - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:1561", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -692,12 +798,14 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMax": 1, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -706,6 +814,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 10, "pointSize": 5, @@ -713,6 +822,7 @@ "type": "linear" }, "showPoints": "never", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -728,7 +838,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] } @@ -750,18 +860,20 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "synapse_build_info{instance=\"$instance\", job=\"master\"} - 1", + "expr": "(\n synapse_build_info{job=\"synapse\"} * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) - 1", "legendFormat": "version {{version}}", "range": true, "refId": "deployed_synapse_versions" @@ -771,69 +883,115 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/max$/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 19 }, - "hiddenSeries": false, "id": 37, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:639", - "alias": "/max$/", - "color": "#890F02", - "fill": 0, - "legend": false + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "12.3.1", "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "process_open_fds{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "hide": false, "interval": "", @@ -844,9 +1002,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "process_max_fds{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "hide": true, "interval": "", @@ -856,45 +1014,11 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Open FDs", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:650", - "format": "none", - "label": "", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:651", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "gridPos": { "h": 1, "w": 24, @@ -904,79 +1028,34 @@ "id": 54, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 27 }, - "hiddenSeries": false, "id": 5, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 3, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:1240", - "alias": "/user/" - }, - { - "$$hashKey": "object:1241", - "alias": "/system/" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(process_cpu_system_seconds_total{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} system ", @@ -987,9 +1066,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(process_cpu_user_seconds_total{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "hide": false, "interval": "", @@ -999,75 +1078,13 @@ "step": 20 } ], - "thresholds": [ - { - "$$hashKey": "object:1278", - "colorMode": "custom", - "fillColor": "rgba(255, 255, 255, 1)", - "line": true, - "lineColor": "rgba(216, 200, 27, 0.27)", - "op": "gt", - "value": 0.5, - "yaxis": "left" - }, - { - "$$hashKey": "object:1279", - "colorMode": "custom", - "fillColor": "rgba(255, 255, 255, 1)", - "line": true, - "lineColor": "rgb(87, 6, 16)", - "op": "gt", - "value": 0.8, - "yaxis": "left" - }, - { - "$$hashKey": "object:1498", - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1, - "yaxis": "left" - } - ], - "timeRegions": [], "title": "CPU", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1250", - "format": "percentunit", - "label": "", - "logBase": 1, - "max": "1.2", - "min": 0, - "show": true - }, - { - "$$hashKey": "object:1251", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan", "fieldConfig": { @@ -1130,8 +1147,6 @@ "y": 27 }, "id": 105, - "interval": "", - "links": [], "options": { "legend": { "calcs": [], @@ -1149,10 +1164,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "histogram_quantile(0.999, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))", + "expr": "histogram_quantile(0.999, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",job=~\"$job\"}[$bucket_size])) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "hide": false, "interval": "", "legendFormat": "{{job}}-{{index}} 99.9%", @@ -1161,10 +1176,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))", + "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",job=~\"$job\"}[$bucket_size])) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1175,10 +1190,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))", + "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",job=~\"$job\"}[$bucket_size])) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1188,9 +1203,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))", + "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",job=~\"$job\"}[$bucket_size])) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} 90%", @@ -1199,9 +1214,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])", + "expr": "(\n\trate(python_twisted_reactor_tick_time_sum{index=~\"$index\",job=~\"$job\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n\tsynapse_server_name_info{server_name=\"$server_name\"}\n) / (\n\trate(python_twisted_reactor_tick_time_count{index=~\"$index\",job=~\"$job\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n\tsynapse_server_name_info{server_name=\"$server_name\"}\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} mean", @@ -1212,64 +1227,32 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 34 }, - "hiddenSeries": false, "id": 34, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "process_resident_memory_bytes{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1280,53 +1263,20 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})", + "expr": "sum by (server_name) (\n process_resident_memory_bytes{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n)", "interval": "", "legendFormat": "total", "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Memory", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "transformations": [], - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1334,54 +1284,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 34 }, - "hiddenSeries": false, "id": 49, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/^up/", - "legend": false, - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "scrape_duration_seconds{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1390,49 +1309,13 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Prometheus scrape time", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "decimals": 0, - "format": "none", - "label": "", - "logBase": 1, - "max": "0", - "min": "-1", - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1440,57 +1323,24 @@ }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 41 }, - "hiddenSeries": false, "id": 53, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:116", - "alias": "/^version .*/", - "lines": true, - "linewidth": 6, - "points": false - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "min_over_time(up{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}-{{index}}", @@ -1500,53 +1350,22 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1", + "expr": "(\n synapse_build_info{job=\"synapse\"} * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) - 1", "hide": false, "legendFormat": "version {{version}}", "range": true, "refId": "deployed_synapse_versions" } ], - "thresholds": [], - "timeRegions": [], "title": "Up", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1554,47 +1373,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 41 }, - "hiddenSeries": false, "id": 120, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_http_server_response_ru_utime_seconds{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "hide": false, "instant": false, @@ -1604,9 +1399,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_background_process_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "hide": false, "instant": false, @@ -1616,55 +1411,13 @@ "refId": "B" } ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1, - "yaxis": "left" - } - ], - "timeRegions": [], "title": "Stacked CPU usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:572", - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:573", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1672,99 +1425,46 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 48 }, - "hiddenSeries": false, "id": 136, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "legendFormat": "{{job}}-{{index}} {{method}}", "range": true, "refId": "A" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "legendFormat": "{{job}}-{{index}} {{method}} (federation)", "range": true, "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Outgoing HTTP request rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:123", - "format": "reqps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:124", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1840,10 +1540,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "synapse_threadpool_working_threads{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_threadpool_working_threads{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "interval": "", "legendFormat": "{{job}}-{{index}} {{name}}", "refId": "A" @@ -1853,24 +1553,11 @@ "type": "timeseries" } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "refId": "A" - } - ], "title": "Process info", "type": "row" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "gridPos": { "h": 1, "w": 24, @@ -1880,20 +1567,8 @@ "id": 56, "panels": [ { - "cards": { - "cardPadding": -1, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1916,14 +1591,7 @@ "x": 0, "y": 28 }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, "id": 85, - "legend": { - "show": false - }, - "links": [], "options": { "calculate": false, "calculation": {}, @@ -1963,13 +1631,12 @@ } }, "pluginVersion": "9.2.2", - "reverseYBuckets": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)", + "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\"}[$bucket_size])) by (le)", "format": "heatmap", "intervalFactor": 1, "legendFormat": "{{le}}", @@ -1977,81 +1644,36 @@ } ], "title": "Event Send Time (Including errors, across all workers)", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "s", - "logBase": 2, - "show": true - }, - "yBucketBound": "auto" + "type": "heatmap" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "", - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 28 }, - "hiddenSeries": false, "id": 33, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size])) without (job,index)", + "expr": "sum(rate(synapse_storage_events_persisted_events_total{server_name=\"$server_name\"}[$bucket_size])) without (job,index)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2061,176 +1683,68 @@ "target": "" } ], - "thresholds": [], - "timeRegions": [], "title": "Events Persisted (all workers)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:102", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:103", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "decimals": 1, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 37 }, - "hiddenSeries": false, "id": 40, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_storage_events_persisted_events_sep_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{origin_type}}", "refId": "D" } ], - "thresholds": [], - "timeRegions": [], "title": "Events/s Local vs Remote", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "decimals": 1, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 37 }, - "hiddenSeries": false, "id": 46, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "sum by(type) (rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -2239,185 +1753,36 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Events/s by Type", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": { - "irc-freenode (local)": "#EAB839" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "${datasource}" - }, - "decimals": 1, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 44 - }, - "hiddenSeries": false, - "id": 44, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "${datasource}" - }, - "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{origin_entity}} ({{origin_type}})", - "refId": "A", - "step": 20 - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Events/s by Origin", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "decimals": 1, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 44 }, - "hiddenSeries": false, "id": 45, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)", + "expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",server_name=\"$server_name\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{origin_entity}} ({{origin_type}})", @@ -2426,47 +1791,13 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Memberships/s by Origin", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:232", - "format": "hertz", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:233", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -2474,57 +1805,25 @@ }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 51 }, - "hiddenSeries": false, "id": 118, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeatDirection": "h", - "seriesOverrides": [ - { - "$$hashKey": "object:316", - "alias": "mean", - "linewidth": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", + "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -2534,9 +1833,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", + "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -2546,9 +1845,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", + "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} 90%", @@ -2557,9 +1856,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", + "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} 50%", @@ -2568,10 +1867,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)", + "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',server_name=\"$server_name\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} mean", @@ -2579,42 +1878,13 @@ "refId": "E" } ], - "thresholds": [], - "timeRegions": [], "title": "Event send time quantiles by worker", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:263", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:264", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "CPU and DB time spent on most expensive state resolution in a room, summed over all workers. This is a very rough proxy for \"how fast is state res\", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).\n", "fieldConfig": { @@ -2692,10 +1962,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))", + "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{server_name=\"$server_name\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -2706,10 +1976,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))", + "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{server_name=\"$server_name\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -2722,24 +1992,11 @@ "type": "timeseries" } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "refId": "A" - } - ], "title": "Event persistence", "type": "row" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "gridPos": { "h": 1, "w": 24, @@ -2749,68 +2006,33 @@ "id": 57, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 2, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, - "hiddenSeries": false, "id": 4, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_http_server_requests_received_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2819,116 +2041,37 @@ "step": 20 } ], - "thresholds": [ - { - "$$hashKey": "object:234", - "colorMode": "custom", - "fill": true, - "fillColor": "rgba(216, 200, 27, 0.27)", - "op": "gt", - "value": 100, - "yaxis": "left" - }, - { - "$$hashKey": "object:235", - "colorMode": "custom", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.22)", - "op": "gt", - "value": 250, - "yaxis": "left" - } - ], - "timeRegions": [], "title": "Request Count by arrival time", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:206", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:207", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, - "hiddenSeries": false, "id": 32, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})", + "expr": "rate(synapse_http_server_requests_received_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{server_name=\"$server_name\",job=~\"$job\",method!=\"OPTIONS\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}", @@ -2937,101 +2080,37 @@ "target": "" } ], - "thresholds": [], - "timeRegions": [], "title": "Top 10 Request Counts", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:305", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:306", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 2, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, - "hiddenSeries": false, "id": 139, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -3040,120 +2119,37 @@ "step": 20 } ], - "thresholds": [ - { - "$$hashKey": "object:135", - "colorMode": "custom", - "fill": true, - "fillColor": "rgba(216, 200, 27, 0.27)", - "op": "gt", - "value": 100, - "yaxis": "left" - }, - { - "$$hashKey": "object:136", - "colorMode": "custom", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.22)", - "op": "gt", - "value": 250, - "yaxis": "left" - } - ], - "timeRegions": [], "title": "Total CPU Usage by Endpoint", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:107", - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:108", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 }, - "hiddenSeries": false, "id": 52, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -3162,119 +2158,37 @@ "step": 20 } ], - "thresholds": [ - { - "$$hashKey": "object:417", - "colorMode": "custom", - "fill": true, - "fillColor": "rgba(216, 200, 27, 0.27)", - "op": "gt", - "value": 100, - "yaxis": "left" - }, - { - "$$hashKey": "object:418", - "colorMode": "custom", - "fill": true, - "fillColor": "rgba(234, 112, 112, 0.22)", - "op": "gt", - "value": 250, - "yaxis": "left" - } - ], - "timeRegions": [], "title": "Average CPU Usage by Endpoint", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:389", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:390", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 }, - "hiddenSeries": false, "id": 7, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -3283,100 +2197,36 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "DB Usage by endpoint", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:488", - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:489", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 2, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 45 }, - "hiddenSeries": false, "id": 47, - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))", + "expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))", "format": "time_series", "hide": false, "interval": "", @@ -3386,43 +2236,12 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Non-sync avg response time", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3430,54 +2249,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 53 }, - "hiddenSeries": false, "id": 103, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "Total", - "color": "rgb(255, 255, 255)", - "fill": 0, - "linewidth": 3 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})", + "expr": "topk(10,synapse_http_server_in_flight_requests_count{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -3486,52 +2274,16 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))", "interval": "", "legendFormat": "Total", "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Requests in flight", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Requests", @@ -3539,10 +2291,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "gridPos": { "h": 1, "w": 24, @@ -3552,12 +2300,8 @@ "id": 97, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3565,48 +2309,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 30 }, - "hiddenSeries": false, "id": 99, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_background_process_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -3614,43 +2333,12 @@ "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "CPU usage by background jobs", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3658,48 +2346,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 30 }, - "hiddenSeries": false, "id": 101, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -3707,43 +2370,12 @@ "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "DB usage by background jobs (including scheduling time)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3751,88 +2383,29 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 39 }, - "hiddenSeries": false, "id": 138, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", + "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}", "legendFormat": "{{job}}-{{index}} {{name}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Background jobs in flight", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Background jobs", @@ -3840,10 +2413,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "gridPos": { "h": 1, "w": 24, @@ -3853,12 +2422,8 @@ "id": 81, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3866,48 +2431,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 31 }, - "hiddenSeries": false, "id": 79, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_client_sent_transactions_total{server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "successful txn rate", @@ -3915,50 +2455,19 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",server_name=\"$server_name\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{server_name=\"$server_name\"}[$bucket_size]))", "legendFormat": "failed txn rate", "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Outgoing federation transaction rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3966,48 +2475,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 31 }, - "hiddenSeries": false, "id": 83, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_federation_server_received_pdus_total{instance=~\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_server_received_pdus_total{server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "pdus", @@ -4015,53 +2499,22 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_federation_server_received_edus_total{instance=~\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_server_received_edus_total{server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "edus", "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Incoming PDU/EDU rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -4069,49 +2522,24 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 40 }, - "hiddenSeries": false, "id": 109, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations_count_total{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations_count_total{server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -4121,53 +2549,22 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_federation_client_sent_edus_total{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_client_sent_edus_total{server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "edus", "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Outgoing PDU/EDU rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -4175,49 +2572,24 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 40 }, - "hiddenSeries": false, "id": 111, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_federation_client_sent_edus_by_type_total{instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_federation_client_sent_edus_by_type_total{server_name=\"$server_name\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -4226,44 +2598,15 @@ "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Outgoing EDUs by type", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:462", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:463", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "Triangular growth may indicate a problem with federation sending from the remote host --- but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains", + "description": "Triangular growth may indicate a problem with federation sending from the remote host --- but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains", "fieldConfig": { "defaults": { "color": { @@ -4335,7 +2678,7 @@ "id": "custom.hideFrom", "value": { "legend": false, - "tooltip": false, + "tooltip": true, "viz": true } } @@ -4366,13 +2709,13 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_received_pdu_time[10m]))) / 60", "instant": false, - "legendFormat": "{{server_name}} ", + "legendFormat": "{{origin_server_name}} ", "range": true, "refId": "A" } @@ -4383,9 +2726,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "Triangular growth may indicate a problem with federation senders on the monitored instance---but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains", + "description": "Triangular growth may indicate a problem with federation senders on the monitored instance---but it may also be the case that everyone is asleep and no messages are being sent.\n\nSee https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_metrics_domains", "fieldConfig": { "defaults": { "color": { @@ -4457,7 +2800,7 @@ "id": "custom.hideFrom", "value": { "legend": false, - "tooltip": false, + "tooltip": true, "viz": true } } @@ -4488,13 +2831,13 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "(time() - max without (job, index, host) (avg_over_time(synapse_federation_last_sent_pdu_time[10m]))) / 60", "instant": false, - "legendFormat": "{{server_name}}", + "legendFormat": "{{destination_server_name}}", "range": true, "refId": "A" } @@ -4503,13 +2846,9 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "The number of events in the in-memory queues ", "fieldConfig": { @@ -4518,48 +2857,25 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 57 }, - "hiddenSeries": false, "id": 142, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_federation_transaction_queue_pending_pdus{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "interval": "", "legendFormat": "pending PDUs {{job}}-{{index}}", "range": true, @@ -4568,56 +2884,20 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_federation_transaction_queue_pending_edus{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "interval": "", "legendFormat": "pending EDUs {{job}}-{{index}}", "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "In-memory federation transmission queues", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:547", - "format": "short", - "label": "events", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:548", - "format": "short", - "label": "", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Number of events queued up on the master process for processing by the federation sender", "fieldConfig": { @@ -4626,48 +2906,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 57 }, - "hiddenSeries": false, "id": 140, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_federation_send_queue_presence_changed_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -4676,9 +2931,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_federation_send_queue_presence_map_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "hide": false, "interval": "", @@ -4688,9 +2943,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_federation_send_queue_presence_destinations_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "hide": false, "interval": "", @@ -4700,9 +2955,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_federation_send_queue_keyed_edu_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "hide": false, "interval": "", @@ -4712,9 +2967,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_federation_send_queue_edus_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "hide": false, "interval": "", @@ -4724,9 +2979,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_federation_send_queue_pos_time_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "hide": false, "interval": "", @@ -4735,52 +2990,12 @@ "refId": "F" } ], - "thresholds": [], - "timeRegions": [], "title": "Outgoing EDU queues on master", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "cards": { - "cardPadding": -1 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -4803,14 +3018,7 @@ "x": 0, "y": 66 }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, "id": 166, - "legend": { - "show": false - }, - "links": [], "options": { "calculate": false, "calculation": {}, @@ -4853,13 +3061,12 @@ } }, "pluginVersion": "9.2.2", - "reverseYBuckets": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)", + "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{server_name=\"$server_name\",name=\"federation_sender\"}[$bucket_size])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -4869,30 +3076,11 @@ } ], "title": "Federation send PDU lag", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 2, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "type": "heatmap" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -4900,90 +3088,23 @@ }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 66 }, - "hiddenSeries": false, "id": 162, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "connected", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "Avg", - "fill": 0, - "linewidth": 3 - }, - { - "alias": "99%", - "color": "#C4162A", - "fillBelowTo": "90%" - }, - { - "alias": "90%", - "color": "#FF7383", - "fillBelowTo": "75%" - }, - { - "alias": "75%", - "color": "#FFEE52", - "fillBelowTo": "50%" - }, - { - "alias": "50%", - "color": "#73BF69", - "fillBelowTo": "25%" - }, - { - "alias": "25%", - "color": "#1F60C4", - "fillBelowTo": "5%" - }, - { - "alias": "5%", - "lines": false - }, - { - "alias": "Average", - "color": "rgb(255, 255, 255)", - "lines": true, - "linewidth": 3 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -4992,9 +3113,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -5003,9 +3124,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -5014,9 +3135,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -5025,98 +3146,38 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "interval": "", "legendFormat": "25%", "refId": "F" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "interval": "", "legendFormat": "5%", "refId": "G" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))", "interval": "", "legendFormat": "Average", "refId": "H" } ], - "thresholds": [ - { - "colorMode": "warning", - "fill": false, - "line": true, - "op": "gt", - "value": 0.25, - "yaxis": "left" - }, - { - "colorMode": "critical", - "fill": false, - "line": true, - "op": "gt", - "value": 1, - "yaxis": "left" - } - ], - "timeRegions": [], "title": "Federation send PDU lag quantiles", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "hertz", - "label": "", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "cards": { - "cardPadding": -1 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -5139,14 +3200,7 @@ "x": 0, "y": 75 }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, "id": 164, - "legend": { - "show": false - }, - "links": [], "options": { "calculate": false, "calculation": {}, @@ -5189,13 +3243,12 @@ } }, "pluginVersion": "9.2.2", - "reverseYBuckets": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)", + "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{server_name=\"$server_name\"}[$bucket_size])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -5205,84 +3258,37 @@ } ], "title": "Handle inbound PDU time", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 2, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "type": "heatmap" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 75 }, - "hiddenSeries": false, "id": 203, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", + "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -5292,101 +3298,38 @@ "step": 4 } ], - "thresholds": [], - "timeRegions": [], "title": "Age of oldest event in staging area", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:92", - "format": "ms", - "logBase": 1, - "min": 0, - "show": true - }, - { - "$$hashKey": "object:93", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 84 }, - "hiddenSeries": false, "id": 202, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", + "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -5396,135 +3339,43 @@ "step": 4 } ], - "thresholds": [], - "timeRegions": [], "title": "Number of events in federation staging area", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:92", - "format": "none", - "logBase": 1, - "min": 0, - "show": true - }, - { - "$$hashKey": "object:93", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 84 }, - "hiddenSeries": false, "id": 205, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_soft_failed_events_total{server_name=\"$server_name\"}[$bucket_size]))", "interval": "", "legendFormat": "soft-failed events", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Soft-failed event rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:131", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:132", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Federation", @@ -5543,7 +3394,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -5619,9 +3470,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(increase(synapse_rate_limit_reject_total{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(increase(synapse_rate_limit_reject_total{server_name=\"$server_name\"}[$bucket_size]))", "refId": "A" } ], @@ -5631,7 +3482,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -5707,9 +3558,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(increase(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(increase(synapse_rate_limit_sleep_total{server_name=\"$server_name\"}[$bucket_size]))", "refId": "A" } ], @@ -5719,7 +3570,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Why is the data zero (0)? https://github.com/matrix-org/synapse/pull/13541#discussion_r951926322", "fieldConfig": { @@ -5797,9 +3648,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(increase(synapse_rate_limit_reject_affected_hosts{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(increase(synapse_rate_limit_reject_affected_hosts{server_name=\"$server_name\"}[$bucket_size]))", "refId": "A" } ], @@ -5809,7 +3660,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "https://github.com/matrix-org/synapse/pull/13541", "fieldConfig": { @@ -5887,9 +3738,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(increase(synapse_rate_limit_sleep_affected_hosts{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(increase(synapse_rate_limit_sleep_affected_hosts{server_name=\"$server_name\"}[$bucket_size]))", "refId": "A" } ], @@ -5897,13 +3748,9 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -5912,106 +3759,24 @@ }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 170 }, - "hiddenSeries": false, "id": 229, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "connected", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:276", - "alias": "Avg", - "fill": 0, - "linewidth": 3 - }, - { - "$$hashKey": "object:277", - "alias": "99%", - "color": "#C4162A", - "fillBelowTo": "90%" - }, - { - "$$hashKey": "object:278", - "alias": "90%", - "color": "#FF7383", - "fillBelowTo": "75%" - }, - { - "$$hashKey": "object:279", - "alias": "75%", - "color": "#FFEE52", - "fillBelowTo": "50%" - }, - { - "$$hashKey": "object:280", - "alias": "50%", - "color": "#73BF69", - "fillBelowTo": "25%" - }, - { - "$$hashKey": "object:281", - "alias": "25%", - "color": "#1F60C4", - "fillBelowTo": "5%" - }, - { - "$$hashKey": "object:282", - "alias": "5%", - "lines": false - }, - { - "$$hashKey": "object:283", - "alias": "Average", - "color": "rgb(255, 255, 255)", - "lines": true, - "linewidth": 3 - }, - { - "$$hashKey": "object:284", - "alias": ">99%", - "color": "#B877D9", - "fill": 3, - "lines": true - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -6021,10 +3786,10 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99%", @@ -6033,9 +3798,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -6044,9 +3809,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "75%", @@ -6054,9 +3819,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "50%", @@ -6064,88 +3829,36 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "legendFormat": "25%", "refId": "F" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) by (le))", "legendFormat": "5%", "refId": "G" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))", "legendFormat": "Average", "refId": "H" } ], - "thresholds": [ - { - "$$hashKey": "object:283", - "colorMode": "warning", - "fill": false, - "line": true, - "op": "gt", - "value": 1, - "yaxis": "left" - }, - { - "$$hashKey": "object:284", - "colorMode": "critical", - "fill": false, - "line": true, - "op": "gt", - "value": 2, - "yaxis": "left" - } - ], - "timeRegions": [], "title": "Rate limit queue wait time Quantiles (all workers)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:255", - "format": "s", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:256", - "format": "hertz", - "label": "", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -6238,15 +3951,15 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_rate_limit_sleep_total{server_name=\"$server_name\"}[$bucket_size]))", "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "50", @@ -6265,10 +3978,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "gridPos": { "h": 1, "w": 24, @@ -6280,7 +3989,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -6342,7 +4051,6 @@ "y": 155 }, "id": 51, - "links": [], "options": { "legend": { "calcs": [], @@ -6359,10 +4067,10 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", + "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6373,10 +4081,10 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", + "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "failed {{job}}-{{index}}", @@ -6389,12 +4097,8 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -6403,89 +4107,29 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 155 }, - "hiddenSeries": false, "id": 134, - "legend": { - "avg": false, - "current": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})", + "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", server_name=\"$server_name\"})", "legendFormat": "{{kind}} {{app_id}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Active pusher instances by app", - "tooltip": { - "shared": false, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Pushes", @@ -6493,10 +4137,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "gridPos": { "h": 1, "w": 24, @@ -6506,13 +4146,9 @@ "id": 219, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "How many entries in current state that we are iterating over while calculating push rules.", "fieldConfig": { @@ -6521,50 +4157,25 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 33 }, - "hiddenSeries": false, "id": 209, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "8.4.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6574,46 +4185,13 @@ "step": 2 } ], - "thresholds": [], - "timeRegions": [], "title": "Iterations over State", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Rate that the cached push rules for a room get invalidated due to underlying push rules being changed. ", "fieldConfig": { @@ -6622,50 +4200,25 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 33 }, - "hiddenSeries": false, "id": 211, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "8.4.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6675,46 +4228,13 @@ "step": 2 } ], - "thresholds": [], - "timeRegions": [], "title": "Push Rule Invalidations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "How often the \"delta optimisation\" happens.\n\nThe delta optimisation is when we update the push rules for a room incrementally after a state change where we know the delta between the old state and the new state.\n\nThis can't happen if we don't the delta or we're calculating push rules from scratch.", "fieldConfig": { @@ -6723,55 +4243,25 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 40 }, - "hiddenSeries": false, "id": 213, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "8.4.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "Number of calls", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",server_name=\"$server_name\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6783,10 +4273,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6795,49 +4285,13 @@ "step": 2 } ], - "thresholds": [], - "timeRegions": [], "title": "Delta Optimisation", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "format": "hertz", - "label": "", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "How often we have the correct cached push rules for a room.", "fieldConfig": { @@ -6846,55 +4300,25 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 40 }, - "hiddenSeries": false, "id": 215, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "8.4.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "Number of calls", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",server_name=\"$server_name\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6906,10 +4330,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6918,47 +4342,13 @@ "step": 2 } ], - "thresholds": [], - "timeRegions": [], "title": "How often we reuse existing calculated push rules", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "format": "hertz", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "How often we have existing cached push rules for the room. \n\nNote that these might be outdated and need to be recalculated if the state has changed.", "fieldConfig": { @@ -6967,55 +4357,25 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 47 }, - "hiddenSeries": false, "id": 217, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "8.4.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "Number of calls", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",server_name=\"$server_name\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -7027,10 +4387,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -7039,47 +4399,8 @@ "step": 2 } ], - "thresholds": [], - "timeRegions": [], "title": "How often we have the RulesForRoom cached", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "format": "hertz", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Push Rule Cache", @@ -7087,10 +4408,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -7101,7 +4418,7 @@ "panels": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -7162,7 +4479,6 @@ "y": 35 }, "id": 48, - "links": [], "options": { "legend": { "calcs": [], @@ -7179,9 +4495,9 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])", + "expr": "rate(synapse_storage_schedule_time_sum{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}-{{index}}", @@ -7193,12 +4509,8 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan", "fieldConfig": { @@ -7207,49 +4519,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, - "hiddenSeries": false, "id": 104, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -7259,9 +4545,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}} {{index}} 95%", @@ -7269,9 +4555,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}} {{index}} 90%", @@ -7279,9 +4565,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_storage_schedule_time_sum{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -7289,99 +4575,36 @@ "refId": "D" } ], - "thresholds": [], - "timeRegions": [], "title": "Db scheduling time quantiles", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, - "hiddenSeries": false, "id": 10, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -7390,98 +4613,36 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Top DB transactions by txn rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, - "hiddenSeries": false, "id": 11, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_storage_transaction_time_sum_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "instant": false, "interval": "", @@ -7491,97 +4652,36 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "DB transactions by total txn time", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, - "hiddenSeries": false, "id": 180, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": false }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_storage_transaction_time_sum_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "instant": false, "interval": "", @@ -7591,43 +4691,12 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Average DB txn time", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -7635,47 +4704,23 @@ }, "overrides": [] }, - "fill": 6, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 49 }, - "hiddenSeries": false, "id": 200, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",server_name=\"$server_name\",job=~\"$job\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99%", @@ -7683,9 +4728,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",server_name=\"$server_name\",job=~\"$job\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "90%", @@ -7693,9 +4738,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",server_name=\"$server_name\",job=~\"$job\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "75%", @@ -7703,57 +4748,17 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",server_name=\"$server_name\",job=~\"$job\"}[$bucket_size])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "50%", "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Time waiting for DB connection quantiles", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:203", - "format": "s", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:204", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Database", @@ -7761,10 +4766,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -7774,65 +4775,32 @@ "id": 59, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 0, "y": 158 }, - "hiddenSeries": false, "id": 12, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -7841,96 +4809,36 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Total CPU Usage by Block", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 12, "y": 158 }, - "hiddenSeries": false, "id": 26, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])", + "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -7939,99 +4847,39 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Average CPU Time per Block", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 0, "y": 171 }, - "hiddenSeries": false, "id": 13, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -8040,100 +4888,37 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Total DB Usage by Block", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:196", - "format": "percentunit", - "logBase": 1, - "min": 0, - "show": true - }, - { - "$$hashKey": "object:197", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "The time each database transaction takes to execute, on average, broken down by metrics block.", - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 12, "y": 171 }, - "hiddenSeries": false, "id": 27, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -8142,95 +4927,36 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Average Database Transaction time, by Block", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 0, "y": 184 }, - "hiddenSeries": false, "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -8239,95 +4965,36 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Average Transactions per Block", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 12, "y": 184 }, - "hiddenSeries": false, "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_util_metrics_block_time_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_time_seconds_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -8336,130 +5003,41 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Average Wallclock Time per Block", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:180", - "format": "s", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:181", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 15, "w": 12, "x": 0, "y": 197 }, - "hiddenSeries": false, "id": 154, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "interval": "", "legendFormat": "{{job}}-{{index}} {{block_name}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Block count", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Per-block metrics", @@ -8467,10 +5045,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -8480,67 +5054,32 @@ "id": 61, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "decimals": 2, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 36 }, - "hiddenSeries": false, "id": 1, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}} {{job}}-{{index}}", @@ -8548,100 +5087,36 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Cache Hit Ratio", - "tooltip": { - "msResolution": true, - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": "1", - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 36 }, - "hiddenSeries": false, "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_util_caches_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_util_caches_cache_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "hide": false, "interval": "", @@ -8651,97 +5126,36 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Cache Size", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 46 }, - "hiddenSeries": false, "id": 38, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -8750,44 +5164,12 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Cache request rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "rps", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -8795,53 +5177,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 46 }, - "hiddenSeries": false, "id": 39, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", + "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -8850,45 +5202,12 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Top 10 cache misses", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "rps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -8896,48 +5215,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 56 }, - "hiddenSeries": false, "id": 65, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.0.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_util_caches_cache_evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_caches_cache_evicted_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -8945,45 +5239,8 @@ "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Cache eviction rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "label": "entries / second", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Caches", @@ -8991,10 +5248,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -9004,12 +5257,8 @@ "id": 148, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9018,88 +5267,34 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, - "hiddenSeries": false, "id": 146, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "7.3.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_util_caches_response_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_util_caches_response_cache_size{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "interval": "", "legendFormat": "{{name}} {{job}}-{{index}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Response cache size", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9108,53 +5303,30 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, - "hiddenSeries": false, "id": 150, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "7.3.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_util_caches_response_cache_hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_caches_response_cache_hits{server_name=\"$server_name\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{server_name=\"$server_name\", job=~\"$job\", index=~\"$index\"}[$bucket_size])", "interval": "", "legendFormat": "{{name}} {{job}}-{{index}}", "refId": "A" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "expr": "", "interval": "", @@ -9162,46 +5334,8 @@ "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Response cache hit rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Response caches", @@ -9209,10 +5343,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -9222,12 +5352,8 @@ "id": 62, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9236,47 +5362,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 30 }, - "hiddenSeries": false, "id": 91, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "7.3.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])", + "expr": "rate(python_gc_time_sum{job=~\"$job\",index=~\"$index\"}[10m]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "instant": false, "intervalFactor": 1, @@ -9284,48 +5386,13 @@ "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Total GC time by bucket (10m smoothing)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "decimals": 3, - "editable": true, - "error": false, "fieldConfig": { "defaults": { "custom": {}, @@ -9333,49 +5400,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 30 }, - "hiddenSeries": false, "id": 21, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "7.3.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])", + "expr": "(\n rate(python_gc_time_sum{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) / (\n rate(python_gc_time_count{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} {{index}} gen {{gen}} ", @@ -9384,43 +5425,12 @@ "target": "" } ], - "thresholds": [], - "timeRegions": [], "title": "Average GC Time Per Collection", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.", "fieldConfig": { @@ -9430,99 +5440,35 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 39 }, - "hiddenSeries": false, "id": 89, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "7.3.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/gen 0$/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", + "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\"} * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} gen {{gen}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Allocation counts", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Gen N-1 GCs since last Gen N GC", - "logBase": 1, - "show": true - }, - { - "format": "short", - "label": "Objects since last Gen 0 GC", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9531,90 +5477,35 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 39 }, - "hiddenSeries": false, "id": 93, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "7.3.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "(\n rate(python_gc_unreachable_total{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n) / (\n rate(python_gc_time_count{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\n synapse_server_name_info{server_name=\"$server_name\"}\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} gen {{gen}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Object counts per collection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9623,99 +5514,36 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 48 }, - "hiddenSeries": false, "id": 95, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "7.3.7", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(python_gc_time_count{job=~\"$job\",index=~\"$index\"}[$bucket_size]) * on (instance, job, index) group_left(server_name)\nsynapse_server_name_info{server_name=\"$server_name\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} gen {{gen}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "GC frequency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "cards": { - "cardPadding": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9729,20 +5557,13 @@ "x": 12, "y": 48 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, "id": 87, - "legend": { - "show": true - }, - "links": [], - "reverseYBuckets": false, + "options": {}, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)", "format": "heatmap", @@ -9752,29 +5573,7 @@ } ], "title": "GC durations", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "heatmap" } ], "title": "GC", @@ -9782,10 +5581,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -9795,13 +5590,9 @@ "id": 63, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9809,48 +5600,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 162 }, - "hiddenSeries": false, "id": 43, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)", + "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}-{{index}} {{command}}", @@ -9858,42 +5624,13 @@ "step": 20 } ], - "thresholds": [], - "timeRegions": [], "title": "Rate of outgoing commands", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:89", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:90", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9955,7 +5692,6 @@ "y": 162 }, "id": 41, - "links": [], "options": { "legend": { "calcs": [], @@ -9973,10 +5709,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "rate(synapse_replication_tcp_resource_stream_updates_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_replication_tcp_resource_stream_updates_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -9991,7 +5727,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10054,7 +5790,6 @@ "y": 169 }, "id": 42, - "links": [], "options": { "legend": { "calcs": [], @@ -10072,10 +5807,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)", + "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -10090,7 +5825,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10154,7 +5889,6 @@ "y": 169 }, "id": 220, - "links": [], "options": { "legend": { "calcs": [], @@ -10172,10 +5906,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "rate(synapse_replication_tcp_protocol_inbound_rdata_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_replication_tcp_protocol_inbound_rdata_count_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -10188,13 +5922,9 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -10203,92 +5933,36 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 176 }, - "hiddenSeries": false, "id": 144, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_replication_tcp_command_queue{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "interval": "", "legendFormat": "{{stream_name}} {{job}}-{{index}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Queued incoming RDATA commands, by stream", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:218", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:219", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10296,93 +5970,35 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 176 }, - "hiddenSeries": false, "id": 115, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} {{reason_type}}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Replication connection close reasons", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:260", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:261", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10390,48 +6006,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 183 }, - "hiddenSeries": false, "id": 113, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", + "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} {{stream_name}}", @@ -10439,54 +6030,17 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", + "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}}", "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Replication connections", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Replication", @@ -10494,10 +6048,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -10507,12 +6057,8 @@ "id": 69, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10520,48 +6066,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 163 }, - "hiddenSeries": false, "id": 67, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "max(synapse_event_persisted_position{server_name=\"$server_name\"}) - on () group_right() synapse_event_processing_positions{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -10569,45 +6090,12 @@ "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Event processing lag", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "events", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10615,48 +6103,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 163 }, - "hiddenSeries": false, "id": 71, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "time()*1000-synapse_event_processing_last_ts{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "hide": false, "interval": "", @@ -10665,44 +6128,12 @@ "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Age of last processed event", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10710,49 +6141,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 172 }, - "hiddenSeries": false, "id": 121, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", "options": { "alertThreshold": true }, - "paceLength": 10, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1", + "expr": "deriv(synapse_event_processing_last_ts{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1", "format": "time_series", "hide": false, "interval": "", @@ -10761,45 +6166,8 @@ "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Event processing catchup rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": "fallbehind(-) / catchup(+): s/sec", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Event processing loop positions", @@ -10807,10 +6175,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -10820,20 +6184,8 @@ "id": 126, "panels": [ { - "cards": { - "cardPadding": 0 - }, - "color": { - "cardColor": "#B877D9", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", - "exponent": 0.5, - "min": 0, - "mode": "opacity" - }, - "dataFormat": "tsbuckets", "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.", "fieldConfig": { @@ -10857,14 +6209,7 @@ "x": 0, "y": 42 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, "id": 122, - "legend": { - "show": true - }, - "links": [], "options": { "calculate": false, "calculation": {}, @@ -10905,13 +6250,12 @@ } }, "pluginVersion": "9.2.2", - "reverseYBuckets": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", + "expr": "synapse_forward_extremities_bucket{server_name=\"$server_name\"} and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0)", "format": "heatmap", "intervalFactor": 1, "legendFormat": "{{le}}", @@ -10919,29 +6263,11 @@ } ], "title": "Number of rooms, by number of forward extremities in room", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "type": "heatmap" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.", "fieldConfig": { @@ -10950,48 +6276,23 @@ }, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, - "hiddenSeries": false, "id": 124, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0", + "expr": "synapse_forward_extremities_bucket{server_name=\"$server_name\"} > 0", "format": "heatmap", "interval": "", "intervalFactor": 1, @@ -10999,52 +6300,12 @@ "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Room counts, by number of extremities", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": "Number of rooms", - "logBase": 10, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "cards": { - "cardPadding": 0 - }, - "color": { - "cardColor": "#5794F2", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", - "exponent": 0.5, - "min": 0, - "mode": "opacity" - }, - "dataFormat": "tsbuckets", "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.", "fieldConfig": { @@ -11068,14 +6329,7 @@ "x": 0, "y": 50 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, "id": 127, - "legend": { - "show": true - }, - "links": [], "options": { "calculate": false, "calculation": {}, @@ -11116,13 +6370,12 @@ } }, "pluginVersion": "9.2.2", - "reverseYBuckets": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", + "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0)", "format": "heatmap", "intervalFactor": 1, "legendFormat": "{{le}}", @@ -11130,29 +6383,11 @@ } ], "title": "Events persisted, by number of forward extremities in room (heatmap)", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "type": "heatmap" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.", "fieldConfig": { @@ -11161,47 +6396,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 }, - "hiddenSeries": false, "id": 128, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", + "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "50%", @@ -11209,9 +6420,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", + "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "75%", @@ -11219,9 +6430,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", + "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "90%", @@ -11229,62 +6440,21 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", + "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99%", "refId": "D" } ], - "thresholds": [], - "timeRegions": [], "title": "Events persisted, by number of forward extremities in room (quantiles)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Number of extremities in room", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "cards": { - "cardPadding": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", - "exponent": 0.5, - "min": 0, - "mode": "opacity" - }, - "dataFormat": "tsbuckets", "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.", "fieldConfig": { @@ -11308,14 +6478,7 @@ "x": 0, "y": 58 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, "id": 129, - "legend": { - "show": true - }, - "links": [], "options": { "calculate": false, "calculation": {}, @@ -11356,13 +6519,12 @@ } }, "pluginVersion": "9.2.2", - "reverseYBuckets": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", + "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0)", "format": "heatmap", "intervalFactor": 1, "legendFormat": "{{le}}", @@ -11370,29 +6532,11 @@ } ], "title": "Events persisted, by number of stale forward extremities in room (heatmap)", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "type": "heatmap" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.", "fieldConfig": { @@ -11401,47 +6545,23 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 58 }, - "hiddenSeries": false, "id": 130, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", + "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "50%", @@ -11449,9 +6569,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", + "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "75%", @@ -11459,9 +6579,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", + "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "90%", @@ -11469,62 +6589,21 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", + "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99%", "refId": "D" } ], - "thresholds": [], - "timeRegions": [], "title": "Events persisted, by number of stale forward extremities in room (quantiles)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Number of stale forward extremities in room", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "cards": { - "cardPadding": 0 - }, - "color": { - "cardColor": "#73BF69", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", - "exponent": 0.5, - "min": 0, - "mode": "opacity" - }, - "dataFormat": "tsbuckets", "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.", "fieldConfig": { @@ -11548,14 +6627,7 @@ "x": 0, "y": 66 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, "id": 131, - "legend": { - "show": true - }, - "links": [], "options": { "calculate": false, "calculation": {}, @@ -11596,13 +6668,12 @@ } }, "pluginVersion": "9.2.2", - "reverseYBuckets": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "heatmap", "interval": "", "intervalFactor": 1, @@ -11611,30 +6682,12 @@ } ], "title": "Number of state resolution performed, by number of state groups involved (heatmap)", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "type": "heatmap" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.", "fieldConfig": { @@ -11643,49 +6696,24 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 66 }, - "hiddenSeries": false, "id": 132, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -11695,9 +6723,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -11706,9 +6734,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -11717,9 +6745,9 @@ }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -11727,148 +6755,60 @@ "refId": "D" } ], - "thresholds": [], - "timeRegions": [], "title": "Number of state resolutions performed, by number of state groups involved (quantiles)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Number of state groups", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 74 }, - "hiddenSeries": false, "id": 179, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "interval": "", "legendFormat": "State res ", "refId": "A" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "interval": "", "legendFormat": "Potential to prune", "refId": "B" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{server_name=\"$server_name\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "interval": "", "legendFormat": "Pruned", "refId": "C" } ], - "thresholds": [], - "timeRegions": [], "title": "Stale extremity dropping", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Extremities", @@ -11876,10 +6816,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -11889,13 +6825,9 @@ "id": 158, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -11903,56 +6835,24 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, - "hiddenSeries": false, "id": 156, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:632", - "alias": "Max", - "color": "#bf1b00", - "fill": 0, - "linewidth": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "max(synapse_admin_mau_max{instance=\"$instance\"})", + "expr": "max(synapse_admin_mau_max{server_name=\"$server_name\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -11963,140 +6863,51 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "max(synapse_admin_mau_current{instance=\"$instance\"})", + "expr": "max(synapse_admin_mau_current{server_name=\"$server_name\"})", "hide": false, "legendFormat": "Current", "range": true, "refId": "C" } ], - "thresholds": [], - "timeRegions": [], "title": "MAU Limits", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:176", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:177", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, - "hiddenSeries": false, "id": 160, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}", + "expr": "synapse_admin_mau_current_mau_by_service{server_name=\"$server_name\"}", "interval": "", "legendFormat": "{{ app_service }}", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "MAU by Appservice", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "MAU", @@ -12104,10 +6915,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -12117,13 +6924,9 @@ "id": 177, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -12131,48 +6934,24 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 44 }, - "hiddenSeries": false, "id": 173, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_notifier_users_woken_by_stream_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_notifier_users_woken_by_stream_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -12183,46 +6962,13 @@ "step": 2 } ], - "thresholds": [], - "timeRegions": [], "title": "Notifier Streams Woken", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:734", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:735", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -12230,48 +6976,24 @@ }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 44 }, - "hiddenSeries": false, "id": 175, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",server_name=\"$server_name\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -12281,47 +7003,8 @@ "step": 2 } ], - "thresholds": [], - "timeRegions": [], "title": "Presence Stream Fetch Type Rates", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:819", - "format": "hertz", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:820", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Notifier", @@ -12329,10 +7012,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -12342,189 +7021,76 @@ "id": 170, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 }, - "hiddenSeries": false, "id": 168, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_appservice_api_sent_events_total{instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_appservice_api_sent_events_total{server_name=\"$server_name\"}[$bucket_size])", "interval": "", "legendFormat": "{{service}}", "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Sent Events rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:177", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:178", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 45 }, - "hiddenSeries": false, "id": 171, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_appservice_api_sent_transactions_total{instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_appservice_api_sent_transactions_total{server_name=\"$server_name\"}[$bucket_size])", "interval": "", "legendFormat": "{{exported_service }} {{ service }}", "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Transactions rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:260", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:261", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Appservices", @@ -12532,10 +7098,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -12545,308 +7107,145 @@ "id": 188, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, - "hiddenSeries": false, "id": 182, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_notified_presence_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "interval": "", "legendFormat": "Notified", "refId": "A" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "interval": "", "legendFormat": "Remote ping", "refId": "B" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_presence_updates_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "interval": "", "legendFormat": "Total updates", "refId": "C" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_federation_presence_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "interval": "", "legendFormat": "Remote updates", "refId": "D" }, { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_bump_active_time_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "interval": "", "legendFormat": "Bump active time", "refId": "E" } ], - "thresholds": [], - "timeRegions": [], "title": "Presence", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 }, - "hiddenSeries": false, "id": 184, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_handler_presence_state_transition_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_state_transition_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "interval": "", "legendFormat": "{{from}} -> {{to}}", "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Presence state transitions", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1090", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1091", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 }, - "hiddenSeries": false, "id": 186, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_handler_presence_notify_reason_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_notify_reason_total{job=~\"$job\",index=~\"$index\",server_name=\"$server_name\"}[$bucket_size])", "interval": "", "legendFormat": "{{reason}}", "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Presence notify reason", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:165", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:166", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "timeseries" } ], "title": "Presence", @@ -12854,10 +7253,6 @@ }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -12869,7 +7264,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -12946,10 +7341,10 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_external_cache_set{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_external_cache_set{job=~\"$job\", server_name=\"$server_name\", index=~\"$index\"}[$bucket_size])", "interval": "", "legendFormat": "{{ cache_name }} {{job}}-{{ index }}", "range": true, @@ -12960,110 +7355,46 @@ "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "", - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 47 }, - "hiddenSeries": false, "id": 193, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, "pluginVersion": "9.2.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size]))", + "expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", server_name=\"$server_name\", index=~\"$index\"}[$bucket_size]))", "interval": "", "legendFormat": "{{ cache_name }} {{job}}-{{ index }}", "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "External Cache Get Rate", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:390", - "format": "hertz", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:391", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "cards": { - "cardPadding": -1 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateInferno", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -13086,14 +7417,7 @@ "x": 0, "y": 55 }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, "id": 195, - "legend": { - "show": false - }, - "links": [], "options": { "calculate": false, "calculation": {}, @@ -13136,13 +7460,12 @@ } }, "pluginVersion": "9.2.2", - "reverseYBuckets": false, "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])) by (le)", + "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",server_name=\"$server_name\",job=~\"$job\"}[$bucket_size])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -13152,27 +7475,12 @@ } ], "title": "External Cache Response Time", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 2, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "type": "heatmap" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -13246,10 +7554,10 @@ "targets": [ { "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\", hit=\"False\"}[$bucket_size])", + "expr": "rate(synapse_external_cache_get{job=~\"$job\", server_name=\"$server_name\", index=~\"$index\", hit=\"False\"}[$bucket_size])", "interval": "", "legendFormat": "{{ cache_name }} {{job}}-{{ index }}", "range": true, @@ -13260,22 +7568,13 @@ "type": "timeseries" } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], "title": "External Cache", "type": "row" } ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", + "preload": false, + "refresh": "", + "schemaVersion": 42, "tags": [ "matrix" ], @@ -13283,44 +7582,30 @@ "list": [ { "current": { - "selected": false, - "text": "default", - "value": "default" + "text": "", + "value": "${DS_PROMETHEUS}", + "selected": true }, - "hide": 0, "includeAll": false, - "multi": false, - "name": "datasource", + "label": "Datasource", + "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { - "allFormat": "glob", "auto": true, "auto_count": 100, "auto_min": "30s", "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_bucket_size" + "text": "$__auto", + "value": "$__auto" }, - "hide": 0, - "includeAll": false, "label": "Bucket Size", - "multi": false, - "multiFormat": "glob", "name": "bucket_size", "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_bucket_size" - }, { "selected": false, "text": "30s", @@ -13353,49 +7638,36 @@ } ], "query": "30s,1m,2m,5m,10m,15m", - "queryValue": "", "refresh": 2, - "skipUrlSync": false, "type": "interval" }, { "current": {}, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "definition": "", - "hide": 0, "includeAll": false, - "multi": false, - "name": "instance", + "name": "server_name", "options": [], "query": { - "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, instance)", + "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, server_name)", "refId": "Prometheus-instance-Variable-Query" }, "refresh": 2, "regex": "", - "skipUrlSync": false, "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allFormat": "regex wildcard", - "allValue": "", "current": {}, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "definition": "", - "hide": 0, - "hideLabel": false, "includeAll": true, "label": "Job", "multi": true, - "multiFormat": "regex values", "name": "job", "options": [], "query": { @@ -13403,29 +7675,19 @@ "refId": "Prometheus-job-Variable-Query" }, "refresh": 2, - "refresh_on_load": false, "regex": "", - "skipUrlSync": false, "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allFormat": "regex wildcard", "allValue": ".*", "current": {}, "datasource": { - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "definition": "", - "hide": 0, - "hideLabel": false, "includeAll": true, - "label": "", "multi": true, - "multiFormat": "regex values", "name": "index", "options": [], "query": { @@ -13433,14 +7695,9 @@ "refId": "Prometheus-index-Variable-Query" }, "refresh": 2, - "refresh_on_load": false, "regex": "", - "skipUrlSync": false, "sort": 3, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" } ] }, @@ -13448,35 +7705,10 @@ "from": "now-3h", "to": "now" }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, + "timepicker": {}, "timezone": "", "title": "Synapse", "uid": "000000012", - "version": 160, + "version": 1, "weekStart": "" } diff --git a/hosts/ildkule/services/monitoring/grafana.nix b/hosts/ildkule/services/monitoring/grafana.nix index 154b3cb..f5c251a 100644 --- a/hosts/ildkule/services/monitoring/grafana.nix +++ b/hosts/ildkule/services/monitoring/grafana.nix @@ -47,13 +47,13 @@ in { { name = "Node Exporter Full"; type = "file"; - url = "https://grafana.com/api/dashboards/1860/revisions/29/download"; + url = "https://grafana.com/api/dashboards/1860/revisions/42/download"; options.path = dashboards/node-exporter-full.json; } { name = "Matrix Synapse"; type = "file"; - url = "https://raw.githubusercontent.com/matrix-org/synapse/develop/contrib/grafana/synapse.json"; + url = "https://github.com/element-hq/synapse/raw/refs/heads/develop/contrib/grafana/synapse.json"; options.path = dashboards/synapse.json; } { @@ -65,7 +65,7 @@ in { { name = "Postgresql"; type = "file"; - url = "https://grafana.com/api/dashboards/9628/revisions/7/download"; + url = "https://grafana.com/api/dashboards/9628/revisions/8/download"; options.path = dashboards/postgres.json; } {