Node health dashboard + fix node-exporter scrape config (role:pod, instance=node name)

manifests/monitoring/grafana-dashboards.yaml

@@ -22,6 +22,66 @@ metadata:
   name: grafana-dashboards
   namespace: monitoring
 data:
+  node-health.json: |
+    {
+      "title": "planck node health",
+      "uid": "plancknodes",
+      "timezone": "browser",
+      "refresh": "10s",
+      "time": { "from": "now-1h", "to": "now" },
+      "panels": [
+        {
+          "id": 1, "type": "timeseries", "title": "CPU temperature (all nodes)",
+          "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8},
+          "targets": [{"expr": "node_thermal_zone_temp", "legendFormat": "{{instance}}"}],
+          "fieldConfig": {"defaults": {"unit": "celsius"}, "overrides": []}
+        },
+        {
+          "id": 2, "type": "timeseries", "title": "CPU usage % (all nodes)",
+          "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8},
+          "targets": [{"expr": "100 * (1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[2m])))", "legendFormat": "{{instance}}"}],
+          "fieldConfig": {"defaults": {"unit": "percent", "min": 0, "max": 100}, "overrides": []}
+        },
+        {
+          "id": 3, "type": "timeseries", "title": "Memory used % (all nodes)",
+          "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8},
+          "targets": [{"expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", "legendFormat": "{{instance}}"}],
+          "fieldConfig": {"defaults": {"unit": "percent", "min": 0, "max": 100}, "overrides": []}
+        },
+        {
+          "id": 4, "type": "timeseries", "title": "Disk used % (root, all nodes)",
+          "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8},
+          "targets": [{"expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})", "legendFormat": "{{instance}}"}],
+          "fieldConfig": {"defaults": {"unit": "percent", "min": 0, "max": 100}, "overrides": []}
+        },
+        {
+          "id": 5, "type": "timeseries", "title": "Network eth0: RX / TX",
+          "gridPos": {"x": 0, "y": 16, "w": 12, "h": 8},
+          "targets": [
+            {"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device=\"eth0\"}[1m]))", "legendFormat": "rx {{instance}}"},
+            {"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device=\"eth0\"}[1m]))", "legendFormat": "tx {{instance}}"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []}
+        },
+        {
+          "id": 6, "type": "timeseries", "title": "SSD I/O: read / write",
+          "gridPos": {"x": 12, "y": 16, "w": 12, "h": 8},
+          "targets": [
+            {"expr": "sum by (instance) (rate(node_disk_read_bytes_total{device=~\"sda.*\"}[1m]))", "legendFormat": "read {{instance}}"},
+            {"expr": "sum by (instance) (rate(node_disk_written_bytes_total{device=~\"sda.*\"}[1m]))", "legendFormat": "write {{instance}}"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []}
+        },
+        {
+          "id": 7, "type": "timeseries", "title": "Load (1 min, all nodes)",
+          "gridPos": {"x": 0, "y": 24, "w": 24, "h": 7},
+          "targets": [{"expr": "node_load1", "legendFormat": "{{instance}}"}],
+          "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}
+        }
+      ],
+      "schemaVersion": 39,
+      "version": 1
+    }
   k6-step-load.json: |
     {
       "title": "k6 Step Load",

manifests/monitoring/prometheus.yaml

@@ -51,16 +51,16 @@ data:
           - targets: ["localhost:9090"]
       - job_name: node-exporter
         kubernetes_sd_configs:
-          - role: node
+          - role: pod
+            selectors:
+              - role: pod
+                label: "app=node-exporter"
         relabel_configs:
-          - source_labels: [__meta_kubernetes_node_address_InternalIP]
-            target_label: instance_ip
-          - action: labelmap
-            regex: __meta_kubernetes_node_label_(.+)
-        metric_relabel_configs:
-          - source_labels: [__name__]
-            regex: go_.*
-            action: drop
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: instance
+          - source_labels: [__meta_kubernetes_pod_container_port_number]
+            action: keep
+            regex: "9100"
 ---
 apiVersion: apps/v1
 kind: StatefulSet