k3s cluster dashboard: kube-state-metrics + kubelet/cAdvisor scraping, fix empty-label relabel bug

manifests/monitoring/grafana-dashboards.yaml

@@ -22,6 +22,72 @@ metadata:
   name: grafana-dashboards
   namespace: monitoring
 data:
+  k3s-cluster.json: |
+    {
+      "title": "planck k3s cluster",
+      "uid": "planckk3s",
+      "timezone": "browser",
+      "refresh": "15s",
+      "time": { "from": "now-3h", "to": "now" },
+      "panels": [
+        {
+          "id": 1, "type": "timeseries", "title": "Pods running by node",
+          "gridPos": {"x": 0, "y": 0, "w": 12, "h": 9},
+          "targets": [{"expr": "count by (node) (kube_pod_status_phase{phase=\"Running\"} == 1)", "legendFormat": "{{node}}"}],
+          "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []}
+        },
+        {
+          "id": 2, "type": "timeseries", "title": "Deployment replicas: ready vs desired",
+          "gridPos": {"x": 12, "y": 0, "w": 12, "h": 9},
+          "targets": [
+            {"expr": "sum by (deployment, namespace) (kube_deployment_status_replicas_ready)", "legendFormat": "ready {{namespace}}/{{deployment}}"},
+            {"expr": "sum by (deployment, namespace) (kube_deployment_spec_replicas)", "legendFormat": "want {{namespace}}/{{deployment}}"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []}
+        },
+        {
+          "id": 3, "type": "timeseries", "title": "Top pods by CPU (cores)",
+          "gridPos": {"x": 0, "y": 9, "w": 12, "h": 9},
+          "targets": [{"expr": "topk(12, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[2m])))", "legendFormat": "{{namespace}}/{{pod}}"}],
+          "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}
+        },
+        {
+          "id": 4, "type": "timeseries", "title": "Top pods by memory (working set)",
+          "gridPos": {"x": 12, "y": 9, "w": 12, "h": 9},
+          "targets": [{"expr": "topk(12, sum by (pod, namespace) (container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}))", "legendFormat": "{{namespace}}/{{pod}}"}],
+          "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}
+        },
+        {
+          "id": 5, "type": "timeseries", "title": "Container restarts (cumulative)",
+          "gridPos": {"x": 0, "y": 18, "w": 12, "h": 9},
+          "targets": [{"expr": "sum by (pod, namespace) (kube_pod_container_status_restarts_total)", "legendFormat": "{{namespace}}/{{pod}}"}],
+          "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []}
+        },
+        {
+          "id": 6, "type": "timeseries", "title": "CPU: requested vs allocatable (per node, top 10 nodes)",
+          "gridPos": {"x": 12, "y": 18, "w": 12, "h": 9},
+          "targets": [
+            {"expr": "topk(10, sum by (node) (kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\"}))", "legendFormat": "requested {{node}}"},
+            {"expr": "kube_node_status_allocatable{resource=\"cpu\", unit=\"core\"}", "legendFormat": "allocatable {{node}}"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []}
+        },
+        {
+          "id": 7, "type": "timeseries", "title": "Nodes Ready",
+          "gridPos": {"x": 0, "y": 27, "w": 12, "h": 7},
+          "targets": [{"expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 1)", "legendFormat": "nodes ready"}],
+          "fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 20}, "overrides": []}
+        },
+        {
+          "id": 8, "type": "timeseries", "title": "Pods not Running (pending/failed/succeeded)",
+          "gridPos": {"x": 12, "y": 27, "w": 12, "h": 7},
+          "targets": [{"expr": "count by (phase) (kube_pod_status_phase == 1) unless on(pod, namespace, phase) label_replace(kube_pod_status_phase{phase=\"Running\"}, \"phase\", \"running\", \"\", \"\") * 0", "legendFormat": "{{phase}}"}],
+          "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []}
+        }
+      ],
+      "schemaVersion": 39,
+      "version": 1
+    }
   node-health.json: |
     {
       "title": "planck node health",

manifests/monitoring/kube-state-metrics.yaml

@@ -0,0 +1,71 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics
+rules:
+  - apiGroups: [""]
+    resources: [pods, nodes, namespaces, services, endpoints, configmaps, secrets, persistentvolumeclaims, persistentvolumes, resourcequotas, limitranges]
+    verbs: [list, watch]
+  - apiGroups: [apps]
+    resources: [deployments, daemonsets, replicasets, statefulsets]
+    verbs: [list, watch]
+  - apiGroups: [batch]
+    resources: [jobs, cronjobs]
+    verbs: [list, watch]
+  - apiGroups: [autoscaling]
+    resources: [horizontalpodautoscalers]
+    verbs: [list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+    namespace: monitoring
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: kube-state-metrics}
+  template:
+    metadata:
+      labels: {app: kube-state-metrics}
+    spec:
+      serviceAccountName: kube-state-metrics
+      containers:
+        - name: kube-state-metrics
+          image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.17.0
+          ports: [{containerPort: 8080}]
+          resources:
+            requests: {cpu: 30m, memory: 64Mi}
+            limits: {memory: 512Mi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+  labels: {app: kube-state-metrics}
+spec:
+  selector: {app: kube-state-metrics}
+  ports:
+    - name: http-metrics
+      port: 8080
+      targetPort: 8080

manifests/monitoring/prometheus.yaml

@@ -61,6 +61,50 @@ data:
           - source_labels: [__meta_kubernetes_pod_container_port_number]
             action: keep
             regex: "9100"
+      - job_name: kube-state-metrics
+        kubernetes_sd_configs:
+          - role: pod
+            selectors:
+              - role: pod
+                label: "app=kube-state-metrics"
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_container_port_number]
+            action: keep
+            regex: "8080"
+      - job_name: kubelet
+        scheme: https
+        tls_config:
+          insecure_skip_verify: true
+        authorization:
+          credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+        kubernetes_sd_configs:
+          - role: node
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_node_name]
+            target_label: instance
+          - target_label: __address__
+            replacement: kubernetes.default.svc:443
+          - source_labels: [__meta_kubernetes_node_name]
+            target_label: __metrics_path__
+            regex: (.+)
+            replacement: /api/v1/nodes/${1}/proxy/metrics
+      - job_name: cadvisor
+        scheme: https
+        tls_config:
+          insecure_skip_verify: true
+        authorization:
+          credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+        kubernetes_sd_configs:
+          - role: node
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_node_name]
+            target_label: instance
+          - target_label: __address__
+            replacement: kubernetes.default.svc:443
+          - source_labels: [__meta_kubernetes_node_name]
+            target_label: __metrics_path__
+            regex: (.+)
+            replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
 ---
 apiVersion: apps/v1
 kind: StatefulSet