diff --git a/manifests/monitoring/grafana-dashboards.yaml b/manifests/monitoring/grafana-dashboards.yaml index 59485ef..36dbe6a 100644 --- a/manifests/monitoring/grafana-dashboards.yaml +++ b/manifests/monitoring/grafana-dashboards.yaml @@ -22,6 +22,72 @@ metadata: name: grafana-dashboards namespace: monitoring data: + k3s-cluster.json: | + { + "title": "planck k3s cluster", + "uid": "planckk3s", + "timezone": "browser", + "refresh": "15s", + "time": { "from": "now-3h", "to": "now" }, + "panels": [ + { + "id": 1, "type": "timeseries", "title": "Pods running by node", + "gridPos": {"x": 0, "y": 0, "w": 12, "h": 9}, + "targets": [{"expr": "count by (node) (kube_pod_status_phase{phase=\"Running\"} == 1)", "legendFormat": "{{node}}"}], + "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []} + }, + { + "id": 2, "type": "timeseries", "title": "Deployment replicas: ready vs desired", + "gridPos": {"x": 12, "y": 0, "w": 12, "h": 9}, + "targets": [ + {"expr": "sum by (deployment, namespace) (kube_deployment_status_replicas_ready)", "legendFormat": "ready {{namespace}}/{{deployment}}"}, + {"expr": "sum by (deployment, namespace) (kube_deployment_spec_replicas)", "legendFormat": "want {{namespace}}/{{deployment}}"} + ], + "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []} + }, + { + "id": 3, "type": "timeseries", "title": "Top pods by CPU (cores)", + "gridPos": {"x": 0, "y": 9, "w": 12, "h": 9}, + "targets": [{"expr": "topk(12, sum by (pod, namespace) (rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[2m])))", "legendFormat": "{{namespace}}/{{pod}}"}], + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []} + }, + { + "id": 4, "type": "timeseries", "title": "Top pods by memory (working set)", + "gridPos": {"x": 12, "y": 9, "w": 12, "h": 9}, + "targets": [{"expr": "topk(12, sum by (pod, namespace) (container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}))", "legendFormat": "{{namespace}}/{{pod}}"}], + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []} + }, + { + "id": 5, "type": "timeseries", "title": "Container restarts (cumulative)", + "gridPos": {"x": 0, "y": 18, "w": 12, "h": 9}, + "targets": [{"expr": "sum by (pod, namespace) (kube_pod_container_status_restarts_total)", "legendFormat": "{{namespace}}/{{pod}}"}], + "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []} + }, + { + "id": 6, "type": "timeseries", "title": "CPU: requested vs allocatable (per node, top 10 nodes)", + "gridPos": {"x": 12, "y": 18, "w": 12, "h": 9}, + "targets": [ + {"expr": "topk(10, sum by (node) (kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\"}))", "legendFormat": "requested {{node}}"}, + {"expr": "kube_node_status_allocatable{resource=\"cpu\", unit=\"core\"}", "legendFormat": "allocatable {{node}}"} + ], + "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []} + }, + { + "id": 7, "type": "timeseries", "title": "Nodes Ready", + "gridPos": {"x": 0, "y": 27, "w": 12, "h": 7}, + "targets": [{"expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 1)", "legendFormat": "nodes ready"}], + "fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 20}, "overrides": []} + }, + { + "id": 8, "type": "timeseries", "title": "Pods not Running (pending/failed/succeeded)", + "gridPos": {"x": 12, "y": 27, "w": 12, "h": 7}, + "targets": [{"expr": "count by (phase) (kube_pod_status_phase == 1) unless on(pod, namespace, phase) label_replace(kube_pod_status_phase{phase=\"Running\"}, \"phase\", \"running\", \"\", \"\") * 0", "legendFormat": "{{phase}}"}], + "fieldConfig": {"defaults": {"unit": "short", "min": 0}, "overrides": []} + } + ], + "schemaVersion": 39, + "version": 1 + } node-health.json: | { "title": "planck node health", diff --git a/manifests/monitoring/kube-state-metrics.yaml b/manifests/monitoring/kube-state-metrics.yaml new file mode 100644 index 0000000..5e14d2d --- /dev/null +++ b/manifests/monitoring/kube-state-metrics.yaml @@ -0,0 +1,71 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-state-metrics + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics +rules: + - apiGroups: [""] + resources: [pods, nodes, namespaces, services, endpoints, configmaps, secrets, persistentvolumeclaims, persistentvolumes, resourcequotas, limitranges] + verbs: [list, watch] + - apiGroups: [apps] + resources: [deployments, daemonsets, replicasets, statefulsets] + verbs: [list, watch] + - apiGroups: [batch] + resources: [jobs, cronjobs] + verbs: [list, watch] + - apiGroups: [autoscaling] + resources: [horizontalpodautoscalers] + verbs: [list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics +subjects: + - kind: ServiceAccount + name: kube-state-metrics + namespace: monitoring +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: {app: kube-state-metrics} + template: + metadata: + labels: {app: kube-state-metrics} + spec: + serviceAccountName: kube-state-metrics + containers: + - name: kube-state-metrics + image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.17.0 + ports: [{containerPort: 8080}] + resources: + requests: {cpu: 30m, memory: 64Mi} + limits: {memory: 512Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-state-metrics + namespace: monitoring + labels: {app: kube-state-metrics} +spec: + selector: {app: kube-state-metrics} + ports: + - name: http-metrics + port: 8080 + targetPort: 8080 diff --git a/manifests/monitoring/prometheus.yaml b/manifests/monitoring/prometheus.yaml index 06865c9..e2b480e 100644 --- a/manifests/monitoring/prometheus.yaml +++ b/manifests/monitoring/prometheus.yaml @@ -61,6 +61,50 @@ data: - source_labels: [__meta_kubernetes_pod_container_port_number] action: keep regex: "9100" + - job_name: kube-state-metrics + kubernetes_sd_configs: + - role: pod + selectors: + - role: pod + label: "app=kube-state-metrics" + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_port_number] + action: keep + regex: "8080" + - job_name: kubelet + scheme: https + tls_config: + insecure_skip_verify: true + authorization: + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - source_labels: [__meta_kubernetes_node_name] + target_label: instance + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + target_label: __metrics_path__ + regex: (.+) + replacement: /api/v1/nodes/${1}/proxy/metrics + - job_name: cadvisor + scheme: https + tls_config: + insecure_skip_verify: true + authorization: + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - source_labels: [__meta_kubernetes_node_name] + target_label: instance + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + target_label: __metrics_path__ + regex: (.+) + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor --- apiVersion: apps/v1 kind: StatefulSet