k6 step-load rig: staircase scenarios, per-step/per-endpoint tagging, live Prometheus remote-write, Grafana dashboard

manifests/loadtest/k6-step-script.yaml

@@ -0,0 +1,94 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: k6-step-script
+  namespace: loadtest
+data:
+  step.js: |
+    // Step-load (staircase) test. Hits a collection of endpoints, stepping
+    // the request rate up over time, tagging every request with its step and
+    // endpoint so latency can be sliced either way in Grafana.
+    //
+    // Env:
+    //   BASE_URL        - target base URL
+    //   ENDPOINTS_JSON  - [{"name":"list","method":"GET","path":"/x","weight":2}, ...]
+    //   START_RPS       - rate of step 1
+    //   STEP_RPS        - rate added each step
+    //   STEP_DURATION   - hold time per step, e.g. "2m"
+    //   STEPS           - number of steps
+    import http from 'k6/http';
+    import { check } from 'k6';
+    import exec from 'k6/execution';
+
+    const BASE = __ENV.BASE_URL;
+    let endpoints;
+    try {
+      endpoints = JSON.parse(__ENV.ENDPOINTS_JSON);
+    } catch (e) {
+      throw new Error('ENDPOINTS_JSON is not valid JSON: ' + e.message);
+    }
+    // expand weights
+    const pool = [];
+    for (const ep of endpoints) {
+      for (let i = 0; i < (ep.weight || 1); i++) pool.push(ep);
+    }
+
+    const startRps = parseInt(__ENV.START_RPS || '50');
+    const stepRps = parseInt(__ENV.STEP_RPS || '50');
+    const stepDurMs = ms(__ENV.STEP_DURATION || '2m');
+    const steps = parseInt(__ENV.STEPS || '5');
+    const rampMs = 5000;
+
+    // Build stage list and per-step [startMs, endMs, label] windows.
+    // Each step: 5s quick ramp to the rate, then a hold.
+    const stages = [];
+    const windows = [];
+    let clock = 0;
+    for (let i = 0; i < steps; i++) {
+      const rate = startRps + i * stepRps;
+      stages.push({ duration: ms2str(rampMs), target: rate });
+      clock += rampMs;
+      const wStart = clock;
+      stages.push({ duration: ms2str(stepDurMs), target: rate });
+      clock += stepDurMs;
+      windows.push([wStart, clock, `step${i + 1} @ ${rate} rps`]);
+    }
+    const totalMs = clock;
+    const maxRate = startRps + (steps - 1) * stepRps;
+    const vus = Math.max(10, Math.ceil(maxRate * 2));
+
+    function ms(s) {
+      const m = /^(\d+)(ms|s|m)$/.exec(String(s));
+      if (!m) throw new Error('bad duration: ' + s);
+      return +m[1] * ({ ms: 1, s: 1000, m: 60000 })[m[2]];
+    }
+    function ms2str(n) { return n + 'ms'; }
+
+    export const options = {
+      discardResponseBodies: true,
+      scenarios: {
+        staircase: {
+          executor: 'ramping-arrival-rate',
+          startRate: 0,
+          preAllocatedVUs: vus,
+          maxVUs: vus,
+          stages: stages,
+        },
+      },
+      thresholds: { http_req_failed: ['rate<0.05'] },
+    };
+
+    export default function () {
+      const elapsed = exec.scenario.progress * totalMs;
+      let step = 'ramp-up';
+      for (const [s, e, label] of windows) {
+        if (elapsed >= s && elapsed < e) { step = label; break; }
+      }
+      const ep = pool[exec.scenario.iterationInTest % pool.length];
+      const url = BASE + ep.path;
+      const params = { tags: { name: ep.name, step: step }, timeout: '15s' };
+      const res = (ep.method || 'GET') === 'POST'
+        ? http.post(url, ep.body || '', params)
+        : http.get(url, params);
+      check(res, { ok: (r) => r.status < 300 });
+    }

manifests/monitoring/grafana-dashboards.yaml

@@ -0,0 +1,86 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-provider
+  namespace: monitoring
+data:
+  dashboards.yaml: |
+    apiVersion: 1
+    providers:
+      - name: default
+        orgId: 1
+        folder: ""
+        type: file
+        disableDeletion: false
+        updateIntervalSeconds: 30
+        options:
+          path: /var/lib/grafana/dashboards
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboards
+  namespace: monitoring
+data:
+  k6-step-load.json: |
+    {
+      "title": "k6 Step Load",
+      "uid": "k6step",
+      "timezone": "browser",
+      "refresh": "5s",
+      "time": { "from": "now-30m", "to": "now" },
+      "panels": [
+        {
+          "id": 1, "type": "timeseries", "title": "Request rate (the staircase)",
+          "gridPos": {"x": 0, "y": 0, "w": 24, "h": 8},
+          "targets": [
+            {"expr": "sum(rate(k6_http_reqs_total[30s]))", "legendFormat": "total rps"},
+            {"expr": "sum by (name) (rate(k6_http_reqs_total[30s]))", "legendFormat": "{{name}}"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}
+        },
+        {
+          "id": 2, "type": "timeseries", "title": "Latency avg / p50 / p95 / p99",
+          "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8},
+          "targets": [
+            {"expr": "k6_http_req_duration_avg", "legendFormat": "avg"},
+            {"expr": "k6_http_req_duration_p50", "legendFormat": "p50"},
+            {"expr": "k6_http_req_duration_p95", "legendFormat": "p95"},
+            {"expr": "k6_http_req_duration_p99", "legendFormat": "p99"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "ms"}, "overrides": []}
+        },
+        {
+          "id": 3, "type": "timeseries", "title": "p95 latency per step",
+          "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8},
+          "targets": [
+            {"expr": "k6_http_req_duration_p95", "legendFormat": "{{step}}"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "ms"}, "overrides": []}
+        },
+        {
+          "id": 4, "type": "timeseries", "title": "p95 latency per endpoint",
+          "gridPos": {"x": 0, "y": 16, "w": 12, "h": 8},
+          "targets": [
+            {"expr": "k6_http_req_duration_p95", "legendFormat": "{{name}}"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "ms"}, "overrides": []}
+        },
+        {
+          "id": 5, "type": "timeseries", "title": "Error rate",
+          "gridPos": {"x": 12, "y": 16, "w": 12, "h": 8},
+          "targets": [
+            {"expr": "sum(rate(k6_http_req_failed_total[30s])) / sum(rate(k6_http_reqs_total[30s]))", "legendFormat": "error fraction"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []}
+        },
+        {
+          "id": 6, "type": "timeseries", "title": "Active VUs",
+          "gridPos": {"x": 0, "y": 24, "w": 24, "h": 6},
+          "targets": [{"expr": "k6_vus", "legendFormat": "vus"}],
+          "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}
+        }
+      ],
+      "schemaVersion": 39,
+      "version": 1
+    }

manifests/monitoring/grafana.yaml

@@ -49,9 +49,15 @@ spec:
             limits: {memory: 512Mi}
           volumeMounts:
             - {name: datasources, mountPath: /etc/grafana/provisioning/datasources}
+            - {name: dashboard-provider, mountPath: /etc/grafana/provisioning/dashboards}
+            - {name: dashboards, mountPath: /var/lib/grafana/dashboards}
       volumes:
         - name: datasources
           configMap: {name: grafana-datasources}
+        - name: dashboard-provider
+          configMap: {name: grafana-dashboard-provider}
+        - name: dashboards
+          configMap: {name: grafana-dashboards}
 ---
 apiVersion: v1
 kind: Service

manifests/monitoring/prometheus.yaml

@@ -84,6 +84,7 @@ spec:
             - --config.file=/etc/prometheus/prometheus.yml
             - --storage.tsdb.retention.time=15d
             - --storage.tsdb.path=/data
+            - --web.enable-remote-write-receiver
           ports:
             - containerPort: 9090
           resources:

scripts/stepload.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# Run a step-load (staircase) test across a collection of endpoints.
+# Usage:
+#   ./scripts/stepload.sh <base-url> <endpoints-json> [start-rps] [step-rps] [step-duration] [steps]
+# Example:
+#   ./scripts/stepload.sh https://api.example.com \
+#     '[{"name":"login","method":"POST","path":"/login"},{"name":"list","path":"/items","weight":3}]' \
+#     25 25 2m 6
+#   -> starts at 25 rps, +25 rps every 2 minutes, 6 steps (ending at 150 rps)
+set -euo pipefail
+
+BASE=${1:?usage: stepload.sh <base-url> <endpoints-json> [start-rps] [step-rps] [step-duration] [steps]}
+EPS=${2:?endpoints-json required - see script header for format}
+START=${3:-50}
+STEP=${4:-50}
+DUR=${5:-2m}
+STEPS=${6:-5}
+NAME="stepload-$(date +%s)"
+PROM_RW="http://prometheus.monitoring.svc:9090/api/v1/write"
+
+kubectl -n loadtest apply -f - <<EOF
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: $NAME
+spec:
+  parallelism: 1
+  ttlSecondsAfterFinished: 1800
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: k6
+          image: grafana/k6:1.4.0
+          command: ["k6", "run", "/scripts/step.js", "-o", "experimental-prometheus-rw"]
+          env:
+            - {name: BASE_URL, value: "$BASE"}
+            - {name: ENDPOINTS_JSON, value: '$EPS'}
+            - {name: START_RPS, value: "$START"}
+            - {name: STEP_RPS, value: "$STEP"}
+            - {name: STEP_DURATION, value: "$DUR"}
+            - {name: STEPS, value: "$STEPS"}
+            - {name: K6_PROMETHEUS_RW_SERVER_URL, value: "$PROM_RW"}
+            - {name: K6_PROMETHEUS_RW_TREND_STATS, value: "avg,p(50),p(95),p(99),min,max"}
+          resources:
+            requests: {cpu: 200m, memory: 256Mi}
+            limits: {memory: 512Mi}
+          volumeMounts:
+            - {name: scripts, mountPath: /scripts}
+      volumes:
+        - name: scripts
+          configMap: {name: k6-step-script}
+EOF
+
+MAX=$(( START + (STEPS - 1) * STEP ))
+echo "Staircase '$NAME': $STEPS steps, $START -> $MAX rps (+$STEP each), $DUR holds"
+echo "Total duration: $STEPS x $DUR (plus 5s ramps per step)"
+echo ""
+echo "Watch live:  Grafana -> the 'k6 Step Load' dashboard (or kubectl -n loadtest logs -f job/$NAME)"
+echo "Logs:        kubectl -n loadtest logs -f job/$NAME"
+echo "Stop early:  kubectl -n loadtest delete job $NAME"