From 97d5cda6f92f75cb559093047158c7628436f98c Mon Sep 17 00:00:00 2001 From: Adam Carr Date: Sun, 20 Sep 2026 11:12:03 -0700 Subject: [PATCH] k6 step-load rig: staircase scenarios, per-step/per-endpoint tagging, live Prometheus remote-write, Grafana dashboard --- manifests/loadtest/k6-step-script.yaml | 94 ++++++++++++++++++++ manifests/monitoring/grafana-dashboards.yaml | 86 ++++++++++++++++++ manifests/monitoring/grafana.yaml | 6 ++ manifests/monitoring/prometheus.yaml | 1 + scripts/stepload.sh | 61 +++++++++++++ 5 files changed, 248 insertions(+) create mode 100644 manifests/loadtest/k6-step-script.yaml create mode 100644 manifests/monitoring/grafana-dashboards.yaml create mode 100755 scripts/stepload.sh diff --git a/manifests/loadtest/k6-step-script.yaml b/manifests/loadtest/k6-step-script.yaml new file mode 100644 index 0000000..c0b9fb4 --- /dev/null +++ b/manifests/loadtest/k6-step-script.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: k6-step-script + namespace: loadtest +data: + step.js: | + // Step-load (staircase) test. Hits a collection of endpoints, stepping + // the request rate up over time, tagging every request with its step and + // endpoint so latency can be sliced either way in Grafana. + // + // Env: + // BASE_URL - target base URL + // ENDPOINTS_JSON - [{"name":"list","method":"GET","path":"/x","weight":2}, ...] + // START_RPS - rate of step 1 + // STEP_RPS - rate added each step + // STEP_DURATION - hold time per step, e.g. "2m" + // STEPS - number of steps + import http from 'k6/http'; + import { check } from 'k6'; + import exec from 'k6/execution'; + + const BASE = __ENV.BASE_URL; + let endpoints; + try { + endpoints = JSON.parse(__ENV.ENDPOINTS_JSON); + } catch (e) { + throw new Error('ENDPOINTS_JSON is not valid JSON: ' + e.message); + } + // expand weights + const pool = []; + for (const ep of endpoints) { + for (let i = 0; i < (ep.weight || 1); i++) pool.push(ep); + } + + const startRps = parseInt(__ENV.START_RPS || '50'); + const stepRps = parseInt(__ENV.STEP_RPS || '50'); + const stepDurMs = ms(__ENV.STEP_DURATION || '2m'); + const steps = parseInt(__ENV.STEPS || '5'); + const rampMs = 5000; + + // Build stage list and per-step [startMs, endMs, label] windows. + // Each step: 5s quick ramp to the rate, then a hold. + const stages = []; + const windows = []; + let clock = 0; + for (let i = 0; i < steps; i++) { + const rate = startRps + i * stepRps; + stages.push({ duration: ms2str(rampMs), target: rate }); + clock += rampMs; + const wStart = clock; + stages.push({ duration: ms2str(stepDurMs), target: rate }); + clock += stepDurMs; + windows.push([wStart, clock, `step${i + 1} @ ${rate} rps`]); + } + const totalMs = clock; + const maxRate = startRps + (steps - 1) * stepRps; + const vus = Math.max(10, Math.ceil(maxRate * 2)); + + function ms(s) { + const m = /^(\d+)(ms|s|m)$/.exec(String(s)); + if (!m) throw new Error('bad duration: ' + s); + return +m[1] * ({ ms: 1, s: 1000, m: 60000 })[m[2]]; + } + function ms2str(n) { return n + 'ms'; } + + export const options = { + discardResponseBodies: true, + scenarios: { + staircase: { + executor: 'ramping-arrival-rate', + startRate: 0, + preAllocatedVUs: vus, + maxVUs: vus, + stages: stages, + }, + }, + thresholds: { http_req_failed: ['rate<0.05'] }, + }; + + export default function () { + const elapsed = exec.scenario.progress * totalMs; + let step = 'ramp-up'; + for (const [s, e, label] of windows) { + if (elapsed >= s && elapsed < e) { step = label; break; } + } + const ep = pool[exec.scenario.iterationInTest % pool.length]; + const url = BASE + ep.path; + const params = { tags: { name: ep.name, step: step }, timeout: '15s' }; + const res = (ep.method || 'GET') === 'POST' + ? http.post(url, ep.body || '', params) + : http.get(url, params); + check(res, { ok: (r) => r.status < 300 }); + } diff --git a/manifests/monitoring/grafana-dashboards.yaml b/manifests/monitoring/grafana-dashboards.yaml new file mode 100644 index 0000000..ef11fbd --- /dev/null +++ b/manifests/monitoring/grafana-dashboards.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-provider + namespace: monitoring +data: + dashboards.yaml: | + apiVersion: 1 + providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards + namespace: monitoring +data: + k6-step-load.json: | + { + "title": "k6 Step Load", + "uid": "k6step", + "timezone": "browser", + "refresh": "5s", + "time": { "from": "now-30m", "to": "now" }, + "panels": [ + { + "id": 1, "type": "timeseries", "title": "Request rate (the staircase)", + "gridPos": {"x": 0, "y": 0, "w": 24, "h": 8}, + "targets": [ + {"expr": "sum(rate(k6_http_reqs_total[30s]))", "legendFormat": "total rps"}, + {"expr": "sum by (name) (rate(k6_http_reqs_total[30s]))", "legendFormat": "{{name}}"} + ], + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []} + }, + { + "id": 2, "type": "timeseries", "title": "Latency avg / p50 / p95 / p99", + "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, + "targets": [ + {"expr": "k6_http_req_duration_avg", "legendFormat": "avg"}, + {"expr": "k6_http_req_duration_p50", "legendFormat": "p50"}, + {"expr": "k6_http_req_duration_p95", "legendFormat": "p95"}, + {"expr": "k6_http_req_duration_p99", "legendFormat": "p99"} + ], + "fieldConfig": {"defaults": {"unit": "ms"}, "overrides": []} + }, + { + "id": 3, "type": "timeseries", "title": "p95 latency per step", + "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, + "targets": [ + {"expr": "k6_http_req_duration_p95", "legendFormat": "{{step}}"} + ], + "fieldConfig": {"defaults": {"unit": "ms"}, "overrides": []} + }, + { + "id": 4, "type": "timeseries", "title": "p95 latency per endpoint", + "gridPos": {"x": 0, "y": 16, "w": 12, "h": 8}, + "targets": [ + {"expr": "k6_http_req_duration_p95", "legendFormat": "{{name}}"} + ], + "fieldConfig": {"defaults": {"unit": "ms"}, "overrides": []} + }, + { + "id": 5, "type": "timeseries", "title": "Error rate", + "gridPos": {"x": 12, "y": 16, "w": 12, "h": 8}, + "targets": [ + {"expr": "sum(rate(k6_http_req_failed_total[30s])) / sum(rate(k6_http_reqs_total[30s]))", "legendFormat": "error fraction"} + ], + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []} + }, + { + "id": 6, "type": "timeseries", "title": "Active VUs", + "gridPos": {"x": 0, "y": 24, "w": 24, "h": 6}, + "targets": [{"expr": "k6_vus", "legendFormat": "vus"}], + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []} + } + ], + "schemaVersion": 39, + "version": 1 + } diff --git a/manifests/monitoring/grafana.yaml b/manifests/monitoring/grafana.yaml index 323ef21..e25a82e 100644 --- a/manifests/monitoring/grafana.yaml +++ b/manifests/monitoring/grafana.yaml @@ -49,9 +49,15 @@ spec: limits: {memory: 512Mi} volumeMounts: - {name: datasources, mountPath: /etc/grafana/provisioning/datasources} + - {name: dashboard-provider, mountPath: /etc/grafana/provisioning/dashboards} + - {name: dashboards, mountPath: /var/lib/grafana/dashboards} volumes: - name: datasources configMap: {name: grafana-datasources} + - name: dashboard-provider + configMap: {name: grafana-dashboard-provider} + - name: dashboards + configMap: {name: grafana-dashboards} --- apiVersion: v1 kind: Service diff --git a/manifests/monitoring/prometheus.yaml b/manifests/monitoring/prometheus.yaml index 28e638f..46cf1a6 100644 --- a/manifests/monitoring/prometheus.yaml +++ b/manifests/monitoring/prometheus.yaml @@ -84,6 +84,7 @@ spec: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.retention.time=15d - --storage.tsdb.path=/data + - --web.enable-remote-write-receiver ports: - containerPort: 9090 resources: diff --git a/scripts/stepload.sh b/scripts/stepload.sh new file mode 100755 index 0000000..54a98d8 --- /dev/null +++ b/scripts/stepload.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Run a step-load (staircase) test across a collection of endpoints. +# Usage: +# ./scripts/stepload.sh [start-rps] [step-rps] [step-duration] [steps] +# Example: +# ./scripts/stepload.sh https://api.example.com \ +# '[{"name":"login","method":"POST","path":"/login"},{"name":"list","path":"/items","weight":3}]' \ +# 25 25 2m 6 +# -> starts at 25 rps, +25 rps every 2 minutes, 6 steps (ending at 150 rps) +set -euo pipefail + +BASE=${1:?usage: stepload.sh [start-rps] [step-rps] [step-duration] [steps]} +EPS=${2:?endpoints-json required - see script header for format} +START=${3:-50} +STEP=${4:-50} +DUR=${5:-2m} +STEPS=${6:-5} +NAME="stepload-$(date +%s)" +PROM_RW="http://prometheus.monitoring.svc:9090/api/v1/write" + +kubectl -n loadtest apply -f - < $MAX rps (+$STEP each), $DUR holds" +echo "Total duration: $STEPS x $DUR (plus 5s ramps per step)" +echo "" +echo "Watch live: Grafana -> the 'k6 Step Load' dashboard (or kubectl -n loadtest logs -f job/$NAME)" +echo "Logs: kubectl -n loadtest logs -f job/$NAME" +echo "Stop early: kubectl -n loadtest delete job $NAME"