From b722631d3254c5cc2882993b8e361913e540cbd8 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 19 Nov 2026 18:13:17 -0800 Subject: [PATCH 1/9] Add opt-in scale comparison framework Signed-off-by: James Sturtevant --- Justfile | 30 + docs/observability.md | 3 + docs/scale-testing.md | 62 ++ k8s/observability.yaml | 2 +- scripts/fixtures/scale-workload/Dockerfile | 13 + scripts/scale-test.py | 712 +++++++++++++++++++++ scripts/tests/test_scale_test.py | 266 ++++++++ 7 files changed, 1087 insertions(+), 1 deletion(-) create mode 100644 docs/scale-testing.md create mode 100644 scripts/fixtures/scale-workload/Dockerfile create mode 100644 scripts/scale-test.py create mode 100644 scripts/tests/test_scale_test.py diff --git a/Justfile b/Justfile index 3e2e54a..4e4c1f5 100644 --- a/Justfile +++ b/Justfile @@ -15,6 +15,7 @@ exe-suffix := if os() == "windows" { ".exe" } else { "" } # Kind release installed by `just install-kind` in every workflow (CI, release # gates, Copilot); providers/kind/templates/kind-config.yaml pins its node image. kind-version := "v0.33.0" +oha-version := "1.17.0" default: @just --list @@ -205,6 +206,18 @@ install-mm: mkdir -p "{{repo-root}}/build/tools"; \ tar --extract --gzip --file "$dir/mm.tar.gz" --directory "{{repo-root}}/build/tools" mm +# Install the pinned oha load generator used by opt-in scale tests. +install-oha: + if [ -x "{{repo-root}}/build/tools/oha" ] && \ + [ "$("{{repo-root}}/build/tools/oha" --version)" = "oha {{oha-version}}" ]; then exit 0; fi; \ + [ "$(uname -sm)" = "Linux x86_64" ] || { echo "install-oha supports linux-amd64 only" >&2; exit 1; }; \ + mkdir -p "{{repo-root}}/build/tools"; \ + curl --fail --silent --show-error --location --output "{{repo-root}}/build/tools/oha.tmp" \ + "https://github.com/hatoo/oha/releases/download/v{{oha-version}}/oha-linux-amd64"; \ + echo "85c6eb98066b8c8bd513b2dd96fafd8f9863249ea653606bae1ca2e3a75c43d2 {{repo-root}}/build/tools/oha.tmp" | sha256sum --check; \ + chmod 0755 "{{repo-root}}/build/tools/oha.tmp"; \ + mv "{{repo-root}}/build/tools/oha.tmp" "{{repo-root}}/build/tools/oha" + prepare-workspace: build-ly if [[ ! -f build/workspace/lampyr.workspace.yaml ]]; then build/ly scaffold workspace build/workspace; fi @@ -346,6 +359,23 @@ release-disruption-gate service url: --service "{{service}}" \ --url "{{url}}" +# Build the normal Kubernetes image from the same Go source as the Lampyr example. +[positional-arguments] +scale-baseline-image image: + docker build --tag "$1" --file scripts/fixtures/scale-workload/Dockerfile . + +# Compare Lampyr with a normal Kubernetes Deployment using platform telemetry. +[positional-arguments] +scale-test context service lampyr-url baseline-image duration="30s" concurrency="1,4,16,64" output="build/scale-results.json": install-oha + python3 scripts/scale-test.py \ + --context "$1" \ + --service "$2" \ + --lampyr-url "$3" \ + --baseline-image "$4" \ + --duration "$5" \ + --concurrency "$6" \ + --output "$7" + # Run the repository test suites used in CI. test: test-unit test-cli examples-check test-preview-release cli::test cli::test-intent helper::test azure::test kind::test k3d::test test-examples-local test-kind-integration test-k3d-integration test-existing-kind-integration test-knative-portability test-walkthrough diff --git a/docs/observability.md b/docs/observability.md index c4bfe2e..1b678a5 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -59,6 +59,9 @@ Uninstall disables fiber-router trace export before it removes Jaeger. Remove the add-on before `ly cluster uninstall` when retaining the Kubernetes cluster; the platform uninstall does not remove optional add-ons. +The [scale framework](scale-testing.md) uses these metrics for repeatable +Lampyr and Kubernetes comparisons. + ## Platform metrics | Metric | Type | Meaning | diff --git a/docs/scale-testing.md b/docs/scale-testing.md new file mode 100644 index 0000000..749e028 --- /dev/null +++ b/docs/scale-testing.md @@ -0,0 +1,62 @@ +--- +title: Scale testing +description: Compare Lampyr with the same application in a Kubernetes Deployment. +--- + +# Scale testing + +The opt-in scale framework compares a Lampyr Service with the same Go +application in a normal Kubernetes Deployment. It uses oha for client latency +and throughput, and queries the optional observability add-on for platform +metrics. It does not run in CI. + +Install the add-on and deploy the committed Go standalone example through +Lampyr. Build and publish the normal Kubernetes image from the same source: + +```bash +just scale-baseline-image REGISTRY/IMAGE:TAG +docker push REGISTRY/IMAGE:TAG +docker buildx imagetools inspect REGISTRY/IMAGE:TAG +``` + +Pass the digest-pinned image and the Lampyr URL: + +```bash +just scale-test \ + CONTEXT \ + SERVICE \ + https://LAMPYR_URL \ + REGISTRY/IMAGE@sha256:DIGEST +``` + +The default sweep runs each target for 30 seconds at concurrency 1, 4, 16, +and 64. The framework: + +- creates a temporary one-replica Kubernetes Deployment and Service; +- warms both targets; +- runs the same client against Lampyr and Kubernetes; +- queries Prometheus for request, Fiber, CPU, and memory measurements; +- writes `build/scale-results.json` after every scenario; and +- writes a concise Markdown comparison beside it. + +The Kubernetes target uses a local Service port-forward, while the Lampyr URL +uses its configured ingress. Record that path difference when interpreting +results. The result metadata records both paths. The framework monitors the +port-forward and stops if it exits, but the tunnel can still limit high +concurrency results. This framework is for repeatable comparisons, not a claim +that the two network paths have identical overhead. + +The report calculates HTTP success from response status codes and transport +errors. Its latency columns include only successful HTTP responses. Platform +error and shed counts come from Lampyr metrics. The runner waits for a final +Prometheus scrape after each load interval. If needed, it waits for up to five +more scrape intervals so short smoke tests still have enough cAdvisor samples. +CPU measurements include that collection interval. + +Use the Lampyr Platform Grafana dashboard while a run is active. The JSON +result remains the source for comparisons across commits or machines. It +records the image digest, namespace, replica count, oha version, Git state, and +a limited node summary without node addresses or machine identifiers. + +Keep the node type, cluster configuration, workload image, Lampyr settings, +duration, and concurrency sweep unchanged when comparing revisions. diff --git a/k8s/observability.yaml b/k8s/observability.yaml index cfd63d3..53798e7 100644 --- a/k8s/observability.yaml +++ b/k8s/observability.yaml @@ -273,7 +273,7 @@ spec: image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.20.0@sha256:42cfe3723a5f058171c627537fb57a3ea0f26e4380fa18555a95cb1a1b4cfc5b args: - --resources=deployments,namespaces,nodes,pods - - --metric-labels-allowlist=pods=[lampyr.dev/home] + - --metric-labels-allowlist=pods=[lampyr.dev/home,serving.knative.dev/revision] ports: - name: metrics containerPort: 8080 diff --git a/scripts/fixtures/scale-workload/Dockerfile b/scripts/fixtures/scale-workload/Dockerfile new file mode 100644 index 0000000..c0a8c05 --- /dev/null +++ b/scripts/fixtures/scale-workload/Dockerfile @@ -0,0 +1,13 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.26.7-bookworm@sha256:e8c859f5632dcfde7b32d2012b4351728f6437930887c2f6a91ea242459e5514 AS build +WORKDIR /src +COPY docs/examples/hello-go-standalone/go.mod docs/examples/hello-go-standalone/go.sum ./ +RUN go mod download +COPY docs/examples/hello-go-standalone/application.go docs/examples/hello-go-standalone/main.go ./ +RUN CGO_ENABLED=0 go build -mod=readonly -trimpath -ldflags="-s -w" -o /service . + +FROM scratch +COPY --from=build /service /service +ENV PORT=8080 +USER 65532:65532 +ENTRYPOINT ["/service"] diff --git a/scripts/scale-test.py b/scripts/scale-test.py new file mode 100644 index 0000000..e28c3a7 --- /dev/null +++ b/scripts/scale-test.py @@ -0,0 +1,712 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import datetime +import json +import math +import os +from pathlib import Path +import re +import socket +import subprocess +import sys +import tempfile +import time +import urllib.parse + + +BASELINE = "lampyr-scale-baseline" +PROMETHEUS_PROXY = ( + "/api/v1/namespaces/lampyr-observability/" + "services/http:prometheus:9090/proxy/api/v1/query" +) + + +def command(*args): + return subprocess.check_output(args, text=True).strip() + + +def duration_seconds(value): + match = re.fullmatch(r"([1-9][0-9]*)(s|m)", value) + if not match: + raise ValueError("duration must be a positive number of seconds or minutes") + amount = int(match.group(1)) + return amount * (60 if match.group(2) == "m" else 1) + + +def active_revision(context, service, namespace): + value = json.loads( + command( + "kubectl", + "--context", + context, + "--namespace", + namespace, + "get", + "service.serving.knative.dev", + service, + "--output", + "json", + ) + ) + revisions = { + target["revisionName"] + for target in value.get("status", {}).get("traffic", []) + if target.get("percent", 0) > 0 + } + if len(revisions) != 1: + raise RuntimeError( + f"service must have exactly one active Revision, got {len(revisions)}" + ) + return next(iter(revisions)) + + +def prometheus_query(context, query, evaluation_time=None): + parameters = {"query": query} + if evaluation_time is not None: + parameters["time"] = f"{evaluation_time:.3f}" + path = PROMETHEUS_PROXY + "?" + urllib.parse.urlencode(parameters) + result = json.loads(command("kubectl", "--context", context, "get", "--raw", path)) + if result.get("status") != "success": + raise RuntimeError(f"Prometheus query failed: {result}") + values = result["data"]["result"] + if not values: + return None + if len(values) != 1: + raise RuntimeError(f"Prometheus query returned {len(values)} series: {query}") + value = float(values[0]["value"][1]) + return value if math.isfinite(value) else None + + +def metric_queries(namespace, revision, window, baseline, window_seconds=30): + revision_match = f'namespace="{namespace}",revision="{revision}"' + running_home = ( + '* on(namespace,pod) group_left() ' + f'(kube_pod_status_phase{{namespace="{namespace}",phase="Running"}} == 1)' + ) + running_router = ( + '* on(namespace,pod) group_left() ' + '(kube_pod_status_phase{namespace="lampyr-system",phase="Running"} == 1)' + ) + home = ( + '* on(namespace,pod) group_left() ' + f'(kube_pod_labels{{namespace="{namespace}",label_lampyr_dev_home="true",' + f'label_serving_knative_dev_revision="{revision}"}} {running_home})' + ) + baseline_pods = f'namespace="{namespace}",pod=~"{baseline}-.*"' + return { + "lampyrRequests": ( + "sum(lampyr_router_request_duration_seconds_count" + f"{{{revision_match}}})" + ), + "lampyrErrors": ( + "sum(lampyr_router_request_duration_seconds_count" + f'{{{revision_match},code!~"2..|3.."}})' + ), + "lampyrRequestP95Seconds": ( + "histogram_quantile(0.95,sum by(le)(increase(" + f"lampyr_router_request_duration_seconds_bucket{{{revision_match}}}" + f"[{window}])))" + ), + "lampyrAcquireP95Seconds": ( + "histogram_quantile(0.95,sum by(le)(increase(" + f"lampyr_router_fiber_acquire_seconds_bucket{{{revision_match}}}" + f"[{window}])))" + ), + "lampyrShed": ( + "sum(lampyr_router_shed_total" f"{{{revision_match}}})" + ), + "lampyrRetries": ( + "sum(lampyr_router_clone_retries_total" f"{{{revision_match}}})" + ), + "lampyrPeakBusyFibers": ( + "max_over_time((sum(lampyr_router_fibers_busy" + f"{{{revision_match}}}))[{window}:5s])" + ), + "lampyrPeakInFlight": ( + "max_over_time((sum(lampyr_router_requests_in_flight" + f"{{{revision_match}}}))[{window}:5s])" + ), + "lampyrRouterCPU": ( + "sum(increase(container_cpu_usage_seconds_total" + f'{{namespace="lampyr-system",container="router"}}[{window}])' + f" {running_router}" + ")" + f" / {window_seconds:.3f}" + ), + "lampyrRouterMemoryBytes": ( + "max_over_time((sum(container_memory_working_set_bytes" + '{namespace="lampyr-system",container="router"} ' + f"{running_router}))" + f"[{window}:5s])" + ), + "lampyrHomeCPU": ( + "(sum(increase(container_cpu_usage_seconds_total" + f'{{namespace="{namespace}",container="fiberd"}}[{window}]) ' + f"{home})) / {window_seconds:.3f}" + ), + "lampyrHomeMemoryBytes": ( + "max_over_time((sum(container_memory_working_set_bytes" + f'{{namespace="{namespace}",container="fiberd"}} {home}))[{window}:5s])' + ), + "baselineCPU": ( + "sum(increase(container_cpu_usage_seconds_total" + f"{{{baseline_pods},container=\"app\"}}[{window}])" + f" {running_home}" + ")" + f" / {window_seconds:.3f}" + ), + "baselineMemoryBytes": ( + "max_over_time((sum(container_memory_working_set_bytes" + f"{{{baseline_pods},container=\"app\"}} {running_home}))" + f"[{window}:5s])" + ), + } + + +COUNTER_METRICS = { + "lampyrRequests", + "lampyrErrors", + "lampyrShed", + "lampyrRetries", +} + + +def counter_snapshot(context, queries): + return { + name: prometheus_query(context, query) + for name, query in queries.items() + if name in COUNTER_METRICS + } + + +def collect_metrics(context, queries, before, evaluation_time): + collected = {} + for name, query in queries.items(): + value = prometheus_query(context, query, evaluation_time) + if name not in COUNTER_METRICS: + collected[name] = value + continue + if name == "lampyrRequests" and (before[name] is None or value is None): + raise RuntimeError( + "Lampyr request metrics are missing; check the observability add-on " + "and requested namespace" + ) + previous = before[name] or 0 + current = value or 0 + if current < previous: + raise RuntimeError(f"Prometheus counter reset during scenario: {name}") + collected[name] = current - previous + return collected + + +def baseline_manifest(image, replicas, namespace): + labels = {"app": BASELINE} + return { + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": BASELINE, "namespace": namespace}, + "spec": { + "replicas": replicas, + "selector": {"matchLabels": labels}, + "template": { + "metadata": {"labels": labels}, + "spec": { + "automountServiceAccountToken": False, + "securityContext": { + "runAsNonRoot": True, + "runAsUser": 65532, + "runAsGroup": 65532, + "seccompProfile": {"type": "RuntimeDefault"}, + }, + "containers": [ + { + "name": "app", + "image": image, + "ports": [{"name": "http", "containerPort": 8080}], + "readinessProbe": { + "httpGet": {"path": "/", "port": "http"} + }, + "resources": { + "requests": { + "cpu": "250m", + "memory": "256Mi", + }, + "limits": {"memory": "256Mi"}, + }, + "securityContext": { + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + "readOnlyRootFilesystem": True, + "runAsNonRoot": True, + }, + } + ] + }, + }, + }, + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": BASELINE, "namespace": namespace}, + "spec": { + "selector": labels, + "ports": [{"name": "http", "port": 80, "targetPort": "http"}], + }, + }, + ], + } + + +def kubectl_manifest(context, action, value, *extra): + return subprocess.run( + ["kubectl", "--context", context, action, "--filename", "-", *extra], + input=json.dumps(value), + text=True, + check=True, + stdout=subprocess.DEVNULL, + ) + + +def free_port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +@contextlib.contextmanager +def baseline_server(context, image, replicas, namespace): + if "@sha256:" not in image: + raise ValueError("baseline image must be digest-pinned") + manifest = baseline_manifest(image, replicas, namespace) + forward = None + try: + kubectl_manifest(context, "apply", manifest) + subprocess.run( + [ + "kubectl", + "--context", + context, + "--namespace", + namespace, + "rollout", + "status", + f"deployment/{BASELINE}", + "--timeout=300s", + ], + check=True, + ) + port = free_port() + forward = subprocess.Popen( + [ + "kubectl", + "--context", + context, + "--namespace", + namespace, + "port-forward", + f"service/{BASELINE}", + f"{port}:80", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + deadline = time.time() + 30 + while time.time() < deadline: + if forward.poll() is not None: + raise RuntimeError("baseline port-forward exited before becoming ready") + with socket.socket() as connection: + if connection.connect_ex(("127.0.0.1", port)) == 0: + break + time.sleep(0.1) + else: + raise RuntimeError("baseline port-forward did not become ready") + yield {"url": f"http://127.0.0.1:{port}/", "process": forward} + finally: + original_error = sys.exc_info()[0] is not None + if forward is not None: + forward.terminate() + try: + forward.wait(timeout=5) + except subprocess.TimeoutExpired: + forward.kill() + forward.wait() + try: + kubectl_manifest( + context, "delete", manifest, "--ignore-not-found=true", "--wait=true" + ) + except subprocess.CalledProcessError as error: + if not original_error: + raise + print(f"warning: baseline cleanup failed: {error}", file=sys.stderr) + + +def require_running(process, name): + return_code = process.poll() + if return_code is not None: + raise RuntimeError(f"{name} exited with status {return_code}") + + +def run_oha(oha, url, duration, concurrency): + output = subprocess.check_output( + [ + str(oha), + "-z", + duration, + "-c", + str(concurrency), + "--wait-ongoing-requests-after-deadline", + "--stats-success-breakdown", + "--no-tui", + "--output-format", + "json", + "--insecure", + url, + ], + text=True, + ) + result = json.loads(output) + client = { + "summary": result["summary"], + "metrics": result.get("metrics", {}), + "statusCodeDistribution": result["statusCodeDistribution"], + "errorDistribution": result["errorDistribution"], + "successfulLatencyMs": { + name: value * 1000 if value is not None else None + for name, value in result["latencyPercentilesSuccessful"].items() + }, + } + client["httpSuccessRate"] = http_success_rate(client) + return client + + +def http_success_rate(client): + statuses = client.get("statusCodeDistribution", {}) + errors = client.get("errorDistribution", {}) + total = sum(statuses.values()) + sum(errors.values()) + if total == 0: + return 0 + successful = sum( + count for code, count in statuses.items() if 200 <= int(code) < 400 + ) + return successful / total + + +def warm_target(oha, url): + output = subprocess.check_output( + [ + str(oha), + "-n", + "20", + "-c", + "1", + "--stats-success-breakdown", + "--no-tui", + "--output-format", + "json", + "--insecure", + url, + ], + text=True, + ) + client = json.loads(output) + if http_success_rate(client) != 1: + raise RuntimeError( + "target warmup did not return only successful HTTP responses: " + f"statuses={client.get('statusCodeDistribution', {})} " + f"errors={client.get('errorDistribution', {})}" + ) + + +def display_number(value, precision=2): + if value is None: + return "n/a" + return f"{value:.{precision}f}" + + +def add_metrics(*values): + if any(value is None for value in values): + return None + return sum(values) + + +def target_resource_names(target): + if target == "lampyr": + return ( + "lampyrRouterCPU", + "lampyrHomeCPU", + "lampyrRouterMemoryBytes", + "lampyrHomeMemoryBytes", + ) + return ("baselineCPU", "baselineMemoryBytes") + + +def target_resources_available(target, metrics): + return all(metrics[name] is not None for name in target_resource_names(target)) + + +def atomic_write(path, content): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", dir=path.parent, prefix=f".{path.name}.", delete=False + ) as handle: + temporary = Path(handle.name) + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def write_json(result, path): + atomic_write(path, json.dumps(result, indent=2, sort_keys=True) + "\n") + + +def write_report(result, path): + lines = [ + "# Lampyr scale comparison", + "", + f"- Commit: `{result['metadata']['commit']}`", + f"- Context: `{result['metadata']['context']}`", + f"- Revision: `{result['metadata']['revision']}`", + f"- Duration per run: `{result['metadata']['duration']}`", + f"- Baseline image: `{result['metadata']['baselineImage']}`", + f"- Lampyr path: {result['metadata']['networkPaths']['lampyr']}", + f"- Kubernetes path: {result['metadata']['networkPaths']['kubernetes']}", + "", + "| Target | Concurrency | Requests/s | HTTP success | p50 ms | p95 ms | p99 ms | Platform errors | Shed | CPU cores | Peak memory MiB |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for scenario in result["scenarios"]: + client = scenario["client"] + metrics = scenario["platform"] + if scenario["target"] == "lampyr": + cpu = add_metrics( + metrics["lampyrRouterCPU"], metrics["lampyrHomeCPU"] + ) + memory = add_metrics( + metrics["lampyrRouterMemoryBytes"], + metrics["lampyrHomeMemoryBytes"], + ) + errors = metrics.get("lampyrErrors") + shed = metrics.get("lampyrShed") + else: + cpu = metrics["baselineCPU"] + memory = metrics["baselineMemoryBytes"] + errors = None + shed = None + latency = client.get("successfulLatencyMs", {}) + lines.append( + "| {target} | {concurrency} | {rps:.2f} | {success:.4f} | " + "{p50} | {p95} | {p99} | {errors} | {shed} | {cpu} | {memory} |".format( + target=scenario["target"], + concurrency=scenario["concurrency"], + rps=client["summary"]["requestsPerSec"], + success=client["httpSuccessRate"], + p50=display_number(latency.get("p50")), + p95=display_number(latency.get("p95")), + p99=display_number(latency.get("p99")), + errors=display_number(errors, 0), + shed=display_number(shed, 0), + cpu=display_number(cpu, 3), + memory=display_number( + memory / (1 << 20) if memory is not None else None + ), + ) + ) + atomic_write(path, "\n".join(lines) + "\n") + + +def node_summary(value): + nodes = [] + for item in value.get("items", []): + metadata = item.get("metadata", {}) + status = item.get("status", {}) + info = status.get("nodeInfo", {}) + labels = metadata.get("labels", {}) + nodes.append( + { + "name": metadata.get("name"), + "instanceType": labels.get("node.kubernetes.io/instance-type"), + "allocatable": status.get("allocatable", {}), + "kubeletVersion": info.get("kubeletVersion"), + "operatingSystem": info.get("operatingSystem"), + "osImage": info.get("osImage"), + "kernelVersion": info.get("kernelVersion"), + "containerRuntimeVersion": info.get("containerRuntimeVersion"), + } + ) + return nodes + + +def safe_url(value): + parsed = urllib.parse.urlsplit(value) + host = parsed.hostname or "" + if ":" in host: + host = f"[{host}]" + netloc = host + if parsed.port is not None: + netloc += f":{parsed.port}" + return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, "", "")) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--context", required=True) + parser.add_argument("--service", required=True) + parser.add_argument("--lampyr-url", required=True) + parser.add_argument("--baseline-image", required=True) + parser.add_argument("--namespace", default="default") + parser.add_argument("--baseline-replicas", type=int, default=1) + parser.add_argument("--duration", default="30s") + parser.add_argument("--concurrency", default="1,4,16,64") + parser.add_argument("--oha", default="build/tools/oha") + parser.add_argument("--output", default="build/scale-results.json") + args = parser.parse_args() + + seconds = duration_seconds(args.duration) + concurrencies = [int(value) for value in args.concurrency.split(",")] + if not concurrencies or any(value < 1 for value in concurrencies): + raise ValueError("concurrency values must be positive integers") + if args.baseline_replicas < 1: + raise ValueError("baseline replicas must be a positive integer") + oha = Path(args.oha) + if not oha.is_file(): + raise FileNotFoundError(f"load generator not found: {oha}") + revision = active_revision(args.context, args.service, args.namespace) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + metadata = { + "context": args.context, + "service": args.service, + "namespace": args.namespace, + "revision": revision, + "lampyrUrl": safe_url(args.lampyr_url), + "baselineImage": args.baseline_image, + "baselineReplicas": args.baseline_replicas, + "duration": args.duration, + "concurrency": concurrencies, + "commit": command("git", "rev-parse", "HEAD"), + "dirty": bool(command("git", "status", "--porcelain")), + "ohaVersion": command(str(oha), "--version"), + "networkPaths": { + "lampyr": "configured ingress URL", + "kubernetes": "kubectl Service port-forward", + }, + "startedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "nodes": node_summary( + json.loads( + command( + "kubectl", + "--context", + args.context, + "get", + "nodes", + "--output", + "json", + ) + ) + ), + } + result = {"schema": 1, "metadata": metadata, "scenarios": []} + with baseline_server( + args.context, args.baseline_image, args.baseline_replicas, args.namespace + ) as baseline: + for target, url in [ + ("lampyr", args.lampyr_url), + ("kubernetes", baseline["url"]), + ]: + if target == "kubernetes": + require_running(baseline["process"], "baseline port-forward") + warm_target(oha, url) + time.sleep(6) + for concurrency in concurrencies: + for target, url in [ + ("lampyr", args.lampyr_url), + ("kubernetes", baseline["url"]), + ]: + if target == "kubernetes": + require_running(baseline["process"], "baseline port-forward") + snapshot_queries = metric_queries( + args.namespace, revision, "1s", BASELINE, 1 + ) + before = counter_snapshot(args.context, snapshot_queries) + if before["lampyrRequests"] is None: + raise RuntimeError( + "Lampyr request metrics are missing after warmup; check the " + "observability add-on and requested namespace" + ) + started = datetime.datetime.now(datetime.timezone.utc) + client = run_oha(oha, url, args.duration, concurrency) + load_ended = datetime.datetime.now(datetime.timezone.utc) + if target == "kubernetes": + require_running(baseline["process"], "baseline port-forward") + platform = None + for wait_seconds in (6, 5, 5, 5, 5, 5): + time.sleep(wait_seconds) + evaluated = time.time() + measured_seconds = evaluated - started.timestamp() + window = f"{math.ceil(measured_seconds * 1000)}ms" + queries = metric_queries( + args.namespace, + revision, + window, + BASELINE, + measured_seconds, + ) + resources = { + name: prometheus_query( + args.context, queries[name], evaluated + ) + for name in target_resource_names(target) + } + if target_resources_available(target, resources): + platform = collect_metrics( + args.context, queries, before, evaluated + ) + break + else: + raise RuntimeError( + f"Prometheus did not collect CPU and memory metrics for {target}" + ) + scenario = { + "target": target, + "concurrency": concurrency, + "startedAt": started.isoformat(), + "loadEndedAt": load_ended.isoformat(), + "metricsEvaluatedAt": datetime.datetime.fromtimestamp( + evaluated, datetime.timezone.utc + ).isoformat(), + "metricsWindowSeconds": measured_seconds, + "client": client, + "platform": platform, + } + result["scenarios"].append(scenario) + write_json(result, output) + write_report(result, output.with_suffix(".md")) + p95 = display_number(client["successfulLatencyMs"].get("p95")) + print( + f"{target} concurrency={concurrency} " + f"rps={client['summary']['requestsPerSec']:.2f} " + f"http_success={client['httpSuccessRate']:.4f} " + f"p95={p95}ms" + ) + result["metadata"]["completedAt"] = datetime.datetime.now( + datetime.timezone.utc + ).isoformat() + write_json(result, output) + write_report(result, output.with_suffix(".md")) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_scale_test.py b/scripts/tests/test_scale_test.py new file mode 100644 index 0000000..85caa3d --- /dev/null +++ b/scripts/tests/test_scale_test.py @@ -0,0 +1,266 @@ +import importlib.util +from pathlib import Path +import tempfile +import unittest +from unittest import mock + + +PATH = Path(__file__).resolve().parents[1] / "scale-test.py" +SPEC = importlib.util.spec_from_file_location("scale_test", PATH) +scale = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(scale) + + +class ScaleTestTests(unittest.TestCase): + def test_duration_seconds(self): + self.assertEqual(scale.duration_seconds("30s"), 30) + self.assertEqual(scale.duration_seconds("2m"), 120) + for invalid in ("0s", "1h", "ten"): + with self.assertRaises(ValueError): + scale.duration_seconds(invalid) + + def test_report_compares_lampyr_and_kubernetes(self): + result = { + "metadata": { + "commit": "abc", + "dirty": False, + "context": "kind-test", + "revision": "demo-00001", + "duration": "30s", + "baselineImage": "example.test/baseline@sha256:abc", + "networkPaths": { + "lampyr": "configured ingress URL", + "kubernetes": "kubectl Service port-forward", + }, + }, + "scenarios": [ + { + "target": "lampyr", + "concurrency": 4, + "client": { + "summary": {"requestsPerSec": 10}, + "httpSuccessRate": 1, + "successfulLatencyMs": { + "p50": 1, + "p95": 2, + "p99": 3, + }, + }, + "platform": { + "lampyrErrors": 0, + "lampyrShed": 0, + "lampyrRouterCPU": 0.1, + "lampyrHomeCPU": 0.2, + "lampyrRouterMemoryBytes": 1 << 20, + "lampyrHomeMemoryBytes": 2 << 20, + "baselineCPU": None, + "baselineMemoryBytes": None, + }, + }, + { + "target": "kubernetes", + "concurrency": 4, + "client": { + "summary": {"requestsPerSec": 20}, + "httpSuccessRate": 1, + "successfulLatencyMs": { + "p50": 1, + "p95": 2, + "p99": 3, + }, + }, + "platform": { + "lampyrErrors": None, + "lampyrShed": None, + "lampyrRouterCPU": None, + "lampyrHomeCPU": None, + "lampyrRouterMemoryBytes": None, + "lampyrHomeMemoryBytes": None, + "baselineCPU": 0.4, + "baselineMemoryBytes": 4 << 20, + }, + }, + ], + } + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "report.md" + scale.write_report(result, output) + report = output.read_text() + self.assertIn("| lampyr | 4 | 10.00", report) + self.assertIn("| kubernetes | 4 | 20.00", report) + self.assertIn("| 0 | 0 | 0.300 | 3.00 |", report) + self.assertIn("kubectl Service port-forward", report) + + def test_report_handles_no_successful_responses(self): + result = { + "metadata": { + "commit": "abc", + "dirty": True, + "context": "kind-test", + "revision": "demo-00001", + "duration": "30s", + "baselineImage": "example.test/baseline@sha256:abc", + "networkPaths": { + "lampyr": "configured ingress URL", + "kubernetes": "kubectl Service port-forward", + }, + }, + "scenarios": [ + { + "target": "lampyr", + "concurrency": 4, + "client": { + "summary": {"requestsPerSec": 10}, + "httpSuccessRate": 0, + "successfulLatencyMs": { + "p50": None, + "p95": None, + "p99": None, + }, + }, + "platform": { + "lampyrErrors": 20, + "lampyrShed": 20, + "lampyrRouterCPU": None, + "lampyrHomeCPU": None, + "lampyrRouterMemoryBytes": None, + "lampyrHomeMemoryBytes": None, + "baselineCPU": None, + "baselineMemoryBytes": None, + }, + } + ], + } + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "report.md" + scale.write_report(result, output) + report = output.read_text() + self.assertIn("| lampyr | 4 | 10.00 | 0.0000 | n/a | n/a | n/a | 20 | 20 |", report) + + def test_report_does_not_turn_missing_metrics_into_zero(self): + result = { + "metadata": { + "commit": "abc", + "dirty": False, + "context": "kind-test", + "revision": "demo-00001", + "duration": "30s", + "baselineImage": "example.test/baseline@sha256:abc", + "networkPaths": { + "lampyr": "configured ingress URL", + "kubernetes": "kubectl Service port-forward", + }, + }, + "scenarios": [ + { + "target": "kubernetes", + "concurrency": 4, + "client": { + "summary": {"requestsPerSec": 20}, + "httpSuccessRate": 1, + "successfulLatencyMs": { + "p50": 1, + "p95": 2, + "p99": 3, + }, + }, + "platform": { + "lampyrErrors": 0, + "lampyrShed": 0, + "lampyrRouterCPU": 0.1, + "lampyrHomeCPU": 0.2, + "lampyrRouterMemoryBytes": 1 << 20, + "lampyrHomeMemoryBytes": 2 << 20, + "baselineCPU": None, + "baselineMemoryBytes": None, + }, + } + ], + } + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "report.md" + scale.write_report(result, output) + report = output.read_text() + self.assertIn("| n/a | n/a | n/a | n/a |", report) + + def test_http_success_rate_includes_transport_errors(self): + client = { + "statusCodeDistribution": {"200": 3, "503": 1}, + "errorDistribution": {"connection refused": 2}, + } + self.assertEqual(scale.http_success_rate(client), 0.5) + + def test_safe_url_removes_credentials_and_query(self): + self.assertEqual( + scale.safe_url("https://user:secret@example.test:8443/path?token=secret"), + "https://example.test:8443/path", + ) + + def test_metric_queries_use_requested_namespace(self): + queries = scale.metric_queries("team-a", "demo-00001", "30s", "baseline") + for name in ( + "lampyrRequestP95Seconds", + "lampyrAcquireP95Seconds", + "lampyrPeakBusyFibers", + "lampyrPeakInFlight", + "baselineCPU", + "baselineMemoryBytes", + ): + self.assertIn('namespace="team-a"', queries[name]) + self.assertIn( + 'label_serving_knative_dev_revision="demo-00001"', + queries["lampyrHomeCPU"], + ) + for name in ( + "lampyrRouterCPU", + "lampyrRouterMemoryBytes", + "lampyrHomeCPU", + "lampyrHomeMemoryBytes", + "baselineCPU", + "baselineMemoryBytes", + ): + self.assertIn("kube_pod_status_phase", queries[name]) + + def test_target_resources_require_cpu_and_memory(self): + metrics = { + "lampyrRouterCPU": 0.1, + "lampyrHomeCPU": 0.2, + "lampyrRouterMemoryBytes": 1, + "lampyrHomeMemoryBytes": 2, + "baselineCPU": None, + "baselineMemoryBytes": None, + } + self.assertTrue(scale.target_resources_available("lampyr", metrics)) + self.assertFalse(scale.target_resources_available("kubernetes", metrics)) + + def test_baseline_manifest_uses_restricted_pod_security(self): + manifest = scale.baseline_manifest( + "example.test/baseline@sha256:abc", 1, "team-a" + ) + deployment = manifest["items"][0] + pod = deployment["spec"]["template"]["spec"] + container = pod["containers"][0] + self.assertEqual(deployment["metadata"]["namespace"], "team-a") + self.assertFalse(pod["automountServiceAccountToken"]) + self.assertEqual(pod["securityContext"]["seccompProfile"]["type"], "RuntimeDefault") + self.assertTrue(container["securityContext"]["runAsNonRoot"]) + self.assertTrue(container["securityContext"]["readOnlyRootFilesystem"]) + + def test_baseline_cleanup_runs_when_rollout_fails(self): + image = "example.test/baseline@sha256:abc" + rollout_error = scale.subprocess.CalledProcessError(1, ["kubectl"]) + with mock.patch.object(scale, "kubectl_manifest") as manifest: + with mock.patch.object( + scale.subprocess, "run", side_effect=rollout_error + ): + with self.assertRaises(scale.subprocess.CalledProcessError): + with scale.baseline_server("test", image, 1, "team-a"): + pass + self.assertEqual( + [call.args[1] for call in manifest.call_args_list], + ["apply", "delete"], + ) + + +if __name__ == "__main__": + unittest.main() From 3547b7017687f94ae6770e9741788f8dd4b5bfb8 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 17:33:45 -0800 Subject: [PATCH 2/9] fixup! Add opt-in scale comparison framework Signed-off-by: James Sturtevant --- Justfile | 1 + scripts/tests/test_scale_test.py | 212 ++++++++++++++++++++++++++++++- 2 files changed, 211 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index 4e4c1f5..5a89dca 100644 --- a/Justfile +++ b/Justfile @@ -278,6 +278,7 @@ check-action-versions: # Run focused unit tests for Lampyr. test-unit: + python3 "{{repo-root}}/scripts/tests/test_scale_test.py" go test ./cmd/... ./internal/... python3 "{{repo-root}}/scripts/tests/test_worker_workspace.py" diff --git a/scripts/tests/test_scale_test.py b/scripts/tests/test_scale_test.py index 85caa3d..42596fb 100644 --- a/scripts/tests/test_scale_test.py +++ b/scripts/tests/test_scale_test.py @@ -1,4 +1,7 @@ +import contextlib import importlib.util +import json +import os from pathlib import Path import tempfile import unittest @@ -12,6 +15,18 @@ class ScaleTestTests(unittest.TestCase): + def main_arguments(self, *extra): + return [ + str(PATH), + "--context", "test", + "--service", "demo", + "--lampyr-url", "https://example.test/", + "--baseline-image", "example.test/baseline@sha256:abc", + "--oha", str(PATH), + "--concurrency", "1", + *extra, + ] + def test_duration_seconds(self): self.assertEqual(scale.duration_seconds("30s"), 30) self.assertEqual(scale.duration_seconds("2m"), 120) @@ -53,6 +68,7 @@ def test_report_compares_lampyr_and_kubernetes(self): "lampyrHomeCPU": 0.2, "lampyrRouterMemoryBytes": 1 << 20, "lampyrHomeMemoryBytes": 2 << 20, + "lampyrMemoryBytes": 2.5 * (1 << 20), "baselineCPU": None, "baselineMemoryBytes": None, }, @@ -88,7 +104,7 @@ def test_report_compares_lampyr_and_kubernetes(self): report = output.read_text() self.assertIn("| lampyr | 4 | 10.00", report) self.assertIn("| kubernetes | 4 | 20.00", report) - self.assertIn("| 0 | 0 | 0.300 | 3.00 |", report) + self.assertIn("| 0 | 0 | 0.300 | 2.50 |", report) self.assertIn("kubectl Service port-forward", report) def test_report_handles_no_successful_responses(self): @@ -125,6 +141,7 @@ def test_report_handles_no_successful_responses(self): "lampyrHomeCPU": None, "lampyrRouterMemoryBytes": None, "lampyrHomeMemoryBytes": None, + "lampyrMemoryBytes": None, "baselineCPU": None, "baselineMemoryBytes": None, }, @@ -227,11 +244,14 @@ def test_target_resources_require_cpu_and_memory(self): "lampyrHomeCPU": 0.2, "lampyrRouterMemoryBytes": 1, "lampyrHomeMemoryBytes": 2, + "lampyrMemoryBytes": 3, "baselineCPU": None, "baselineMemoryBytes": None, } self.assertTrue(scale.target_resources_available("lampyr", metrics)) self.assertFalse(scale.target_resources_available("kubernetes", metrics)) + metrics["lampyrMemoryBytes"] = None + self.assertFalse(scale.target_resources_available("lampyr", metrics)) def test_baseline_manifest_uses_restricted_pod_security(self): manifest = scale.baseline_manifest( @@ -258,8 +278,196 @@ def test_baseline_cleanup_runs_when_rollout_fails(self): pass self.assertEqual( [call.args[1] for call in manifest.call_args_list], - ["apply", "delete"], + ["create", "create", "delete"], + ) + + def test_baseline_does_not_delete_resources_when_creation_fails(self): + failure = scale.subprocess.CalledProcessError(1, ["kubectl"]) + with mock.patch.object(scale, "kubectl_manifest", side_effect=failure) as manifest: + with self.assertRaises(scale.subprocess.CalledProcessError): + with scale.baseline_server( + "test", "example.test/baseline@sha256:abc", 1, "team-a" + ): + pass + self.assertEqual(manifest.call_count, 1) + + def test_baseline_partial_creation_cleans_only_created_resources(self): + failure = scale.subprocess.CalledProcessError(1, ["kubectl"]) + with mock.patch.object( + scale, "kubectl_manifest", side_effect=[None, failure, None] + ) as manifest, mock.patch.object( + scale.subprocess, "run", side_effect=AssertionError("unexpected rollout") + ): + with self.assertRaises(scale.subprocess.CalledProcessError): + with scale.baseline_server( + "test", "example.test/baseline@sha256:abc", 1, "team-a" + ): + pass + self.assertEqual( + [call.args[1] for call in manifest.call_args_list], + ["create", "create", "delete"], + ) + cleanup = manifest.call_args.args[2]["items"] + self.assertEqual(cleanup, [manifest.call_args_list[0].args[2]]) + self.assertEqual(cleanup[0]["kind"], "Deployment") + + def test_baseline_runs_have_distinct_names_and_selectors(self): + failure = scale.subprocess.CalledProcessError(1, ["kubectl"]) + with mock.patch.object(scale, "kubectl_manifest") as manifest, mock.patch.object( + scale.subprocess, "run", side_effect=failure + ): + for _ in range(2): + with self.assertRaises(scale.subprocess.CalledProcessError): + with scale.baseline_server( + "test", "example.test/baseline@sha256:abc", 1, "team-a" + ): + pass + deployments = [] + for call in manifest.call_args_list: + if call.args[1] in ("apply", "create"): + value = call.args[2] + deployments.extend( + item for item in value.get("items", [value]) + if item["kind"] == "Deployment" + ) + self.assertEqual(len(deployments), 2) + self.assertNotEqual( + deployments[0]["metadata"]["name"], deployments[1]["metadata"]["name"] ) + self.assertNotEqual( + deployments[0]["spec"]["selector"], deployments[1]["spec"]["selector"] + ) + + def test_counter_resets_are_rejected_when_aggregate_increases(self): + queries = scale.metric_queries("team-a", "demo-00001", "30s", "baseline") + before = {name: 100 for name in scale.COUNTER_METRICS} + for name in scale.COUNTER_METRICS: + series = queries[name].removeprefix("sum(").removesuffix(")") + + def query_result(context, query, evaluation_time): + if "resets(" in query: + return int(f"resets({series}[" in query) + return 150 if query in (queries[key] for key in before) else 1 + + with self.subTest(metric=name), mock.patch.object( + scale, "prometheus_query", side_effect=query_result + ): + with self.assertRaisesRegex(RuntimeError, f"counter reset.*{name}"): + scale.collect_metrics("test", queries, before, 42) + + def test_load_and_warmup_bound_stalled_requests_with_the_same_timeout(self): + result = json.dumps({ + "summary": {"requestsPerSec": 1}, + "statusCodeDistribution": {"200": 1}, + "errorDistribution": {}, + "latencyPercentilesSuccessful": {"p95": 0.1}, + }) + with mock.patch.object( + scale.subprocess, "check_output", return_value=result + ) as execute: + scale.run_oha(Path("/oha"), "http://example.test/", "30s", 1) + scale.warm_target(Path("/oha"), "http://example.test/") + timeouts = [] + for call in execute.call_args_list: + arguments = call.args[0] + self.assertIn("-t", arguments) + value = arguments[arguments.index("-t") + 1] + self.assertGreater(scale.duration_seconds(value), 0) + self.assertGreater(call.kwargs["timeout"], 0) + timeouts.append(value) + self.assertEqual(timeouts[0], timeouts[1]) + + def test_main_rejects_multiple_baseline_replicas_and_short_cpu_windows(self): + for arguments, message in ( + (("--baseline-replicas", "2"), "one replica"), + (("--duration", "1s"), "at least 30"), + ): + with self.subTest(arguments=arguments), mock.patch.object( + scale.sys, "argv", self.main_arguments(*arguments) + ), mock.patch.object( + scale, "active_revision", return_value="demo-00001" + ) as revision, mock.patch.object( + scale, "command", return_value="{}" + ), mock.patch.object( + scale, "baseline_server", side_effect=AssertionError("unsafe configuration") + ): + with self.assertRaisesRegex(ValueError, message): + scale.main() + revision.assert_not_called() + + def test_relative_oha_path_executes_the_requested_local_binary(self): + execute = scale.command + with tempfile.TemporaryDirectory() as temporary: + executable = Path(temporary) / "oha" + executable.write_text("#!/bin/sh\necho 'oha test'\n") + executable.chmod(0o755) + + def metadata_command(*arguments): + return execute(*arguments) if "--version" in arguments else "{}" + + with contextlib.chdir(temporary), mock.patch.dict( + os.environ, {"PATH": os.defpath} + ), mock.patch.object( + scale.sys, "argv", self.main_arguments("--oha", "./oha") + ), mock.patch.object( + scale, "active_revision", return_value="demo-00001" + ), mock.patch.object( + scale, "command", side_effect=metadata_command + ) as command, mock.patch.object( + scale, "baseline_server", side_effect=RuntimeError("metadata collected") + ): + with self.assertRaisesRegex(RuntimeError, "metadata collected"): + scale.main() + version = next( + call.args for call in command.call_args_list if "--version" in call.args + ) + self.assertEqual(version[0], str(executable)) + + def test_scenario_waits_for_every_router_post_load_scrape(self): + clock = scale.datetime.datetime.fromtimestamp(400, scale.datetime.timezone.utc) + baseline = { + "name": "baseline-owned", + "url": "http://localhost/", + "process": mock.Mock(poll=mock.Mock(return_value=None)), + } + client = { + "summary": {"requestsPerSec": 1}, + "httpSuccessRate": 1, + "successfulLatencyMs": {"p95": 1}, + } + with mock.patch.object( + scale.sys, "argv", self.main_arguments() + ), mock.patch.object( + scale, "active_revision", return_value="demo-00001" + ), mock.patch.object( + scale, "command", return_value="{}" + ), mock.patch.object( + scale, "baseline_server", return_value=contextlib.nullcontext(baseline) + ), mock.patch.object( + scale, "warm_target" + ), mock.patch.object( + scale, "run_oha", return_value=client + ), mock.patch.object( + scale, "prometheus_query", return_value=1 + ), mock.patch.object( + scale, "router_scrapes", create=True, + side_effect=[{"router-1": 399, "router-2": 399}] + + [{"router-1": 420, "router-2": 399}] * 6, + ), mock.patch.object( + scale.datetime, "datetime", wraps=scale.datetime.datetime + ) as datetime, mock.patch.object( + scale.time, "time", return_value=420 + ), mock.patch.object( + scale.time, "sleep" + ), mock.patch.object( + scale, "write_json" + ) as write, mock.patch.object( + scale, "write_report" + ), mock.patch.object(scale, "print", create=True): + datetime.now.return_value = clock + with self.assertRaisesRegex(RuntimeError, "fresh.*router"): + scale.main() + write.assert_not_called() if __name__ == "__main__": From 3a9ea5009cc76d309ee672dd47ba9e77e407f700 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 17:50:38 -0800 Subject: [PATCH 3/9] fixup! Add opt-in scale comparison framework Signed-off-by: James Sturtevant --- docs/scale-testing.md | 17 +-- scripts/scale-test.py | 173 ++++++++++++++++++++++--------- scripts/tests/test_scale_test.py | 2 +- 3 files changed, 135 insertions(+), 57 deletions(-) diff --git a/docs/scale-testing.md b/docs/scale-testing.md index 749e028..d4212b8 100644 --- a/docs/scale-testing.md +++ b/docs/scale-testing.md @@ -8,7 +8,7 @@ description: Compare Lampyr with the same application in a Kubernetes Deployment The opt-in scale framework compares a Lampyr Service with the same Go application in a normal Kubernetes Deployment. It uses oha for client latency and throughput, and queries the optional observability add-on for platform -metrics. It does not run in CI. +metrics. The live comparison does not run in CI. Install the add-on and deploy the committed Go standalone example through Lampyr. Build and publish the normal Kubernetes image from the same source: @@ -32,7 +32,7 @@ just scale-test \ The default sweep runs each target for 30 seconds at concurrency 1, 4, 16, and 64. The framework: -- creates a temporary one-replica Kubernetes Deployment and Service; +- creates a temporary one-replica Deployment and Service with unique names; - warms both targets; - runs the same client against Lampyr and Kubernetes; - queries Prometheus for request, Fiber, CPU, and memory measurements; @@ -46,14 +46,19 @@ port-forward and stops if it exits, but the tunnel can still limit high concurrency results. This framework is for repeatable comparisons, not a claim that the two network paths have identical overhead. +The runner requires one baseline replica and a duration of at least 30 seconds. +Both targets use a 10-second request timeout, including warmup. Cleanup removes +only resources this run created. + The report calculates HTTP success from response status codes and transport errors. Its latency columns include only successful HTTP responses. Platform error and shed counts come from Lampyr metrics. The runner waits for a final -Prometheus scrape after each load interval. If needed, it waits for up to five -more scrape intervals so short smoke tests still have enough cAdvisor samples. -CPU measurements include that collection interval. +healthy Prometheus scrape from every router after each load interval. It can +wait for up to five more scrape intervals. Counter resets reject the scenario. +The measurement window includes the pre-load router scrape and the final +collection interval. Peak memory is the maximum combined usage at one time. -Use the Lampyr Platform Grafana dashboard while a run is active. The JSON +Use the Lampyr Platform Perses dashboard while a run is active. The JSON result remains the source for comparisons across commits or machines. It records the image digest, namespace, replica count, oha version, Git state, and a limited node summary without node addresses or machine identifiers. diff --git a/scripts/scale-test.py b/scripts/scale-test.py index e28c3a7..80537f2 100644 --- a/scripts/scale-test.py +++ b/scripts/scale-test.py @@ -14,9 +14,11 @@ import tempfile import time import urllib.parse +import uuid BASELINE = "lampyr-scale-baseline" +REQUEST_TIMEOUT = "10s" PROMETHEUS_PROXY = ( "/api/v1/namespaces/lampyr-observability/" "services/http:prometheus:9090/proxy/api/v1/query" @@ -62,7 +64,7 @@ def active_revision(context, service, namespace): return next(iter(revisions)) -def prometheus_query(context, query, evaluation_time=None): +def prometheus_results(context, query, evaluation_time=None): parameters = {"query": query} if evaluation_time is not None: parameters["time"] = f"{evaluation_time:.3f}" @@ -70,7 +72,11 @@ def prometheus_query(context, query, evaluation_time=None): result = json.loads(command("kubectl", "--context", context, "get", "--raw", path)) if result.get("status") != "success": raise RuntimeError(f"Prometheus query failed: {result}") - values = result["data"]["result"] + return result["data"]["result"] + + +def prometheus_query(context, query, evaluation_time=None): + values = prometheus_results(context, query, evaluation_time) if not values: return None if len(values) != 1: @@ -79,6 +85,24 @@ def prometheus_query(context, query, evaluation_time=None): return value if math.isfinite(value) else None +def router_scrapes(context, evaluation_time): + values = prometheus_results( + context, + 'timestamp(up{job="fiber-router"}) * (up{job="fiber-router"} == bool 1)', + evaluation_time, + ) + if not values: + raise RuntimeError("Prometheus router targets are missing") + scrapes = {} + for item in values: + timestamp = float(item["value"][1]) + if not math.isfinite(timestamp): + raise RuntimeError("Prometheus router scrape timestamp is not finite") + identity = tuple(sorted(item["metric"].items())) + scrapes[identity] = timestamp + return scrapes + + def metric_queries(namespace, revision, window, baseline, window_seconds=30): revision_match = f'namespace="{namespace}",revision="{revision}"' running_home = ( @@ -95,14 +119,35 @@ def metric_queries(namespace, revision, window, baseline, window_seconds=30): f'label_serving_knative_dev_revision="{revision}"}} {running_home})' ) baseline_pods = f'namespace="{namespace}",pod=~"{baseline}-.*"' - return { + counter_series = { "lampyrRequests": ( - "sum(lampyr_router_request_duration_seconds_count" - f"{{{revision_match}}})" + "lampyr_router_request_duration_seconds_count" + f"{{{revision_match}}}" ), "lampyrErrors": ( - "sum(lampyr_router_request_duration_seconds_count" - f'{{{revision_match},code!~"2..|3.."}})' + "lampyr_router_request_duration_seconds_count" + f'{{{revision_match},code!~"2..|3.."}}' + ), + "lampyrShed": f"lampyr_router_shed_total{{{revision_match}}}", + "lampyrRetries": f"lampyr_router_clone_retries_total{{{revision_match}}}", + } + router_memory = ( + "sum(container_memory_working_set_bytes" + '{namespace="lampyr-system",container="router"} ' + f"{running_router})" + ) + home_memory = ( + "sum(container_memory_working_set_bytes" + f'{{namespace="{namespace}",container="fiberd"}} {home})' + ) + return { + **{name: f"sum({series})" for name, series in counter_series.items()}, + **{ + f"{name}Resets": f"sum(resets({series}[{window}]))" + for name, series in counter_series.items() + }, + "lampyrMemoryBytes": ( + f"max_over_time((({router_memory}) + ({home_memory}))[{window}:5s])" ), "lampyrRequestP95Seconds": ( "histogram_quantile(0.95,sum by(le)(increase(" @@ -114,12 +159,6 @@ def metric_queries(namespace, revision, window, baseline, window_seconds=30): f"lampyr_router_fiber_acquire_seconds_bucket{{{revision_match}}}" f"[{window}])))" ), - "lampyrShed": ( - "sum(lampyr_router_shed_total" f"{{{revision_match}}})" - ), - "lampyrRetries": ( - "sum(lampyr_router_clone_retries_total" f"{{{revision_match}}})" - ), "lampyrPeakBusyFibers": ( "max_over_time((sum(lampyr_router_fibers_busy" f"{{{revision_match}}}))[{window}:5s])" @@ -136,10 +175,7 @@ def metric_queries(namespace, revision, window, baseline, window_seconds=30): f" / {window_seconds:.3f}" ), "lampyrRouterMemoryBytes": ( - "max_over_time((sum(container_memory_working_set_bytes" - '{namespace="lampyr-system",container="router"} ' - f"{running_router}))" - f"[{window}:5s])" + f"max_over_time(({router_memory})[{window}:5s])" ), "lampyrHomeCPU": ( "(sum(increase(container_cpu_usage_seconds_total" @@ -147,8 +183,7 @@ def metric_queries(namespace, revision, window, baseline, window_seconds=30): f"{home})) / {window_seconds:.3f}" ), "lampyrHomeMemoryBytes": ( - "max_over_time((sum(container_memory_working_set_bytes" - f'{{namespace="{namespace}",container="fiberd"}} {home}))[{window}:5s])' + f"max_over_time(({home_memory})[{window}:5s])" ), "baselineCPU": ( "sum(increase(container_cpu_usage_seconds_total" @@ -173,9 +208,9 @@ def metric_queries(namespace, revision, window, baseline, window_seconds=30): } -def counter_snapshot(context, queries): +def counter_snapshot(context, queries, evaluation_time=None): return { - name: prometheus_query(context, query) + name: prometheus_query(context, query, evaluation_time) for name, query in queries.items() if name in COUNTER_METRICS } @@ -183,8 +218,17 @@ def counter_snapshot(context, queries): def collect_metrics(context, queries, before, evaluation_time): collected = {} + reset_metrics = {f"{name}Resets": name for name in COUNTER_METRICS} for name, query in queries.items(): value = prometheus_query(context, query, evaluation_time) + if name in reset_metrics: + counter = reset_metrics[name] + if value is None and before[counter] is not None: + raise RuntimeError(f"Prometheus reset metrics are missing: {counter}") + if value is not None and value > 0: + raise RuntimeError(f"Prometheus counter reset during scenario: {counter}") + collected[name] = value + continue if name not in COUNTER_METRICS: collected[name] = value continue @@ -201,8 +245,8 @@ def collect_metrics(context, queries, before, evaluation_time): return collected -def baseline_manifest(image, replicas, namespace): - labels = {"app": BASELINE} +def baseline_manifest(image, replicas, namespace, name): + labels = {"app": name} return { "apiVersion": "v1", "kind": "List", @@ -210,7 +254,7 @@ def baseline_manifest(image, replicas, namespace): { "apiVersion": "apps/v1", "kind": "Deployment", - "metadata": {"name": BASELINE, "namespace": namespace}, + "metadata": {"name": name, "namespace": namespace}, "spec": { "replicas": replicas, "selector": {"matchLabels": labels}, @@ -254,7 +298,7 @@ def baseline_manifest(image, replicas, namespace): { "apiVersion": "v1", "kind": "Service", - "metadata": {"name": BASELINE, "namespace": namespace}, + "metadata": {"name": name, "namespace": namespace}, "spec": { "selector": labels, "ports": [{"name": "http", "port": 80, "targetPort": "http"}], @@ -284,10 +328,14 @@ def free_port(): def baseline_server(context, image, replicas, namespace): if "@sha256:" not in image: raise ValueError("baseline image must be digest-pinned") - manifest = baseline_manifest(image, replicas, namespace) + name = f"{BASELINE}-{uuid.uuid4().hex}" + manifest = baseline_manifest(image, replicas, namespace, name) + created = [] forward = None try: - kubectl_manifest(context, "apply", manifest) + for resource in manifest["items"]: + kubectl_manifest(context, "create", resource) + created.append(resource) subprocess.run( [ "kubectl", @@ -297,7 +345,7 @@ def baseline_server(context, image, replicas, namespace): namespace, "rollout", "status", - f"deployment/{BASELINE}", + f"deployment/{name}", "--timeout=300s", ], check=True, @@ -311,7 +359,7 @@ def baseline_server(context, image, replicas, namespace): "--namespace", namespace, "port-forward", - f"service/{BASELINE}", + f"service/{name}", f"{port}:80", ], stdout=subprocess.DEVNULL, @@ -327,7 +375,7 @@ def baseline_server(context, image, replicas, namespace): time.sleep(0.1) else: raise RuntimeError("baseline port-forward did not become ready") - yield {"url": f"http://127.0.0.1:{port}/", "process": forward} + yield {"name": name, "url": f"http://127.0.0.1:{port}/", "process": forward} finally: original_error = sys.exc_info()[0] is not None if forward is not None: @@ -337,14 +385,19 @@ def baseline_server(context, image, replicas, namespace): except subprocess.TimeoutExpired: forward.kill() forward.wait() - try: - kubectl_manifest( - context, "delete", manifest, "--ignore-not-found=true", "--wait=true" - ) - except subprocess.CalledProcessError as error: - if not original_error: - raise - print(f"warning: baseline cleanup failed: {error}", file=sys.stderr) + if created: + try: + kubectl_manifest( + context, + "delete", + {"apiVersion": "v1", "kind": "List", "items": created}, + "--ignore-not-found=true", + "--wait=true", + ) + except subprocess.CalledProcessError as error: + if not original_error: + raise + print(f"warning: baseline cleanup failed: {error}", file=sys.stderr) def require_running(process, name): @@ -361,6 +414,8 @@ def run_oha(oha, url, duration, concurrency): duration, "-c", str(concurrency), + "-t", + REQUEST_TIMEOUT, "--wait-ongoing-requests-after-deadline", "--stats-success-breakdown", "--no-tui", @@ -370,6 +425,7 @@ def run_oha(oha, url, duration, concurrency): url, ], text=True, + timeout=duration_seconds(duration) + duration_seconds(REQUEST_TIMEOUT) + 5, ) result = json.loads(output) client = { @@ -406,6 +462,8 @@ def warm_target(oha, url): "20", "-c", "1", + "-t", + REQUEST_TIMEOUT, "--stats-success-breakdown", "--no-tui", "--output-format", @@ -414,6 +472,7 @@ def warm_target(oha, url): url, ], text=True, + timeout=20 * duration_seconds(REQUEST_TIMEOUT) + 5, ) client = json.loads(output) if http_success_rate(client) != 1: @@ -443,6 +502,7 @@ def target_resource_names(target): "lampyrHomeCPU", "lampyrRouterMemoryBytes", "lampyrHomeMemoryBytes", + "lampyrMemoryBytes", ) return ("baselineCPU", "baselineMemoryBytes") @@ -494,10 +554,7 @@ def write_report(result, path): cpu = add_metrics( metrics["lampyrRouterCPU"], metrics["lampyrHomeCPU"] ) - memory = add_metrics( - metrics["lampyrRouterMemoryBytes"], - metrics["lampyrHomeMemoryBytes"], - ) + memory = metrics.get("lampyrMemoryBytes") errors = metrics.get("lampyrErrors") shed = metrics.get("lampyrShed") else: @@ -575,12 +632,14 @@ def main(): args = parser.parse_args() seconds = duration_seconds(args.duration) + if seconds < 30: + raise ValueError("duration must be at least 30s to measure CPU across scrapes") concurrencies = [int(value) for value in args.concurrency.split(",")] if not concurrencies or any(value < 1 for value in concurrencies): raise ValueError("concurrency values must be positive integers") - if args.baseline_replicas < 1: - raise ValueError("baseline replicas must be a positive integer") - oha = Path(args.oha) + if args.baseline_replicas != 1: + raise ValueError("baseline must use one replica with Service port-forward") + oha = Path(args.oha).resolve() if not oha.is_file(): raise FileNotFoundError(f"load generator not found: {oha}") revision = active_revision(args.context, args.service, args.namespace) @@ -595,6 +654,7 @@ def main(): "baselineImage": args.baseline_image, "baselineReplicas": args.baseline_replicas, "duration": args.duration, + "requestTimeout": REQUEST_TIMEOUT, "concurrency": concurrencies, "commit": command("git", "rev-parse", "HEAD"), "dirty": bool(command("git", "status", "--porcelain")), @@ -622,6 +682,7 @@ def main(): with baseline_server( args.context, args.baseline_image, args.baseline_replicas, args.namespace ) as baseline: + metadata["baselineName"] = baseline["name"] for target, url in [ ("lampyr", args.lampyr_url), ("kubernetes", baseline["url"]), @@ -638,9 +699,14 @@ def main(): if target == "kubernetes": require_running(baseline["process"], "baseline port-forward") snapshot_queries = metric_queries( - args.namespace, revision, "1s", BASELINE, 1 + args.namespace, revision, "1s", baseline["name"], 1 ) - before = counter_snapshot(args.context, snapshot_queries) + snapshot_time = time.time() + before_scrapes = router_scrapes(args.context, snapshot_time) + if any(timestamp <= 0 for timestamp in before_scrapes.values()): + raise RuntimeError("Prometheus router targets are not healthy") + measurement_started = min(before_scrapes.values()) - 0.001 + before = counter_snapshot(args.context, snapshot_queries, snapshot_time) if before["lampyrRequests"] is None: raise RuntimeError( "Lampyr request metrics are missing after warmup; check the " @@ -655,13 +721,19 @@ def main(): for wait_seconds in (6, 5, 5, 5, 5, 5): time.sleep(wait_seconds) evaluated = time.time() - measured_seconds = evaluated - started.timestamp() + scrapes = router_scrapes(args.context, evaluated) + if scrapes.keys() != before_scrapes.keys() or any( + timestamp < load_ended.timestamp() + for timestamp in scrapes.values() + ): + continue + measured_seconds = evaluated - measurement_started window = f"{math.ceil(measured_seconds * 1000)}ms" queries = metric_queries( args.namespace, revision, window, - BASELINE, + baseline["name"], measured_seconds, ) resources = { @@ -677,7 +749,8 @@ def main(): break else: raise RuntimeError( - f"Prometheus did not collect CPU and memory metrics for {target}" + f"Prometheus did not collect fresh post-load router scrapes " + f"and CPU and memory metrics for {target}" ) scenario = { "target": target, diff --git a/scripts/tests/test_scale_test.py b/scripts/tests/test_scale_test.py index 42596fb..b3290c3 100644 --- a/scripts/tests/test_scale_test.py +++ b/scripts/tests/test_scale_test.py @@ -255,7 +255,7 @@ def test_target_resources_require_cpu_and_memory(self): def test_baseline_manifest_uses_restricted_pod_security(self): manifest = scale.baseline_manifest( - "example.test/baseline@sha256:abc", 1, "team-a" + "example.test/baseline@sha256:abc", 1, "team-a", "baseline-owned" ) deployment = manifest["items"][0] pod = deployment["spec"]["template"]["spec"] From c1c4ee666e95cf49abf477d646f5d0a619f36d36 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:07:10 -0800 Subject: [PATCH 4/9] fixup! Add opt-in scale comparison framework Signed-off-by: James Sturtevant --- scripts/tests/test_scale_test.py | 126 +++++++++++++++++++++++++++++-- 1 file changed, 121 insertions(+), 5 deletions(-) diff --git a/scripts/tests/test_scale_test.py b/scripts/tests/test_scale_test.py index b3290c3..956adaa 100644 --- a/scripts/tests/test_scale_test.py +++ b/scripts/tests/test_scale_test.py @@ -1,4 +1,5 @@ import contextlib +import hashlib import importlib.util import json import os @@ -381,6 +382,8 @@ def test_main_rejects_multiple_baseline_replicas_and_short_cpu_windows(self): for arguments, message in ( (("--baseline-replicas", "2"), "one replica"), (("--duration", "1s"), "at least 30"), + (("--output", "results.md"), "JSON and Markdown"), + (("--output", "results.MD"), "JSON and Markdown"), ): with self.subTest(arguments=arguments), mock.patch.object( scale.sys, "argv", self.main_arguments(*arguments) @@ -423,7 +426,8 @@ def metadata_command(*arguments): ) self.assertEqual(version[0], str(executable)) - def test_scenario_waits_for_every_router_post_load_scrape(self): + @contextlib.contextmanager + def scenario_environment(self, routers, resources): clock = scale.datetime.datetime.fromtimestamp(400, scale.datetime.timezone.utc) baseline = { "name": "baseline-owned", @@ -448,11 +452,12 @@ def test_scenario_waits_for_every_router_post_load_scrape(self): ), mock.patch.object( scale, "run_oha", return_value=client ), mock.patch.object( - scale, "prometheus_query", return_value=1 + scale, "prometheus_query", + side_effect=lambda context, query, evaluation_time=None: 0 if "resets(" in query else 1, + ), mock.patch.object( + scale, "router_scrapes", side_effect=routers, ), mock.patch.object( - scale, "router_scrapes", create=True, - side_effect=[{"router-1": 399, "router-2": 399}] - + [{"router-1": 420, "router-2": 399}] * 6, + scale, "resource_samples", create=True, side_effect=resources, ), mock.patch.object( scale.datetime, "datetime", wraps=scale.datetime.datetime ) as datetime, mock.patch.object( @@ -465,10 +470,121 @@ def test_scenario_waits_for_every_router_post_load_scrape(self): scale, "write_report" ), mock.patch.object(scale, "print", create=True): datetime.now.return_value = clock + yield write + + def test_scenario_waits_for_every_router_post_load_scrape(self): + routers = [{"router-1": 400, "router-2": 400}] + routers += [{"router-1": 420, "router-2": 399}] * 6 + resources = [{"cadvisor": 400, "kube-state": 400}] + resources += [{"cadvisor": 420, "kube-state": 420}] * 6 + with self.scenario_environment(routers, resources) as write: with self.assertRaisesRegex(RuntimeError, "fresh.*router"): scale.main() write.assert_not_called() + def test_scenario_rejects_stale_resource_samples(self): + for source in ("cadvisor", "kube-state"): + before = {"cadvisor": 400, "kube-state": 400} + after = {"cadvisor": 420, "kube-state": 420} + after[source] = 399 + routers = [{"router-1": 400}] + [{"router-1": 420}] * 20 + resources = [before] + [after] * 20 + with self.subTest(source=source), self.scenario_environment( + routers, resources + ) as write: + with self.assertRaisesRegex(RuntimeError, "fresh.*resource"): + scale.main() + write.assert_not_called() + + def test_installer_uses_unique_downloads_for_concurrent_invocations(self): + justfile = (PATH.parents[1] / "Justfile").read_text() + recipe = justfile.split("\ninstall-oha:\n", 1)[1].split("\n\n", 1)[0] + checksums = scale.re.findall(r"[0-9a-f]{64}", recipe) + self.assertEqual(len(checksums), 1) + payload = b"#!/bin/sh\necho 'oha 1.17.0'\n" + digest = hashlib.sha256(payload).hexdigest() + recipe = recipe.replace(checksums[0], digest) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + bin_path = root / "bin" + bin_path.mkdir() + (root / "payload").write_bytes(payload) + (root / "Justfile").write_text( + 'set shell := ["bash", "-euo", "pipefail", "-c"]\n' + 'repo-root := justfile_directory()\noha-version := "1.17.0"\n' + f"install-oha:\n{recipe}\n" + ) + scripts = { + "uname": "#!/bin/sh\necho 'Linux x86_64'\n", + "sha256sum": ( + "#!/usr/bin/env python3\n" + "import hashlib, pathlib, sys\n" + "expected, filename = sys.stdin.read().strip().split(' ', 1)\n" + "try:\n" + " actual = hashlib.sha256(pathlib.Path(filename).read_bytes()).hexdigest()\n" + "except OSError:\n" + " print('checksum input missing', file=sys.stderr)\n" + " sys.exit(1)\n" + "sys.exit(0 if actual == expected else 1)\n" + ), + "curl": ( + "#!/bin/bash\nset -euo pipefail\n" + "while (($#)); do\n" + " if [[ $1 == --output ]]; then output=$2; shift 2; else shift; fi\n" + "done\n" + 'cp "$TEST_ROOT/payload" "$output"\n' + "wait_for() {\n" + ' for ((i=0; i<500; i++)); do [[ -e $1 ]] && return 0; sleep 0.01; done\n' + " return 1\n" + "}\n" + 'if [[ $TEST_INSTALLER_ID == first ]]; then\n' + ' touch "$TEST_ROOT/first"\n' + ' wait_for "$TEST_ROOT/second"\n' + "else\n" + ' touch "$TEST_ROOT/second"\n' + ' wait_for "$TEST_ROOT/build/tools/oha"\n' + "fi\n" + ), + } + for name, content in scripts.items(): + executable = bin_path / name + executable.write_text(content) + executable.chmod(0o755) + environment = { + **os.environ, + "PATH": f"{bin_path}{os.pathsep}{os.environ['PATH']}", + "TEST_ROOT": str(root), + } + processes = [] + try: + for identifier in ("first", "second"): + processes.append(scale.subprocess.Popen( + ["just", "--justfile", str(root / "Justfile"), "install-oha"], + cwd=root, + env={**environment, "TEST_INSTALLER_ID": identifier}, + stdout=scale.subprocess.PIPE, + stderr=scale.subprocess.PIPE, + text=True, + )) + if identifier == "first": + deadline = scale.time.monotonic() + 5 + while not (root / "first").exists(): + self.assertLess(scale.time.monotonic(), deadline) + scale.time.sleep(0.01) + for process in processes: + stdout, stderr = process.communicate(timeout=10) + self.assertEqual(process.returncode, 0, stdout + stderr) + self.assertEqual((root / "build/tools/oha").read_bytes(), payload) + finally: + for process in processes: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=3) + except scale.subprocess.TimeoutExpired: + process.kill() + process.wait() + if __name__ == "__main__": unittest.main() From 607f8ebb5466769a163ad8bef9467f50677bc394 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:19:35 -0800 Subject: [PATCH 5/9] fixup! Add opt-in scale comparison framework Signed-off-by: James Sturtevant --- Justfile | 10 +- docs/scale-testing.md | 12 +- scripts/scale-test.py | 209 +++++++++++++++++++++---------- scripts/tests/test_scale_test.py | 32 +++++ 4 files changed, 190 insertions(+), 73 deletions(-) diff --git a/Justfile b/Justfile index 5a89dca..3a22da8 100644 --- a/Justfile +++ b/Justfile @@ -212,11 +212,13 @@ install-oha: [ "$("{{repo-root}}/build/tools/oha" --version)" = "oha {{oha-version}}" ]; then exit 0; fi; \ [ "$(uname -sm)" = "Linux x86_64" ] || { echo "install-oha supports linux-amd64 only" >&2; exit 1; }; \ mkdir -p "{{repo-root}}/build/tools"; \ - curl --fail --silent --show-error --location --output "{{repo-root}}/build/tools/oha.tmp" \ + temporary=$(mktemp "{{repo-root}}/build/tools/oha.XXXXXXXX"); \ + trap 'rm -f "$temporary"' EXIT; \ + curl --fail --silent --show-error --location --output "$temporary" \ "https://github.com/hatoo/oha/releases/download/v{{oha-version}}/oha-linux-amd64"; \ - echo "85c6eb98066b8c8bd513b2dd96fafd8f9863249ea653606bae1ca2e3a75c43d2 {{repo-root}}/build/tools/oha.tmp" | sha256sum --check; \ - chmod 0755 "{{repo-root}}/build/tools/oha.tmp"; \ - mv "{{repo-root}}/build/tools/oha.tmp" "{{repo-root}}/build/tools/oha" + echo "85c6eb98066b8c8bd513b2dd96fafd8f9863249ea653606bae1ca2e3a75c43d2 $temporary" | sha256sum --check; \ + chmod 0755 "$temporary"; \ + mv "$temporary" "{{repo-root}}/build/tools/oha" prepare-workspace: build-ly if [[ ! -f build/workspace/lampyr.workspace.yaml ]]; then build/ly scaffold workspace build/workspace; fi diff --git a/docs/scale-testing.md b/docs/scale-testing.md index d4212b8..8d5641e 100644 --- a/docs/scale-testing.md +++ b/docs/scale-testing.md @@ -49,14 +49,18 @@ that the two network paths have identical overhead. The runner requires one baseline replica and a duration of at least 30 seconds. Both targets use a 10-second request timeout, including warmup. Cleanup removes only resources this run created. +Custom JSON output names must not use the `.md` extension. The report calculates HTTP success from response status codes and transport errors. Its latency columns include only successful HTTP responses. Platform error and shed counts come from Lampyr metrics. The runner waits for a final -healthy Prometheus scrape from every router after each load interval. It can -wait for up to five more scrape intervals. Counter resets reject the scenario. -The measurement window includes the pre-load router scrape and the final -collection interval. Peak memory is the maximum combined usage at one time. +healthy Prometheus scrape from every router after each load interval. It also +requires fresh cAdvisor samples from the measured containers and fresh +kube-state-metrics inputs. Missing or changed input series reject the scenario. +It can wait for up to five more scrape intervals. Counter resets reject the +scenario. The measurement window includes the pre-load router and resource +samples after warmup, and the final collection interval. Peak memory is the +maximum combined usage at one time. Use the Lampyr Platform Perses dashboard while a run is active. The JSON result remains the source for comparisons across commits or machines. It diff --git a/scripts/scale-test.py b/scripts/scale-test.py index 80537f2..1266135 100644 --- a/scripts/scale-test.py +++ b/scripts/scale-test.py @@ -103,22 +103,100 @@ def router_scrapes(context, evaluation_time): return scrapes +def resource_inputs(namespace, revision, baseline): + baseline_pods = f'namespace="{namespace}",pod=~"{baseline}-.*"' + phase = f'kube_pod_status_phase{{namespace="{namespace}",phase="Running"}}' + return { + "lampyrRouter": { + "cpu": 'container_cpu_usage_seconds_total{namespace="lampyr-system",container="router"}', + "memory": 'container_memory_working_set_bytes{namespace="lampyr-system",container="router"}', + "phase": 'kube_pod_status_phase{namespace="lampyr-system",phase="Running"}', + }, + "lampyrHome": { + "cpu": f'container_cpu_usage_seconds_total{{namespace="{namespace}",container="fiberd"}}', + "memory": f'container_memory_working_set_bytes{{namespace="{namespace}",container="fiberd"}}', + "phase": phase, + "labels": ( + f'kube_pod_labels{{namespace="{namespace}",label_lampyr_dev_home="true",' + f'label_serving_knative_dev_revision="{revision}"}}' + ), + }, + "baseline": { + "cpu": f'container_cpu_usage_seconds_total{{{baseline_pods},container="app"}}', + "memory": f'container_memory_working_set_bytes{{{baseline_pods},container="app"}}', + "phase": phase, + }, + } + + +def selected_resource(selector, inputs): + membership = f'({inputs["phase"]} == 1)' + if "labels" in inputs: + membership = ( + f'({inputs["labels"]} * on(namespace,pod) group_left() {membership})' + ) + return f"{selector} * on(namespace,pod) group_left() {membership}" + + +def resource_sample_queries(namespace, revision, baseline, target): + inputs = resource_inputs(namespace, revision, baseline) + prefixes = { + "lampyr": ("lampyrRouter", "lampyrHome"), + "kubernetes": ("baseline",), + }[target] + queries = { + "kubeState": ( + 'timestamp(up{job="kube-state-metrics"}) ' + '* (up{job="kube-state-metrics"} == bool 1)' + ), + } + for prefix in prefixes: + resource = inputs[prefix] + selected_cpu = selected_resource(resource["cpu"], resource) + selected_memory = selected_resource(resource["memory"], resource) + selected = f"({selected_cpu}) or ({selected_memory})" + pods = f"group by(namespace,pod) ({selected})" + nodes = f"group by(job,instance,node) ({selected})" + for kind in ("cpu", "memory"): + queries[f"{prefix}{kind.title()}"] = selected_resource( + f'timestamp({resource[kind]})', resource + ) + for kind in ("phase", "labels"): + if kind in resource: + queries[f"{prefix}{kind.title()}"] = ( + f'timestamp({resource[kind]}) ' + f"* on(namespace,pod) group_left() ({pods})" + ) + queries[f"{prefix}Cadvisor"] = ( + 'timestamp(up{job="kubernetes-cadvisor"}) ' + '* (up{job="kubernetes-cadvisor"} == bool 1) ' + f"and on(job,instance,node) ({nodes})" + ) + return queries + + +def resource_samples(context, queries, evaluation_time): + query = " or ".join( + f'label_replace(({expression}), "scale_input", "{name}", "", "")' + for name, expression in queries.items() + ) + values = prometheus_results(context, query, evaluation_time) + samples = {} + sources = set() + for item in values: + source = item["metric"].get("scale_input") + if source not in queries: + raise RuntimeError(f"Prometheus returned an unknown resource input: {source}") + timestamp = float(item["value"][1]) + if not math.isfinite(timestamp): + raise RuntimeError("Prometheus resource sample timestamp is not finite") + sources.add(source) + samples[tuple(sorted(item["metric"].items()))] = timestamp + return samples if sources == queries.keys() else None + + def metric_queries(namespace, revision, window, baseline, window_seconds=30): revision_match = f'namespace="{namespace}",revision="{revision}"' - running_home = ( - '* on(namespace,pod) group_left() ' - f'(kube_pod_status_phase{{namespace="{namespace}",phase="Running"}} == 1)' - ) - running_router = ( - '* on(namespace,pod) group_left() ' - '(kube_pod_status_phase{namespace="lampyr-system",phase="Running"} == 1)' - ) - home = ( - '* on(namespace,pod) group_left() ' - f'(kube_pod_labels{{namespace="{namespace}",label_lampyr_dev_home="true",' - f'label_serving_knative_dev_revision="{revision}"}} {running_home})' - ) - baseline_pods = f'namespace="{namespace}",pod=~"{baseline}-.*"' counter_series = { "lampyrRequests": ( "lampyr_router_request_duration_seconds_count" @@ -131,24 +209,12 @@ def metric_queries(namespace, revision, window, baseline, window_seconds=30): "lampyrShed": f"lampyr_router_shed_total{{{revision_match}}}", "lampyrRetries": f"lampyr_router_clone_retries_total{{{revision_match}}}", } - router_memory = ( - "sum(container_memory_working_set_bytes" - '{namespace="lampyr-system",container="router"} ' - f"{running_router})" - ) - home_memory = ( - "sum(container_memory_working_set_bytes" - f'{{namespace="{namespace}",container="fiberd"}} {home})' - ) - return { + queries = { **{name: f"sum({series})" for name, series in counter_series.items()}, **{ f"{name}Resets": f"sum(resets({series}[{window}]))" for name, series in counter_series.items() }, - "lampyrMemoryBytes": ( - f"max_over_time((({router_memory}) + ({home_memory}))[{window}:5s])" - ), "lampyrRequestP95Seconds": ( "histogram_quantile(0.95,sum by(le)(increase(" f"lampyr_router_request_duration_seconds_bucket{{{revision_match}}}" @@ -167,37 +233,20 @@ def metric_queries(namespace, revision, window, baseline, window_seconds=30): "max_over_time((sum(lampyr_router_requests_in_flight" f"{{{revision_match}}}))[{window}:5s])" ), - "lampyrRouterCPU": ( - "sum(increase(container_cpu_usage_seconds_total" - f'{{namespace="lampyr-system",container="router"}}[{window}])' - f" {running_router}" - ")" - f" / {window_seconds:.3f}" - ), - "lampyrRouterMemoryBytes": ( - f"max_over_time(({router_memory})[{window}:5s])" - ), - "lampyrHomeCPU": ( - "(sum(increase(container_cpu_usage_seconds_total" - f'{{namespace="{namespace}",container="fiberd"}}[{window}]) ' - f"{home})) / {window_seconds:.3f}" - ), - "lampyrHomeMemoryBytes": ( - f"max_over_time(({home_memory})[{window}:5s])" - ), - "baselineCPU": ( - "sum(increase(container_cpu_usage_seconds_total" - f"{{{baseline_pods},container=\"app\"}}[{window}])" - f" {running_home}" - ")" - f" / {window_seconds:.3f}" - ), - "baselineMemoryBytes": ( - "max_over_time((sum(container_memory_working_set_bytes" - f"{{{baseline_pods},container=\"app\"}} {running_home}))" - f"[{window}:5s])" - ), } + memory = {} + for prefix, resource in resource_inputs(namespace, revision, baseline).items(): + cpu = selected_resource(f'increase({resource["cpu"]}[{window}])', resource) + memory[prefix] = f'sum({selected_resource(resource["memory"], resource)})' + queries[f"{prefix}CPU"] = f"sum({cpu}) / {window_seconds:.3f}" + queries[f"{prefix}MemoryBytes"] = ( + f"max_over_time(({memory[prefix]})[{window}:5s])" + ) + queries["lampyrMemoryBytes"] = ( + f'max_over_time((({memory["lampyrRouter"]}) + ' + f'({memory["lampyrHome"]}))[{window}:5s])' + ) + return queries COUNTER_METRICS = { @@ -639,11 +688,13 @@ def main(): raise ValueError("concurrency values must be positive integers") if args.baseline_replicas != 1: raise ValueError("baseline must use one replica with Service port-forward") + output = Path(args.output) + if output.suffix.lower() == ".md": + raise ValueError("JSON and Markdown output files must be distinct; do not use .md") oha = Path(args.oha).resolve() if not oha.is_file(): raise FileNotFoundError(f"load generator not found: {oha}") revision = active_revision(args.context, args.service, args.namespace) - output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) metadata = { "context": args.context, @@ -690,6 +741,7 @@ def main(): if target == "kubernetes": require_running(baseline["process"], "baseline port-forward") warm_target(oha, url) + warmup_ended = datetime.datetime.now(datetime.timezone.utc).timestamp() time.sleep(6) for concurrency in concurrencies: for target, url in [ @@ -701,11 +753,32 @@ def main(): snapshot_queries = metric_queries( args.namespace, revision, "1s", baseline["name"], 1 ) - snapshot_time = time.time() - before_scrapes = router_scrapes(args.context, snapshot_time) - if any(timestamp <= 0 for timestamp in before_scrapes.values()): - raise RuntimeError("Prometheus router targets are not healthy") - measurement_started = min(before_scrapes.values()) - 0.001 + sample_queries = resource_sample_queries( + args.namespace, revision, baseline["name"], target + ) + for wait_seconds in (0, 5, 5, 5, 5, 5, 5): + if wait_seconds: + time.sleep(wait_seconds) + snapshot_time = time.time() + before_scrapes = router_scrapes(args.context, snapshot_time) + before_resources = resource_samples( + args.context, sample_queries, snapshot_time + ) + if before_resources is not None and all( + timestamp >= warmup_ended + for timestamp in ( + *before_scrapes.values(), *before_resources.values() + ) + ): + break + else: + raise RuntimeError( + f"Prometheus did not collect fresh pre-load router and " + f"resource samples for {target}" + ) + measurement_started = min( + *before_scrapes.values(), *before_resources.values() + ) - 0.001 before = counter_snapshot(args.context, snapshot_queries, snapshot_time) if before["lampyrRequests"] is None: raise RuntimeError( @@ -727,6 +800,12 @@ def main(): for timestamp in scrapes.values() ): continue + samples = resource_samples(args.context, sample_queries, evaluated) + if samples is None or samples.keys() != before_resources.keys() or any( + timestamp < load_ended.timestamp() + for timestamp in samples.values() + ): + continue measured_seconds = evaluated - measurement_started window = f"{math.ceil(measured_seconds * 1000)}ms" queries = metric_queries( @@ -749,8 +828,8 @@ def main(): break else: raise RuntimeError( - f"Prometheus did not collect fresh post-load router scrapes " - f"and CPU and memory metrics for {target}" + f"Prometheus did not collect fresh post-load router and " + f"resource samples with CPU and memory metrics for {target}" ) scenario = { "target": target, diff --git a/scripts/tests/test_scale_test.py b/scripts/tests/test_scale_test.py index 956adaa..725915e 100644 --- a/scripts/tests/test_scale_test.py +++ b/scripts/tests/test_scale_test.py @@ -496,6 +496,38 @@ def test_scenario_rejects_stale_resource_samples(self): scale.main() write.assert_not_called() + def test_scenario_rejects_disappeared_resource_series(self): + routers = [{"router-1": 420}] * 20 + resources = [{"container-1": 400, "container-2": 400}] + resources += [{"container-1": 420}] * 20 + with self.scenario_environment(routers, resources) as write: + with self.assertRaisesRegex(RuntimeError, "fresh.*resource"): + scale.main() + write.assert_not_called() + + def test_scenario_accepts_fresh_resource_samples(self): + with self.scenario_environment( + [{"router-1": 420}] * 20, + [{"container-1": 420, "kube-state": 420}] * 20, + ) as write: + scale.main() + self.assertEqual(len(write.call_args.args[0]["scenarios"]), 2) + + def test_resource_samples_require_every_input_family(self): + queries = {"cpu": "cpu_timestamp", "phase": "phase_timestamp"} + complete = [ + {"metric": {"scale_input": "cpu", "pod": "one"}, "value": [450, "420"]}, + {"metric": {"scale_input": "phase", "pod": "one"}, "value": [450, "410"]}, + ] + with mock.patch.object(scale, "prometheus_results", return_value=complete): + samples = scale.resource_samples("context", queries, 450) + self.assertEqual(sorted(samples.values()), [410, 420]) + for partial in ([], complete[:1]): + with self.subTest(partial=partial), mock.patch.object( + scale, "prometheus_results", return_value=partial + ): + self.assertIsNone(scale.resource_samples("context", queries, 450)) + def test_installer_uses_unique_downloads_for_concurrent_invocations(self): justfile = (PATH.parents[1] / "Justfile").read_text() recipe = justfile.split("\ninstall-oha:\n", 1)[1].split("\n\n", 1)[0] From 6160db4f16447d1098a9758c4491f4a6c8504c56 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:20:12 -0800 Subject: [PATCH 6/9] fixup! Add opt-in scale comparison framework Signed-off-by: James Sturtevant --- scripts/tests/test_scale_test.py | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/scripts/tests/test_scale_test.py b/scripts/tests/test_scale_test.py index 725915e..ad1782b 100644 --- a/scripts/tests/test_scale_test.py +++ b/scripts/tests/test_scale_test.py @@ -528,6 +528,49 @@ def test_resource_samples_require_every_input_family(self): ): self.assertIsNone(scale.resource_samples("context", queries, 450)) + def test_resource_samples_require_cpu_and_memory_for_each_container(self): + queries = { + "routersCpu": "cpu_timestamp", + "routersMemory": "memory_timestamp", + "routersPhase": "phase_timestamp", + } + complete = [] + for number in (1, 2): + labels = { + "namespace": "lampyr-system", "pod": f"router-{number}", + "container": "router", "id": f"/container-{number}", + "job": "kubernetes-cadvisor", "instance": f"node-{number}:10250", + "node": f"node-{number}", + } + for source in queries: + metric = {**labels, "scale_input": source} + if source == "routersCpu": + metric["cpu"] = "total" + complete.append({"metric": metric, "value": [450, "420"]}) + with mock.patch.object(scale, "prometheus_results", return_value=complete): + self.assertIsNotNone(scale.resource_samples("context", queries, 450)) + for missing in ("routersCpu", "routersMemory"): + partial = [ + item for item in complete + if not ( + item["metric"]["scale_input"] == missing + and item["metric"]["pod"] == "router-2" + ) + ] + with self.subTest(missing=missing), mock.patch.object( + scale, "prometheus_results", return_value=partial + ): + self.assertIsNone(scale.resource_samples("context", queries, 450)) + mismatched = [ + { + **item, + "metric": {**item["metric"], "id": "/different-container"}, + } if item["metric"]["scale_input"] == "routersCpu" else item + for item in complete + ] + with mock.patch.object(scale, "prometheus_results", return_value=mismatched): + self.assertIsNone(scale.resource_samples("context", queries, 450)) + def test_installer_uses_unique_downloads_for_concurrent_invocations(self): justfile = (PATH.parents[1] / "Justfile").read_text() recipe = justfile.split("\ninstall-oha:\n", 1)[1].split("\n\n", 1)[0] From 2bcd427687da1d589078e182d8e6c3d52a452af2 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:30:11 -0800 Subject: [PATCH 7/9] fixup! Add opt-in scale comparison framework Signed-off-by: James Sturtevant --- scripts/scale-test.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/scale-test.py b/scripts/scale-test.py index 1266135..e430dbc 100644 --- a/scripts/scale-test.py +++ b/scripts/scale-test.py @@ -183,6 +183,7 @@ def resource_samples(context, queries, evaluation_time): values = prometheus_results(context, query, evaluation_time) samples = {} sources = set() + containers = {} for item in values: source = item["metric"].get("scale_input") if source not in queries: @@ -192,7 +193,19 @@ def resource_samples(context, queries, evaluation_time): raise RuntimeError("Prometheus resource sample timestamp is not finite") sources.add(source) samples[tuple(sorted(item["metric"].items()))] = timestamp - return samples if sources == queries.keys() else None + identity = tuple(sorted( + (label, value) for label, value in item["metric"].items() + if label not in {"scale_input", "__name__", "cpu"} + )) + containers.setdefault(source, set()).add(identity) + if sources != queries.keys(): + return None + for source in queries: + if source.endswith("Cpu"): + memory = source.removesuffix("Cpu") + "Memory" + if containers[source] != containers[memory]: + return None + return samples def metric_queries(namespace, revision, window, baseline, window_seconds=30): From 270afb49ea29c203010c859f791003d743b19b81 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:32:42 -0800 Subject: [PATCH 8/9] fixup! Add opt-in scale comparison framework Signed-off-by: James Sturtevant --- scripts/tests/test_scale_test.py | 43 ++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_scale_test.py b/scripts/tests/test_scale_test.py index ad1782b..3d33014 100644 --- a/scripts/tests/test_scale_test.py +++ b/scripts/tests/test_scale_test.py @@ -427,7 +427,7 @@ def metadata_command(*arguments): self.assertEqual(version[0], str(executable)) @contextlib.contextmanager - def scenario_environment(self, routers, resources): + def scenario_environment(self, routers, resources, inventory=None): clock = scale.datetime.datetime.fromtimestamp(400, scale.datetime.timezone.utc) baseline = { "name": "baseline-owned", @@ -439,6 +439,10 @@ def scenario_environment(self, routers, resources): "httpSuccessRate": 1, "successfulLatencyMs": {"p95": 1}, } + coverage = ( + contextlib.nullcontext() if inventory is not None else + mock.patch.object(scale, "resource_coverage", create=True, return_value=True) + ) with mock.patch.object( scale.sys, "argv", self.main_arguments() ), mock.patch.object( @@ -458,6 +462,8 @@ def scenario_environment(self, routers, resources): scale, "router_scrapes", side_effect=routers, ), mock.patch.object( scale, "resource_samples", create=True, side_effect=resources, + ), mock.patch.object( + scale, "resource_inventory", create=True, return_value=inventory or {}, ), mock.patch.object( scale.datetime, "datetime", wraps=scale.datetime.datetime ) as datetime, mock.patch.object( @@ -468,7 +474,7 @@ def scenario_environment(self, routers, resources): scale, "write_json" ) as write, mock.patch.object( scale, "write_report" - ), mock.patch.object(scale, "print", create=True): + ), mock.patch.object(scale, "print", create=True), coverage: datetime.now.return_value = clock yield write @@ -513,6 +519,39 @@ def test_scenario_accepts_fresh_resource_samples(self): scale.main() self.assertEqual(len(write.call_args.args[0]["scenarios"]), 2) + def test_scenario_rejects_missing_router_membership_at_both_endpoints(self): + inventory = { + "lampyrRouter": { + ("lampyr-system", "router-1"), ("lampyr-system", "router-2"), + }, + "lampyrHome": {("team-a", "home-1")}, + } + queries = scale.resource_sample_queries( + "team-a", "demo-00001", "baseline-owned", "lampyr" + ) + for missing in ("membership", "router-target"): + samples = {} + for source in queries: + pods = ( + [("lampyr-system", f"router-{number}") for number in ( + (1,) if missing == "membership" else (1, 2) + )] if source.startswith("lampyrRouter") + else [("team-a", "home-1")] + ) + for namespace, pod in pods: + labels = {"scale_input": source, "namespace": namespace, "pod": pod} + samples[tuple(sorted(labels.items()))] = 420 + routers = { + (("pod", f"router-{number}"),): 420 + for number in ((1, 2) if missing == "membership" else (1,)) + } + with self.subTest(missing=missing), self.scenario_environment( + [routers] * 20, [samples] * 20, inventory, + ) as write: + with self.assertRaisesRegex(RuntimeError, "fresh.*resource"): + scale.main() + write.assert_not_called() + def test_resource_samples_require_every_input_family(self): queries = {"cpu": "cpu_timestamp", "phase": "phase_timestamp"} complete = [ From f128d176fb53c391bdf2bfe3cda2d7ee161fee11 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 20 Nov 2026 18:41:56 -0800 Subject: [PATCH 9/9] fixup! Add opt-in scale comparison framework Signed-off-by: James Sturtevant --- docs/scale-testing.md | 2 + scripts/scale-test.py | 72 ++++++++++++++++++++++++++++++-- scripts/tests/test_scale_test.py | 59 ++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 4 deletions(-) diff --git a/docs/scale-testing.md b/docs/scale-testing.md index 8d5641e..841cabe 100644 --- a/docs/scale-testing.md +++ b/docs/scale-testing.md @@ -50,6 +50,7 @@ The runner requires one baseline replica and a duration of at least 30 seconds. Both targets use a 10-second request timeout, including warmup. Cleanup removes only resources this run created. Custom JSON output names must not use the `.md` extension. +The context must allow read access to the measured Pods. The report calculates HTTP success from response status codes and transport errors. Its latency columns include only successful HTTP responses. Platform @@ -57,6 +58,7 @@ error and shed counts come from Lampyr metrics. The runner waits for a final healthy Prometheus scrape from every router after each load interval. It also requires fresh cAdvisor samples from the measured containers and fresh kube-state-metrics inputs. Missing or changed input series reject the scenario. +Coverage must match the running Pods from the Kubernetes API. It can wait for up to five more scrape intervals. Counter resets reject the scenario. The measurement window includes the pre-load router and resource samples after warmup, and the final collection interval. Peak memory is the diff --git a/scripts/scale-test.py b/scripts/scale-test.py index e430dbc..99c1e52 100644 --- a/scripts/scale-test.py +++ b/scripts/scale-test.py @@ -108,11 +108,15 @@ def resource_inputs(namespace, revision, baseline): phase = f'kube_pod_status_phase{{namespace="{namespace}",phase="Running"}}' return { "lampyrRouter": { + "namespace": "lampyr-system", + "selector": "app=fiber-router", "cpu": 'container_cpu_usage_seconds_total{namespace="lampyr-system",container="router"}', "memory": 'container_memory_working_set_bytes{namespace="lampyr-system",container="router"}', "phase": 'kube_pod_status_phase{namespace="lampyr-system",phase="Running"}', }, "lampyrHome": { + "namespace": namespace, + "selector": f"lampyr.dev/home=true,serving.knative.dev/revision={revision}", "cpu": f'container_cpu_usage_seconds_total{{namespace="{namespace}",container="fiberd"}}', "memory": f'container_memory_working_set_bytes{{namespace="{namespace}",container="fiberd"}}', "phase": phase, @@ -122,6 +126,8 @@ def resource_inputs(namespace, revision, baseline): ), }, "baseline": { + "namespace": namespace, + "selector": f"app={baseline}", "cpu": f'container_cpu_usage_seconds_total{{{baseline_pods},container="app"}}', "memory": f'container_memory_working_set_bytes{{{baseline_pods},container="app"}}', "phase": phase, @@ -208,6 +214,51 @@ def resource_samples(context, queries, evaluation_time): return samples +def resource_inventory(context, namespace, revision, baseline, target): + inputs = resource_inputs(namespace, revision, baseline) + prefixes = { + "lampyr": ("lampyrRouter", "lampyrHome"), + "kubernetes": ("lampyrRouter", "baseline"), + }[target] + inventory = {} + for prefix in prefixes: + resource = inputs[prefix] + pods = json.loads(command( + "kubectl", "--context", context, "--namespace", resource["namespace"], + "get", "pods", "--selector", resource["selector"], + "--field-selector", "status.phase=Running", "--output", "json", + )) + inventory[prefix] = { + (pod["metadata"]["namespace"], pod["metadata"]["name"]) + for pod in pods["items"] + } + return inventory + + +def resource_coverage(samples, queries, inventory, scrapes): + if not samples: + return False + observed = {} + for identity in samples: + labels = dict(identity) + if "namespace" in labels and "pod" in labels: + observed.setdefault(labels["scale_input"], set()).add( + (labels["namespace"], labels["pod"]) + ) + for prefix, pods in inventory.items(): + if not pods: + return False + for kind in ("Cpu", "Memory", "Phase", "Labels"): + source = prefix + kind + if source in queries and observed.get(source) != pods: + return False + routers = { + ("lampyr-system", dict(identity).get("pod")) + for identity in scrapes + } + return routers == inventory["lampyrRouter"] + + def metric_queries(namespace, revision, window, baseline, window_seconds=30): revision_match = f'namespace="{namespace}",revision="{revision}"' counter_series = { @@ -773,11 +824,16 @@ def main(): if wait_seconds: time.sleep(wait_seconds) snapshot_time = time.time() + before_inventory = resource_inventory( + args.context, args.namespace, revision, baseline["name"], target + ) before_scrapes = router_scrapes(args.context, snapshot_time) before_resources = resource_samples( args.context, sample_queries, snapshot_time ) - if before_resources is not None and all( + if resource_coverage( + before_resources, sample_queries, before_inventory, before_scrapes + ) and all( timestamp >= warmup_ended for timestamp in ( *before_scrapes.values(), *before_resources.values() @@ -814,9 +870,17 @@ def main(): ): continue samples = resource_samples(args.context, sample_queries, evaluated) - if samples is None or samples.keys() != before_resources.keys() or any( - timestamp < load_ended.timestamp() - for timestamp in samples.values() + inventory = resource_inventory( + args.context, args.namespace, revision, baseline["name"], target + ) + if ( + not resource_coverage(samples, sample_queries, inventory, scrapes) + or inventory != before_inventory + or samples.keys() != before_resources.keys() + or any( + timestamp < load_ended.timestamp() + for timestamp in samples.values() + ) ): continue measured_seconds = evaluated - measurement_started diff --git a/scripts/tests/test_scale_test.py b/scripts/tests/test_scale_test.py index 3d33014..7a9c771 100644 --- a/scripts/tests/test_scale_test.py +++ b/scripts/tests/test_scale_test.py @@ -567,6 +567,65 @@ def test_resource_samples_require_every_input_family(self): ): self.assertIsNone(scale.resource_samples("context", queries, 450)) + def test_resource_inventory_selects_the_revision_and_unique_baseline(self): + names = { + "app=fiber-router": "router-1", + "lampyr.dev/home=true,serving.knative.dev/revision=demo-00001": "home-1", + "app=baseline-owned": "baseline-owned-one", + } + + def pods(*arguments): + namespace = arguments[arguments.index("--namespace") + 1] + selector = arguments[arguments.index("--selector") + 1] + return json.dumps({"items": [{ + "metadata": {"namespace": namespace, "name": names[selector]}, + }]}) + + with mock.patch.object(scale, "command", side_effect=pods): + lampyr = scale.resource_inventory( + "context", "team-a", "demo-00001", "baseline-owned", "lampyr" + ) + baseline = scale.resource_inventory( + "context", "team-a", "demo-00001", "baseline-owned", "kubernetes" + ) + self.assertEqual(lampyr, { + "lampyrRouter": {("lampyr-system", "router-1")}, + "lampyrHome": {("team-a", "home-1")}, + }) + self.assertEqual(baseline, { + "lampyrRouter": {("lampyr-system", "router-1")}, + "baseline": {("team-a", "baseline-owned-one")}, + }) + + def test_resource_coverage_accepts_complete_target_inventories(self): + routers = {(("pod", "router-1"),): 420} + for target, prefix, pod in ( + ("lampyr", "lampyrHome", "home-1"), + ("kubernetes", "baseline", "baseline-owned-one"), + ): + queries = scale.resource_sample_queries( + "team-a", "demo-00001", "baseline-owned", target + ) + inventory = { + "lampyrRouter": {("lampyr-system", "router-1")}, + prefix: {("team-a", pod)}, + } + samples = {} + for source in queries: + namespace, name = ( + ("lampyr-system", "router-1") if source.startswith("lampyrRouter") + else ("team-a", pod) + ) + labels = {"scale_input": source, "namespace": namespace, "pod": name} + samples[tuple(sorted(labels.items()))] = 420 + with self.subTest(target=target): + self.assertTrue(scale.resource_coverage( + samples, queries, inventory, routers + )) + self.assertFalse(scale.resource_coverage( + None, queries, inventory, routers + )) + def test_resource_samples_require_cpu_and_memory_for_each_container(self): queries = { "routersCpu": "cpu_timestamp",