diff --git a/Justfile b/Justfile index 3e2e54a..3a22da8 100644 --- a/Justfile +++ b/Justfile @@ -15,6 +15,7 @@ exe-suffix := if os() == "windows" { ".exe" } else { "" } # Kind release installed by `just install-kind` in every workflow (CI, release # gates, Copilot); providers/kind/templates/kind-config.yaml pins its node image. kind-version := "v0.33.0" +oha-version := "1.17.0" default: @just --list @@ -205,6 +206,20 @@ install-mm: mkdir -p "{{repo-root}}/build/tools"; \ tar --extract --gzip --file "$dir/mm.tar.gz" --directory "{{repo-root}}/build/tools" mm +# Install the pinned oha load generator used by opt-in scale tests. +install-oha: + if [ -x "{{repo-root}}/build/tools/oha" ] && \ + [ "$("{{repo-root}}/build/tools/oha" --version)" = "oha {{oha-version}}" ]; then exit 0; fi; \ + [ "$(uname -sm)" = "Linux x86_64" ] || { echo "install-oha supports linux-amd64 only" >&2; exit 1; }; \ + mkdir -p "{{repo-root}}/build/tools"; \ + temporary=$(mktemp "{{repo-root}}/build/tools/oha.XXXXXXXX"); \ + trap 'rm -f "$temporary"' EXIT; \ + curl --fail --silent --show-error --location --output "$temporary" \ + "https://github.com/hatoo/oha/releases/download/v{{oha-version}}/oha-linux-amd64"; \ + echo "85c6eb98066b8c8bd513b2dd96fafd8f9863249ea653606bae1ca2e3a75c43d2 $temporary" | sha256sum --check; \ + chmod 0755 "$temporary"; \ + mv "$temporary" "{{repo-root}}/build/tools/oha" + prepare-workspace: build-ly if [[ ! -f build/workspace/lampyr.workspace.yaml ]]; then build/ly scaffold workspace build/workspace; fi @@ -265,6 +280,7 @@ check-action-versions: # Run focused unit tests for Lampyr. test-unit: + python3 "{{repo-root}}/scripts/tests/test_scale_test.py" go test ./cmd/... ./internal/... python3 "{{repo-root}}/scripts/tests/test_worker_workspace.py" @@ -346,6 +362,23 @@ release-disruption-gate service url: --service "{{service}}" \ --url "{{url}}" +# Build the normal Kubernetes image from the same Go source as the Lampyr example. +[positional-arguments] +scale-baseline-image image: + docker build --tag "$1" --file scripts/fixtures/scale-workload/Dockerfile . + +# Compare Lampyr with a normal Kubernetes Deployment using platform telemetry. +[positional-arguments] +scale-test context service lampyr-url baseline-image duration="30s" concurrency="1,4,16,64" output="build/scale-results.json": install-oha + python3 scripts/scale-test.py \ + --context "$1" \ + --service "$2" \ + --lampyr-url "$3" \ + --baseline-image "$4" \ + --duration "$5" \ + --concurrency "$6" \ + --output "$7" + # Run the repository test suites used in CI. test: test-unit test-cli examples-check test-preview-release cli::test cli::test-intent helper::test azure::test kind::test k3d::test test-examples-local test-kind-integration test-k3d-integration test-existing-kind-integration test-knative-portability test-walkthrough diff --git a/docs/observability.md b/docs/observability.md index c4bfe2e..1b678a5 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -59,6 +59,9 @@ Uninstall disables fiber-router trace export before it removes Jaeger. Remove the add-on before `ly cluster uninstall` when retaining the Kubernetes cluster; the platform uninstall does not remove optional add-ons. +The [scale framework](scale-testing.md) uses these metrics for repeatable +Lampyr and Kubernetes comparisons. + ## Platform metrics | Metric | Type | Meaning | diff --git a/docs/scale-testing.md b/docs/scale-testing.md new file mode 100644 index 0000000..841cabe --- /dev/null +++ b/docs/scale-testing.md @@ -0,0 +1,73 @@ +--- +title: Scale testing +description: Compare Lampyr with the same application in a Kubernetes Deployment. +--- + +# Scale testing + +The opt-in scale framework compares a Lampyr Service with the same Go +application in a normal Kubernetes Deployment. It uses oha for client latency +and throughput, and queries the optional observability add-on for platform +metrics. The live comparison does not run in CI. + +Install the add-on and deploy the committed Go standalone example through +Lampyr. Build and publish the normal Kubernetes image from the same source: + +```bash +just scale-baseline-image REGISTRY/IMAGE:TAG +docker push REGISTRY/IMAGE:TAG +docker buildx imagetools inspect REGISTRY/IMAGE:TAG +``` + +Pass the digest-pinned image and the Lampyr URL: + +```bash +just scale-test \ + CONTEXT \ + SERVICE \ + https://LAMPYR_URL \ + REGISTRY/IMAGE@sha256:DIGEST +``` + +The default sweep runs each target for 30 seconds at concurrency 1, 4, 16, +and 64. The framework: + +- creates a temporary one-replica Deployment and Service with unique names; +- warms both targets; +- runs the same client against Lampyr and Kubernetes; +- queries Prometheus for request, Fiber, CPU, and memory measurements; +- writes `build/scale-results.json` after every scenario; and +- writes a concise Markdown comparison beside it. + +The Kubernetes target uses a local Service port-forward, while the Lampyr URL +uses its configured ingress. Record that path difference when interpreting +results. The result metadata records both paths. The framework monitors the +port-forward and stops if it exits, but the tunnel can still limit high +concurrency results. This framework is for repeatable comparisons, not a claim +that the two network paths have identical overhead. + +The runner requires one baseline replica and a duration of at least 30 seconds. +Both targets use a 10-second request timeout, including warmup. Cleanup removes +only resources this run created. +Custom JSON output names must not use the `.md` extension. +The context must allow read access to the measured Pods. + +The report calculates HTTP success from response status codes and transport +errors. Its latency columns include only successful HTTP responses. Platform +error and shed counts come from Lampyr metrics. The runner waits for a final +healthy Prometheus scrape from every router after each load interval. It also +requires fresh cAdvisor samples from the measured containers and fresh +kube-state-metrics inputs. Missing or changed input series reject the scenario. +Coverage must match the running Pods from the Kubernetes API. +It can wait for up to five more scrape intervals. Counter resets reject the +scenario. The measurement window includes the pre-load router and resource +samples after warmup, and the final collection interval. Peak memory is the +maximum combined usage at one time. + +Use the Lampyr Platform Perses dashboard while a run is active. The JSON +result remains the source for comparisons across commits or machines. It +records the image digest, namespace, replica count, oha version, Git state, and +a limited node summary without node addresses or machine identifiers. + +Keep the node type, cluster configuration, workload image, Lampyr settings, +duration, and concurrency sweep unchanged when comparing revisions. diff --git a/k8s/observability.yaml b/k8s/observability.yaml index cfd63d3..53798e7 100644 --- a/k8s/observability.yaml +++ b/k8s/observability.yaml @@ -273,7 +273,7 @@ spec: image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.20.0@sha256:42cfe3723a5f058171c627537fb57a3ea0f26e4380fa18555a95cb1a1b4cfc5b args: - --resources=deployments,namespaces,nodes,pods - - --metric-labels-allowlist=pods=[lampyr.dev/home] + - --metric-labels-allowlist=pods=[lampyr.dev/home,serving.knative.dev/revision] ports: - name: metrics containerPort: 8080 diff --git a/scripts/fixtures/scale-workload/Dockerfile b/scripts/fixtures/scale-workload/Dockerfile new file mode 100644 index 0000000..c0a8c05 --- /dev/null +++ b/scripts/fixtures/scale-workload/Dockerfile @@ -0,0 +1,13 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.26.7-bookworm@sha256:e8c859f5632dcfde7b32d2012b4351728f6437930887c2f6a91ea242459e5514 AS build +WORKDIR /src +COPY docs/examples/hello-go-standalone/go.mod docs/examples/hello-go-standalone/go.sum ./ +RUN go mod download +COPY docs/examples/hello-go-standalone/application.go docs/examples/hello-go-standalone/main.go ./ +RUN CGO_ENABLED=0 go build -mod=readonly -trimpath -ldflags="-s -w" -o /service . + +FROM scratch +COPY --from=build /service /service +ENV PORT=8080 +USER 65532:65532 +ENTRYPOINT ["/service"] diff --git a/scripts/scale-test.py b/scripts/scale-test.py new file mode 100644 index 0000000..99c1e52 --- /dev/null +++ b/scripts/scale-test.py @@ -0,0 +1,941 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import datetime +import json +import math +import os +from pathlib import Path +import re +import socket +import subprocess +import sys +import tempfile +import time +import urllib.parse +import uuid + + +BASELINE = "lampyr-scale-baseline" +REQUEST_TIMEOUT = "10s" +PROMETHEUS_PROXY = ( + "/api/v1/namespaces/lampyr-observability/" + "services/http:prometheus:9090/proxy/api/v1/query" +) + + +def command(*args): + return subprocess.check_output(args, text=True).strip() + + +def duration_seconds(value): + match = re.fullmatch(r"([1-9][0-9]*)(s|m)", value) + if not match: + raise ValueError("duration must be a positive number of seconds or minutes") + amount = int(match.group(1)) + return amount * (60 if match.group(2) == "m" else 1) + + +def active_revision(context, service, namespace): + value = json.loads( + command( + "kubectl", + "--context", + context, + "--namespace", + namespace, + "get", + "service.serving.knative.dev", + service, + "--output", + "json", + ) + ) + revisions = { + target["revisionName"] + for target in value.get("status", {}).get("traffic", []) + if target.get("percent", 0) > 0 + } + if len(revisions) != 1: + raise RuntimeError( + f"service must have exactly one active Revision, got {len(revisions)}" + ) + return next(iter(revisions)) + + +def prometheus_results(context, query, evaluation_time=None): + parameters = {"query": query} + if evaluation_time is not None: + parameters["time"] = f"{evaluation_time:.3f}" + path = PROMETHEUS_PROXY + "?" + urllib.parse.urlencode(parameters) + result = json.loads(command("kubectl", "--context", context, "get", "--raw", path)) + if result.get("status") != "success": + raise RuntimeError(f"Prometheus query failed: {result}") + return result["data"]["result"] + + +def prometheus_query(context, query, evaluation_time=None): + values = prometheus_results(context, query, evaluation_time) + if not values: + return None + if len(values) != 1: + raise RuntimeError(f"Prometheus query returned {len(values)} series: {query}") + value = float(values[0]["value"][1]) + return value if math.isfinite(value) else None + + +def router_scrapes(context, evaluation_time): + values = prometheus_results( + context, + 'timestamp(up{job="fiber-router"}) * (up{job="fiber-router"} == bool 1)', + evaluation_time, + ) + if not values: + raise RuntimeError("Prometheus router targets are missing") + scrapes = {} + for item in values: + timestamp = float(item["value"][1]) + if not math.isfinite(timestamp): + raise RuntimeError("Prometheus router scrape timestamp is not finite") + identity = tuple(sorted(item["metric"].items())) + scrapes[identity] = timestamp + return scrapes + + +def resource_inputs(namespace, revision, baseline): + baseline_pods = f'namespace="{namespace}",pod=~"{baseline}-.*"' + phase = f'kube_pod_status_phase{{namespace="{namespace}",phase="Running"}}' + return { + "lampyrRouter": { + "namespace": "lampyr-system", + "selector": "app=fiber-router", + "cpu": 'container_cpu_usage_seconds_total{namespace="lampyr-system",container="router"}', + "memory": 'container_memory_working_set_bytes{namespace="lampyr-system",container="router"}', + "phase": 'kube_pod_status_phase{namespace="lampyr-system",phase="Running"}', + }, + "lampyrHome": { + "namespace": namespace, + "selector": f"lampyr.dev/home=true,serving.knative.dev/revision={revision}", + "cpu": f'container_cpu_usage_seconds_total{{namespace="{namespace}",container="fiberd"}}', + "memory": f'container_memory_working_set_bytes{{namespace="{namespace}",container="fiberd"}}', + "phase": phase, + "labels": ( + f'kube_pod_labels{{namespace="{namespace}",label_lampyr_dev_home="true",' + f'label_serving_knative_dev_revision="{revision}"}}' + ), + }, + "baseline": { + "namespace": namespace, + "selector": f"app={baseline}", + "cpu": f'container_cpu_usage_seconds_total{{{baseline_pods},container="app"}}', + "memory": f'container_memory_working_set_bytes{{{baseline_pods},container="app"}}', + "phase": phase, + }, + } + + +def selected_resource(selector, inputs): + membership = f'({inputs["phase"]} == 1)' + if "labels" in inputs: + membership = ( + f'({inputs["labels"]} * on(namespace,pod) group_left() {membership})' + ) + return f"{selector} * on(namespace,pod) group_left() {membership}" + + +def resource_sample_queries(namespace, revision, baseline, target): + inputs = resource_inputs(namespace, revision, baseline) + prefixes = { + "lampyr": ("lampyrRouter", "lampyrHome"), + "kubernetes": ("baseline",), + }[target] + queries = { + "kubeState": ( + 'timestamp(up{job="kube-state-metrics"}) ' + '* (up{job="kube-state-metrics"} == bool 1)' + ), + } + for prefix in prefixes: + resource = inputs[prefix] + selected_cpu = selected_resource(resource["cpu"], resource) + selected_memory = selected_resource(resource["memory"], resource) + selected = f"({selected_cpu}) or ({selected_memory})" + pods = f"group by(namespace,pod) ({selected})" + nodes = f"group by(job,instance,node) ({selected})" + for kind in ("cpu", "memory"): + queries[f"{prefix}{kind.title()}"] = selected_resource( + f'timestamp({resource[kind]})', resource + ) + for kind in ("phase", "labels"): + if kind in resource: + queries[f"{prefix}{kind.title()}"] = ( + f'timestamp({resource[kind]}) ' + f"* on(namespace,pod) group_left() ({pods})" + ) + queries[f"{prefix}Cadvisor"] = ( + 'timestamp(up{job="kubernetes-cadvisor"}) ' + '* (up{job="kubernetes-cadvisor"} == bool 1) ' + f"and on(job,instance,node) ({nodes})" + ) + return queries + + +def resource_samples(context, queries, evaluation_time): + query = " or ".join( + f'label_replace(({expression}), "scale_input", "{name}", "", "")' + for name, expression in queries.items() + ) + values = prometheus_results(context, query, evaluation_time) + samples = {} + sources = set() + containers = {} + for item in values: + source = item["metric"].get("scale_input") + if source not in queries: + raise RuntimeError(f"Prometheus returned an unknown resource input: {source}") + timestamp = float(item["value"][1]) + if not math.isfinite(timestamp): + raise RuntimeError("Prometheus resource sample timestamp is not finite") + sources.add(source) + samples[tuple(sorted(item["metric"].items()))] = timestamp + identity = tuple(sorted( + (label, value) for label, value in item["metric"].items() + if label not in {"scale_input", "__name__", "cpu"} + )) + containers.setdefault(source, set()).add(identity) + if sources != queries.keys(): + return None + for source in queries: + if source.endswith("Cpu"): + memory = source.removesuffix("Cpu") + "Memory" + if containers[source] != containers[memory]: + return None + return samples + + +def resource_inventory(context, namespace, revision, baseline, target): + inputs = resource_inputs(namespace, revision, baseline) + prefixes = { + "lampyr": ("lampyrRouter", "lampyrHome"), + "kubernetes": ("lampyrRouter", "baseline"), + }[target] + inventory = {} + for prefix in prefixes: + resource = inputs[prefix] + pods = json.loads(command( + "kubectl", "--context", context, "--namespace", resource["namespace"], + "get", "pods", "--selector", resource["selector"], + "--field-selector", "status.phase=Running", "--output", "json", + )) + inventory[prefix] = { + (pod["metadata"]["namespace"], pod["metadata"]["name"]) + for pod in pods["items"] + } + return inventory + + +def resource_coverage(samples, queries, inventory, scrapes): + if not samples: + return False + observed = {} + for identity in samples: + labels = dict(identity) + if "namespace" in labels and "pod" in labels: + observed.setdefault(labels["scale_input"], set()).add( + (labels["namespace"], labels["pod"]) + ) + for prefix, pods in inventory.items(): + if not pods: + return False + for kind in ("Cpu", "Memory", "Phase", "Labels"): + source = prefix + kind + if source in queries and observed.get(source) != pods: + return False + routers = { + ("lampyr-system", dict(identity).get("pod")) + for identity in scrapes + } + return routers == inventory["lampyrRouter"] + + +def metric_queries(namespace, revision, window, baseline, window_seconds=30): + revision_match = f'namespace="{namespace}",revision="{revision}"' + counter_series = { + "lampyrRequests": ( + "lampyr_router_request_duration_seconds_count" + f"{{{revision_match}}}" + ), + "lampyrErrors": ( + "lampyr_router_request_duration_seconds_count" + f'{{{revision_match},code!~"2..|3.."}}' + ), + "lampyrShed": f"lampyr_router_shed_total{{{revision_match}}}", + "lampyrRetries": f"lampyr_router_clone_retries_total{{{revision_match}}}", + } + queries = { + **{name: f"sum({series})" for name, series in counter_series.items()}, + **{ + f"{name}Resets": f"sum(resets({series}[{window}]))" + for name, series in counter_series.items() + }, + "lampyrRequestP95Seconds": ( + "histogram_quantile(0.95,sum by(le)(increase(" + f"lampyr_router_request_duration_seconds_bucket{{{revision_match}}}" + f"[{window}])))" + ), + "lampyrAcquireP95Seconds": ( + "histogram_quantile(0.95,sum by(le)(increase(" + f"lampyr_router_fiber_acquire_seconds_bucket{{{revision_match}}}" + f"[{window}])))" + ), + "lampyrPeakBusyFibers": ( + "max_over_time((sum(lampyr_router_fibers_busy" + f"{{{revision_match}}}))[{window}:5s])" + ), + "lampyrPeakInFlight": ( + "max_over_time((sum(lampyr_router_requests_in_flight" + f"{{{revision_match}}}))[{window}:5s])" + ), + } + memory = {} + for prefix, resource in resource_inputs(namespace, revision, baseline).items(): + cpu = selected_resource(f'increase({resource["cpu"]}[{window}])', resource) + memory[prefix] = f'sum({selected_resource(resource["memory"], resource)})' + queries[f"{prefix}CPU"] = f"sum({cpu}) / {window_seconds:.3f}" + queries[f"{prefix}MemoryBytes"] = ( + f"max_over_time(({memory[prefix]})[{window}:5s])" + ) + queries["lampyrMemoryBytes"] = ( + f'max_over_time((({memory["lampyrRouter"]}) + ' + f'({memory["lampyrHome"]}))[{window}:5s])' + ) + return queries + + +COUNTER_METRICS = { + "lampyrRequests", + "lampyrErrors", + "lampyrShed", + "lampyrRetries", +} + + +def counter_snapshot(context, queries, evaluation_time=None): + return { + name: prometheus_query(context, query, evaluation_time) + for name, query in queries.items() + if name in COUNTER_METRICS + } + + +def collect_metrics(context, queries, before, evaluation_time): + collected = {} + reset_metrics = {f"{name}Resets": name for name in COUNTER_METRICS} + for name, query in queries.items(): + value = prometheus_query(context, query, evaluation_time) + if name in reset_metrics: + counter = reset_metrics[name] + if value is None and before[counter] is not None: + raise RuntimeError(f"Prometheus reset metrics are missing: {counter}") + if value is not None and value > 0: + raise RuntimeError(f"Prometheus counter reset during scenario: {counter}") + collected[name] = value + continue + if name not in COUNTER_METRICS: + collected[name] = value + continue + if name == "lampyrRequests" and (before[name] is None or value is None): + raise RuntimeError( + "Lampyr request metrics are missing; check the observability add-on " + "and requested namespace" + ) + previous = before[name] or 0 + current = value or 0 + if current < previous: + raise RuntimeError(f"Prometheus counter reset during scenario: {name}") + collected[name] = current - previous + return collected + + +def baseline_manifest(image, replicas, namespace, name): + labels = {"app": name} + return { + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": name, "namespace": namespace}, + "spec": { + "replicas": replicas, + "selector": {"matchLabels": labels}, + "template": { + "metadata": {"labels": labels}, + "spec": { + "automountServiceAccountToken": False, + "securityContext": { + "runAsNonRoot": True, + "runAsUser": 65532, + "runAsGroup": 65532, + "seccompProfile": {"type": "RuntimeDefault"}, + }, + "containers": [ + { + "name": "app", + "image": image, + "ports": [{"name": "http", "containerPort": 8080}], + "readinessProbe": { + "httpGet": {"path": "/", "port": "http"} + }, + "resources": { + "requests": { + "cpu": "250m", + "memory": "256Mi", + }, + "limits": {"memory": "256Mi"}, + }, + "securityContext": { + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + "readOnlyRootFilesystem": True, + "runAsNonRoot": True, + }, + } + ] + }, + }, + }, + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": name, "namespace": namespace}, + "spec": { + "selector": labels, + "ports": [{"name": "http", "port": 80, "targetPort": "http"}], + }, + }, + ], + } + + +def kubectl_manifest(context, action, value, *extra): + return subprocess.run( + ["kubectl", "--context", context, action, "--filename", "-", *extra], + input=json.dumps(value), + text=True, + check=True, + stdout=subprocess.DEVNULL, + ) + + +def free_port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +@contextlib.contextmanager +def baseline_server(context, image, replicas, namespace): + if "@sha256:" not in image: + raise ValueError("baseline image must be digest-pinned") + name = f"{BASELINE}-{uuid.uuid4().hex}" + manifest = baseline_manifest(image, replicas, namespace, name) + created = [] + forward = None + try: + for resource in manifest["items"]: + kubectl_manifest(context, "create", resource) + created.append(resource) + subprocess.run( + [ + "kubectl", + "--context", + context, + "--namespace", + namespace, + "rollout", + "status", + f"deployment/{name}", + "--timeout=300s", + ], + check=True, + ) + port = free_port() + forward = subprocess.Popen( + [ + "kubectl", + "--context", + context, + "--namespace", + namespace, + "port-forward", + f"service/{name}", + f"{port}:80", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + deadline = time.time() + 30 + while time.time() < deadline: + if forward.poll() is not None: + raise RuntimeError("baseline port-forward exited before becoming ready") + with socket.socket() as connection: + if connection.connect_ex(("127.0.0.1", port)) == 0: + break + time.sleep(0.1) + else: + raise RuntimeError("baseline port-forward did not become ready") + yield {"name": name, "url": f"http://127.0.0.1:{port}/", "process": forward} + finally: + original_error = sys.exc_info()[0] is not None + if forward is not None: + forward.terminate() + try: + forward.wait(timeout=5) + except subprocess.TimeoutExpired: + forward.kill() + forward.wait() + if created: + try: + kubectl_manifest( + context, + "delete", + {"apiVersion": "v1", "kind": "List", "items": created}, + "--ignore-not-found=true", + "--wait=true", + ) + except subprocess.CalledProcessError as error: + if not original_error: + raise + print(f"warning: baseline cleanup failed: {error}", file=sys.stderr) + + +def require_running(process, name): + return_code = process.poll() + if return_code is not None: + raise RuntimeError(f"{name} exited with status {return_code}") + + +def run_oha(oha, url, duration, concurrency): + output = subprocess.check_output( + [ + str(oha), + "-z", + duration, + "-c", + str(concurrency), + "-t", + REQUEST_TIMEOUT, + "--wait-ongoing-requests-after-deadline", + "--stats-success-breakdown", + "--no-tui", + "--output-format", + "json", + "--insecure", + url, + ], + text=True, + timeout=duration_seconds(duration) + duration_seconds(REQUEST_TIMEOUT) + 5, + ) + result = json.loads(output) + client = { + "summary": result["summary"], + "metrics": result.get("metrics", {}), + "statusCodeDistribution": result["statusCodeDistribution"], + "errorDistribution": result["errorDistribution"], + "successfulLatencyMs": { + name: value * 1000 if value is not None else None + for name, value in result["latencyPercentilesSuccessful"].items() + }, + } + client["httpSuccessRate"] = http_success_rate(client) + return client + + +def http_success_rate(client): + statuses = client.get("statusCodeDistribution", {}) + errors = client.get("errorDistribution", {}) + total = sum(statuses.values()) + sum(errors.values()) + if total == 0: + return 0 + successful = sum( + count for code, count in statuses.items() if 200 <= int(code) < 400 + ) + return successful / total + + +def warm_target(oha, url): + output = subprocess.check_output( + [ + str(oha), + "-n", + "20", + "-c", + "1", + "-t", + REQUEST_TIMEOUT, + "--stats-success-breakdown", + "--no-tui", + "--output-format", + "json", + "--insecure", + url, + ], + text=True, + timeout=20 * duration_seconds(REQUEST_TIMEOUT) + 5, + ) + client = json.loads(output) + if http_success_rate(client) != 1: + raise RuntimeError( + "target warmup did not return only successful HTTP responses: " + f"statuses={client.get('statusCodeDistribution', {})} " + f"errors={client.get('errorDistribution', {})}" + ) + + +def display_number(value, precision=2): + if value is None: + return "n/a" + return f"{value:.{precision}f}" + + +def add_metrics(*values): + if any(value is None for value in values): + return None + return sum(values) + + +def target_resource_names(target): + if target == "lampyr": + return ( + "lampyrRouterCPU", + "lampyrHomeCPU", + "lampyrRouterMemoryBytes", + "lampyrHomeMemoryBytes", + "lampyrMemoryBytes", + ) + return ("baselineCPU", "baselineMemoryBytes") + + +def target_resources_available(target, metrics): + return all(metrics[name] is not None for name in target_resource_names(target)) + + +def atomic_write(path, content): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", dir=path.parent, prefix=f".{path.name}.", delete=False + ) as handle: + temporary = Path(handle.name) + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def write_json(result, path): + atomic_write(path, json.dumps(result, indent=2, sort_keys=True) + "\n") + + +def write_report(result, path): + lines = [ + "# Lampyr scale comparison", + "", + f"- Commit: `{result['metadata']['commit']}`", + f"- Context: `{result['metadata']['context']}`", + f"- Revision: `{result['metadata']['revision']}`", + f"- Duration per run: `{result['metadata']['duration']}`", + f"- Baseline image: `{result['metadata']['baselineImage']}`", + f"- Lampyr path: {result['metadata']['networkPaths']['lampyr']}", + f"- Kubernetes path: {result['metadata']['networkPaths']['kubernetes']}", + "", + "| Target | Concurrency | Requests/s | HTTP success | p50 ms | p95 ms | p99 ms | Platform errors | Shed | CPU cores | Peak memory MiB |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for scenario in result["scenarios"]: + client = scenario["client"] + metrics = scenario["platform"] + if scenario["target"] == "lampyr": + cpu = add_metrics( + metrics["lampyrRouterCPU"], metrics["lampyrHomeCPU"] + ) + memory = metrics.get("lampyrMemoryBytes") + errors = metrics.get("lampyrErrors") + shed = metrics.get("lampyrShed") + else: + cpu = metrics["baselineCPU"] + memory = metrics["baselineMemoryBytes"] + errors = None + shed = None + latency = client.get("successfulLatencyMs", {}) + lines.append( + "| {target} | {concurrency} | {rps:.2f} | {success:.4f} | " + "{p50} | {p95} | {p99} | {errors} | {shed} | {cpu} | {memory} |".format( + target=scenario["target"], + concurrency=scenario["concurrency"], + rps=client["summary"]["requestsPerSec"], + success=client["httpSuccessRate"], + p50=display_number(latency.get("p50")), + p95=display_number(latency.get("p95")), + p99=display_number(latency.get("p99")), + errors=display_number(errors, 0), + shed=display_number(shed, 0), + cpu=display_number(cpu, 3), + memory=display_number( + memory / (1 << 20) if memory is not None else None + ), + ) + ) + atomic_write(path, "\n".join(lines) + "\n") + + +def node_summary(value): + nodes = [] + for item in value.get("items", []): + metadata = item.get("metadata", {}) + status = item.get("status", {}) + info = status.get("nodeInfo", {}) + labels = metadata.get("labels", {}) + nodes.append( + { + "name": metadata.get("name"), + "instanceType": labels.get("node.kubernetes.io/instance-type"), + "allocatable": status.get("allocatable", {}), + "kubeletVersion": info.get("kubeletVersion"), + "operatingSystem": info.get("operatingSystem"), + "osImage": info.get("osImage"), + "kernelVersion": info.get("kernelVersion"), + "containerRuntimeVersion": info.get("containerRuntimeVersion"), + } + ) + return nodes + + +def safe_url(value): + parsed = urllib.parse.urlsplit(value) + host = parsed.hostname or "" + if ":" in host: + host = f"[{host}]" + netloc = host + if parsed.port is not None: + netloc += f":{parsed.port}" + return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, "", "")) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--context", required=True) + parser.add_argument("--service", required=True) + parser.add_argument("--lampyr-url", required=True) + parser.add_argument("--baseline-image", required=True) + parser.add_argument("--namespace", default="default") + parser.add_argument("--baseline-replicas", type=int, default=1) + parser.add_argument("--duration", default="30s") + parser.add_argument("--concurrency", default="1,4,16,64") + parser.add_argument("--oha", default="build/tools/oha") + parser.add_argument("--output", default="build/scale-results.json") + args = parser.parse_args() + + seconds = duration_seconds(args.duration) + if seconds < 30: + raise ValueError("duration must be at least 30s to measure CPU across scrapes") + concurrencies = [int(value) for value in args.concurrency.split(",")] + if not concurrencies or any(value < 1 for value in concurrencies): + raise ValueError("concurrency values must be positive integers") + if args.baseline_replicas != 1: + raise ValueError("baseline must use one replica with Service port-forward") + output = Path(args.output) + if output.suffix.lower() == ".md": + raise ValueError("JSON and Markdown output files must be distinct; do not use .md") + oha = Path(args.oha).resolve() + if not oha.is_file(): + raise FileNotFoundError(f"load generator not found: {oha}") + revision = active_revision(args.context, args.service, args.namespace) + output.parent.mkdir(parents=True, exist_ok=True) + metadata = { + "context": args.context, + "service": args.service, + "namespace": args.namespace, + "revision": revision, + "lampyrUrl": safe_url(args.lampyr_url), + "baselineImage": args.baseline_image, + "baselineReplicas": args.baseline_replicas, + "duration": args.duration, + "requestTimeout": REQUEST_TIMEOUT, + "concurrency": concurrencies, + "commit": command("git", "rev-parse", "HEAD"), + "dirty": bool(command("git", "status", "--porcelain")), + "ohaVersion": command(str(oha), "--version"), + "networkPaths": { + "lampyr": "configured ingress URL", + "kubernetes": "kubectl Service port-forward", + }, + "startedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "nodes": node_summary( + json.loads( + command( + "kubectl", + "--context", + args.context, + "get", + "nodes", + "--output", + "json", + ) + ) + ), + } + result = {"schema": 1, "metadata": metadata, "scenarios": []} + with baseline_server( + args.context, args.baseline_image, args.baseline_replicas, args.namespace + ) as baseline: + metadata["baselineName"] = baseline["name"] + for target, url in [ + ("lampyr", args.lampyr_url), + ("kubernetes", baseline["url"]), + ]: + if target == "kubernetes": + require_running(baseline["process"], "baseline port-forward") + warm_target(oha, url) + warmup_ended = datetime.datetime.now(datetime.timezone.utc).timestamp() + time.sleep(6) + for concurrency in concurrencies: + for target, url in [ + ("lampyr", args.lampyr_url), + ("kubernetes", baseline["url"]), + ]: + if target == "kubernetes": + require_running(baseline["process"], "baseline port-forward") + snapshot_queries = metric_queries( + args.namespace, revision, "1s", baseline["name"], 1 + ) + sample_queries = resource_sample_queries( + args.namespace, revision, baseline["name"], target + ) + for wait_seconds in (0, 5, 5, 5, 5, 5, 5): + if wait_seconds: + time.sleep(wait_seconds) + snapshot_time = time.time() + before_inventory = resource_inventory( + args.context, args.namespace, revision, baseline["name"], target + ) + before_scrapes = router_scrapes(args.context, snapshot_time) + before_resources = resource_samples( + args.context, sample_queries, snapshot_time + ) + if resource_coverage( + before_resources, sample_queries, before_inventory, before_scrapes + ) and all( + timestamp >= warmup_ended + for timestamp in ( + *before_scrapes.values(), *before_resources.values() + ) + ): + break + else: + raise RuntimeError( + f"Prometheus did not collect fresh pre-load router and " + f"resource samples for {target}" + ) + measurement_started = min( + *before_scrapes.values(), *before_resources.values() + ) - 0.001 + before = counter_snapshot(args.context, snapshot_queries, snapshot_time) + if before["lampyrRequests"] is None: + raise RuntimeError( + "Lampyr request metrics are missing after warmup; check the " + "observability add-on and requested namespace" + ) + started = datetime.datetime.now(datetime.timezone.utc) + client = run_oha(oha, url, args.duration, concurrency) + load_ended = datetime.datetime.now(datetime.timezone.utc) + if target == "kubernetes": + require_running(baseline["process"], "baseline port-forward") + platform = None + for wait_seconds in (6, 5, 5, 5, 5, 5): + time.sleep(wait_seconds) + evaluated = time.time() + scrapes = router_scrapes(args.context, evaluated) + if scrapes.keys() != before_scrapes.keys() or any( + timestamp < load_ended.timestamp() + for timestamp in scrapes.values() + ): + continue + samples = resource_samples(args.context, sample_queries, evaluated) + inventory = resource_inventory( + args.context, args.namespace, revision, baseline["name"], target + ) + if ( + not resource_coverage(samples, sample_queries, inventory, scrapes) + or inventory != before_inventory + or samples.keys() != before_resources.keys() + or any( + timestamp < load_ended.timestamp() + for timestamp in samples.values() + ) + ): + continue + measured_seconds = evaluated - measurement_started + window = f"{math.ceil(measured_seconds * 1000)}ms" + queries = metric_queries( + args.namespace, + revision, + window, + baseline["name"], + measured_seconds, + ) + resources = { + name: prometheus_query( + args.context, queries[name], evaluated + ) + for name in target_resource_names(target) + } + if target_resources_available(target, resources): + platform = collect_metrics( + args.context, queries, before, evaluated + ) + break + else: + raise RuntimeError( + f"Prometheus did not collect fresh post-load router and " + f"resource samples with CPU and memory metrics for {target}" + ) + scenario = { + "target": target, + "concurrency": concurrency, + "startedAt": started.isoformat(), + "loadEndedAt": load_ended.isoformat(), + "metricsEvaluatedAt": datetime.datetime.fromtimestamp( + evaluated, datetime.timezone.utc + ).isoformat(), + "metricsWindowSeconds": measured_seconds, + "client": client, + "platform": platform, + } + result["scenarios"].append(scenario) + write_json(result, output) + write_report(result, output.with_suffix(".md")) + p95 = display_number(client["successfulLatencyMs"].get("p95")) + print( + f"{target} concurrency={concurrency} " + f"rps={client['summary']['requestsPerSec']:.2f} " + f"http_success={client['httpSuccessRate']:.4f} " + f"p95={p95}ms" + ) + result["metadata"]["completedAt"] = datetime.datetime.now( + datetime.timezone.utc + ).isoformat() + write_json(result, output) + write_report(result, output.with_suffix(".md")) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_scale_test.py b/scripts/tests/test_scale_test.py new file mode 100644 index 0000000..7a9c771 --- /dev/null +++ b/scripts/tests/test_scale_test.py @@ -0,0 +1,763 @@ +import contextlib +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + + +PATH = Path(__file__).resolve().parents[1] / "scale-test.py" +SPEC = importlib.util.spec_from_file_location("scale_test", PATH) +scale = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(scale) + + +class ScaleTestTests(unittest.TestCase): + def main_arguments(self, *extra): + return [ + str(PATH), + "--context", "test", + "--service", "demo", + "--lampyr-url", "https://example.test/", + "--baseline-image", "example.test/baseline@sha256:abc", + "--oha", str(PATH), + "--concurrency", "1", + *extra, + ] + + def test_duration_seconds(self): + self.assertEqual(scale.duration_seconds("30s"), 30) + self.assertEqual(scale.duration_seconds("2m"), 120) + for invalid in ("0s", "1h", "ten"): + with self.assertRaises(ValueError): + scale.duration_seconds(invalid) + + def test_report_compares_lampyr_and_kubernetes(self): + result = { + "metadata": { + "commit": "abc", + "dirty": False, + "context": "kind-test", + "revision": "demo-00001", + "duration": "30s", + "baselineImage": "example.test/baseline@sha256:abc", + "networkPaths": { + "lampyr": "configured ingress URL", + "kubernetes": "kubectl Service port-forward", + }, + }, + "scenarios": [ + { + "target": "lampyr", + "concurrency": 4, + "client": { + "summary": {"requestsPerSec": 10}, + "httpSuccessRate": 1, + "successfulLatencyMs": { + "p50": 1, + "p95": 2, + "p99": 3, + }, + }, + "platform": { + "lampyrErrors": 0, + "lampyrShed": 0, + "lampyrRouterCPU": 0.1, + "lampyrHomeCPU": 0.2, + "lampyrRouterMemoryBytes": 1 << 20, + "lampyrHomeMemoryBytes": 2 << 20, + "lampyrMemoryBytes": 2.5 * (1 << 20), + "baselineCPU": None, + "baselineMemoryBytes": None, + }, + }, + { + "target": "kubernetes", + "concurrency": 4, + "client": { + "summary": {"requestsPerSec": 20}, + "httpSuccessRate": 1, + "successfulLatencyMs": { + "p50": 1, + "p95": 2, + "p99": 3, + }, + }, + "platform": { + "lampyrErrors": None, + "lampyrShed": None, + "lampyrRouterCPU": None, + "lampyrHomeCPU": None, + "lampyrRouterMemoryBytes": None, + "lampyrHomeMemoryBytes": None, + "baselineCPU": 0.4, + "baselineMemoryBytes": 4 << 20, + }, + }, + ], + } + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "report.md" + scale.write_report(result, output) + report = output.read_text() + self.assertIn("| lampyr | 4 | 10.00", report) + self.assertIn("| kubernetes | 4 | 20.00", report) + self.assertIn("| 0 | 0 | 0.300 | 2.50 |", report) + self.assertIn("kubectl Service port-forward", report) + + def test_report_handles_no_successful_responses(self): + result = { + "metadata": { + "commit": "abc", + "dirty": True, + "context": "kind-test", + "revision": "demo-00001", + "duration": "30s", + "baselineImage": "example.test/baseline@sha256:abc", + "networkPaths": { + "lampyr": "configured ingress URL", + "kubernetes": "kubectl Service port-forward", + }, + }, + "scenarios": [ + { + "target": "lampyr", + "concurrency": 4, + "client": { + "summary": {"requestsPerSec": 10}, + "httpSuccessRate": 0, + "successfulLatencyMs": { + "p50": None, + "p95": None, + "p99": None, + }, + }, + "platform": { + "lampyrErrors": 20, + "lampyrShed": 20, + "lampyrRouterCPU": None, + "lampyrHomeCPU": None, + "lampyrRouterMemoryBytes": None, + "lampyrHomeMemoryBytes": None, + "lampyrMemoryBytes": None, + "baselineCPU": None, + "baselineMemoryBytes": None, + }, + } + ], + } + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "report.md" + scale.write_report(result, output) + report = output.read_text() + self.assertIn("| lampyr | 4 | 10.00 | 0.0000 | n/a | n/a | n/a | 20 | 20 |", report) + + def test_report_does_not_turn_missing_metrics_into_zero(self): + result = { + "metadata": { + "commit": "abc", + "dirty": False, + "context": "kind-test", + "revision": "demo-00001", + "duration": "30s", + "baselineImage": "example.test/baseline@sha256:abc", + "networkPaths": { + "lampyr": "configured ingress URL", + "kubernetes": "kubectl Service port-forward", + }, + }, + "scenarios": [ + { + "target": "kubernetes", + "concurrency": 4, + "client": { + "summary": {"requestsPerSec": 20}, + "httpSuccessRate": 1, + "successfulLatencyMs": { + "p50": 1, + "p95": 2, + "p99": 3, + }, + }, + "platform": { + "lampyrErrors": 0, + "lampyrShed": 0, + "lampyrRouterCPU": 0.1, + "lampyrHomeCPU": 0.2, + "lampyrRouterMemoryBytes": 1 << 20, + "lampyrHomeMemoryBytes": 2 << 20, + "baselineCPU": None, + "baselineMemoryBytes": None, + }, + } + ], + } + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "report.md" + scale.write_report(result, output) + report = output.read_text() + self.assertIn("| n/a | n/a | n/a | n/a |", report) + + def test_http_success_rate_includes_transport_errors(self): + client = { + "statusCodeDistribution": {"200": 3, "503": 1}, + "errorDistribution": {"connection refused": 2}, + } + self.assertEqual(scale.http_success_rate(client), 0.5) + + def test_safe_url_removes_credentials_and_query(self): + self.assertEqual( + scale.safe_url("https://user:secret@example.test:8443/path?token=secret"), + "https://example.test:8443/path", + ) + + def test_metric_queries_use_requested_namespace(self): + queries = scale.metric_queries("team-a", "demo-00001", "30s", "baseline") + for name in ( + "lampyrRequestP95Seconds", + "lampyrAcquireP95Seconds", + "lampyrPeakBusyFibers", + "lampyrPeakInFlight", + "baselineCPU", + "baselineMemoryBytes", + ): + self.assertIn('namespace="team-a"', queries[name]) + self.assertIn( + 'label_serving_knative_dev_revision="demo-00001"', + queries["lampyrHomeCPU"], + ) + for name in ( + "lampyrRouterCPU", + "lampyrRouterMemoryBytes", + "lampyrHomeCPU", + "lampyrHomeMemoryBytes", + "baselineCPU", + "baselineMemoryBytes", + ): + self.assertIn("kube_pod_status_phase", queries[name]) + + def test_target_resources_require_cpu_and_memory(self): + metrics = { + "lampyrRouterCPU": 0.1, + "lampyrHomeCPU": 0.2, + "lampyrRouterMemoryBytes": 1, + "lampyrHomeMemoryBytes": 2, + "lampyrMemoryBytes": 3, + "baselineCPU": None, + "baselineMemoryBytes": None, + } + self.assertTrue(scale.target_resources_available("lampyr", metrics)) + self.assertFalse(scale.target_resources_available("kubernetes", metrics)) + metrics["lampyrMemoryBytes"] = None + self.assertFalse(scale.target_resources_available("lampyr", metrics)) + + def test_baseline_manifest_uses_restricted_pod_security(self): + manifest = scale.baseline_manifest( + "example.test/baseline@sha256:abc", 1, "team-a", "baseline-owned" + ) + deployment = manifest["items"][0] + pod = deployment["spec"]["template"]["spec"] + container = pod["containers"][0] + self.assertEqual(deployment["metadata"]["namespace"], "team-a") + self.assertFalse(pod["automountServiceAccountToken"]) + self.assertEqual(pod["securityContext"]["seccompProfile"]["type"], "RuntimeDefault") + self.assertTrue(container["securityContext"]["runAsNonRoot"]) + self.assertTrue(container["securityContext"]["readOnlyRootFilesystem"]) + + def test_baseline_cleanup_runs_when_rollout_fails(self): + image = "example.test/baseline@sha256:abc" + rollout_error = scale.subprocess.CalledProcessError(1, ["kubectl"]) + with mock.patch.object(scale, "kubectl_manifest") as manifest: + with mock.patch.object( + scale.subprocess, "run", side_effect=rollout_error + ): + with self.assertRaises(scale.subprocess.CalledProcessError): + with scale.baseline_server("test", image, 1, "team-a"): + pass + self.assertEqual( + [call.args[1] for call in manifest.call_args_list], + ["create", "create", "delete"], + ) + + def test_baseline_does_not_delete_resources_when_creation_fails(self): + failure = scale.subprocess.CalledProcessError(1, ["kubectl"]) + with mock.patch.object(scale, "kubectl_manifest", side_effect=failure) as manifest: + with self.assertRaises(scale.subprocess.CalledProcessError): + with scale.baseline_server( + "test", "example.test/baseline@sha256:abc", 1, "team-a" + ): + pass + self.assertEqual(manifest.call_count, 1) + + def test_baseline_partial_creation_cleans_only_created_resources(self): + failure = scale.subprocess.CalledProcessError(1, ["kubectl"]) + with mock.patch.object( + scale, "kubectl_manifest", side_effect=[None, failure, None] + ) as manifest, mock.patch.object( + scale.subprocess, "run", side_effect=AssertionError("unexpected rollout") + ): + with self.assertRaises(scale.subprocess.CalledProcessError): + with scale.baseline_server( + "test", "example.test/baseline@sha256:abc", 1, "team-a" + ): + pass + self.assertEqual( + [call.args[1] for call in manifest.call_args_list], + ["create", "create", "delete"], + ) + cleanup = manifest.call_args.args[2]["items"] + self.assertEqual(cleanup, [manifest.call_args_list[0].args[2]]) + self.assertEqual(cleanup[0]["kind"], "Deployment") + + def test_baseline_runs_have_distinct_names_and_selectors(self): + failure = scale.subprocess.CalledProcessError(1, ["kubectl"]) + with mock.patch.object(scale, "kubectl_manifest") as manifest, mock.patch.object( + scale.subprocess, "run", side_effect=failure + ): + for _ in range(2): + with self.assertRaises(scale.subprocess.CalledProcessError): + with scale.baseline_server( + "test", "example.test/baseline@sha256:abc", 1, "team-a" + ): + pass + deployments = [] + for call in manifest.call_args_list: + if call.args[1] in ("apply", "create"): + value = call.args[2] + deployments.extend( + item for item in value.get("items", [value]) + if item["kind"] == "Deployment" + ) + self.assertEqual(len(deployments), 2) + self.assertNotEqual( + deployments[0]["metadata"]["name"], deployments[1]["metadata"]["name"] + ) + self.assertNotEqual( + deployments[0]["spec"]["selector"], deployments[1]["spec"]["selector"] + ) + + def test_counter_resets_are_rejected_when_aggregate_increases(self): + queries = scale.metric_queries("team-a", "demo-00001", "30s", "baseline") + before = {name: 100 for name in scale.COUNTER_METRICS} + for name in scale.COUNTER_METRICS: + series = queries[name].removeprefix("sum(").removesuffix(")") + + def query_result(context, query, evaluation_time): + if "resets(" in query: + return int(f"resets({series}[" in query) + return 150 if query in (queries[key] for key in before) else 1 + + with self.subTest(metric=name), mock.patch.object( + scale, "prometheus_query", side_effect=query_result + ): + with self.assertRaisesRegex(RuntimeError, f"counter reset.*{name}"): + scale.collect_metrics("test", queries, before, 42) + + def test_load_and_warmup_bound_stalled_requests_with_the_same_timeout(self): + result = json.dumps({ + "summary": {"requestsPerSec": 1}, + "statusCodeDistribution": {"200": 1}, + "errorDistribution": {}, + "latencyPercentilesSuccessful": {"p95": 0.1}, + }) + with mock.patch.object( + scale.subprocess, "check_output", return_value=result + ) as execute: + scale.run_oha(Path("/oha"), "http://example.test/", "30s", 1) + scale.warm_target(Path("/oha"), "http://example.test/") + timeouts = [] + for call in execute.call_args_list: + arguments = call.args[0] + self.assertIn("-t", arguments) + value = arguments[arguments.index("-t") + 1] + self.assertGreater(scale.duration_seconds(value), 0) + self.assertGreater(call.kwargs["timeout"], 0) + timeouts.append(value) + self.assertEqual(timeouts[0], timeouts[1]) + + def test_main_rejects_multiple_baseline_replicas_and_short_cpu_windows(self): + for arguments, message in ( + (("--baseline-replicas", "2"), "one replica"), + (("--duration", "1s"), "at least 30"), + (("--output", "results.md"), "JSON and Markdown"), + (("--output", "results.MD"), "JSON and Markdown"), + ): + with self.subTest(arguments=arguments), mock.patch.object( + scale.sys, "argv", self.main_arguments(*arguments) + ), mock.patch.object( + scale, "active_revision", return_value="demo-00001" + ) as revision, mock.patch.object( + scale, "command", return_value="{}" + ), mock.patch.object( + scale, "baseline_server", side_effect=AssertionError("unsafe configuration") + ): + with self.assertRaisesRegex(ValueError, message): + scale.main() + revision.assert_not_called() + + def test_relative_oha_path_executes_the_requested_local_binary(self): + execute = scale.command + with tempfile.TemporaryDirectory() as temporary: + executable = Path(temporary) / "oha" + executable.write_text("#!/bin/sh\necho 'oha test'\n") + executable.chmod(0o755) + + def metadata_command(*arguments): + return execute(*arguments) if "--version" in arguments else "{}" + + with contextlib.chdir(temporary), mock.patch.dict( + os.environ, {"PATH": os.defpath} + ), mock.patch.object( + scale.sys, "argv", self.main_arguments("--oha", "./oha") + ), mock.patch.object( + scale, "active_revision", return_value="demo-00001" + ), mock.patch.object( + scale, "command", side_effect=metadata_command + ) as command, mock.patch.object( + scale, "baseline_server", side_effect=RuntimeError("metadata collected") + ): + with self.assertRaisesRegex(RuntimeError, "metadata collected"): + scale.main() + version = next( + call.args for call in command.call_args_list if "--version" in call.args + ) + self.assertEqual(version[0], str(executable)) + + @contextlib.contextmanager + def scenario_environment(self, routers, resources, inventory=None): + clock = scale.datetime.datetime.fromtimestamp(400, scale.datetime.timezone.utc) + baseline = { + "name": "baseline-owned", + "url": "http://localhost/", + "process": mock.Mock(poll=mock.Mock(return_value=None)), + } + client = { + "summary": {"requestsPerSec": 1}, + "httpSuccessRate": 1, + "successfulLatencyMs": {"p95": 1}, + } + coverage = ( + contextlib.nullcontext() if inventory is not None else + mock.patch.object(scale, "resource_coverage", create=True, return_value=True) + ) + with mock.patch.object( + scale.sys, "argv", self.main_arguments() + ), mock.patch.object( + scale, "active_revision", return_value="demo-00001" + ), mock.patch.object( + scale, "command", return_value="{}" + ), mock.patch.object( + scale, "baseline_server", return_value=contextlib.nullcontext(baseline) + ), mock.patch.object( + scale, "warm_target" + ), mock.patch.object( + scale, "run_oha", return_value=client + ), mock.patch.object( + scale, "prometheus_query", + side_effect=lambda context, query, evaluation_time=None: 0 if "resets(" in query else 1, + ), mock.patch.object( + scale, "router_scrapes", side_effect=routers, + ), mock.patch.object( + scale, "resource_samples", create=True, side_effect=resources, + ), mock.patch.object( + scale, "resource_inventory", create=True, return_value=inventory or {}, + ), mock.patch.object( + scale.datetime, "datetime", wraps=scale.datetime.datetime + ) as datetime, mock.patch.object( + scale.time, "time", return_value=420 + ), mock.patch.object( + scale.time, "sleep" + ), mock.patch.object( + scale, "write_json" + ) as write, mock.patch.object( + scale, "write_report" + ), mock.patch.object(scale, "print", create=True), coverage: + datetime.now.return_value = clock + yield write + + def test_scenario_waits_for_every_router_post_load_scrape(self): + routers = [{"router-1": 400, "router-2": 400}] + routers += [{"router-1": 420, "router-2": 399}] * 6 + resources = [{"cadvisor": 400, "kube-state": 400}] + resources += [{"cadvisor": 420, "kube-state": 420}] * 6 + with self.scenario_environment(routers, resources) as write: + with self.assertRaisesRegex(RuntimeError, "fresh.*router"): + scale.main() + write.assert_not_called() + + def test_scenario_rejects_stale_resource_samples(self): + for source in ("cadvisor", "kube-state"): + before = {"cadvisor": 400, "kube-state": 400} + after = {"cadvisor": 420, "kube-state": 420} + after[source] = 399 + routers = [{"router-1": 400}] + [{"router-1": 420}] * 20 + resources = [before] + [after] * 20 + with self.subTest(source=source), self.scenario_environment( + routers, resources + ) as write: + with self.assertRaisesRegex(RuntimeError, "fresh.*resource"): + scale.main() + write.assert_not_called() + + def test_scenario_rejects_disappeared_resource_series(self): + routers = [{"router-1": 420}] * 20 + resources = [{"container-1": 400, "container-2": 400}] + resources += [{"container-1": 420}] * 20 + with self.scenario_environment(routers, resources) as write: + with self.assertRaisesRegex(RuntimeError, "fresh.*resource"): + scale.main() + write.assert_not_called() + + def test_scenario_accepts_fresh_resource_samples(self): + with self.scenario_environment( + [{"router-1": 420}] * 20, + [{"container-1": 420, "kube-state": 420}] * 20, + ) as write: + scale.main() + self.assertEqual(len(write.call_args.args[0]["scenarios"]), 2) + + def test_scenario_rejects_missing_router_membership_at_both_endpoints(self): + inventory = { + "lampyrRouter": { + ("lampyr-system", "router-1"), ("lampyr-system", "router-2"), + }, + "lampyrHome": {("team-a", "home-1")}, + } + queries = scale.resource_sample_queries( + "team-a", "demo-00001", "baseline-owned", "lampyr" + ) + for missing in ("membership", "router-target"): + samples = {} + for source in queries: + pods = ( + [("lampyr-system", f"router-{number}") for number in ( + (1,) if missing == "membership" else (1, 2) + )] if source.startswith("lampyrRouter") + else [("team-a", "home-1")] + ) + for namespace, pod in pods: + labels = {"scale_input": source, "namespace": namespace, "pod": pod} + samples[tuple(sorted(labels.items()))] = 420 + routers = { + (("pod", f"router-{number}"),): 420 + for number in ((1, 2) if missing == "membership" else (1,)) + } + with self.subTest(missing=missing), self.scenario_environment( + [routers] * 20, [samples] * 20, inventory, + ) as write: + with self.assertRaisesRegex(RuntimeError, "fresh.*resource"): + scale.main() + write.assert_not_called() + + def test_resource_samples_require_every_input_family(self): + queries = {"cpu": "cpu_timestamp", "phase": "phase_timestamp"} + complete = [ + {"metric": {"scale_input": "cpu", "pod": "one"}, "value": [450, "420"]}, + {"metric": {"scale_input": "phase", "pod": "one"}, "value": [450, "410"]}, + ] + with mock.patch.object(scale, "prometheus_results", return_value=complete): + samples = scale.resource_samples("context", queries, 450) + self.assertEqual(sorted(samples.values()), [410, 420]) + for partial in ([], complete[:1]): + with self.subTest(partial=partial), mock.patch.object( + scale, "prometheus_results", return_value=partial + ): + self.assertIsNone(scale.resource_samples("context", queries, 450)) + + def test_resource_inventory_selects_the_revision_and_unique_baseline(self): + names = { + "app=fiber-router": "router-1", + "lampyr.dev/home=true,serving.knative.dev/revision=demo-00001": "home-1", + "app=baseline-owned": "baseline-owned-one", + } + + def pods(*arguments): + namespace = arguments[arguments.index("--namespace") + 1] + selector = arguments[arguments.index("--selector") + 1] + return json.dumps({"items": [{ + "metadata": {"namespace": namespace, "name": names[selector]}, + }]}) + + with mock.patch.object(scale, "command", side_effect=pods): + lampyr = scale.resource_inventory( + "context", "team-a", "demo-00001", "baseline-owned", "lampyr" + ) + baseline = scale.resource_inventory( + "context", "team-a", "demo-00001", "baseline-owned", "kubernetes" + ) + self.assertEqual(lampyr, { + "lampyrRouter": {("lampyr-system", "router-1")}, + "lampyrHome": {("team-a", "home-1")}, + }) + self.assertEqual(baseline, { + "lampyrRouter": {("lampyr-system", "router-1")}, + "baseline": {("team-a", "baseline-owned-one")}, + }) + + def test_resource_coverage_accepts_complete_target_inventories(self): + routers = {(("pod", "router-1"),): 420} + for target, prefix, pod in ( + ("lampyr", "lampyrHome", "home-1"), + ("kubernetes", "baseline", "baseline-owned-one"), + ): + queries = scale.resource_sample_queries( + "team-a", "demo-00001", "baseline-owned", target + ) + inventory = { + "lampyrRouter": {("lampyr-system", "router-1")}, + prefix: {("team-a", pod)}, + } + samples = {} + for source in queries: + namespace, name = ( + ("lampyr-system", "router-1") if source.startswith("lampyrRouter") + else ("team-a", pod) + ) + labels = {"scale_input": source, "namespace": namespace, "pod": name} + samples[tuple(sorted(labels.items()))] = 420 + with self.subTest(target=target): + self.assertTrue(scale.resource_coverage( + samples, queries, inventory, routers + )) + self.assertFalse(scale.resource_coverage( + None, queries, inventory, routers + )) + + def test_resource_samples_require_cpu_and_memory_for_each_container(self): + queries = { + "routersCpu": "cpu_timestamp", + "routersMemory": "memory_timestamp", + "routersPhase": "phase_timestamp", + } + complete = [] + for number in (1, 2): + labels = { + "namespace": "lampyr-system", "pod": f"router-{number}", + "container": "router", "id": f"/container-{number}", + "job": "kubernetes-cadvisor", "instance": f"node-{number}:10250", + "node": f"node-{number}", + } + for source in queries: + metric = {**labels, "scale_input": source} + if source == "routersCpu": + metric["cpu"] = "total" + complete.append({"metric": metric, "value": [450, "420"]}) + with mock.patch.object(scale, "prometheus_results", return_value=complete): + self.assertIsNotNone(scale.resource_samples("context", queries, 450)) + for missing in ("routersCpu", "routersMemory"): + partial = [ + item for item in complete + if not ( + item["metric"]["scale_input"] == missing + and item["metric"]["pod"] == "router-2" + ) + ] + with self.subTest(missing=missing), mock.patch.object( + scale, "prometheus_results", return_value=partial + ): + self.assertIsNone(scale.resource_samples("context", queries, 450)) + mismatched = [ + { + **item, + "metric": {**item["metric"], "id": "/different-container"}, + } if item["metric"]["scale_input"] == "routersCpu" else item + for item in complete + ] + with mock.patch.object(scale, "prometheus_results", return_value=mismatched): + self.assertIsNone(scale.resource_samples("context", queries, 450)) + + def test_installer_uses_unique_downloads_for_concurrent_invocations(self): + justfile = (PATH.parents[1] / "Justfile").read_text() + recipe = justfile.split("\ninstall-oha:\n", 1)[1].split("\n\n", 1)[0] + checksums = scale.re.findall(r"[0-9a-f]{64}", recipe) + self.assertEqual(len(checksums), 1) + payload = b"#!/bin/sh\necho 'oha 1.17.0'\n" + digest = hashlib.sha256(payload).hexdigest() + recipe = recipe.replace(checksums[0], digest) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + bin_path = root / "bin" + bin_path.mkdir() + (root / "payload").write_bytes(payload) + (root / "Justfile").write_text( + 'set shell := ["bash", "-euo", "pipefail", "-c"]\n' + 'repo-root := justfile_directory()\noha-version := "1.17.0"\n' + f"install-oha:\n{recipe}\n" + ) + scripts = { + "uname": "#!/bin/sh\necho 'Linux x86_64'\n", + "sha256sum": ( + "#!/usr/bin/env python3\n" + "import hashlib, pathlib, sys\n" + "expected, filename = sys.stdin.read().strip().split(' ', 1)\n" + "try:\n" + " actual = hashlib.sha256(pathlib.Path(filename).read_bytes()).hexdigest()\n" + "except OSError:\n" + " print('checksum input missing', file=sys.stderr)\n" + " sys.exit(1)\n" + "sys.exit(0 if actual == expected else 1)\n" + ), + "curl": ( + "#!/bin/bash\nset -euo pipefail\n" + "while (($#)); do\n" + " if [[ $1 == --output ]]; then output=$2; shift 2; else shift; fi\n" + "done\n" + 'cp "$TEST_ROOT/payload" "$output"\n' + "wait_for() {\n" + ' for ((i=0; i<500; i++)); do [[ -e $1 ]] && return 0; sleep 0.01; done\n' + " return 1\n" + "}\n" + 'if [[ $TEST_INSTALLER_ID == first ]]; then\n' + ' touch "$TEST_ROOT/first"\n' + ' wait_for "$TEST_ROOT/second"\n' + "else\n" + ' touch "$TEST_ROOT/second"\n' + ' wait_for "$TEST_ROOT/build/tools/oha"\n' + "fi\n" + ), + } + for name, content in scripts.items(): + executable = bin_path / name + executable.write_text(content) + executable.chmod(0o755) + environment = { + **os.environ, + "PATH": f"{bin_path}{os.pathsep}{os.environ['PATH']}", + "TEST_ROOT": str(root), + } + processes = [] + try: + for identifier in ("first", "second"): + processes.append(scale.subprocess.Popen( + ["just", "--justfile", str(root / "Justfile"), "install-oha"], + cwd=root, + env={**environment, "TEST_INSTALLER_ID": identifier}, + stdout=scale.subprocess.PIPE, + stderr=scale.subprocess.PIPE, + text=True, + )) + if identifier == "first": + deadline = scale.time.monotonic() + 5 + while not (root / "first").exists(): + self.assertLess(scale.time.monotonic(), deadline) + scale.time.sleep(0.01) + for process in processes: + stdout, stderr = process.communicate(timeout=10) + self.assertEqual(process.returncode, 0, stdout + stderr) + self.assertEqual((root / "build/tools/oha").read_bytes(), payload) + finally: + for process in processes: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=3) + except scale.subprocess.TimeoutExpired: + process.kill() + process.wait() + + +if __name__ == "__main__": + unittest.main()