diff --git a/.github/workflows/sdk-compliance.yml b/.github/workflows/sdk-compliance.yml index 21d08059..a9c0a810 100644 --- a/.github/workflows/sdk-compliance.yml +++ b/.github/workflows/sdk-compliance.yml @@ -12,20 +12,39 @@ on: - main jobs: - compliance: - name: PostHog SDK compliance tests (capture v0) - uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@03d972e49be84402c491324320b0a0f38c2ddc53 - with: - adapter-dockerfile: "sdk_compliance_adapter/Dockerfile" - adapter-context: "." - test-harness-version: "0.10.0" - report-name: "sdk-compliance-report-v0" + adapter-tests: + name: Adapter regression tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Install SDK and adapter test dependencies + run: python -m pip install ".[zstd]" -r sdk_compliance_adapter/requirements.txt pytest + - name: Test adapter + run: python -m pytest -q sdk_compliance_adapter/test_adapter.py - compliance-v1: - name: PostHog SDK compliance tests (capture v1) + compliance: + name: PostHog SDK compliance (${{ matrix.profile }}, async) + strategy: + fail-fast: false + matrix: + include: + - profile: v0-gzip + dockerfile: sdk_compliance_adapter/Dockerfile + - profile: v1-gzip + dockerfile: sdk_compliance_adapter/Dockerfile.v1 + - profile: v1-deflate + dockerfile: sdk_compliance_adapter/Dockerfile.v1-deflate + - profile: v1-zstd + dockerfile: sdk_compliance_adapter/Dockerfile.v1-zstd uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@03d972e49be84402c491324320b0a0f38c2ddc53 with: - adapter-dockerfile: "sdk_compliance_adapter/Dockerfile.v1" + adapter-dockerfile: ${{ matrix.dockerfile }} adapter-context: "." - test-harness-version: "0.10.0" - report-name: "sdk-compliance-report-v1" + test-harness-version: "1.0.0" + sdk-type: server + concurrency: 1 + continue-on-error: true + report-name: sdk-compliance-report-python-${{ matrix.profile }}-async diff --git a/sdk_compliance_adapter/CONTRIBUTING.md b/sdk_compliance_adapter/CONTRIBUTING.md index a1e0fe16..4b7262fa 100644 --- a/sdk_compliance_adapter/CONTRIBUTING.md +++ b/sdk_compliance_adapter/CONTRIBUTING.md @@ -35,7 +35,7 @@ docker run -d --name sdk-adapter --network test-network -p 8080:8080 posthog-pyt docker run --rm \ --name test-harness \ --network test-network \ - ghcr.io/posthog/sdk-test-harness:0.10.0 \ + ghcr.io/posthog/sdk-test-harness:1.0.0 \ run --adapter-url http://sdk-adapter:8080 --mock-url http://test-harness:8081 # Cleanup diff --git a/sdk_compliance_adapter/Dockerfile.v1 b/sdk_compliance_adapter/Dockerfile.v1 index 6891837a..91c02c93 100644 --- a/sdk_compliance_adapter/Dockerfile.v1 +++ b/sdk_compliance_adapter/Dockerfile.v1 @@ -7,7 +7,7 @@ COPY posthog/ /app/sdk/posthog/ COPY setup.py pyproject.toml README.md LICENSE /app/sdk/ # Install the SDK from source -RUN cd /app/sdk && pip install --no-cache-dir -e . +RUN cd /app/sdk && pip install --no-cache-dir -e ".[zstd]" # Install adapter dependencies RUN pip install --no-cache-dir flask python-dateutil diff --git a/sdk_compliance_adapter/Dockerfile.v1-deflate b/sdk_compliance_adapter/Dockerfile.v1-deflate new file mode 100644 index 00000000..217537a7 --- /dev/null +++ b/sdk_compliance_adapter/Dockerfile.v1-deflate @@ -0,0 +1,26 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Copy the SDK source code +COPY posthog/ /app/sdk/posthog/ +COPY setup.py pyproject.toml README.md LICENSE /app/sdk/ + +# Install the SDK from source +RUN cd /app/sdk && pip install --no-cache-dir -e ".[zstd]" + +# Install adapter dependencies +RUN pip install --no-cache-dir flask python-dateutil + +# Copy adapter code +COPY sdk_compliance_adapter/adapter.py /app/adapter.py + +# Select the capture-v1 protocol; same adapter code, different runtime mode. +ENV CAPTURE_MODE=v1 +ENV CAPTURE_COMPRESSION=deflate + +# Expose port 8080 +EXPOSE 8080 + +# Run the adapter +CMD ["python", "/app/adapter.py"] diff --git a/sdk_compliance_adapter/Dockerfile.v1-zstd b/sdk_compliance_adapter/Dockerfile.v1-zstd new file mode 100644 index 00000000..885ce729 --- /dev/null +++ b/sdk_compliance_adapter/Dockerfile.v1-zstd @@ -0,0 +1,26 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Copy the SDK source code +COPY posthog/ /app/sdk/posthog/ +COPY setup.py pyproject.toml README.md LICENSE /app/sdk/ + +# Install the SDK from source +RUN cd /app/sdk && pip install --no-cache-dir -e ".[zstd]" + +# Install adapter dependencies +RUN pip install --no-cache-dir flask python-dateutil + +# Copy adapter code +COPY sdk_compliance_adapter/adapter.py /app/adapter.py + +# Select the capture-v1 protocol; same adapter code, different runtime mode. +ENV CAPTURE_MODE=v1 +ENV CAPTURE_COMPRESSION=zstd + +# Expose port 8080 +EXPOSE 8080 + +# Run the adapter +CMD ["python", "/app/adapter.py"] diff --git a/sdk_compliance_adapter/README.md b/sdk_compliance_adapter/README.md index 5cd730c3..6480670b 100644 --- a/sdk_compliance_adapter/README.md +++ b/sdk_compliance_adapter/README.md @@ -1,41 +1,85 @@ # PostHog Python SDK Test Adapter -This adapter wraps the posthog-python SDK for compliance testing with the [PostHog SDK Test Harness](https://github.com/PostHog/posthog-sdk-test-harness). +This Flask adapter exercises the repository's public `posthog.Client` with the +[PostHog SDK Test Harness](https://github.com/PostHog/posthog-sdk-test-harness). +Capture, AI capture, UUID generation, timestamp normalization, retries, flag +results and flag-called events remain SDK-owned. -## What is This? +## Profiles -This is a simple Flask app that: -1. Wraps the posthog-python SDK -2. Exposes a REST API for the test harness to control -3. Tracks internal SDK state for test assertions +CI uses harness **1.0.0**, server wire format and sequential execution against +these asynchronous-client profiles. Every capability-selected suite runs; +compliance assertions remain advisory. Each profile has a distinct report artifact. -## Contributing +| Profile | Dockerfile | Encoding when init enables compression | Selected cases | +| --- | --- | --- | --- | +| v0-gzip | `Dockerfile` | gzip | 30 capture + 5 AI + 17 flags = 52 | +| v1-gzip | `Dockerfile.v1` | gzip | 95 capture + 5 AI + 17 flags = 117 | +| v1-deflate | `Dockerfile.v1-deflate` | deflate | 94 capture + 5 AI + 17 flags = 116 | +| v1-zstd | `Dockerfile.v1-zstd` | zstd | 94 capture + 5 AI + 17 flags = 116 | -See [CONTRIBUTING.md](CONTRIBUTING.md) for local build and compliance test instructions. +`CAPTURE_MODE=v1` selects V1 analytics. `CAPTURE_COMPRESSION` selects gzip +(default), deflate or zstd for that process's compression-enabled V1 calls. +Health advertises only that analytics encoding. V1 builds install the SDK's +optional `zstd` extra. V0 analytics and dedicated AI capture use the legacy gzip +boolean; AI always sends to `/i/v0/ai/batch/`, independently of analytics mode. -## Adapter Implementation +Init maps `enable_compression: false` (or omission) to explicit +`CaptureCompression.NONE`; true maps to the profile codec. This public argument +takes precedence over `POSTHOG_CAPTURE_COMPRESSION`, so SDK environment settings +cannot silently override harness enable/disable controls. Ordinary tests that +omit the compression setting send uncompressed analytics in every profile. -See [adapter.py](adapter.py) for the implementation. +## Results and applicability -The adapter implements the standard SDK adapter interface defined in the [test harness CONTRACT](https://github.com/PostHog/posthog-sdk-test-harness/blob/main/CONTRACT.yaml): +- GeoIP omission preserves the SDK's native `disable_geoip=True`. The unchanged + `feature_flags.request_payload.disable_geoip_omitted_defaults_to_false` assertion + expects false and therefore fails in each profile. Explicit overrides still work. +- In V1, `capture_ai.routing.capture_does_not_reroute_ai_named_events` asserts + `/batch` for an ordinary capture; the SDK correctly uses the configured V1 + analytics endpoint instead. The dedicated AI endpoint is tested separately. +- The pinned mock decodes gzip but not deflate/zstd. Those codec profiles add the + supported header tests, not full compressed-body/partial-response coverage. + Their uncompressed tests still exercise the real SDK; a passing header does + not prove delivery. Gzip's header and decompression tests are capability-gated + out for deflate/zstd, which accounts for the one-case count difference. +- `sync_mode` uses separate synchronous-per-event sending. The current CLI and + reusable workflow select only whole suites, not its applicable single-event + subset. Multi-event queue batching and partial-batch fixtures do not apply; + this matrix does not claim sync-mode coverage. +- Brotli is not supported. Client-wire cases are not applicable to this server SDK. -- `GET /health` - Return SDK information -- `POST /init` - Initialize SDK with config -- `POST /capture` - Capture an event -- `POST /flush` - Flush pending events -- `GET /state` - Return internal state -- `POST /reset` - Reset SDK state +Expected assertion totals are **51/52** for V0, **115/117** for V1 gzip and +**114/116** for each additional V1 codec, with the mismatches above retained in +reports rather than skipped. -### Key Implementation Details +## Adapter interface -**Request Tracking**: The adapter monkey-patches `batch_post` to track all HTTP requests made by the SDK, including retries. +The adapter exposes `/health`, `/init`, `/capture`, `/capture_ai`, `/identify`, +`/flush`, `/get_feature_flag`, `/state` and `/reset`. -**State Management**: Thread-safe state tracking for events captured vs sent, retry attempts, and errors. +Flush uses public `Client.flush(timeout_seconds=None)` to wait for queue drain; +the harness owns the operation timeout. Drain means each event was processed or +permanently failed, not necessarily delivered. Feature flag evaluation also +drains SDK-generated side-effect events before returning. -**UUID Tracking**: Extracts and tracks UUIDs from batches to verify deduplication. +Transport observers delegate to the original SDK transport. Existing `/state` +counters are diagnostic only: they do not fully account for terminal rejection, +flag side effects or retry statuses. `events_flushed` is a cumulative diagnostic +counter, not a delivery receipt. Wire assertions use the harness mock's requests, +not these counters. -## Documentation +## Local validation -For complete documentation on the test harness and how to implement adapters, see: -- [PostHog SDK Test Harness](https://github.com/PostHog/posthog-sdk-test-harness) -- [Adapter Implementation Guide](https://github.com/PostHog/posthog-sdk-test-harness/blob/main/ADAPTER_GUIDE.md) +See [CONTRIBUTING.md](CONTRIBUTING.md) for Docker instructions. Native adapter +regression tests also exercise SDK requests against loopback, independently +decode gzip/deflate/zstd, and verify init controls, UUIDs, UTC conversion, AI +routing, native GeoIP configuration and unbounded drain forwarding: + +```bash +python -m pip install '.[zstd]' -r sdk_compliance_adapter/requirements.txt pytest +python -m pytest -q sdk_compliance_adapter/test_adapter.py +``` + +The local Compose harness targets V0 only. To test another profile manually, +build its Dockerfile and run the same harness command against that adapter. diff --git a/sdk_compliance_adapter/adapter.py b/sdk_compliance_adapter/adapter.py index b9c0de33..8c130377 100644 --- a/sdk_compliance_adapter/adapter.py +++ b/sdk_compliance_adapter/adapter.py @@ -33,6 +33,15 @@ # v0), mirroring the v0/v1 Dockerfile split. One process speaks one mode and # advertises it via /health capabilities. CAPTURE_MODE = os.environ.get("CAPTURE_MODE", "") +# Compression-enabled init calls use this process's codec. Explicit NONE on +# disabled calls also overrides inherited POSTHOG_CAPTURE_COMPRESSION settings. +CAPTURE_COMPRESSION = CaptureCompression(os.environ.get("CAPTURE_COMPRESSION", "gzip")) +if CAPTURE_COMPRESSION not in ( + CaptureCompression.GZIP, + CaptureCompression.DEFLATE, + CaptureCompression.ZSTD, +): + raise ValueError("CAPTURE_COMPRESSION must be gzip, deflate, or zstd") def is_v1() -> bool: @@ -305,7 +314,7 @@ def patched_post_v1( def health(): """Health check endpoint""" capabilities = ( - ["capture_v1", "capture_ai_v0", "encoding_gzip"] + ["capture_v1", "capture_ai_v0", f"encoding_{CAPTURE_COMPRESSION.value}"] if is_v1() else ["capture_v0", "capture_ai_v0", "encoding_gzip"] ) @@ -335,10 +344,9 @@ def init(): flush_interval_ms = data.get("flush_interval_ms", 500) max_retries = data.get("max_retries", 3) enable_compression = data.get("enable_compression", False) - # Compliance tests assert the request-level default when callers omit - # disable_geoip, so the adapter default keeps geoip-enabled /flags - # requests while still allowing per-call overrides. - disable_geoip = data.get("disable_geoip", False) + client_options = {} + if "disable_geoip" in data: + client_options["disable_geoip"] = data["disable_geoip"] historical_migration = data.get("historical_migration", False) if not api_key: @@ -361,18 +369,21 @@ def init(): gzip=enable_compression, max_retries=max_retries, debug=False, - disable_geoip=disable_geoip, historical_migration=historical_migration, capture_mode=capture_mode, + capture_compression=( + CAPTURE_COMPRESSION if enable_compression else CaptureCompression.NONE + ), + **client_options, ) state.client = client logger.info( - f"Initialized SDK with api_key={api_key[:10]}..., host={host}, " + f"Initialized SDK with host={host}, " f"flush_at={flush_at}, flush_interval={flush_interval}, " f"max_retries={max_retries}, gzip={enable_compression}, " - f"capture_mode={capture_mode}, disable_geoip={disable_geoip}, " + f"capture_mode={capture_mode}, disable_geoip={client.disable_geoip}, " f"historical_migration={historical_migration}" ) @@ -536,12 +547,9 @@ def flush(): if not state.client: return jsonify({"error": "SDK not initialized"}), 400 - # Flush and wait - state.client.flush() - - # Wait a bit for flush to complete - # The flush() method triggers queue.join() which blocks until all items are processed - time.sleep(0.5) + # The adapter contract requires a drain, not the SDK's default 10s budget. + # The harness owns the timeout for this blocking operation. + state.client.flush(timeout_seconds=None) logger.info("Flushed pending events") @@ -598,7 +606,7 @@ def get_feature_flag(): # the adapter action returns. Otherwise the harness may reset mock-server # state for the next test while the background consumer is still flushing, # leaking the previous test's event into the next test. - state.client.flush() + state.client.flush(timeout_seconds=None) logger.info(f"Feature flag {key} for {distinct_id}: {value}") diff --git a/sdk_compliance_adapter/docker-compose.yml b/sdk_compliance_adapter/docker-compose.yml index e6519de5..afa922e0 100644 --- a/sdk_compliance_adapter/docker-compose.yml +++ b/sdk_compliance_adapter/docker-compose.yml @@ -23,7 +23,7 @@ services: # Test harness test-harness: - image: ghcr.io/posthog/sdk-test-harness:0.10.0 + image: ghcr.io/posthog/sdk-test-harness:1.0.0 command: ["run", "--adapter-url", "http://sdk-adapter:8080", "--mock-url", "http://test-harness:8081"] networks: - test-network diff --git a/sdk_compliance_adapter/test_adapter.py b/sdk_compliance_adapter/test_adapter.py new file mode 100644 index 00000000..15ca6479 --- /dev/null +++ b/sdk_compliance_adapter/test_adapter.py @@ -0,0 +1,195 @@ +"""Regression tests for adapter configuration and the real SDK transport.""" + +import gzip +import importlib.util +import json +import os +import threading +import zlib +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from unittest.mock import Mock + +import pytest +import zstandard + +import posthog.capture_v1 +import posthog.consumer +import posthog.request +from posthog import CaptureCompression + + +@pytest.fixture +def adapter_factory(monkeypatch): + # Importing the adapter installs passive transport observers. Restore these + # module globals afterwards so other SDK tests retain their normal transport. + for module, name in ( + (posthog.request, "batch_post"), + (posthog.consumer, "batch_post"), + (posthog.capture_v1, "_post_v1"), + ): + monkeypatch.setattr(module, name, getattr(module, name)) + adapters = [] + + def load(mode="v1", codec="gzip"): + monkeypatch.setenv("CAPTURE_MODE", mode) + monkeypatch.setenv("CAPTURE_COMPRESSION", codec) + spec = importlib.util.spec_from_file_location( + "compliance_adapter", Path(__file__).with_name("adapter.py") + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + adapters.append(module) + return module + + yield load + for adapter in adapters: + adapter.state.reset() + + +@pytest.fixture +def receiver(): + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + raw = self.rfile.read(int(self.headers["Content-Length"])) + encoding = self.headers.get("Content-Encoding") + decoders = { + "gzip": gzip.decompress, + "deflate": zlib.decompress, + "zstd": zstandard.ZstdDecompressor().decompress, + } + body = json.loads(decoders[encoding](raw) if encoding else raw) + requests.append((self.path, encoding, body)) + response = json.dumps( + { + "results": { + event["uuid"]: {"result": "ok"} for event in body["batch"] + } + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + def log_message(self, *args): + pass + + # An explicit override lets concurrent local lanes reserve separate ports. + server = HTTPServer( + ("127.0.0.1", int(os.environ.get("ADAPTER_TEST_MOCK_PORT", "0"))), Handler + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", requests + finally: + server.shutdown() + thread.join() + server.server_close() + + +@pytest.mark.parametrize("codec", ["gzip", "deflate", "zstd"]) +@pytest.mark.parametrize("enabled", [True, False, None]) +def test_v1_compression_init_overrides_sdk_environment( + adapter_factory, monkeypatch, receiver, codec, enabled +): + monkeypatch.setenv("POSTHOG_CAPTURE_COMPRESSION", "deflate") + adapter = adapter_factory(codec=codec) + http = adapter.app.test_client() + host, requests = receiver + config = {"api_key": "test-key", "host": host, "flush_at": 1, "max_retries": 0} + if enabled is not None: + config["enable_compression"] = enabled + assert http.post("/init", json=config).status_code == 200 + expected = CaptureCompression(codec) if enabled else CaptureCompression.NONE + assert adapter.state.client.capture_compression == expected + assert http.get("/health").json["capabilities"] == [ + "capture_v1", + "capture_ai_v0", + f"encoding_{codec}", + ] + captured = http.post( + "/capture", + json={ + "distinct_id": "test-user", + "event": "test-event", + "timestamp": "2024-01-02T03:04:05+05:30", + }, + ) + assert captured.status_code == 200 + assert http.post("/flush").status_code == 200 + assert len(requests) == 1 + path, encoding, body = requests[0] + assert path == "/i/v1/analytics/events" + assert encoding == (codec if enabled else None) + event = body["batch"][0] + assert event["uuid"] == captured.json["uuid"] + assert event["timestamp"] == "2024-01-01T21:34:05+00:00" + + +@pytest.mark.parametrize("mode", ["v0", "v1"]) +def test_ai_uses_legacy_gzip_independent_of_analytics_codec( + adapter_factory, receiver, mode +): + adapter = adapter_factory(mode=mode, codec="zstd") + http = adapter.app.test_client() + host, requests = receiver + assert ( + http.post( + "/init", + json={ + "api_key": "test-key", + "host": host, + "enable_compression": True, + "flush_at": 1, + }, + ).status_code + == 200 + ) + captured = http.post( + "/capture_ai", json={"distinct_id": "test-user", "event": "$ai_generation"} + ) + assert captured.status_code == 200 + assert http.post("/flush").status_code == 200 + assert len(requests) == 1 + path, encoding, body = requests[0] + assert path == "/i/v0/ai/batch/" + assert encoding == "gzip" + assert body["batch"][0]["uuid"] == captured.json["uuid"] + if mode == "v0": + assert http.get("/health").json["capabilities"] == [ + "capture_v0", + "capture_ai_v0", + "encoding_gzip", + ] + + +@pytest.mark.parametrize("disable_geoip", [True, False, None]) +def test_init_preserves_native_geoip_default(adapter_factory, receiver, disable_geoip): + adapter = adapter_factory() + host, _ = receiver + config = {"api_key": "test-key", "host": host} + if disable_geoip is not None: + config["disable_geoip"] = disable_geoip + assert adapter.app.test_client().post("/init", json=config).status_code == 200 + assert adapter.state.client.disable_geoip is ( + True if disable_geoip is None else disable_geoip + ) + + +@pytest.mark.parametrize("route", ["/flush", "/get_feature_flag"]) +def test_flush_uses_public_unbounded_drain(adapter_factory, route): + adapter = adapter_factory() + client = Mock() + client.get_feature_flag.return_value = "variant" + adapter.state.client = client + response = adapter.app.test_client().post( + route, json={"key": "test-flag", "distinct_id": "test-user"} + ) + assert response.status_code == 200 + client.flush.assert_called_once_with(timeout_seconds=None)