From 8b70bde1d0a3208ab87868b280b1148df62e142e Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Wed, 11 Nov 2026 11:14:47 -0800 Subject: [PATCH 1/6] test(pgbouncer): assert tenant context reset on backend reuse --- scripts/test-pgbouncer-pooling.sh | 55 ++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/test-pgbouncer-pooling.sh b/scripts/test-pgbouncer-pooling.sh index fdfcba2..4ad8dc2 100644 --- a/scripts/test-pgbouncer-pooling.sh +++ b/scripts/test-pgbouncer-pooling.sh @@ -77,6 +77,59 @@ run_tenant() { -c "$sql" } +# Exercise backend-local state across actual PgBouncer transaction releases. +# Proxima's tenant identity is authenticated at startup and mapped to a dedicated +# PostgreSQL role; this synthetic GUC is a probe for PostgreSQL/PgBouncer cleanup, +# not a claim that Proxima stores its tenant identity in a session variable. +assert_context_cleared_on_reused_backend() { + local label="$1" + local token="$2" + local password="$3" + local begin_sql="$4" + local end_sql="$5" + local setup_result before_pid reuse_result reused_pid context + + setup_result="$(run_tenant "$token" "$password" \ + "$begin_sql SELECT pg_backend_pid(); $end_sql")" + before_pid="$(printf '%s\\n' "$setup_result" | tail -n 1)" + if [[ ! "$before_pid" =~ ^[0-9]+$ ]]; then + echo "FAIL: $label did not return a valid backend PID (got: $before_pid)" >&2 + exit 1 + fi + + reuse_result="$(run_tenant "$token" "$password" \ + "SELECT pg_backend_pid() || '|' || COALESCE(NULLIF(current_setting('proxima.tenant_id', true), ''), '')")" + reused_pid="${reuse_result%%|*}" + context="${reuse_result#*|}" + + test "$reused_pid" = "$before_pid" || { + echo "FAIL: $label did not inspect the same reused backend (before=$before_pid after=$reused_pid)" >&2 + exit 1 + } + test "$context" = "" || { + echo "FAIL: $label leaked tenant context '$context' onto reused backend $reused_pid" >&2 + exit 1 + } + + echo "PASS: $label cleared before backend $reused_pid was reused" +} + +# SET LOCAL semantics: transaction-scoped identity must disappear on COMMIT. +assert_context_cleared_on_reused_backend \ + "COMMIT transaction-local context" "$token_a" tenant-a-password \ + "BEGIN; SELECT set_config('proxima.tenant_id', 'tenant_a', true);" "COMMIT" + +# SET LOCAL semantics must also disappear when the transaction rolls back. +assert_context_cleared_on_reused_backend \ + "ROLLBACK transaction-local context" "$token_a" tenant-a-password \ + "BEGIN; SELECT set_config('proxima.tenant_id', 'tenant_a', true);" "ROLLBACK" + +# A session-scoped setting survives COMMIT in PostgreSQL. Its disappearance on +# the same backend proves PgBouncer's configured DISCARD ALL reset ran before reuse. +assert_context_cleared_on_reused_backend \ + "PgBouncer DISCARD ALL session-state reset" "$token_a" tenant-a-password \ + "BEGIN; SELECT set_config('proxima.tenant_id', 'tenant_a', false);" "COMMIT" + # Confirm tenant A and tenant B each receive only their own data. test "$(run_tenant "$token_a" tenant-a-password \ "SELECT string_agg(secret, ',' ORDER BY secret) FROM pgbouncer_tenant_test.records")" = "A-secret" @@ -132,6 +185,6 @@ if PGPASSWORD=tenant-a-password PGOPTIONS='' \ exit 1 fi -echo "PASS: PgBouncer transaction pooling reused each tenant role's backend, kept tenant pools separate, denied cross-tenant reads/writes, and rejected invalid/missing tenant context." +echo "PASS: PgBouncer transaction pooling reused each tenant role's backend, cleared COMMIT/ROLLBACK transaction-local context, reset session state with DISCARD ALL before backend reuse, denied cross-tenant reads/writes, and rejected invalid/missing tenant context." echo "Tenant A backend PID: $tenant_a_pid" echo "Tenant B backend PID: $tenant_b_pid" From 6ddd32432be2896acc0b8325b34daeedbb64592a Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Wed, 11 Nov 2026 12:28:13 -0800 Subject: [PATCH 2/6] fix(pgbouncer): parse backend PID from test output --- scripts/test-pgbouncer-pooling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-pgbouncer-pooling.sh b/scripts/test-pgbouncer-pooling.sh index 4ad8dc2..4954168 100644 --- a/scripts/test-pgbouncer-pooling.sh +++ b/scripts/test-pgbouncer-pooling.sh @@ -91,7 +91,7 @@ assert_context_cleared_on_reused_backend() { setup_result="$(run_tenant "$token" "$password" \ "$begin_sql SELECT pg_backend_pid(); $end_sql")" - before_pid="$(printf '%s\\n' "$setup_result" | tail -n 1)" + before_pid="$(printf '%s\n' "$setup_result" | tail -n 1)" if [[ ! "$before_pid" =~ ^[0-9]+$ ]]; then echo "FAIL: $label did not return a valid backend PID (got: $before_pid)" >&2 exit 1 From 65e35afb2bf899e4a5a197eff583491f3b14f695 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Wed, 11 Nov 2026 12:57:59 -0800 Subject: [PATCH 3/6] fix: capture backend PID across PgBouncer transaction boundaries --- scripts/test-pgbouncer-pooling.sh | 120 +++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 3 deletions(-) diff --git a/scripts/test-pgbouncer-pooling.sh b/scripts/test-pgbouncer-pooling.sh index 4954168..c7b4efc 100644 --- a/scripts/test-pgbouncer-pooling.sh +++ b/scripts/test-pgbouncer-pooling.sh @@ -77,6 +77,24 @@ run_tenant() { -c "$sql" } +# Send each statement separately over one client connection. PgBouncer's +# transaction pool releases the server backend between these commands while +# the client remains connected. Keeping BEGIN/set_config, PID inspection, and +# COMMIT/ROLLBACK in separate -c requests also makes psql print the PID result +# instead of only the final transaction command tag. +run_tenant_transaction() { + local token="$1" + local password="$2" + local begin_sql="$3" + local end_sql="$4" + PGPASSWORD="$password" PGOPTIONS="-c proxima_tenant_token=$token" \ + psql -X -A -t -v ON_ERROR_STOP=1 \ + "host=127.0.0.1 port=6432 dbname=appdb user=proxima sslmode=disable" \ + -c "$begin_sql" \ + -c "SELECT pg_backend_pid()" \ + -c "$end_sql" +} + # Exercise backend-local state across actual PgBouncer transaction releases. # Proxima's tenant identity is authenticated at startup and mapped to a dedicated # PostgreSQL role; this synthetic GUC is a probe for PostgreSQL/PgBouncer cleanup, @@ -89,9 +107,105 @@ assert_context_cleared_on_reused_backend() { local end_sql="$5" local setup_result before_pid reuse_result reused_pid context - setup_result="$(run_tenant "$token" "$password" \ - "$begin_sql SELECT pg_backend_pid(); $end_sql")" - before_pid="$(printf '%s\n' "$setup_result" | tail -n 1)" + setup_result="$(run_tenant_transaction "$token" "$password" "$begin_sql" "$end_sql")" + before_pid="$(printf '%s\n' "$setup_result" | grep -E '^[0-9]+ + if [[ ! "$before_pid" =~ ^[0-9]+$ ]]; then + echo "FAIL: $label did not return a valid backend PID (got: $before_pid)" >&2 + exit 1 + fi + + reuse_result="$(run_tenant "$token" "$password" \ + "SELECT pg_backend_pid() || '|' || COALESCE(NULLIF(current_setting('proxima.tenant_id', true), ''), '')")" + reused_pid="${reuse_result%%|*}" + context="${reuse_result#*|}" + + test "$reused_pid" = "$before_pid" || { + echo "FAIL: $label did not inspect the same reused backend (before=$before_pid after=$reused_pid)" >&2 + exit 1 + } + test "$context" = "" || { + echo "FAIL: $label leaked tenant context '$context' onto reused backend $reused_pid" >&2 + exit 1 + } + + echo "PASS: $label cleared before backend $reused_pid was reused" +} + +# SET LOCAL semantics: transaction-scoped identity must disappear on COMMIT. +assert_context_cleared_on_reused_backend \ + "COMMIT transaction-local context" "$token_a" tenant-a-password \ + "BEGIN; SELECT set_config('proxima.tenant_id', 'tenant_a', true);" "COMMIT" + +# SET LOCAL semantics must also disappear when the transaction rolls back. +assert_context_cleared_on_reused_backend \ + "ROLLBACK transaction-local context" "$token_a" tenant-a-password \ + "BEGIN; SELECT set_config('proxima.tenant_id', 'tenant_a', true);" "ROLLBACK" + +# A session-scoped setting survives COMMIT in PostgreSQL. Its disappearance on +# the same backend proves PgBouncer's configured DISCARD ALL reset ran before reuse. +assert_context_cleared_on_reused_backend \ + "PgBouncer DISCARD ALL session-state reset" "$token_a" tenant-a-password \ + "BEGIN; SELECT set_config('proxima.tenant_id', 'tenant_a', false);" "COMMIT" + +# Confirm tenant A and tenant B each receive only their own data. +test "$(run_tenant "$token_a" tenant-a-password \ + "SELECT string_agg(secret, ',' ORDER BY secret) FROM pgbouncer_tenant_test.records")" = "A-secret" +test "$(run_tenant "$token_b" tenant-b-password \ + "SELECT string_agg(secret, ',' ORDER BY secret) FROM pgbouncer_tenant_test.records")" = "B-secret" + +# Record backend PID reuse over five sequential connections for each tenant role. +tenant_a_pid="" +for attempt in $(seq 1 5); do + result="$(run_tenant "$token_a" tenant-a-password \ + "SELECT current_user || '|' || pg_backend_pid()")" + role="${result%%|*}" + pid="${result##*|}" + test "$role" = "proxima_tenant_tenant_a" + if [[ -z "$tenant_a_pid" ]]; then tenant_a_pid="$pid"; else test "$pid" = "$tenant_a_pid"; fi +done + +tenant_b_pid="" +for attempt in $(seq 1 5); do + result="$(run_tenant "$token_b" tenant-b-password \ + "SELECT current_user || '|' || pg_backend_pid()")" + role="${result%%|*}" + pid="${result##*|}" + test "$role" = "proxima_tenant_tenant_b" + if [[ -z "$tenant_b_pid" ]]; then tenant_b_pid="$pid"; else test "$pid" = "$tenant_b_pid"; fi +done + +# Different tenant roles must occupy different PgBouncer backend pools. +test "$tenant_a_pid" != "$tenant_b_pid" + +# Cross-tenant reads are filtered and cross-tenant writes are rejected by RLS. +test "$(run_tenant "$token_a" tenant-a-password \ + "SELECT count(*) FROM pgbouncer_tenant_test.records WHERE tenant_id='tenant_b'")" = "0" +test "$(run_tenant "$token_b" tenant-b-password \ + "SELECT count(*) FROM pgbouncer_tenant_test.records WHERE tenant_id='tenant_a'")" = "0" + +if run_tenant "$token_a" tenant-a-password \ + "INSERT INTO pgbouncer_tenant_test.records (tenant_id, secret) VALUES ('tenant_b', 'forbidden')" >/dev/null 2>&1; then + echo "FAIL: Tenant A cross-tenant insert unexpectedly succeeded" >&2 + exit 1 +fi + +if run_tenant "invalid-token" tenant-a-password "SELECT 1" >/dev/null 2>&1; then + echo "FAIL: invalid tenant token was accepted" >&2 + exit 1 +fi + +if PGPASSWORD=tenant-a-password PGOPTIONS='' \ + psql -X -A -t -v ON_ERROR_STOP=1 \ + "host=127.0.0.1 port=6432 dbname=appdb user=proxima sslmode=disable" \ + -c "SELECT 1" >/dev/null 2>&1; then + echo "FAIL: missing tenant token was accepted" >&2 + exit 1 +fi + +echo "PASS: PgBouncer transaction pooling reused each tenant role's backend, cleared COMMIT/ROLLBACK transaction-local context, reset session state with DISCARD ALL before backend reuse, denied cross-tenant reads/writes, and rejected invalid/missing tenant context." +echo "Tenant A backend PID: $tenant_a_pid" +echo "Tenant B backend PID: $tenant_b_pid" + | tail -n 1)" if [[ ! "$before_pid" =~ ^[0-9]+$ ]]; then echo "FAIL: $label did not return a valid backend PID (got: $before_pid)" >&2 exit 1 From 4b23424d848dff3be2bed2dc28ace84135495292 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Wed, 11 Nov 2026 13:08:21 -0800 Subject: [PATCH 4/6] Fix PgBouncer pooling test script syntax and PID parsing --- scripts/test-pgbouncer-pooling.sh | 100 +----------------------------- 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/scripts/test-pgbouncer-pooling.sh b/scripts/test-pgbouncer-pooling.sh index c7b4efc..bb643c5 100644 --- a/scripts/test-pgbouncer-pooling.sh +++ b/scripts/test-pgbouncer-pooling.sh @@ -107,7 +107,7 @@ assert_context_cleared_on_reused_backend() { local end_sql="$5" local setup_result before_pid reuse_result reused_pid context - setup_result="$(run_tenant_transaction "$token" "$password" "$begin_sql" "$end_sql")" + before_pid="$(printf '%s\n' "$setup_result" | grep -E '^[0-9]+$' | head -n 1)" before_pid="$(printf '%s\n' "$setup_result" | grep -E '^[0-9]+ if [[ ! "$before_pid" =~ ^[0-9]+$ ]]; then echo "FAIL: $label did not return a valid backend PID (got: $before_pid)" >&2 @@ -204,101 +204,3 @@ fi echo "PASS: PgBouncer transaction pooling reused each tenant role's backend, cleared COMMIT/ROLLBACK transaction-local context, reset session state with DISCARD ALL before backend reuse, denied cross-tenant reads/writes, and rejected invalid/missing tenant context." echo "Tenant A backend PID: $tenant_a_pid" -echo "Tenant B backend PID: $tenant_b_pid" - | tail -n 1)" - if [[ ! "$before_pid" =~ ^[0-9]+$ ]]; then - echo "FAIL: $label did not return a valid backend PID (got: $before_pid)" >&2 - exit 1 - fi - - reuse_result="$(run_tenant "$token" "$password" \ - "SELECT pg_backend_pid() || '|' || COALESCE(NULLIF(current_setting('proxima.tenant_id', true), ''), '')")" - reused_pid="${reuse_result%%|*}" - context="${reuse_result#*|}" - - test "$reused_pid" = "$before_pid" || { - echo "FAIL: $label did not inspect the same reused backend (before=$before_pid after=$reused_pid)" >&2 - exit 1 - } - test "$context" = "" || { - echo "FAIL: $label leaked tenant context '$context' onto reused backend $reused_pid" >&2 - exit 1 - } - - echo "PASS: $label cleared before backend $reused_pid was reused" -} - -# SET LOCAL semantics: transaction-scoped identity must disappear on COMMIT. -assert_context_cleared_on_reused_backend \ - "COMMIT transaction-local context" "$token_a" tenant-a-password \ - "BEGIN; SELECT set_config('proxima.tenant_id', 'tenant_a', true);" "COMMIT" - -# SET LOCAL semantics must also disappear when the transaction rolls back. -assert_context_cleared_on_reused_backend \ - "ROLLBACK transaction-local context" "$token_a" tenant-a-password \ - "BEGIN; SELECT set_config('proxima.tenant_id', 'tenant_a', true);" "ROLLBACK" - -# A session-scoped setting survives COMMIT in PostgreSQL. Its disappearance on -# the same backend proves PgBouncer's configured DISCARD ALL reset ran before reuse. -assert_context_cleared_on_reused_backend \ - "PgBouncer DISCARD ALL session-state reset" "$token_a" tenant-a-password \ - "BEGIN; SELECT set_config('proxima.tenant_id', 'tenant_a', false);" "COMMIT" - -# Confirm tenant A and tenant B each receive only their own data. -test "$(run_tenant "$token_a" tenant-a-password \ - "SELECT string_agg(secret, ',' ORDER BY secret) FROM pgbouncer_tenant_test.records")" = "A-secret" -test "$(run_tenant "$token_b" tenant-b-password \ - "SELECT string_agg(secret, ',' ORDER BY secret) FROM pgbouncer_tenant_test.records")" = "B-secret" - -# Record backend PID reuse over five sequential connections for each tenant role. -tenant_a_pid="" -for attempt in $(seq 1 5); do - result="$(run_tenant "$token_a" tenant-a-password \ - "SELECT current_user || '|' || pg_backend_pid()")" - role="${result%%|*}" - pid="${result##*|}" - test "$role" = "proxima_tenant_tenant_a" - if [[ -z "$tenant_a_pid" ]]; then tenant_a_pid="$pid"; else test "$pid" = "$tenant_a_pid"; fi -done - -tenant_b_pid="" -for attempt in $(seq 1 5); do - result="$(run_tenant "$token_b" tenant-b-password \ - "SELECT current_user || '|' || pg_backend_pid()")" - role="${result%%|*}" - pid="${result##*|}" - test "$role" = "proxima_tenant_tenant_b" - if [[ -z "$tenant_b_pid" ]]; then tenant_b_pid="$pid"; else test "$pid" = "$tenant_b_pid"; fi -done - -# Different tenant roles must occupy different PgBouncer backend pools. -test "$tenant_a_pid" != "$tenant_b_pid" - -# Cross-tenant reads are filtered and cross-tenant writes are rejected by RLS. -test "$(run_tenant "$token_a" tenant-a-password \ - "SELECT count(*) FROM pgbouncer_tenant_test.records WHERE tenant_id='tenant_b'")" = "0" -test "$(run_tenant "$token_b" tenant-b-password \ - "SELECT count(*) FROM pgbouncer_tenant_test.records WHERE tenant_id='tenant_a'")" = "0" - -if run_tenant "$token_a" tenant-a-password \ - "INSERT INTO pgbouncer_tenant_test.records (tenant_id, secret) VALUES ('tenant_b', 'forbidden')" >/dev/null 2>&1; then - echo "FAIL: Tenant A cross-tenant insert unexpectedly succeeded" >&2 - exit 1 -fi - -if run_tenant "invalid-token" tenant-a-password "SELECT 1" >/dev/null 2>&1; then - echo "FAIL: invalid tenant token was accepted" >&2 - exit 1 -fi - -if PGPASSWORD=tenant-a-password PGOPTIONS='' \ - psql -X -A -t -v ON_ERROR_STOP=1 \ - "host=127.0.0.1 port=6432 dbname=appdb user=proxima sslmode=disable" \ - -c "SELECT 1" >/dev/null 2>&1; then - echo "FAIL: missing tenant token was accepted" >&2 - exit 1 -fi - -echo "PASS: PgBouncer transaction pooling reused each tenant role's backend, cleared COMMIT/ROLLBACK transaction-local context, reset session state with DISCARD ALL before backend reuse, denied cross-tenant reads/writes, and rejected invalid/missing tenant context." -echo "Tenant A backend PID: $tenant_a_pid" -echo "Tenant B backend PID: $tenant_b_pid" From 12913d5d2470f77681b7a696c9af2d6a2989ee6a Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Wed, 11 Nov 2026 13:08:31 -0800 Subject: [PATCH 5/6] Repair PgBouncer transaction PID capture --- scripts/test-pgbouncer-pooling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-pgbouncer-pooling.sh b/scripts/test-pgbouncer-pooling.sh index bb643c5..ca037f1 100644 --- a/scripts/test-pgbouncer-pooling.sh +++ b/scripts/test-pgbouncer-pooling.sh @@ -107,8 +107,8 @@ assert_context_cleared_on_reused_backend() { local end_sql="$5" local setup_result before_pid reuse_result reused_pid context + setup_result="$(run_tenant_transaction "$token" "$password" "$begin_sql" "$end_sql")" before_pid="$(printf '%s\n' "$setup_result" | grep -E '^[0-9]+$' | head -n 1)" - before_pid="$(printf '%s\n' "$setup_result" | grep -E '^[0-9]+ if [[ ! "$before_pid" =~ ^[0-9]+$ ]]; then echo "FAIL: $label did not return a valid backend PID (got: $before_pid)" >&2 exit 1 From 4374f93a145f207352bbf4e4a1637684198bf551 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Wed, 11 Nov 2026 13:40:07 -0800 Subject: [PATCH 6/6] ci: run PgBouncer transaction cleanup integration test --- .github/workflows/tenant-evidence.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tenant-evidence.yml b/.github/workflows/tenant-evidence.yml index acb8b1e..be4b79d 100644 --- a/.github/workflows/tenant-evidence.yml +++ b/.github/workflows/tenant-evidence.yml @@ -36,6 +36,14 @@ jobs: run: | set -o pipefail cargo test -p proxima-engine -- --nocapture 2>&1 | tee tenant-evidence.log + - name: Install PostgreSQL client for PgBouncer integration test + run: | + sudo apt-get update + sudo apt-get install -y postgresql-client + - name: Run PgBouncer transaction-pooling cleanup test + run: | + set -o pipefail + bash scripts/test-pgbouncer-pooling.sh 2>&1 | tee -a tenant-evidence.log - name: Publish test evidence if: always() uses: actions/upload-artifact@v4