From 013081585c44f8c3b4f078940f1c7d6cc4f68e5a Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Tue, 10 Nov 2026 18:39:44 +0900 Subject: [PATCH 01/29] docs(python): record one method for each query operator The engine already derives order and match terms, and the Go SDK uses them. Python's query always asks for equality. ADR-0003 records the call shape for the other operators: eq, lt, lte, gt, gte, between, and matches, with query removed. Match answers can contain rows that do not match, so matches returns an object with target_limit and verify. ADR-0002 now points to ADR-0003 where its op= plan was replaced. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- ...002-a-query-names-its-field-by-a-string.md | 3 + ...0003-one-method-for-each-query-operator.md | 76 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 languages/python/docs/adr/0003-one-method-for-each-query-operator.md diff --git a/languages/python/docs/adr/0002-a-query-names-its-field-by-a-string.md b/languages/python/docs/adr/0002-a-query-names-its-field-by-a-string.md index 8058ca685..91e9a66a8 100644 --- a/languages/python/docs/adr/0002-a-query-names-its-field-by-a-string.md +++ b/languages/python/docs/adr/0002-a-query-names-its-field-by-a-string.md @@ -3,6 +3,7 @@ status: accepted date: 2026-11-02 ticket: "#1391, the first ticket that adds `query`, as the spec (#1341) says" extends: ADR-0001 +amended_by: ADR-0003 --- # A query names its field by a string @@ -53,6 +54,8 @@ A does not block B. A later version can accept a string or an attribute in the s An operator can come later as a keyword argument, such as `op=`. A caller who passes none still gets equality. +> **Replaced by ADR-0003.** The paragraph above is no longer the plan. ADR-0003 gives each operator its own method, such as `eq` and `gte`, and removes `query`. The field is still named by a string, as this ADR decided. + ## Consequences - `query_unknown_field` and `query_unsupported` are raised in Python, by `Cipher.query`, before the extension is called. diff --git a/languages/python/docs/adr/0003-one-method-for-each-query-operator.md b/languages/python/docs/adr/0003-one-method-for-each-query-operator.md new file mode 100644 index 000000000..2c7905094 --- /dev/null +++ b/languages/python/docs/adr/0003-one-method-for-each-query-operator.md @@ -0,0 +1,76 @@ +--- +status: accepted +date: 2026-11-10 +ticket: "none yet" +extends: ADR-0002 +--- + +# One method for each query operator + +`Cipher` has one method for each operator the engine supports on a field. `query` is removed. + +```python +term = users.eq("email", "a@example.com") +term = users.gte("age", 30) +term = users.between("age", 18, 65) +q = users.matches("name", "ali") +``` + +The field is still named by a string, as ADR-0002 decided. + +This Architecture Decision Record (ADR) records the call shape for equality, order, and match queries. It replaces the last paragraph of ADR-0002's decision. That paragraph said that an operator could come later as `op=`. + +## Background + +The engine already derives order and match terms. The Go SDK uses them: its generated code has `Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between`, and `Matches` (`languages/golang/encrypt/doc.go`). Python's core calls the same engine function, `dynamic::query_term`, but always with `OpSpec::Eq` (`crates/core/src/query.rs`). So Python needs a way to name the operator. + +Equality and order answers are exact. A match answer is not. The match index stores a Bloom filter, so the database can return rows that do not contain the search text. The application must decrypt the rows, then remove those that do not match. Go's `MatchQuery` has `TargetLimit` and `Verify` for this. + +## The options + +### A. An operator argument + +```python +users.query("age", 30, op="gte") +``` + +- Today's calls do not change, because `op` defaults to `"eq"`. +- With `op` typed as a `Literal`, a type checker finds a misspelt operator. + +### B. One method for each operator (chosen) + +```python +users.gte("age", 30) +``` + +- It is the usual Python style for a small, fixed set of operations. +- A type checker finds a misspelt method. + +Neither option lets a type checker find a field that does not support the operator, such as `users.gte("email", x)`. The field is a string, so a type checker cannot know its indexes. Both options raise `QueryError` with `query_unsupported` at run time, before any request. Go finds this at compile time, because its generator writes only the methods each field supports. Python has no generator. + +So the choice is about readability, not safety. + +## Decision + +B. + +- `eq`, `lt`, `lte`, `gt`, and `gte` take a field name and a value. Each returns the term as a `str`. +- `between(field, low, high)` returns one `str` term that contains both bounds. Both bounds are included. A range the wrong way round is not refused. It matches no row, as SQL's `BETWEEN` does. +- `query` is removed. The package is not released, so no user code breaks. Keeping it would give two ways to make an equality term. +- `matches(field, text)` returns a match query object, not a `str`. The object has: + - `term`: the term as a `str`. + - `target_limit(want, rows=..., histogram=..., confidence=...)`: the number of rows to fetch so that at least `want` of them pass `verify`, at the given confidence. The default confidence is 99%. It returns `None` when the engine cannot work out a finite number. The caller then fetches page by page. + - `verify(values)`: takes the decrypted values of the field as a list of `str`, and returns one `bool` for each, in order. + +The method names are the engine's operator names, as in Go. + +`verify` takes strings, not decrypted objects. It works the same for every class kind, and the match query object does not need to read a field from an object. + +## Consequences + +- The match query object contains the search text as plaintext until Python frees it. `verify` needs the text, because the engine cuts it into tokens again. Go has the same limit. The object must never show the text in `repr` or in an error. +- `verify` is not an exact substring check. It returns `True` when the value contains every 3-character token of the search text, in any order. +- `target_limit` needs statistics from the database: a row count, and for a finite answer a histogram of the stored filters' set-bit counts. `packages/stack-encrypt/src/op/stats.rs` gives the SQL for both. +- `between` needs a second SQL cast on each order marker. The SQL is `eql_v4.between(col, %s::eql_v4.query_integer_range_ope)`, not `BETWEEN`. The Python markers in `_eql.py` do not name the range casts yet, so `eql-codegen py-eql` must add them. +- The README, the tests, and the customer directory example change from `query` to `eq`. +- The later SQLAlchemy layer should hide `target_limit` and `verify`. ORM users do not expect a search to return rows that do not match. From e99ea8532192601b582222ef34460908bc298ebe Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Tue, 10 Nov 2026 18:40:13 +0900 Subject: [PATCH 02/29] feat(python): one query method for each order operator The engine already derives order terms, but Python's query always asked for equality. Cipher now has eq, lt, lte, gt and gte, as ADR-0003 decides, and query is removed. Core passes the operator to the engine. Cipher checks a field against the operators the engine lists for it, not against the marker's indexes, so the two cannot disagree. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- languages/python/README.md | 18 ++- languages/python/crates/core/src/query.rs | 125 ++++++++++++++---- .../python/crates/engine/src/heap_hygiene.rs | 3 +- languages/python/crates/engine/src/lib.rs | 1 + languages/python/crates/engine/src/query.rs | 51 +++++-- languages/python/docs/scorecard.md | 12 +- .../python/examples/customer-directory/app.py | 2 +- .../customer-directory/app_sqlalchemy.py | 2 +- .../python/python/stackencrypt/_cipher.py | 90 +++++++++---- .../python/python/stackencrypt/_engine.pyi | 10 +- .../python/python/stackencrypt/_errors.py | 10 +- languages/python/tests/test_encrypt_live.py | 44 +++++- languages/python/tests/test_query.py | 76 +++++++++-- languages/python/tests/test_values.py | 10 +- .../tests_deterministic/test_go_rows.py | 2 +- 16 files changed, 344 insertions(+), 114 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ebaf7ba75..1fac4da02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and `query` against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The package has no runtime dependency, so nothing else is locked. The middle three are for the declaration tests. psycopg is for the live tests. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, the call shape of `query`), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The package has no runtime dependency, so nothing else is locked. The middle three are for the declaration tests. psycopg is for the live tests. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `languages/python/examples/customer-directory`: An internal FastAPI example of the Python SDK, written twice: with psycopg (`app.py`) and with SQLAlchemy (`app_sqlalchemy.py`). It has its own `pyproject.toml` and `uv.lock`, and installs `stackencrypt` from this checkout. Its tests need real credentials, and CI does not run them. diff --git a/languages/python/README.md b/languages/python/README.md index df24aa079..4ed3de26a 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -89,10 +89,10 @@ A passthrough `datetime` field needs a `timestamptz` column. psycopg reads a `ti ## Search by equality -`cipher.query` takes a field's name and a value, and returns a query term as a `str`. Cast the term to the field's query type in the SQL: +`cipher.eq` takes a field's name and a value, and returns a query term as a `str`. Cast the term to the field's query type in the SQL: ```python -term = cipher.query("email", "a@example.com") +term = cipher.eq("email", "a@example.com") with psycopg.connect(DATABASE_URL) as conn: with conn.cursor(row_factory=dict_row) as cur: cur.execute( @@ -104,9 +104,15 @@ with psycopg.connect(DATABASE_URL) as conn: The query type is the marker's `query_sql_domain`. For `TextEq` it is `eql_v4.query_text_eq`. The `users_email_eq` index from "Create the table" lets PostgreSQL find the row without reading every row. -`query` sends no request to CipherStash after the first operation of the process. The term is made from a key that the first operation loads. +A query method sends no request to CipherStash after the first operation of the process. The term is made from a key that the first operation loads. -`query` makes equality terms only. A field whose marker has no `eq` index raises `stackencrypt.QueryError` with the code `query_unsupported`. Examples are `Text`, `Passthrough`, `IntegerOrd`, and `TextMatch`. A name that is not a declared field raises `QueryError` with the code `query_unknown_field`. The value is checked as `encrypt` checks it. +A field that does not support the operator raises `stackencrypt.QueryError` with the code `query_unsupported`. For `eq`, examples are `Text`, `Passthrough`, `IntegerOrd`, and `TextMatch`. A name that is not a declared field raises `QueryError` with the code `query_unknown_field`. The value is checked as `encrypt` checks it. + +## Search by order + +A field whose marker has an order index, such as `IntegerOrd`, supports `lt`, `lte`, `gt`, and `gte`. Each takes a field's name and a value, and returns a term as a `str`. Use the SQL operator of the same meaning: `<`, `<=`, `>`, or `>=`. For example, `cipher.gte("level", 3)` returns a term for `WHERE level >= %s::eql_v4.query_integer_ord`. + +The errors are the same as for `eq`. A `TextEq` field, for example, does not support `gte`. ## Declare a SQLAlchemy relationship @@ -186,7 +192,7 @@ A model field cannot be a `uuid.UUID`. The decorator refuses it with the code `d ## What is not supported yet -- Match, order, and range queries. `query` makes equality terms only. +- Match and range queries. The query methods are `eq`, `lt`, `lte`, `gt`, and `gte`. - Encrypted JSON documents. - Lock contexts and identity-bound encryption. - More than one keyset, and a choice of keyset. @@ -194,7 +200,7 @@ A model field cannot be a `uuid.UUID`. The decorator refuses it with the code `d ## Use it from several threads -`encrypt`, `decrypt`, and `query` are synchronous. There is no async form yet. +`encrypt`, `decrypt`, and the query methods are synchronous. There is no async form yet. Each call releases the GIL while the encryption runs. The GIL is CPython's global interpreter lock. So other Python threads keep running during a call, and calls from several threads run at the same time. Within one call, the requests to CipherStash go out one at a time. diff --git a/languages/python/crates/core/src/query.rs b/languages/python/crates/core/src/query.rs index 818596bf5..47fb7d1db 100644 --- a/languages/python/crates/core/src/query.rs +++ b/languages/python/crates/core/src/query.rs @@ -1,12 +1,15 @@ -//! `query`: the engine's equality query term for one field and one value. +//! `query`: the engine's query term for one field, one operator, and one +//! value. //! //! The term is the EQL query value the engine derives through the field's //! EQL type, as JSON text. The SQL casts it to the type's query domain, such -//! as `eql_v4.query_text_eq`, and EQL's `=` compares it with the stored -//! values' `eq_term`. A term is derived from the index key in the loaded -//! keyset, so no ZeroKMS request is made once the shared client is built. +//! as `eql_v4.query_text_eq`, and EQL's operator compares it with the stored +//! values' term for that index. A term is derived from the index key in the +//! loaded keyset, so no ZeroKMS request is made once the shared client is +//! built. -use stack_encrypt::dynamic::{self, OpSpec, Operand}; +pub use stack_encrypt::dynamic::OpSpec; +use stack_encrypt::dynamic::{self, Operand}; use stack_encrypt::kms::{DataKeySource, IndexKeySource}; use stack_encrypt::KeysetCipher; @@ -17,25 +20,26 @@ use crate::refusal::Refusal; use crate::targets; use crate::values::{Kind, Typed, Value}; -/// The equality query term for `value` on the sealed field `field` of +/// The query term for `op` and `value` on the sealed field `field` of /// `plan`, as JSON text. /// -/// Checks first, with no I/O, that the field answers equality. Then builds -/// the shared client on the first call in the process. The caller has -/// released the GIL, and has checked that `value` fits the field's kind. +/// Checks first, with no I/O, that the field answers `op`. Then builds the +/// shared client on the first call in the process. The caller has released +/// the GIL, and has checked that `value` fits the field's kind. /// /// # Errors /// -/// `query_unsupported` for a field that answers no equality query, before -/// any request. Then the client's refusal, or the engine's for the query. +/// `query_unsupported` for a field that does not answer `op`, before any +/// request. Then the client's refusal, or the engine's for the query. pub fn query( plan: &ModelPlan, field: &str, + op: OpSpec, kind: Kind, value: Value<'_>, ) -> Result { - answers_equality(plan, field)?; - client::run!(|cipher| query_with(&cipher.default_keyset(), plan, field, kind, value)) + answers(plan, field, op)?; + client::run!(|cipher| query_with(&cipher.default_keyset(), plan, field, op, kind, value)) } /// [`query`] under `keyset`. @@ -47,6 +51,7 @@ pub async fn query_with( keyset: &KeysetCipher<'_, K>, plan: &ModelPlan, field: &str, + op: OpSpec, kind: Kind, value: Value<'_>, ) -> Result @@ -57,7 +62,7 @@ where keyset, plan.plan(), field, - OpSpec::Eq, + op, Typed(Some(kind), value), &targets::resolver(), ) @@ -65,33 +70,42 @@ where .await .map_err(|error| Refusal::of(&error))?; let Operand::One(json) = term.operand() else { - return Err(engine_fault("the equality term is not one operand")); + return Err(engine_fault("the query term is not one operand")); }; // A copy of the term's bytes, about a hundred: `QueryTerm` has no // accessor that gives up its operand. - String::from_utf8(json.clone()).map_err(|_| engine_fault("the equality term is not UTF-8")) + String::from_utf8(json.clone()).map_err(|_| engine_fault("the query term is not UTF-8")) } -/// Whether `field` answers an equality query, from the plan alone: the -/// engine answers one through the field's `eq` index only, and a type with -/// only an order or match index answers none. `Cipher.query` checks the -/// marker's indexes the same way. This check is for a caller that did not. +/// The operators `field` answers, by their engine names, from the plan +/// alone. A passthrough field answers none. /// /// # Errors /// -/// `query_unsupported`, or the engine's refusal of the field. -pub fn answers_equality(plan: &ModelPlan, field: &str) -> Result<(), Refusal> { +/// The engine's refusal of the field. +pub fn operators(plan: &ModelPlan, field: &str) -> Result, Refusal> { let operators = dynamic::field_operators(plan.plan(), field, &targets::resolver()) .map_err(|error| plan::refusal(&error))?; - if operators + Ok(operators .iter() - .any(|(operator, _)| *operator == OpSpec::Eq) - { + .map(|(operator, _)| operator.name()) + .collect()) +} + +/// Whether `field` answers `op`, from the plan alone: the engine answers an +/// operator through one of the field's indexes. `Cipher` checks the same +/// list first. This check is for a caller that did not. +/// +/// # Errors +/// +/// `query_unsupported`, or the engine's refusal of the field. +pub fn answers(plan: &ModelPlan, field: &str, op: OpSpec) -> Result<(), Refusal> { + if operators(plan, field)?.contains(&op.name()) { return Ok(()); } Err(Refusal { code: "query_unsupported".to_owned(), - message: format!("field {field:?} has no eq index, so it answers no equality query."), + message: format!("field {field:?} has no index that answers {:?}.", op.name()), field: Some(field.to_owned()), reason: None, }) @@ -111,6 +125,7 @@ mod tests { &cipher.default_keyset(), &users(), "email", + OpSpec::Eq, Kind::String, Value::Text(email), )) @@ -148,6 +163,57 @@ mod tests { ); } + fn levels() -> ModelPlan { + ModelPlan::new( + "users", + vec![Field { + name: "level".to_owned(), + eql: Some(("IntegerOrd".to_owned(), Kind::Int32)), + }], + ) + .unwrap() + } + + const ORDER: [OpSpec; 4] = [OpSpec::Lt, OpSpec::Lte, OpSpec::Gt, OpSpec::Gte]; + + #[test] + fn an_order_field_answers_every_order_operator() { + for op in ORDER { + answers(&levels(), "level", op).unwrap(); + } + } + + #[test] + fn an_equality_field_answers_no_order_operator() { + for op in ORDER { + let refused = answers(&users(), "email", op).unwrap_err(); + assert_eq!(refused.code, "query_unsupported", "{op:?}"); + assert_eq!(refused.field.as_deref(), Some("email")); + assert!(refused.message.contains(op.name()), "{}", refused.message); + } + } + + #[test] + fn an_order_term_is_derived_for_the_operator_asked() { + let cipher = cipher(); + for op in ORDER { + let json = run(query_with( + &cipher.default_keyset(), + &levels(), + "level", + op, + Kind::Int32, + Value::Int(3), + )) + .unwrap(); + let term: serde_json::Value = serde_json::from_str(&json).unwrap(); + // `op` is the order term EQL's `<` and `>` compare. The query + // carries no ciphertext. + assert!(term.get("op").is_some(), "{op:?}: {term}"); + assert!(term.get("c").is_none(), "{op:?}: {term}"); + } + } + #[test] fn a_field_with_only_an_order_or_match_index_answers_no_equality() { for eql in ["IntegerOrd", "IntegerOrdOre", "TextMatch"] { @@ -165,7 +231,7 @@ mod tests { ) .unwrap(); - let refused = answers_equality(&plan, "rank").unwrap_err(); + let refused = answers(&plan, "rank", OpSpec::Eq).unwrap_err(); assert_eq!(refused.code, "query_unsupported", "{eql}"); assert_eq!(refused.field.as_deref(), Some("rank")); @@ -174,13 +240,13 @@ mod tests { #[test] fn a_field_with_an_eq_index_answers_equality() { - answers_equality(&users(), "email").unwrap(); + answers(&users(), "email", OpSpec::Eq).unwrap(); } #[test] fn a_passthrough_field_answers_no_equality() { assert_eq!( - answers_equality(&users(), "age").unwrap_err().code, + answers(&users(), "age", OpSpec::Eq).unwrap_err().code, "query_unsupported" ); } @@ -193,6 +259,7 @@ mod tests { &cipher.default_keyset(), &users(), "age", + OpSpec::Eq, Kind::Int64, Value::Int(1), )) diff --git a/languages/python/crates/engine/src/heap_hygiene.rs b/languages/python/crates/engine/src/heap_hygiene.rs index b7bf4c5c6..0a142dd40 100644 --- a/languages/python/crates/engine/src/heap_hygiene.rs +++ b/languages/python/crates/engine/src/heap_hygiene.rs @@ -39,7 +39,7 @@ use stack_kms::FakeDataKeySource; use stackencrypt_core::decrypt::{decrypt_with, Row, Rows}; use stackencrypt_core::encrypt::{encrypt_with, Column}; -use stackencrypt_core::query::query_with; +use stackencrypt_core::query::{query_with, OpSpec}; use crate::encrypt::{build_plan, field_value, input_frame, Plan}; use crate::values::to_python; @@ -329,6 +329,7 @@ fn query_frees_no_copy_of_the_plaintext_unwiped() { &cipher.default_keyset(), &plan.plan, &field.name, + OpSpec::Eq, engine_kind, read.value(), )) diff --git a/languages/python/crates/engine/src/lib.rs b/languages/python/crates/engine/src/lib.rs index 277c986ec..1173b2eec 100644 --- a/languages/python/crates/engine/src/lib.rs +++ b/languages/python/crates/engine/src/lib.rs @@ -112,6 +112,7 @@ fn _engine(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(encrypt::encrypt, module)?)?; module.add_function(wrap_pyfunction!(decrypt::decrypt, module)?)?; module.add_function(wrap_pyfunction!(query::query, module)?)?; + module.add_function(wrap_pyfunction!(query::operators, module)?)?; module.add_function(wrap_pyfunction!(encrypt::client_builds, module)?)?; module.add_function(wrap_pyfunction!(encrypt::request_count, module)?)?; module.add_function(wrap_pyfunction!(encrypt::requests, module)?)?; diff --git a/languages/python/crates/engine/src/query.rs b/languages/python/crates/engine/src/query.rs index 70cebf698..08b7adc3f 100644 --- a/languages/python/crates/engine/src/query.rs +++ b/languages/python/crates/engine/src/query.rs @@ -1,18 +1,19 @@ -//! `query` at the boundary: one value checked as `encrypt` checks it, and -//! the engine's equality term back as a `str`. +//! The query functions at the boundary: one value checked as `encrypt` +//! checks it, and the engine's term for one operator back as a `str`. //! -//! `Cipher.query` has already refused a name that is not a declared field, -//! and a field with no `eq` index (the spec in #1341, "Errors"). Core -//! checks the second again from the plan, before the client is built. +//! `Cipher` has already refused a name that is not a declared field, and a +//! field that does not answer the operator (the spec in #1341, "Errors"). +//! Core checks the second again from the plan, before the client is built. use pyo3::prelude::*; -use stackencrypt_core::query as core; +use stackencrypt_core::query::{self as core, OpSpec}; use stackencrypt_core::refusal::Refusal; use crate::encrypt::{field_value, Plan}; use crate::errors::raise; -/// The equality query term for `value` on the sealed field `field`. +/// The query term for the operator named `operator` and `value` on the +/// sealed field `field`. /// /// The value is checked first, by exact type and range, with no request. /// Then the shared client is built on the first call in the process, and @@ -23,8 +24,12 @@ pub fn query( py: Python<'_>, plan: &Plan, field: &str, + operator: &str, value: &Bound<'_, PyAny>, ) -> PyResult { + let Some(op) = OpSpec::parse(operator) else { + return Err(raise(py, unknown_operator(field, operator))); + }; let found = plan .plan .fields() @@ -39,17 +44,43 @@ pub fn query( }; let read = field_value(value, declared, kind)?.map_err(|refused| raise(py, refused))?; let value = read.value(); - py.detach(|| core::query(&plan.plan, field, engine_kind, value)) + py.detach(|| core::query(&plan.plan, field, op, engine_kind, value)) .map_err(|refused| raise(py, refused)) } +/// The names of the operators the field `field` answers, from the plan +/// alone, with no request. A passthrough field answers none. +#[pyfunction] +pub fn operators(py: Python<'_>, plan: &Plan, field: &str) -> PyResult> { + let sealed = plan + .plan + .fields() + .iter() + .any(|candidate| candidate.name == field && candidate.eql.is_some()); + if !sealed { + return Ok(Vec::new()); + } + core::operators(&plan.plan, field).map_err(|refused| raise(py, refused)) +} + +fn unknown_operator(field: &str, operator: &str) -> Refusal { + Refusal { + code: "stack_encrypt_unknown".to_owned(), + message: format!( + "the operator {operator:?} reached the engine's query, but the engine has no such \ + operator. Cipher should have refused it. Report this message as a stackencrypt bug." + ), + field: Some(field.to_owned()), + reason: None, + } +} + fn not_sealed(field: &str) -> Refusal { Refusal { code: "stack_encrypt_unknown".to_owned(), message: format!( "field {field:?} reached the engine's query, but it is not a sealed field of the \ - plan. Cipher.query should have refused it. Report this message as a stackencrypt \ - bug." + plan. Cipher should have refused it. Report this message as a stackencrypt bug." ), field: Some(field.to_owned()), reason: None, diff --git a/languages/python/docs/scorecard.md b/languages/python/docs/scorecard.md index 22246a6be..5003254e1 100644 --- a/languages/python/docs/scorecard.md +++ b/languages/python/docs/scorecard.md @@ -23,8 +23,8 @@ Each principle is marked: ## G1. One engine, and the SDK never executes: held -- `Cipher.encrypt`, `decrypt`, and `query` each make one call into the extension, in `python/stackencrypt/_cipher.py`. The Python layer has no loop over rows, and no loop over a model's values. -- The Python layer does loop over the declared fields in two places. `Cipher` lists them once, to build the plan. `query` searches them to name a field it refuses. Neither loop touches a value. +- `Cipher.encrypt`, `decrypt`, and each query method make one call into the extension, in `python/stackencrypt/_cipher.py`. The Python layer has no loop over rows, and no loop over a model's values. +- The Python layer does loop over the declared fields in two places. `Cipher` lists them once, to build the plan. A query method searches them to name a field it refuses. Neither loop touches a value. - The extension hands the engine one frame for the whole call, and reads back the engine's whole output. Encryption, decryption, and term derivation all run in `stack-encrypt`. - The declaration check asks the engine. When the class statement runs, the decorator calls `_engine.check_plan`, and the engine refuses a plan it cannot run. `Cipher` later builds the same plan with `_engine.build_plan`. - The EQL type rules are not copied. `crates/core` compiles the Go guest's resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place. `python/stackencrypt/_eql.py` is generated from the EQL catalog, and `tests-python.yml` fails on a hand edit. @@ -46,15 +46,15 @@ Each principle is marked: ## G4. One declaration serves the write, the query and the read: held -- `Cipher(User)` builds one plan from the declaration. `encrypt`, `query`, and `decrypt` all use that plan. -- No call takes a context or an index choice. `query` takes a field name and a value only (Python ADR-0002, under `docs/adr/`). -- `test_a_query_term_finds_the_one_matching_row` in `tests/test_encrypt_live.py` runs `encrypt`, `query`, and `decrypt` on one `Cipher` against PostgreSQL. The README test runs the same three steps. +- `Cipher(User)` builds one plan from the declaration. `encrypt`, the query methods, and `decrypt` all use that plan. +- No call takes a context or an index choice. A query method, such as `eq` or `gte`, takes a field name and a value only (Python ADR-0002 and ADR-0003, under `docs/adr/`). The engine picks the index that supports the operator. +- `test_a_query_term_finds_the_one_matching_row` in `tests/test_encrypt_live.py` runs `encrypt`, `eq`, and `decrypt` on one `Cipher` against PostgreSQL. The README test runs the same three steps. - A context that changes per caller, such as a tenant, is not supported yet. The README lists it. ## G5. Batch by default: held - `encrypt` and `decrypt` take a list. One call makes one key request for every 500 values, in sequence. `tests/test_encrypt_live.py` counts 3 requests for 1001 values, each way. -- `query` is the single-value call the principle allows: one field, one search value. It makes no key request after the first call of the process. +- Each query method is the single-value call the principle allows: one field, one search value. It makes no key request after the first call of the process. - A model's fields of different types go in one request. ## G6. A second way in has to earn its place: held diff --git a/languages/python/examples/customer-directory/app.py b/languages/python/examples/customer-directory/app.py index 12da7908e..1040037e3 100644 --- a/languages/python/examples/customer-directory/app.py +++ b/languages/python/examples/customer-directory/app.py @@ -76,7 +76,7 @@ def add_customer(customer: Customer) -> Customer: @app.post("/customers/search", response_model=Customer) def find_customer(search: Search) -> Customer: # The email is in the body so that it stays out of URLs and access logs. - term = cipher.query("email", search.email) + term = cipher.eq("email", search.email) with psycopg.connect(DATABASE_URL) as conn: with conn.cursor(row_factory=dict_row) as cur: cur.execute(f"{SELECT_CUSTOMER} WHERE email = %s::eql_v4.query_text_eq", (term,)) diff --git a/languages/python/examples/customer-directory/app_sqlalchemy.py b/languages/python/examples/customer-directory/app_sqlalchemy.py index 30c9765fd..32385331f 100644 --- a/languages/python/examples/customer-directory/app_sqlalchemy.py +++ b/languages/python/examples/customer-directory/app_sqlalchemy.py @@ -114,7 +114,7 @@ def add_customer(body: CustomerBody) -> CustomerBody: @app.post("/customers/search", response_model=CustomerBody) def find_customer(search: Search) -> Customer: # The email is in the body so that it stays out of URLs and access logs. - term = sa.literal(cipher.query("email", search.email), sa.Text) + term = sa.literal(cipher.eq("email", search.email), sa.Text) query = sa.select(*COLUMNS).where( Customer.email == sa.cast(term, EqlType("eql_v4.query_text_eq")) ) diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index 4d538ec0d..bec3815e5 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -1,7 +1,7 @@ """``Cipher``: encrypts and decrypts the instances of one declared class. It checks the class and the credentials, and builds the model's plan once. -``encrypt``, ``decrypt`` and ``query`` are here. +``encrypt``, ``decrypt`` and the query methods are here. """ from typing import Dict, Generic, List, Mapping, Sequence, Type, TypeVar @@ -94,11 +94,11 @@ def decrypt(self, rows: Sequence[Mapping[str, object]]) -> List[T]: """ return _engine.decrypt(self._plan, self._cls, list(rows)) - def query(self, field: str, value: object) -> str: + def eq(self, field: str, value: object) -> str: """The equality query term for ``value`` on the declared field ``field``, named as a ``str``:: - term = users.query("email", "a@example.com") + term = users.eq("email", "a@example.com") cur.execute( "SELECT * FROM users WHERE email = %s::eql_v4.query_text_eq", (term,), @@ -106,15 +106,50 @@ def query(self, field: str, value: object) -> str: The term is a ``str``. The SQL casts it to the field marker's ``query_sql_domain``. A term is derived from the keyset already - loaded: after the shared client is built, ``query`` makes no request. + loaded: after the shared client is built, a query method makes no + request. A name that is not a declared field raises ``QueryError`` - (``query_unknown_field``). A field whose marker has no ``eq`` index - raises ``QueryError`` (``query_unsupported``). Examples are ``Text``, - ``Passthrough``, ``IntegerOrd`` and ``TextMatch``. The value is checked as ``encrypt`` checks - it, and a value that does not fit raises ``InvalidValueError``. All - three are raised before any request. + (``query_unknown_field``). A field that does not support the + operator raises ``QueryError`` (``query_unsupported``). Examples for + ``eq`` are ``Text``, ``Passthrough``, ``IntegerOrd`` and + ``TextMatch``. The value is checked as ``encrypt`` checks it, and a + value that does not fit raises ``InvalidValueError``. All three are + raised before any request. ``lt``, ``lte``, ``gt`` and ``gte`` work + the same way. """ + return self._term("eq", field, value) + + def lt(self, field: str, value: object) -> str: + """The query term for "less than ``value``" on ``field``:: + + term = users.lt("level", 3) + cur.execute( + "SELECT * FROM users WHERE level < %s::eql_v4.query_integer_ord", + (term,), + ) + + The field's marker needs an order index, such as ``IntegerOrd``. + The errors are those of ``eq``. + """ + return self._term("lt", field, value) + + def lte(self, field: str, value: object) -> str: + """The query term for "less than or equal to ``value``" on + ``field``. SQL: ``<=``. See ``lt``.""" + return self._term("lte", field, value) + + def gt(self, field: str, value: object) -> str: + """The query term for "greater than ``value``" on ``field``. SQL: + ``>``. See ``lt``.""" + return self._term("gt", field, value) + + def gte(self, field: str, value: object) -> str: + """The query term for "greater than or equal to ``value``" on + ``field``. SQL: ``>=``. See ``lt``.""" + return self._term("gte", field, value) + + def _term(self, op: str, field: str, value: object) -> str: declared = next( (f for f in self._declaration.fields if f.name == field), None ) @@ -122,26 +157,35 @@ def query(self, field: str, value: object) -> str: raise QueryError( "query_unknown_field", f"{field!r} is not a declared field of " - f"{self._cls.__qualname__!r}. {self._queryable()}", + f"{self._cls.__qualname__!r}. {self._supporting(op)}", field=field if isinstance(field, str) else None, ) - if not _has_equality(declared): + if op not in self._operators(declared): # Passthrough names no EQL type, so it has no `name` attribute. marker = declared.marker.__name__ raise QueryError( "query_unsupported", - f"field {field!r} is declared {marker}, which has no eq " - f"index, so it answers no equality query. {self._queryable()}", + f"field {field!r} is declared {marker}, which does not " + f"support {op}. {self._supporting(op)}", field=field, reason=marker, ) - return _engine.query(self._plan, field, value) - - def _queryable(self) -> str: - names = [repr(f.name) for f in self._declaration.fields if _has_equality(f)] + return _engine.query(self._plan, field, op, value) + + def _operators(self, field: DeclaredField) -> List[str]: + """The operators the engine supports on ``field``, from the plan, + with no request. A passthrough field supports none.""" + if field.marker is Passthrough: + return [] + return _engine.operators(self._plan, field.name) + + def _supporting(self, op: str) -> str: + names = [ + repr(f.name) for f in self._declaration.fields if op in self._operators(f) + ] if not names: - return "The class has no field with an equality query." - return "Fields with an equality query: " + ", ".join(names) + "." + return f"The class has no field that supports {op}." + return f"Fields that support {op}: " + ", ".join(names) + "." @property def declaration(self) -> Declaration: @@ -152,14 +196,6 @@ def __repr__(self) -> str: return f"Cipher({self._cls.__qualname__})" -def _has_equality(field: DeclaredField) -> bool: - """Whether ``query`` answers ``field``: the engine answers an equality - query through the ``eq`` index alone (``OpSpec::answered_by`` in - ``stack-encrypt``). A marker with only an order or match index has a - query type, but not for equality.""" - return field.marker is not Passthrough and "eq" in field.marker.indexes - - def _not_decorated(cls: object) -> DeclarationError: if not isinstance(cls, type): return DeclarationError( diff --git a/languages/python/python/stackencrypt/_engine.pyi b/languages/python/python/stackencrypt/_engine.pyi index 9e9598d2a..623b9a09c 100644 --- a/languages/python/python/stackencrypt/_engine.pyi +++ b/languages/python/python/stackencrypt/_engine.pyi @@ -35,11 +35,17 @@ def decrypt(plan: Plan, cls: type, rows: Sequence[Any]) -> List[Any]: keyword arguments. Raises ``DecryptError`` or ``InvalidValueError`` before any request for what the rows themselves can answer.""" -def query(plan: Plan, field: str, value: Any) -> str: - """The equality query term for ``value`` on the sealed field ``field``. +def query(plan: Plan, field: str, operator: str, value: Any) -> str: + """The query term for the operator named ``operator``, such as + ``"eq"`` or ``"gte"``, and ``value`` on the sealed field ``field``. Raises ``InvalidValueError`` before any request for a value that does not fit the field.""" +def operators(plan: Plan, field: str) -> List[str]: + """The names of the operators the engine supports on the field + ``field``, from the plan, with no request. A passthrough field has + none.""" + def client_builds() -> int: """How many times the shared engine client has been built. For tests.""" diff --git a/languages/python/python/stackencrypt/_errors.py b/languages/python/python/stackencrypt/_errors.py index 4837ac930..9d4c33f19 100644 --- a/languages/python/python/stackencrypt/_errors.py +++ b/languages/python/python/stackencrypt/_errors.py @@ -70,11 +70,11 @@ class DecryptError(StashError): class QueryError(StashError): - """A ``query`` call the declared class cannot answer, raised before any - request. + """A query method call the declared class cannot answer, such as ``eq`` + or ``gte``, raised before any request. ``query_unknown_field``: the name is not a declared field. - ``query_unsupported``: the field's marker has no ``eq`` index, such as - ``Text``, ``Passthrough`` or ``IntegerOrd``. ``reason`` is the marker's - name. + ``query_unsupported``: the field does not support the operator. For + ``eq``, examples are ``Text``, ``Passthrough`` and ``IntegerOrd``. For + ``gte``, an example is ``TextEq``. ``reason`` is the marker's name. """ diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py index 6d16c542b..a3a85362c 100644 --- a/languages/python/tests/test_encrypt_live.py +++ b/languages/python/tests/test_encrypt_live.py @@ -1,4 +1,4 @@ -"""``Cipher.encrypt``, ``Cipher.decrypt`` and ``Cipher.query`` against ZeroKMS +"""``Cipher.encrypt``, ``Cipher.decrypt`` and the query methods against ZeroKMS and a PostgreSQL with the EQL v4 bundle. These tests need CipherStash credentials (``CS_CLIENT_ACCESS_KEY``, @@ -24,7 +24,7 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship import stackencrypt -from stackencrypt import Cipher, DecryptError, Ignore, Passthrough, TextEq +from stackencrypt import Cipher, DecryptError, Ignore, IntegerOrd, Passthrough, TextEq from stackencrypt import _engine, _eql pytestmark = LIVE @@ -264,7 +264,7 @@ def test_a_query_term_finds_the_one_matching_row(): # The README's query: the term cast to the marker's query domain. cur.execute( f"SELECT * FROM users WHERE email = %s::{TextEq.query_sql_domain}", - (cipher.query("email", "a@example.com"),), + (cipher.eq("email", "a@example.com"),), ) found = cipher.decrypt(cur.fetchall()) @@ -289,21 +289,53 @@ def test_a_query_uses_the_eq_term_index(): cur.execute("SET LOCAL enable_seqscan = off") cur.execute( "EXPLAIN SELECT * FROM users WHERE email = %s::eql_v4.query_text_eq", - (cipher.query("email", "user7@example.com"),), + (cipher.eq("email", "user7@example.com"),), ) plan = "\n".join(line for (line,) in cur.fetchall()) assert "users_email_eq" in plan, plan +@stackencrypt.model("levels") +@dataclasses.dataclass +class Level: + n: Annotated[int, IntegerOrd] + + +ORDER_SQL = {"lt": "<", "lte": "<=", "gt": ">", "gte": ">="} +# The rows each operator finds against 3, among 1 to 5. +ORDER_FOUND = {"lt": [1, 2], "lte": [1, 2, 3], "gt": [4, 5], "gte": [3, 4, 5]} + + +@needs_database +@pytest.mark.parametrize("op", sorted(ORDER_SQL)) +def test_an_order_term_finds_the_rows_on_its_side(op): + psycopg = pytest.importorskip("psycopg") + from psycopg.rows import dict_row + + cipher = Cipher(Level) + rows = cipher.encrypt([Level(n=n) for n in [5, 1, 4, 2, 3]]) + with psycopg.connect(DATABASE_URL) as conn, conn.cursor(row_factory=dict_row) as cur: + cur.execute("CREATE TEMPORARY TABLE levels (n public.eql_v4_integer_ord)") + for row in rows: + cur.execute("INSERT INTO levels (n) VALUES (%s)", (row["n"],)) + cur.execute( + f"SELECT n FROM levels WHERE n {ORDER_SQL[op]} %s::{IntegerOrd.query_sql_domain}", + (getattr(cipher, op)("n", 3),), + ) + found = sorted(level.n for level in cipher.decrypt(cur.fetchall())) + + assert found == ORDER_FOUND[op] + + def test_100_queries_make_no_request(): cipher = Cipher(User) # The first call in the process may build the client and load the # keyset. Warm it up first, so the count below is the queries alone. - cipher.query("email", "warm@example.com") + cipher.eq("email", "warm@example.com") before = _engine.request_count() - terms = {cipher.query("email", f"user{i}@example.com") for i in range(100)} + terms = {cipher.eq("email", f"user{i}@example.com") for i in range(100)} assert _engine.request_count() == before assert len(terms) == 100 diff --git a/languages/python/tests/test_query.py b/languages/python/tests/test_query.py index db9271efc..14247b295 100644 --- a/languages/python/tests/test_query.py +++ b/languages/python/tests/test_query.py @@ -1,7 +1,8 @@ -"""``Cipher.query`` before any request: the field checks and the value checks. +"""The query methods before any request: the field checks and the value checks. -A name that is not a declared field, a field that answers no query, and a -value that does not fit are all refused before the shared client is built. +``eq``, ``lt``, ``lte``, ``gt`` and ``gte`` (ADR-0003). A name that is not a +declared field, a field that does not support the operator, and a value that +does not fit are all refused before the shared client is built. So these tests need fake credentials only, and make no request. The live search, the index, and the request count are in ``test_encrypt_live.py``. """ @@ -39,12 +40,21 @@ class User: notes: Annotated[str, TextMatch] -def refused(field, value, error=QueryError): +ORDER = ["lt", "lte", "gt", "gte"] + + +def refused(field, value, error=QueryError, op="eq"): with pytest.raises(error) as raised: - Cipher(User).query(field, value) + getattr(Cipher(User), op)(field, value) return raised.value +def test_query_is_gone(): + # ADR-0003: one method for each operator, and no second way to make an + # equality term. + assert not hasattr(Cipher, "query") + + def test_a_query_error_is_a_stash_error(): assert issubclass(QueryError, StashError) @@ -54,7 +64,7 @@ def test_an_unknown_name_lists_the_fields_with_a_query(untouched): assert error.code == "query_unknown_field" assert error.field == "emial" assert "'emial'" in error.message - assert "Fields with an equality query: 'email', 'rank'." in error.message + assert "Fields that support eq: 'email', 'rank'." in error.message def test_a_field_that_is_not_a_str_is_an_unknown_name(untouched): @@ -63,6 +73,12 @@ def test_a_field_that_is_not_a_str_is_an_unknown_name(untouched): assert error.field is None +def test_an_unknown_name_lists_the_fields_with_the_operator(untouched): + error = refused("lvl", 3, op="gte") + assert error.code == "query_unknown_field" + assert "Fields that support gte: 'level'." in error.message + + def test_a_text_field_answers_no_query(untouched): error = refused("bio", "hello") assert error.code == "query_unsupported" @@ -81,15 +97,45 @@ def test_a_passthrough_field_answers_no_query(untouched): ("field", "value", "marker"), [("level", 3, "IntegerOrd"), ("notes", "hello", "TextMatch")], ) -def test_a_field_with_a_query_type_but_no_eq_index_answers_no_query( +def test_a_field_with_a_query_type_but_no_eq_index_answers_no_eq( untouched, field, value, marker ): - # The marker has a query type, for order or match. `query` answers - # equality only, which the engine answers through the eq index alone. + # The marker has a query type, for order or match. The engine answers + # eq through the eq index alone. error = refused(field, value) assert error.code == "query_unsupported" assert error.field == field assert error.reason == marker + assert "does not support eq" in error.message + + +@pytest.mark.parametrize("op", ORDER) +@pytest.mark.parametrize( + ("field", "value", "marker"), + [("email", "a@example.com", "TextEq"), ("notes", "hello", "TextMatch")], +) +def test_a_field_with_no_order_index_answers_no_order_operator( + untouched, op, field, value, marker +): + error = refused(field, value, op=op) + assert error.code == "query_unsupported" + assert error.field == field + assert error.reason == marker + assert f"does not support {op}" in error.message + + +@pytest.mark.parametrize("op", ORDER) +def test_a_passthrough_field_answers_no_order_operator(untouched, op): + error = refused("age", 3, op=op) + assert error.code == "query_unsupported" + assert error.reason == "Passthrough" + + +@pytest.mark.parametrize("op", ORDER) +def test_an_order_value_of_another_type_is_the_wrong_type(untouched, op): + error = refused("level", "3", InvalidValueError, op=op) + assert error.code == "value_wrong_type" + assert error.field == "level" def test_a_class_with_no_queryable_field_says_so(untouched): @@ -99,8 +145,8 @@ class Note: body: Annotated[str, Text] with pytest.raises(QueryError) as raised: - Cipher(Note).query("body", "x") - assert "The class has no field with an equality query." in raised.value.message + Cipher(Note).eq("body", "x") + assert "The class has no field that supports eq." in raised.value.message def test_a_value_of_another_type_is_the_wrong_type(untouched): @@ -132,9 +178,13 @@ def test_the_message_never_contains_the_value(untouched): assert "123456789" not in str(error) -def test_a_query_that_passes_every_check_reaches_the_engine(): +@pytest.mark.parametrize( + ("op", "field", "value"), + [("eq", "email", "a@example.com")] + [(op, "level", 3) for op in ORDER], +) +def test_a_query_that_passes_every_check_reaches_the_engine(op, field, value): # Refused later by the engine for want of a client key, not by the # checks: they passed. with pytest.raises(StashError) as raised: - Cipher(User).query("email", "a@example.com") + getattr(Cipher(User), op)(field, value) assert not isinstance(raised.value, (QueryError, InvalidValueError)) diff --git a/languages/python/tests/test_values.py b/languages/python/tests/test_values.py index ad68d52e0..a9a4ed9ec 100644 --- a/languages/python/tests/test_values.py +++ b/languages/python/tests/test_values.py @@ -1,4 +1,4 @@ -"""The value rules, through ``encrypt`` and ``query`` alike. +"""The value rules, through ``encrypt`` and ``eq`` alike. Every value is checked by exact type and range before any request, by one function that both calls use. Each case here runs through both, and the @@ -62,20 +62,20 @@ def through_encrypt(marker: Any, value: Any) -> None: Cipher(stackencrypt.model("rows")(cls)).encrypt([cls(x=value)]) -def through_query(marker: Any, value: Any) -> None: +def through_eq(marker: Any, value: Any) -> None: cls = dataclasses.make_dataclass( "Row", [("x", Annotated[PYTHON_TYPE[marker], marker])] ) - Cipher(stackencrypt.model("rows")(cls)).query("x", value) + Cipher(stackencrypt.model("rows")(cls)).eq("x", value) def calls(cases: List[Tuple[Any, ...]]) -> List[Any]: - """Each case through ``encrypt``, and through ``query`` where the + """Each case through ``encrypt``, and through ``eq`` where the marker has an ``eq`` index. ``Boolean`` has none.""" return [ pytest.param(call, *case, id=f"{call.__name__}-{case[0].__name__}-{case[1]!r}") for case in cases - for call in (through_encrypt, through_query) + for call in (through_encrypt, through_eq) if call is through_encrypt or "eq" in case[0].indexes ] diff --git a/languages/python/tests_deterministic/test_go_rows.py b/languages/python/tests_deterministic/test_go_rows.py index e6cd2d7d5..429ff4463 100644 --- a/languages/python/tests_deterministic/test_go_rows.py +++ b/languages/python/tests_deterministic/test_go_rows.py @@ -231,7 +231,7 @@ def test_each_equality_term_is_the_term_the_go_sdk_stored(fixture, cls, ids): for row in rows_of(fixture, ids): model = model_of(cls, row) for field in queried: - term = json.loads(cipher.query(field.name, getattr(model, field.name))) + term = json.loads(cipher.eq(field.name, getattr(model, field.name))) assert term["hm"] == row["stored"][field.name]["hm"], (row["id"], field.name) From b0ad5335f1e75cd62e645de193477bddca222667 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Tue, 10 Nov 2026 18:46:42 +0900 Subject: [PATCH 03/29] docs(python): between waits for the engine The engine refuses a between term for an EQL order type, with the reason operator_not_answered, and the Go SDK has no Between for these types either. Python leaves between out and stays at Go's level. A range search uses gte and lte. Building the range in Python would copy EQL's range format with nothing to keep the copy correct. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 2 ++ ...0003-one-method-for-each-query-operator.md | 20 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 4ed3de26a..4d7eeccb9 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -114,6 +114,8 @@ A field whose marker has an order index, such as `IntegerOrd`, supports `lt`, `l The errors are the same as for `eq`. A `TextEq` field, for example, does not support `gte`. +There is no `between` yet. For a range, use `gte` and `lte` together: `WHERE level >= %s::eql_v4.query_integer_ord AND level <= %s::eql_v4.query_integer_ord`. + ## Declare a SQLAlchemy relationship A SQLAlchemy relationship is not a column. Every annotated field needs a marker, so write a relationship in one of two ways: diff --git a/languages/python/docs/adr/0003-one-method-for-each-query-operator.md b/languages/python/docs/adr/0003-one-method-for-each-query-operator.md index 2c7905094..b1e0f8227 100644 --- a/languages/python/docs/adr/0003-one-method-for-each-query-operator.md +++ b/languages/python/docs/adr/0003-one-method-for-each-query-operator.md @@ -55,7 +55,7 @@ So the choice is about readability, not safety. B. - `eq`, `lt`, `lte`, `gt`, and `gte` take a field name and a value. Each returns the term as a `str`. -- `between(field, low, high)` returns one `str` term that contains both bounds. Both bounds are included. A range the wrong way round is not refused. It matches no row, as SQL's `BETWEEN` does. +- `between(field, low, high)` will return one `str` term that contains both bounds. Both bounds are included. A range the wrong way round is not refused. It matches no row, as SQL's `BETWEEN` does. `between` waits for the engine. See "Amendment, 2026-11-10". - `query` is removed. The package is not released, so no user code breaks. Keeping it would give two ways to make an equality term. - `matches(field, text)` returns a match query object, not a `str`. The object has: - `term`: the term as a `str`. @@ -71,6 +71,22 @@ The method names are the engine's operator names, as in Go. - The match query object contains the search text as plaintext until Python frees it. `verify` needs the text, because the engine cuts it into tokens again. Go has the same limit. The object must never show the text in `repr` or in an error. - `verify` is not an exact substring check. It returns `True` when the value contains every 3-character token of the search text, in any order. - `target_limit` needs statistics from the database: a row count, and for a finite answer a histogram of the stored filters' set-bit counts. `packages/stack-encrypt/src/op/stats.rs` gives the SQL for both. -- `between` needs a second SQL cast on each order marker. The SQL is `eql_v4.between(col, %s::eql_v4.query_integer_range_ope)`, not `BETWEEN`. The Python markers in `_eql.py` do not name the range casts yet, so `eql-codegen py-eql` must add them. +- `between` needs a second SQL cast on each order marker. The SQL is `eql_v4.between(col, %s::eql_v4.query_integer_range_ope)`, not `BETWEEN`. The Python markers in `_eql.py` do not name the range casts yet. Go's type table does not name them either. - The README, the tests, and the customer directory example change from `query` to `eq`. - The later SQLAlchemy layer should hide `target_limit` and `verify`. ORM users do not expect a search to return rows that do not match. + +## Amendment, 2026-11-10: `between` waits for the engine + +The engine does not build a `between` term for an EQL type yet. A core test asked it for one on an `IntegerOrd` field. The engine refused, with the reason `operator_not_answered`. The comment at the top of `packages/eql/crates/eql-codegen/src/range.rs` says the same. The EQL test `packages/eql/tests/encryption/tests/v4_range.rs` builds its ranges by hand from two single-bound terms. + +The Go SDK has the same limit. Its generator writes only the methods the engine supports, so a Go field of an EQL order type has no `Between`. + +So Python leaves out `between` for now, and stays at the same level as Go. A range search uses `gte` and `lte`: + +```sql +WHERE level >= %s::eql_v4.query_integer_ord AND level <= %s::eql_v4.query_integer_ord +``` + +We did not choose to build the range in Python from two single-bound terms. Python would then contain its own copy of EQL's range format. No test would keep that copy the same as EQL's. + +`between` comes when the engine supports it for EQL types. That is a change to `packages/stack-encrypt`. It would give `between` to Go too. The call shape above stays the plan. From deacf8ad42e4a1da21665fb40284821d51440ca8 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 09:40:55 +0900 Subject: [PATCH 04/29] feat(python): match search with matches, verify and target_limit The engine answers a match on a TextMatch field, but the answer can hold rows that do not contain the text. Cipher.matches returns a MatchQuery, as ADR-0003 decides: its term goes in the SQL with @@, verify removes the false positives after decrypt, and target_limit sizes the fetch from the stored filters' histogram, or returns None when no finite fetch can be promised. MatchQuery keeps the search text for verify, and its repr never shows it. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 12 +- languages/python/crates/core/src/query.rs | 182 ++++++++++++++++++ languages/python/crates/engine/src/lib.rs | 2 + languages/python/crates/engine/src/query.rs | 88 +++++++++ .../python/python/stackencrypt/__init__.py | 3 +- .../python/python/stackencrypt/_cipher.py | 79 +++++++- .../python/python/stackencrypt/_engine.pyi | 16 ++ languages/python/tests/test_encrypt_live.py | 66 ++++++- languages/python/tests/test_query.py | 28 ++- 9 files changed, 470 insertions(+), 6 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 4d7eeccb9..1bf8000a8 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -116,6 +116,16 @@ The errors are the same as for `eq`. A `TextEq` field, for example, does not sup There is no `between` yet. For a range, use `gte` and `lte` together: `WHERE level >= %s::eql_v4.query_integer_ord AND level <= %s::eql_v4.query_integer_ord`. +## Search by match + +A field whose marker has a match index, such as `TextMatch`, supports `matches`. It takes a field's name and the text to find, and returns a `MatchQuery`. Its `term` goes in the SQL with `@@`: `WHERE notes @@ %s::eql_v4.query_text_match`. + +A match search can return rows that do not hold the text. So decrypt the rows, then pass the field's values to `q.verify`. It returns one `bool` for each value, in order. Keep the rows that are `True`. `verify` checks that a value holds every 3-character piece of the text. The pieces can be in any order, so it is not an exact substring check. + +To size the `LIMIT`, call `q.target_limit(want, rows=..., histogram=...)`. It returns how many rows to fetch so that `want` of them pass `verify`, 99 times in 100. It returns `None` when it cannot promise a number. Then fetch page by page. Without a `histogram`, it always returns `None`. The method's docstring gives the SQL that reads the histogram. + +A `MatchQuery` keeps the search text, because `verify` needs it. Python cannot wipe a `str`, so the text stays in memory until Python frees the object. `repr` does not show it. + ## Declare a SQLAlchemy relationship A SQLAlchemy relationship is not a column. Every annotated field needs a marker, so write a relationship in one of two ways: @@ -194,7 +204,7 @@ A model field cannot be a `uuid.UUID`. The decorator refuses it with the code `d ## What is not supported yet -- Match and range queries. The query methods are `eq`, `lt`, `lte`, `gt`, and `gte`. +- Range queries with `between`. Use `gte` and `lte` together. - Encrypted JSON documents. - Lock contexts and identity-bound encryption. - More than one keyset, and a choice of keyset. diff --git a/languages/python/crates/core/src/query.rs b/languages/python/crates/core/src/query.rs index 47fb7d1db..a563a6666 100644 --- a/languages/python/crates/core/src/query.rs +++ b/languages/python/crates/core/src/query.rs @@ -11,6 +11,7 @@ pub use stack_encrypt::dynamic::OpSpec; use stack_encrypt::dynamic::{self, Operand}; use stack_encrypt::kms::{DataKeySource, IndexKeySource}; +use stack_encrypt::op::{Limit, Stats}; use stack_encrypt::KeysetCipher; use crate::client; @@ -77,6 +78,79 @@ where String::from_utf8(json.clone()).map_err(|_| engine_fault("the query term is not UTF-8")) } +/// For each of `values`, a page of decrypted values of `field`, whether it +/// holds every token of the match text `text`, in order. Runs in the engine +/// with no key and no request. The rows reported `false` are the false +/// positives of a match search. +/// +/// # Errors +/// +/// `query_unsupported` for a field that does not answer `matches`, or the +/// engine's refusal. +pub fn verify( + plan: &ModelPlan, + field: &str, + text: &str, + values: &[&str], +) -> Result, Refusal> { + answers(plan, field, OpSpec::Matches)?; + dynamic::verify( + plan.plan(), + field, + OpSpec::Matches, + Typed(Some(Kind::String), Value::Text(text)), + values + .iter() + .map(|value| Typed(Some(Kind::String), Value::Text(value))), + &targets::resolver(), + ) + .map_err(|error| plan::refusal(&error)) +} + +/// How many rows to fetch for the match term `term` on `field` so that, +/// at the confidence, `want` of them pass [`verify`]: `None` when the +/// engine cannot promise a finite number. `rows` is the rows the search +/// scans. `histogram` is the stored filters' set-bit counts, as `(bits, +/// rows)` pairs; without one, the limit is unbounded. `confidence` is +/// 0.99 when `None`. Uses no key and makes no request. +/// +/// # Errors +/// +/// `query_unsupported` for a field that does not answer `matches`, or the +/// engine's refusal of a term it would not derive. +pub fn target_limit( + plan: &ModelPlan, + field: &str, + term: &str, + want: u64, + rows: u64, + histogram: Option<&[(u32, u64)]>, + confidence: Option, +) -> Result, Refusal> { + answers(plan, field, OpSpec::Matches)?; + let mut stats = match histogram { + Some(histogram) => Stats::measured(rows, histogram.iter().copied()), + None => Stats::declared(rows), + }; + if let Some(confidence) = confidence { + stats = stats.with_confidence(confidence); + } + let limit = dynamic::target_limit( + plan.plan(), + field, + OpSpec::Matches, + term.as_bytes(), + want, + &stats, + &targets::resolver(), + ) + .map_err(|error| plan::refusal(&error))?; + Ok(match limit { + Limit::Rows(rows) => Some(rows), + Limit::Unbounded => None, + }) +} + /// The operators `field` answers, by their engine names, from the plan /// alone. A passthrough field answers none. /// @@ -214,6 +288,114 @@ mod tests { } } + fn notes() -> ModelPlan { + ModelPlan::new( + "users", + vec![Field { + name: "notes".to_owned(), + eql: Some(("TextMatch".to_owned(), Kind::String)), + }], + ) + .unwrap() + } + + fn match_term(cipher: &StackCipher, text: &str) -> String { + run(query_with( + &cipher.default_keyset(), + ¬es(), + "notes", + OpSpec::Matches, + Kind::String, + Value::Text(text), + )) + .unwrap() + } + + #[test] + fn a_match_term_is_a_filter_with_no_ciphertext() { + let term: serde_json::Value = serde_json::from_str(&match_term(&cipher(), "ali")).unwrap(); + // `bf` is the Bloom filter EQL's `@@` compares. + assert!( + term["bf"].as_array().is_some_and(|bits| !bits.is_empty()), + "{term}" + ); + assert!(term.get("c").is_none(), "{term}"); + } + + #[test] + fn verify_keeps_the_values_that_hold_every_token_of_the_text() { + let verified = verify( + ¬es(), + "notes", + "ali", + &["Alice Smith", "Bob Jones", "Natalie"], + ) + .unwrap(); + assert_eq!(verified, [true, false, true]); + } + + #[test] + fn verify_on_a_field_with_no_match_index_is_unsupported() { + let refused = verify(&users(), "email", "ali", &["alice"]).unwrap_err(); + assert_eq!(refused.code, "query_unsupported"); + assert_eq!(refused.field.as_deref(), Some("email")); + } + + #[test] + fn with_no_histogram_the_limit_is_unbounded() { + let term = match_term(&cipher(), "ali"); + let limit = target_limit(¬es(), "notes", &term, 10, 10_000, None, None).unwrap(); + assert_eq!(limit, None); + } + + #[test] + fn with_empty_stored_filters_the_limit_is_the_rows_wanted() { + // Every stored filter sets no bit, so no row passes by chance, and + // the fetch needs no margin. + let term = match_term(&cipher(), "ali"); + let limit = target_limit( + ¬es(), + "notes", + &term, + 10, + 10_000, + Some(&[(0, 10_000)]), + None, + ) + .unwrap(); + assert_eq!(limit, Some(10)); + } + + #[test] + fn fuller_stored_filters_need_a_larger_fetch() { + let cipher = cipher(); + let term = match_term(&cipher, "ali"); + let sparse = target_limit( + ¬es(), + "notes", + &term, + 10, + 10_000, + Some(&[(20, 10_000)]), + None, + ) + .unwrap() + .unwrap(); + let full = target_limit( + ¬es(), + "notes", + &term, + 10, + 10_000, + Some(&[(400, 10_000)]), + None, + ) + .unwrap() + .unwrap(); + assert!(sparse >= 10, "{sparse}"); + assert!(full > sparse, "{full} <= {sparse}"); + } + #[test] fn a_field_with_only_an_order_or_match_index_answers_no_equality() { for eql in ["IntegerOrd", "IntegerOrdOre", "TextMatch"] { diff --git a/languages/python/crates/engine/src/lib.rs b/languages/python/crates/engine/src/lib.rs index 1173b2eec..38f839607 100644 --- a/languages/python/crates/engine/src/lib.rs +++ b/languages/python/crates/engine/src/lib.rs @@ -113,6 +113,8 @@ fn _engine(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(decrypt::decrypt, module)?)?; module.add_function(wrap_pyfunction!(query::query, module)?)?; module.add_function(wrap_pyfunction!(query::operators, module)?)?; + module.add_function(wrap_pyfunction!(query::verify, module)?)?; + module.add_function(wrap_pyfunction!(query::target_limit, module)?)?; module.add_function(wrap_pyfunction!(encrypt::client_builds, module)?)?; module.add_function(wrap_pyfunction!(encrypt::request_count, module)?)?; module.add_function(wrap_pyfunction!(encrypt::requests, module)?)?; diff --git a/languages/python/crates/engine/src/query.rs b/languages/python/crates/engine/src/query.rs index 08b7adc3f..05ae86ff2 100644 --- a/languages/python/crates/engine/src/query.rs +++ b/languages/python/crates/engine/src/query.rs @@ -6,11 +6,13 @@ //! Core checks the second again from the plan, before the client is built. use pyo3::prelude::*; +use pyo3::types::PyList; use stackencrypt_core::query::{self as core, OpSpec}; use stackencrypt_core::refusal::Refusal; use crate::encrypt::{field_value, Plan}; use crate::errors::raise; +use crate::values::Read; /// The query term for the operator named `operator` and `value` on the /// sealed field `field`. @@ -48,6 +50,92 @@ pub fn query( .map_err(|refused| raise(py, refused)) } +/// For each of `values`, decrypted values of the match field `field`, +/// whether it holds every token of the match text `text`. Each value is +/// read as `encrypt` reads the field's values. Runs with the GIL released, +/// with no key and no request. +#[pyfunction] +pub fn verify( + py: Python<'_>, + plan: &Plan, + field: &str, + text: &Bound<'_, PyAny>, + values: &Bound<'_, PyList>, +) -> PyResult> { + let found = plan + .plan + .fields() + .iter() + .zip(&plan.kinds) + .find(|(candidate, _)| candidate.name == field && candidate.eql.is_some()); + let Some((declared, &kind)) = found else { + return Err(raise(py, not_sealed(field))); + }; + let text = field_value(text, declared, kind)?.map_err(|refused| raise(py, refused))?; + let items: Vec> = values.iter().collect(); + let mut read: Vec> = Vec::with_capacity(items.len()); + for item in &items { + read.push(field_value(item, declared, kind)?.map_err(|refused| raise(py, refused))?); + } + let texts: Vec<&str> = read + .iter() + .map(as_text) + .collect::>() + .ok_or_else(|| raise(py, not_text(field)))?; + let Some(query) = as_text(&text) else { + return Err(raise(py, not_text(field))); + }; + py.detach(|| core::verify(&plan.plan, field, query, &texts)) + .map_err(|refused| raise(py, refused)) +} + +/// How many rows to fetch for the match term `term` on `field` so that +/// `want` of them verify: `None` when no finite number can be promised. +/// Uses no key and makes no request. +#[pyfunction] +#[pyo3(signature = (plan, field, term, want, rows, histogram=None, confidence=None))] +#[allow(clippy::too_many_arguments)] +pub fn target_limit( + py: Python<'_>, + plan: &Plan, + field: &str, + term: &str, + want: u64, + rows: u64, + histogram: Option>, + confidence: Option, +) -> PyResult> { + core::target_limit( + &plan.plan, + field, + term, + want, + rows, + histogram.as_deref(), + confidence, + ) + .map_err(|refused| raise(py, refused)) +} + +fn as_text<'r>(read: &'r Read<'_>) -> Option<&'r str> { + match read.value() { + stackencrypt_core::values::Value::Text(text) => Some(text), + _ => None, + } +} + +fn not_text(field: &str) -> Refusal { + Refusal { + code: "stack_encrypt_unknown".to_owned(), + message: format!( + "field {field:?} reached the engine's verify, but it is not a text field. Cipher \ + should have refused it. Report this message as a stackencrypt bug." + ), + field: Some(field.to_owned()), + reason: None, + } +} + /// The names of the operators the field `field` answers, from the plan /// alone, with no request. A passthrough field answers none. #[pyfunction] diff --git a/languages/python/python/stackencrypt/__init__.py b/languages/python/python/stackencrypt/__init__.py index d31ff4df8..515c8f8e5 100644 --- a/languages/python/python/stackencrypt/__init__.py +++ b/languages/python/python/stackencrypt/__init__.py @@ -8,7 +8,7 @@ """ from stackencrypt import _engine, _eql -from stackencrypt._cipher import Cipher +from stackencrypt._cipher import Cipher, MatchQuery from stackencrypt._declaration import model # The markers for the EQL types the engine produces today. `_eql.__all__` # lists them; a marker the engine cannot produce yet stays in `_eql` only. @@ -35,6 +35,7 @@ "DecryptError", "Ignore", "InvalidValueError", + "MatchQuery", "Passthrough", "QueryError", "StashError", diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index bec3815e5..938ba981e 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -4,7 +4,7 @@ ``encrypt``, ``decrypt`` and the query methods are here. """ -from typing import Dict, Generic, List, Mapping, Sequence, Type, TypeVar +from typing import Dict, Generic, List, Mapping, Optional, Sequence, Tuple, Type, TypeVar from stackencrypt import _engine from stackencrypt._markers import Passthrough @@ -149,6 +149,29 @@ def gte(self, field: str, value: object) -> str: ``field``. SQL: ``>=``. See ``lt``.""" return self._term("gte", field, value) + def matches(self, field: str, text: str) -> "MatchQuery": + """A match search for ``text`` on the field ``field``, whose marker + has a match index, such as ``TextMatch``:: + + q = users.matches("notes", "ali") + limit = q.target_limit(10, rows=row_count, histogram=histogram) + cur.execute( + "SELECT ... WHERE notes @@ %s::eql_v4.query_text_match LIMIT %s", + (q.term, limit), + ) + people = users.decrypt(cur.fetchall()) + keep = q.verify([p.notes for p in people]) + + A match search can return rows that do not hold the text. Decrypt + the rows, then keep those ``verify`` reports ``True``. + + The returned object keeps ``text``, because ``verify`` needs it. + Python cannot wipe a ``str``, so the text stays in memory until + Python frees it. ``repr`` never shows it. The errors are those of + ``eq``. + """ + return MatchQuery(self, field, text, self._term("matches", field, text)) + def _term(self, op: str, field: str, value: object) -> str: declared = next( (f for f in self._declaration.fields if f.name == field), None @@ -196,6 +219,60 @@ def __repr__(self) -> str: return f"Cipher({self._cls.__qualname__})" +class MatchQuery: + """A match search on one field, made by ``Cipher.matches``. + + ``term`` is the term as a ``str``. Cast it to the marker's + ``query_sql_domain`` and compare it with ``@@``. The object keeps the + search text for ``verify``. ``repr`` never shows the text. + """ + + __slots__ = ("_plan", "_field", "_text", "term") + + def __init__(self, cipher: "Cipher[object]", field: str, text: str, term: str) -> None: + self._plan = cipher._plan + self._field = field + self._text = text + self.term = term + + def target_limit( + self, + want: int, + *, + rows: int, + histogram: Optional[Sequence[Tuple[int, int]]] = None, + confidence: Optional[float] = None, + ) -> Optional[int]: + """How many rows to fetch so that ``want`` of them pass ``verify``, + at the confidence (0.99 by default): the number for the SQL + ``LIMIT``. ``None`` when no finite number can be promised: then + fetch page by page. + + ``rows`` is the number of rows the search reads, such as the + table's row count. ``histogram`` is the set-bit counts of the stored + filters, as ``(bits, rows)`` pairs:: + + SELECT cardinality(eql_v4.match_term(notes)), count(*) + FROM users WHERE notes IS NOT NULL GROUP BY 1 + + Without a histogram, the result is ``None``. Makes no request. + """ + pairs = None if histogram is None else [(int(b), int(r)) for b, r in histogram] + return _engine.target_limit( + self._plan, self._field, self.term, want, rows, pairs, confidence + ) + + def verify(self, values: Sequence[str]) -> List[bool]: + """For each decrypted value of the field, whether it holds every + 3-character token of the search text, in the same order. Keep the + rows that are ``True``. This is not an exact substring check: the + tokens can be in any order. Makes no request.""" + return _engine.verify(self._plan, self._field, self._text, list(values)) + + def __repr__(self) -> str: + return f"MatchQuery(field={self._field!r})" + + def _not_decorated(cls: object) -> DeclarationError: if not isinstance(cls, type): return DeclarationError( diff --git a/languages/python/python/stackencrypt/_engine.pyi b/languages/python/python/stackencrypt/_engine.pyi index 623b9a09c..70c2e8f89 100644 --- a/languages/python/python/stackencrypt/_engine.pyi +++ b/languages/python/python/stackencrypt/_engine.pyi @@ -46,6 +46,22 @@ def operators(plan: Plan, field: str) -> List[str]: ``field``, from the plan, with no request. A passthrough field has none.""" +def verify(plan: Plan, field: str, text: str, values: List[str]) -> List[bool]: + """For each value, whether it holds every token of the match text + ``text`` on the match field ``field``. No key and no request.""" + +def target_limit( + plan: Plan, + field: str, + term: str, + want: int, + rows: int, + histogram: Optional[List[Tuple[int, int]]] = None, + confidence: Optional[float] = None, +) -> Optional[int]: + """The rows to fetch so that ``want`` of them verify, or ``None`` when + no finite number can be promised. No key and no request.""" + def client_builds() -> int: """How many times the shared engine client has been built. For tests.""" diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py index a3a85362c..1ac17b91e 100644 --- a/languages/python/tests/test_encrypt_live.py +++ b/languages/python/tests/test_encrypt_live.py @@ -24,7 +24,7 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship import stackencrypt -from stackencrypt import Cipher, DecryptError, Ignore, IntegerOrd, Passthrough, TextEq +from stackencrypt import Cipher, DecryptError, Ignore, IntegerOrd, Passthrough, TextEq, TextMatch from stackencrypt import _engine, _eql pytestmark = LIVE @@ -328,6 +328,70 @@ def test_an_order_term_finds_the_rows_on_its_side(op): assert found == ORDER_FOUND[op] +@stackencrypt.model("notes") +@dataclasses.dataclass +class Note: + body: Annotated[str, TextMatch] + + +NOTES = ["Alice Smith", "Bob Jones", "Natalie Wood", "Carol King"] + + +def _notes_table(cur, cipher): + cur.execute("CREATE TEMPORARY TABLE notes (body public.eql_v4_text_match)") + for row in cipher.encrypt([Note(body=body) for body in NOTES]): + cur.execute("INSERT INTO notes (body) VALUES (%s)", (row["body"],)) + + +@needs_database +def test_a_match_search_and_verify_find_the_rows_that_hold_the_text(): + psycopg = pytest.importorskip("psycopg") + from psycopg.rows import dict_row + + cipher = Cipher(Note) + q = cipher.matches("body", "ali") + with psycopg.connect(DATABASE_URL) as conn, conn.cursor(row_factory=dict_row) as cur: + _notes_table(cur, cipher) + cur.execute( + f"SELECT body FROM notes WHERE body @@ %s::{TextMatch.query_sql_domain}", + (q.term,), + ) + candidates = cipher.decrypt(cur.fetchall()) + + bodies = [note.body for note in candidates] + kept = sorted(body for body, ok in zip(bodies, q.verify(bodies)) if ok) + # The server's answer can hold extra rows, but never misses a match. + assert {"Alice Smith", "Natalie Wood"} <= set(bodies) + assert kept == ["Alice Smith", "Natalie Wood"] + + +@needs_database +def test_target_limit_reads_a_measured_histogram(): + psycopg = pytest.importorskip("psycopg") + + cipher = Cipher(Note) + q = cipher.matches("body", "ali") + with psycopg.connect(DATABASE_URL) as conn, conn.cursor() as cur: + _notes_table(cur, cipher) + # The query in MatchQuery.target_limit's docstring. + cur.execute( + "SELECT cardinality(eql_v4.match_term(body)), count(*) FROM notes " + "WHERE body IS NOT NULL GROUP BY 1" + ) + histogram = cur.fetchall() + + assert q.target_limit(2, rows=len(NOTES)) is None + limit = q.target_limit(2, rows=len(NOTES), histogram=histogram) + assert isinstance(limit, int) and limit >= 2, limit + + +def test_a_match_query_never_shows_its_text(): + q = Cipher(Note).matches("body", "zebracorn") + assert "zebracorn" not in repr(q) + assert "zebracorn" not in str(q) + assert not hasattr(q, "__dict__") + + def test_100_queries_make_no_request(): cipher = Cipher(User) # The first call in the process may build the client and load the diff --git a/languages/python/tests/test_query.py b/languages/python/tests/test_query.py index 14247b295..4ef86c21a 100644 --- a/languages/python/tests/test_query.py +++ b/languages/python/tests/test_query.py @@ -1,6 +1,6 @@ """The query methods before any request: the field checks and the value checks. -``eq``, ``lt``, ``lte``, ``gt`` and ``gte`` (ADR-0003). A name that is not a +``eq``, ``lt``, ``lte``, ``gt``, ``gte`` and ``matches`` (ADR-0003). A name that is not a declared field, a field that does not support the operator, and a value that does not fit are all refused before the shared client is built. So these tests need fake credentials only, and make no request. The live @@ -178,9 +178,33 @@ def test_the_message_never_contains_the_value(untouched): assert "123456789" not in str(error) +@pytest.mark.parametrize( + ("field", "value", "marker"), + [("email", "a@example.com", "TextEq"), ("level", 3, "IntegerOrd")], +) +def test_a_field_with_no_match_index_answers_no_matches(untouched, field, value, marker): + error = refused(field, value, op="matches") + assert error.code == "query_unsupported" + assert error.reason == marker + assert "does not support matches" in error.message + assert "Fields that support matches: 'notes'." in error.message + + +def test_matches_takes_text_only(untouched): + error = refused("notes", 3, InvalidValueError, op="matches") + assert error.code == "value_wrong_type" + assert error.field == "notes" + + +def test_matches_message_never_contains_the_text(untouched): + error = refused("email", "zebracorn", op="matches") + assert "zebracorn" not in str(error) + + @pytest.mark.parametrize( ("op", "field", "value"), - [("eq", "email", "a@example.com")] + [(op, "level", 3) for op in ORDER], + [("eq", "email", "a@example.com"), ("matches", "notes", "hello")] + + [(op, "level", 3) for op in ORDER], ) def test_a_query_that_passes_every_check_reaches_the_engine(op, field, value): # Refused later by the engine for want of a client key, not by the From 31cfe645440aab94904e9336d47d810c31d87630 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 10:36:20 +0900 Subject: [PATCH 05/29] test(python): heap hygiene covers verify verify reads the search text and each decrypted value into Rust before it checks them. Those copies are plaintext. The heap test now calls the verify binding that Cipher.verify uses, and fails if a block freed during the call still holds the text. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .../python/crates/engine/src/heap_hygiene.rs | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/languages/python/crates/engine/src/heap_hygiene.rs b/languages/python/crates/engine/src/heap_hygiene.rs index 0a142dd40..a95ef9cf5 100644 --- a/languages/python/crates/engine/src/heap_hygiene.rs +++ b/languages/python/crates/engine/src/heap_hygiene.rs @@ -1,5 +1,5 @@ -//! No heap block the extension frees during `encrypt`, `query` or `decrypt` -//! still contains the plaintext. +//! No heap block the extension frees during `encrypt`, `query`, `verify` or +//! `decrypt` still contains the plaintext. //! //! The scope is the extension's own copies: what this crate and the engine //! allocate in Rust while they read a Python value, write the input frame, @@ -33,7 +33,7 @@ use std::ffi::c_void; use std::sync::Mutex; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyString, PyType}; +use pyo3::types::{PyDict, PyList, PyString, PyType}; use stack_encrypt::StackCipher; use stack_kms::FakeDataKeySource; @@ -384,6 +384,37 @@ fn decrypt_frees_no_copy_of_the_plaintext_unwiped() { }); } +#[test] +fn verify_frees_no_copy_of_the_plaintext_unwiped() { + let _serial = serial(); + Python::initialize(); + Python::attach(|py| { + let plan = build_plan( + py, + "notes", + vec![( + "body".into(), + Some("TextMatch".into()), + Some("string".into()), + py.get_type::(), + )], + ) + .unwrap(); + // The Python objects are made before the watch: they are the + // caller's, not the extension's. + let text = PyString::new(py, &marked("a note")).into_any(); + let values = + PyList::new(py, [marked("a note in full"), "nothing here".to_owned()]).unwrap(); + + // The binding `Cipher.verify` calls, as Python calls it. + let kept = watched("verify", || { + crate::query::verify(py, &plan, "body", &text, &values).unwrap() + }); + + assert_eq!(kept, [true, false]); + }); +} + /// The spy is not blind: a copy of each needle freed unwiped fails the /// watch. If frees went unseen (another thread, an exclusion matching too /// much), the three tests above would pass whatever the extension did. From 283b118f7b733114af813486a54b83a6235ab4f6 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 11:32:23 +0900 Subject: [PATCH 06/29] feat(python): encrypt_rows for rows that contain only some fields The SQLAlchemy layer will encrypt each statement's parameters in one batch. An UPDATE contains only the changed columns, plus keys such as the primary key for the WHERE. encrypt takes whole models, so it cannot take those rows. encrypt_rows takes dicts and builds a plan for the fields they contain. The engine refuses a row that leaves out a field of the plan, and a value from the smaller plan opens and searches under the full plan. Every row in one call must contain the same declared fields, so that all values are checked before the one batch of key requests. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/.gitignore | 4 + languages/python/README.md | 10 ++ languages/python/docs/scorecard.md | 2 + .../python/python/stackencrypt/_cipher.py | 104 +++++++++++++++--- .../python/python/stackencrypt/_errors.py | 5 +- languages/python/tests/test_encrypt_live.py | 66 ++++++++++- languages/python/tests/test_encrypt_rows.py | 77 +++++++++++++ 7 files changed, 251 insertions(+), 17 deletions(-) create mode 100644 languages/python/tests/test_encrypt_rows.py diff --git a/languages/python/.gitignore b/languages/python/.gitignore index 3b035eb39..0b2be11d2 100644 --- a/languages/python/.gitignore +++ b/languages/python/.gitignore @@ -4,3 +4,7 @@ *.so *.pyd .pytest_cache/ +# The CS_* credentials for `mise run test:live`, if `stash env` writes them +# here. The file contains a client key and an access key, so it must never +# be committed. +.stackencrypt-live.env diff --git a/languages/python/README.md b/languages/python/README.md index 1bf8000a8..ad42a0c13 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -126,6 +126,16 @@ To size the `LIMIT`, call `q.target_limit(want, rows=..., histogram=...)`. It re A `MatchQuery` keeps the search text, because `verify` needs it. Python cannot wipe a `str`, so the text stays in memory until Python frees the object. `repr` does not show it. +## Encrypt only some fields + +To update one column, you do not need a whole model. `cipher.encrypt_rows` takes a list of dicts from field name to value. Each dict can contain only some of the declared fields. For example, `cipher.encrypt_rows([{"email": "b@example.com", "id": 1}])` returns `[{"email": '{"c": ...}', "id": 1}]`. + +- Each declared field's value is replaced, as `encrypt` writes it. +- Every other key, such as `id`, is kept unchanged. +- Every row in one call must contain the same declared fields. Then one call sends one batch of key requests. + +The values are checked as `encrypt` checks them. A row that is not a mapping raises `InvalidValueError` with the code `value_row_shape`. So do rows that contain different declared fields. + ## Declare a SQLAlchemy relationship A SQLAlchemy relationship is not a column. Every annotated field needs a marker, so write a relationship in one of two ways: diff --git a/languages/python/docs/scorecard.md b/languages/python/docs/scorecard.md index 5003254e1..dcc62837b 100644 --- a/languages/python/docs/scorecard.md +++ b/languages/python/docs/scorecard.md @@ -24,6 +24,7 @@ Each principle is marked: ## G1. One engine, and the SDK never executes: held - `Cipher.encrypt`, `decrypt`, and each query method make one call into the extension, in `python/stackencrypt/_cipher.py`. The Python layer has no loop over rows, and no loop over a model's values. +- `Cipher.encrypt_rows` is the exception. It loops over the rows in Python. For each row, it checks that the row is a mapping with the same declared fields as row 0. It then puts those fields' values on an object for the extension. It does not read, convert, or check a value. The extension does that, as for `encrypt`. - The Python layer does loop over the declared fields in two places. `Cipher` lists them once, to build the plan. A query method searches them to name a field it refuses. Neither loop touches a value. - The extension hands the engine one frame for the whole call, and reads back the engine's whole output. Encryption, decryption, and term derivation all run in `stack-encrypt`. - The declaration check asks the engine. When the class statement runs, the decorator calls `_engine.check_plan`, and the engine refuses a plan it cannot run. `Cipher` later builds the same plan with `_engine.build_plan`. @@ -47,6 +48,7 @@ Each principle is marked: ## G4. One declaration serves the write, the query and the read: held - `Cipher(User)` builds one plan from the declaration. `encrypt`, the query methods, and `decrypt` all use that plan. +- `encrypt_rows` uses a smaller plan with only the fields a row contains. It is built from the same declaration, with the same table, field names, and markers. `test_a_row_updated_with_some_fields_decrypts_and_searches_with_the_full_model` in `tests/test_encrypt_live.py` shows that its values decrypt and search under the full plan. - No call takes a context or an index choice. A query method, such as `eq` or `gte`, takes a field name and a value only (Python ADR-0002 and ADR-0003, under `docs/adr/`). The engine picks the index that supports the operator. - `test_a_query_term_finds_the_one_matching_row` in `tests/test_encrypt_live.py` runs `encrypt`, `eq`, and `decrypt` on one `Cipher` against PostgreSQL. The README test runs the same three steps. - A context that changes per caller, such as a tenant, is not supported yet. The README lists it. diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index 938ba981e..01977b12a 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -4,7 +4,19 @@ ``encrypt``, ``decrypt`` and the query methods are here. """ -from typing import Dict, Generic, List, Mapping, Optional, Sequence, Tuple, Type, TypeVar +from types import SimpleNamespace +from typing import ( + Dict, + FrozenSet, + Generic, + List, + Mapping, + Optional, + Sequence, + Tuple, + Type, + TypeVar, +) from stackencrypt import _engine from stackencrypt._markers import Passthrough @@ -14,7 +26,12 @@ check_unchanged, declaration_of, ) -from stackencrypt._errors import AuthenticationError, DeclarationError, QueryError +from stackencrypt._errors import ( + AuthenticationError, + DeclarationError, + InvalidValueError, + QueryError, +) T = TypeVar("T") @@ -48,18 +65,11 @@ def __init__(self, cls: Type[T]) -> None: raise AuthenticationError(code, message, field=field, reason=reason) self._cls = cls self._declaration = declaration - self._plan = _engine.build_plan( - declaration.table, - [ - ( - field.name, - None if field.marker is Passthrough else field.marker.name, - None if field.marker is Passthrough else field.marker.plaintext, - field.value_type, - ) - for field in declaration.fields - ], - ) + self._plan = _build_plan(declaration, declaration.fields) + self._planned = frozenset(field.name for field in declaration.fields) + # encrypt_rows's plans, one for each set of fields it has seen. Two + # threads may build the same plan at once; either one is kept. + self._row_plans: Dict[FrozenSet[str], object] = {} def encrypt(self, models: Sequence[object]) -> List[Dict[str, object]]: """Encrypt ``models``, instances of the declared class. @@ -75,6 +85,56 @@ def encrypt(self, models: Sequence[object]) -> List[Dict[str, object]]: """ return _engine.encrypt(self._plan, list(models)) + def encrypt_rows(self, rows: Sequence[Mapping[str, object]]) -> List[Dict[str, object]]: + """Encrypt ``rows``, mappings from field name to value, that may + hold only some of the declared fields:: + + users.encrypt_rows([{"email": "a@example.com", "users_id": 1}]) + + Returns one new dict per row. Each declared field's value is + replaced as ``encrypt`` writes it: EQL JSON as a ``str`` for a + sealed field, and the value itself for a passthrough field. Every + other key, such as an ``Ignore`` field or a key that SQLAlchemy adds + for a ``WHERE``, is kept unchanged. + + Every row must hold the same declared fields, so that one call + sends one batch of key requests. Rows that hold none are returned + unchanged with no request. + + Every value is checked first, as ``encrypt`` checks it. A row that + is not a mapping, or that holds other declared fields than row 0, + raises ``InvalidValueError`` with the code ``value_row_shape``. No + request is made when any row fails. The input rows are not changed. + """ + rows = list(rows) + names: Optional[FrozenSet[str]] = None + for at, row in enumerate(rows): + if not isinstance(row, Mapping): + raise InvalidValueError( + "value_row_shape", + f"row {at} is a {type(row).__name__}, not a mapping of field names to values.", + ) + these = frozenset(name for name in row if name in self._planned) + if names is None: + names = these + elif these != names: + raise InvalidValueError( + "value_row_shape", + f"row {at} holds other declared fields than row 0. " + f"Row 0 holds {sorted(names)}; row {at} holds {sorted(these)}. " + "Encrypt rows with different fields in separate calls.", + ) + if not names: + return [dict(row) for row in rows] + plan = self._row_plans.get(names) + if plan is None: + fields = [field for field in self._declaration.fields if field.name in names] + plan = self._row_plans[names] = _build_plan(self._declaration, fields) + sealed = _engine.encrypt( + plan, [SimpleNamespace(**{name: row[name] for name in names}) for row in rows] + ) + return [{**row, **columns} for row, columns in zip(rows, sealed)] + def decrypt(self, rows: Sequence[Mapping[str, object]]) -> List[T]: """Decrypt ``rows`` into instances of the declared class. @@ -292,3 +352,19 @@ def _not_decorated(cls: object) -> DeclarationError: f"class {cls.__name__!r} is not declared. {detail} Apply " f'@stackencrypt.model("...") to {cls.__name__!r} itself.', ) + + +def _build_plan(declaration: Declaration, fields: Sequence[DeclaredField]) -> object: + """The engine's plan for ``fields`` of ``declaration``, in their order.""" + return _engine.build_plan( + declaration.table, + [ + ( + field.name, + None if field.marker is Passthrough else field.marker.name, + None if field.marker is Passthrough else field.marker.plaintext, + field.value_type, + ) + for field in fields + ], + ) diff --git a/languages/python/python/stackencrypt/_errors.py b/languages/python/python/stackencrypt/_errors.py index 9d4c33f19..10547d1c6 100644 --- a/languages/python/python/stackencrypt/_errors.py +++ b/languages/python/python/stackencrypt/_errors.py @@ -49,8 +49,9 @@ class InvalidValueError(StashError, ValueError): """A model's value that does not fit its field, refused before any request. The code is one of ``value_none``, ``value_wrong_type``, - ``value_out_of_range``, ``value_not_representable`` and - ``value_naive_datetime``. ``field`` names the field. ``reason`` is the + ``value_out_of_range``, ``value_not_representable``, + ``value_naive_datetime``, and ``value_row_shape``. The last is for a row + that ``encrypt_rows`` cannot read. ``field`` names the field. ``reason`` is the field's kind, such as ``int32``, or ``passthrough``. The message never contains the value. """ diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py index 1ac17b91e..49574ed05 100644 --- a/languages/python/tests/test_encrypt_live.py +++ b/languages/python/tests/test_encrypt_live.py @@ -24,7 +24,16 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship import stackencrypt -from stackencrypt import Cipher, DecryptError, Ignore, IntegerOrd, Passthrough, TextEq, TextMatch +from stackencrypt import ( + Cipher, + DecryptError, + Ignore, + IntegerOrd, + Passthrough, + Text, + TextEq, + TextMatch, +) from stackencrypt import _engine, _eql pytestmark = LIVE @@ -507,3 +516,58 @@ def test_nan_infinity_and_negative_zero_round_trip(value): else: assert opened == value assert math.copysign(1.0, opened) == math.copysign(1.0, value) + + +@stackencrypt.model("people") +@dataclasses.dataclass +class Person: + email: Annotated[str, TextEq] + name: Annotated[str, Text] + age: Annotated[int, Passthrough] + + +@needs_database +def test_a_row_updated_with_some_fields_decrypts_and_searches_with_the_full_model(): + psycopg = pytest.importorskip("psycopg") + from psycopg.rows import dict_row + + people = Cipher(Person) + [row] = people.encrypt([Person(email="old@example.com", name="Ann", age=30)]) + # As an UPDATE statement's parameters: one declared field, and the key + # SQLAlchemy adds for the WHERE. + [change] = people.encrypt_rows([{"email": "new@example.com", "people_id": 1}]) + + assert set(change) == {"email", "people_id"} + assert change["people_id"] == 1 + with psycopg.connect(DATABASE_URL) as conn, conn.cursor(row_factory=dict_row) as cur: + cur.execute( + "CREATE TEMPORARY TABLE people (id int PRIMARY KEY, " + "email public.eql_v4_text_eq, name public.eql_v4_text, age int)" + ) + cur.execute( + "INSERT INTO people (id, email, name, age) VALUES (1, %s, %s, %s)", + (row["email"], row["name"], row["age"]), + ) + cur.execute( + "UPDATE people SET email = %(email)s WHERE id = %(people_id)s", change + ) + cur.execute( + "SELECT email, name, age FROM people WHERE email = %s::eql_v4.query_text_eq", + (people.eq("email", "new@example.com"),), + ) + found = people.decrypt(cur.fetchall()) + + assert [dataclasses.astuple(person) for person in found] == [("new@example.com", "Ann", 30)] + + +def test_100_rows_with_one_field_make_one_key_request(): + people = Cipher(Person) + # The first call in the process may load the keyset. Warm the client up + # first, so the count below is the key requests alone. + people.encrypt_rows([{"email": "warm@example.com"}]) + before = _engine.request_count() + + rows = people.encrypt_rows([{"email": f"user{i}@example.com"} for i in range(100)]) + + assert _engine.request_count() - before == 1 + assert len(rows) == 100 diff --git a/languages/python/tests/test_encrypt_rows.py b/languages/python/tests/test_encrypt_rows.py new file mode 100644 index 000000000..e4d5a7ca9 --- /dev/null +++ b/languages/python/tests/test_encrypt_rows.py @@ -0,0 +1,77 @@ +"""``Cipher.encrypt_rows`` before any request: the row rules. + +These tests need fake credentials only, and make no request. The live round +trip is in ``test_encrypt_live.py``. +""" + +import dataclasses +from typing import Annotated + +import pytest + +import stackencrypt +from stackencrypt import Cipher, Ignore, InvalidValueError, Passthrough, Text, TextEq + +SECRET = "secret@example.com" + +pytestmark = pytest.mark.usefixtures("credentials") + + +@stackencrypt.model("people") +@dataclasses.dataclass +class Person: + email: Annotated[str, TextEq] + name: Annotated[str, Text] + age: Annotated[int, Passthrough] + id: Annotated[int, Ignore] = 0 + + +@pytest.fixture +def people(): + return Cipher(Person) + + +def refused(cipher, rows, error=InvalidValueError): + with pytest.raises(error) as raised: + cipher.encrypt_rows(rows) + assert SECRET not in str(raised.value) + return raised.value + + +def test_an_empty_list_is_an_empty_list_with_no_request(people, untouched): + assert people.encrypt_rows([]) == [] + + +def test_rows_with_no_declared_field_come_back_unchanged_with_no_request(people, untouched): + rows = [{"people_id": 1}, {"people_id": 2, "id": 7}] + assert people.encrypt_rows(rows) == rows + + +def test_a_row_that_is_not_a_mapping_is_refused(people, untouched): + error = refused(people, [{"email": SECRET}, ("email", SECRET)]) + assert error.code == "value_row_shape" + assert "row 1" in error.message + + +def test_rows_with_different_fields_are_refused(people, untouched): + error = refused(people, [{"email": SECRET}, {"email": SECRET, "name": "Ann"}]) + assert error.code == "value_row_shape" + assert "row 1" in error.message + + +def test_a_value_of_the_wrong_type_names_the_field(people, untouched): + error = refused(people, [{"email": SECRET, "age": "3"}]) + assert error.code == "value_wrong_type" + assert error.field == "age" + + +def test_a_none_value_is_refused(people, untouched): + error = refused(people, [{"email": None}]) + assert error.code == "value_none" + assert error.field == "email" + + +def test_the_input_rows_are_not_changed(people, untouched): + rows = [{"email": SECRET, "age": "3"}] + refused(people, rows) + assert rows == [{"email": SECRET, "age": "3"}] From cd69444739bfbee1dfe1c1ca339940300160c081 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 12:20:30 +0900 Subject: [PATCH 07/29] feat(python): decrypt_rows, the reverse of encrypt_rows The SQLAlchemy layer will decrypt the objects a query loads, in one batch per class, and write the plaintext back to their attributes. It needs the values, not new model instances, and a query can load only some of the columns. decrypt_rows takes dicts of stored values and returns dicts of plaintext, with the same rules as encrypt_rows. It passes dict to the engine in place of the model class, so it needs no change to the extension. The two methods share the row-shape check and the plan for each set of fields. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 4 +- languages/python/docs/scorecard.md | 4 +- .../python/python/stackencrypt/_cipher.py | 67 ++++++++++--- .../python/python/stackencrypt/_errors.py | 3 +- languages/python/tests/test_decrypt_rows.py | 96 +++++++++++++++++++ languages/python/tests/test_encrypt_live.py | 35 +++++++ 6 files changed, 194 insertions(+), 15 deletions(-) create mode 100644 languages/python/tests/test_decrypt_rows.py diff --git a/languages/python/README.md b/languages/python/README.md index ad42a0c13..ab58ba5cf 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -126,7 +126,7 @@ To size the `LIMIT`, call `q.target_limit(want, rows=..., histogram=...)`. It re A `MatchQuery` keeps the search text, because `verify` needs it. Python cannot wipe a `str`, so the text stays in memory until Python frees the object. `repr` does not show it. -## Encrypt only some fields +## Encrypt or decrypt only some fields To update one column, you do not need a whole model. `cipher.encrypt_rows` takes a list of dicts from field name to value. Each dict can contain only some of the declared fields. For example, `cipher.encrypt_rows([{"email": "b@example.com", "id": 1}])` returns `[{"email": '{"c": ...}', "id": 1}]`. @@ -136,6 +136,8 @@ To update one column, you do not need a whole model. `cipher.encrypt_rows` takes The values are checked as `encrypt` checks them. A row that is not a mapping raises `InvalidValueError` with the code `value_row_shape`. So do rows that contain different declared fields. +`cipher.decrypt_rows` is the reverse. It takes dicts of stored values that contain some of the declared fields, and returns dicts with the plaintext. Other keys are kept unchanged. It follows the same rules, and raises the same errors as `decrypt`. A row that is not a mapping raises `DecryptError` with the code `decrypt_row_shape`. So do rows that contain different declared fields. + ## Declare a SQLAlchemy relationship A SQLAlchemy relationship is not a column. Every annotated field needs a marker, so write a relationship in one of two ways: diff --git a/languages/python/docs/scorecard.md b/languages/python/docs/scorecard.md index dcc62837b..c574f040f 100644 --- a/languages/python/docs/scorecard.md +++ b/languages/python/docs/scorecard.md @@ -24,7 +24,7 @@ Each principle is marked: ## G1. One engine, and the SDK never executes: held - `Cipher.encrypt`, `decrypt`, and each query method make one call into the extension, in `python/stackencrypt/_cipher.py`. The Python layer has no loop over rows, and no loop over a model's values. -- `Cipher.encrypt_rows` is the exception. It loops over the rows in Python. For each row, it checks that the row is a mapping with the same declared fields as row 0. It then puts those fields' values on an object for the extension. It does not read, convert, or check a value. The extension does that, as for `encrypt`. +- `Cipher.encrypt_rows` and `decrypt_rows` are the exceptions. They loop over the rows in Python. For each row, they check that the row is a mapping with the same declared fields as row 0. They then pass those fields' values to the extension. They do not read, convert, or check a value. The extension does that, as for `encrypt` and `decrypt`. - The Python layer does loop over the declared fields in two places. `Cipher` lists them once, to build the plan. A query method searches them to name a field it refuses. Neither loop touches a value. - The extension hands the engine one frame for the whole call, and reads back the engine's whole output. Encryption, decryption, and term derivation all run in `stack-encrypt`. - The declaration check asks the engine. When the class statement runs, the decorator calls `_engine.check_plan`, and the engine refuses a plan it cannot run. `Cipher` later builds the same plan with `_engine.build_plan`. @@ -48,7 +48,7 @@ Each principle is marked: ## G4. One declaration serves the write, the query and the read: held - `Cipher(User)` builds one plan from the declaration. `encrypt`, the query methods, and `decrypt` all use that plan. -- `encrypt_rows` uses a smaller plan with only the fields a row contains. It is built from the same declaration, with the same table, field names, and markers. `test_a_row_updated_with_some_fields_decrypts_and_searches_with_the_full_model` in `tests/test_encrypt_live.py` shows that its values decrypt and search under the full plan. +- `encrypt_rows` and `decrypt_rows` use a smaller plan with only the fields a row contains. It is built from the same declaration, with the same table, field names, and markers. `test_a_row_updated_with_some_fields_decrypts_and_searches_with_the_full_model` in `tests/test_encrypt_live.py` shows that its values decrypt and search under the full plan. - No call takes a context or an index choice. A query method, such as `eq` or `gte`, takes a field name and a value only (Python ADR-0002 and ADR-0003, under `docs/adr/`). The engine picks the index that supports the operator. - `test_a_query_term_finds_the_one_matching_row` in `tests/test_encrypt_live.py` runs `encrypt`, `eq`, and `decrypt` on one `Cipher` against PostgreSQL. The README test runs the same three steps. - A context that changes per caller, such as a tenant, is not supported yet. The README lists it. diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index 01977b12a..417c800c2 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -29,8 +29,10 @@ from stackencrypt._errors import ( AuthenticationError, DeclarationError, + DecryptError, InvalidValueError, QueryError, + StashError, ) @@ -107,33 +109,76 @@ def encrypt_rows(self, rows: Sequence[Mapping[str, object]]) -> List[Dict[str, o request is made when any row fails. The input rows are not changed. """ rows = list(rows) + names = self._row_fields(rows, InvalidValueError, "value_row_shape", "Encrypt") + if not names: + return [dict(row) for row in rows] + sealed = _engine.encrypt( + self._row_plan(names), + [SimpleNamespace(**{name: row[name] for name in names}) for row in rows], + ) + return [{**row, **columns} for row, columns in zip(rows, sealed)] + + def decrypt_rows(self, rows: Sequence[Mapping[str, object]]) -> List[Dict[str, object]]: + """Decrypt ``rows``, mappings from field name to stored value, that + may hold only some of the declared fields:: + + users.decrypt_rows([{"email": '{"c": ...}', "id": 1}]) + + The reverse of ``encrypt_rows``. Returns one new dict per row. Each + declared field's value is replaced with its plaintext. A sealed + value may be EQL JSON as a ``str``, as UTF-8 ``bytes``, or as a + parsed ``dict``. Every other key is kept unchanged. + + Every row must hold the same declared fields, so that one call + sends one batch of key requests. Rows that hold none are returned + unchanged with no request. + + Every row is checked first, as ``decrypt`` checks it, with the same + errors. A row that is not a mapping, or that holds other declared + fields than row 0, raises ``DecryptError`` with the code + ``decrypt_row_shape``. No request is made when any row fails. + """ + rows = list(rows) + names = self._row_fields(rows, DecryptError, "decrypt_row_shape", "Decrypt") + if not names: + return [dict(row) for row in rows] + opened = _engine.decrypt( + self._row_plan(names), dict, [{name: row[name] for name in names} for row in rows] + ) + return [{**row, **values} for row, values in zip(rows, opened)] + + def _row_fields( + self, rows: List[Mapping[str, object]], error: Type[StashError], code: str, verb: str + ) -> FrozenSet[str]: + """The declared fields every row holds. Raises ``error`` with + ``code`` for a row that is not a mapping, or that holds other + declared fields than row 0.""" names: Optional[FrozenSet[str]] = None for at, row in enumerate(rows): if not isinstance(row, Mapping): - raise InvalidValueError( - "value_row_shape", + raise error( + code, f"row {at} is a {type(row).__name__}, not a mapping of field names to values.", ) these = frozenset(name for name in row if name in self._planned) if names is None: names = these elif these != names: - raise InvalidValueError( - "value_row_shape", + raise error( + code, f"row {at} holds other declared fields than row 0. " f"Row 0 holds {sorted(names)}; row {at} holds {sorted(these)}. " - "Encrypt rows with different fields in separate calls.", + f"{verb} rows with different fields in separate calls.", ) - if not names: - return [dict(row) for row in rows] + return names or frozenset() + + def _row_plan(self, names: FrozenSet[str]) -> object: + """The plan for the declared fields ``names``, in declaration order.""" plan = self._row_plans.get(names) if plan is None: fields = [field for field in self._declaration.fields if field.name in names] plan = self._row_plans[names] = _build_plan(self._declaration, fields) - sealed = _engine.encrypt( - plan, [SimpleNamespace(**{name: row[name] for name in names}) for row in rows] - ) - return [{**row, **columns} for row, columns in zip(rows, sealed)] + return plan def decrypt(self, rows: Sequence[Mapping[str, object]]) -> List[T]: """Decrypt ``rows`` into instances of the declared class. diff --git a/languages/python/python/stackencrypt/_errors.py b/languages/python/python/stackencrypt/_errors.py index 10547d1c6..cb4ec980d 100644 --- a/languages/python/python/stackencrypt/_errors.py +++ b/languages/python/python/stackencrypt/_errors.py @@ -61,7 +61,8 @@ class DecryptError(StashError): """A stored row that ``decrypt`` cannot turn into a model. Before any key request: ``decrypt_row_shape`` (a row that is not a - mapping), ``decrypt_missing_column`` (a declared field's column is not in + mapping, or, for ``decrypt_rows``, rows that hold different declared + fields), ``decrypt_missing_column`` (a declared field's column is not in the row), ``decrypt_malformed`` (the column value is not an EQL value), and ``decrypt_foreign_row`` (the value was stored under another table or column). After the key requests: ``decrypt_kind_mismatch`` (the value was diff --git a/languages/python/tests/test_decrypt_rows.py b/languages/python/tests/test_decrypt_rows.py new file mode 100644 index 000000000..be545bcb3 --- /dev/null +++ b/languages/python/tests/test_decrypt_rows.py @@ -0,0 +1,96 @@ +"""``Cipher.decrypt_rows`` before any request: the row rules. + +These tests need fake credentials only, and make no request. The live round +trip is in ``test_encrypt_live.py``. +""" + +import dataclasses +import json +from typing import Annotated + +import pytest + +import stackencrypt +from stackencrypt import ( + Cipher, + DecryptError, + Ignore, + InvalidValueError, + Passthrough, + StashError, + Text, + TextEq, +) + +# The `i` of a value stored under people.email, and the rest of an EQL value +# left out: the checks before any request read only `i`. +STORED = json.dumps({"i": {"t": "people", "c": "email"}, "v": 2}) + +pytestmark = pytest.mark.usefixtures("credentials") + + +@stackencrypt.model("people") +@dataclasses.dataclass +class Person: + email: Annotated[str, TextEq] + name: Annotated[str, Text] + age: Annotated[int, Passthrough] + id: Annotated[int, Ignore] = 0 + + +@pytest.fixture +def people(): + return Cipher(Person) + + +def refused(cipher, rows, error=DecryptError): + with pytest.raises(error) as raised: + cipher.decrypt_rows(rows) + return raised.value + + +def test_an_empty_list_is_an_empty_list_with_no_request(people, untouched): + assert people.decrypt_rows([]) == [] + + +def test_rows_with_no_declared_field_come_back_unchanged_with_no_request(people, untouched): + rows = [{"people_id": 1}, {"people_id": 2, "id": 7}] + assert people.decrypt_rows(rows) == rows + + +def test_a_row_that_is_not_a_mapping_is_refused(people, untouched): + error = refused(people, [{"email": STORED}, (STORED,)]) + assert error.code == "decrypt_row_shape" + assert "row 1" in error.message + + +def test_rows_with_different_fields_are_refused(people, untouched): + error = refused(people, [{"email": STORED}, {"email": STORED, "age": 3}]) + assert error.code == "decrypt_row_shape" + assert "row 1" in error.message + + +@pytest.mark.parametrize("value", [STORED[:10], 42, None], ids=["truncated", "int", "null"]) +def test_a_value_that_is_not_eql_json_is_malformed(people, untouched, value): + error = refused(people, [{"email": value}]) + assert error.code == "decrypt_malformed" + assert error.field == "email" + + +def test_a_value_stored_under_another_column_is_a_foreign_row(people, untouched): + error = refused(people, [{"name": STORED}]) + assert error.code == "decrypt_foreign_row" + assert error.field == "name" + + +def test_a_passthrough_of_the_wrong_type_is_refused(people, untouched): + error = refused(people, [{"email": STORED, "age": "3"}], InvalidValueError) + assert error.field == "age" + + +def test_a_row_that_passes_every_check_reaches_the_engine(people): + # Refused later by the engine for want of a client key, not by the row + # checks: they passed. + with pytest.raises(StashError) as raised: + people.decrypt_rows([{"email": STORED, "people_id": 1}]) + assert not isinstance(raised.value, (DecryptError, InvalidValueError)) diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py index 49574ed05..cab201374 100644 --- a/languages/python/tests/test_encrypt_live.py +++ b/languages/python/tests/test_encrypt_live.py @@ -571,3 +571,38 @@ def test_100_rows_with_one_field_make_one_key_request(): assert _engine.request_count() - before == 1 assert len(rows) == 100 + + +def test_decrypt_rows_opens_values_from_encrypt_and_from_encrypt_rows(): + people = Cipher(Person) + [whole] = people.encrypt([Person(email="a@example.com", name="Ann", age=30)]) + [part] = people.encrypt_rows([{"email": "b@example.com", "people_id": 2}]) + + opened = people.decrypt_rows( + [{"email": whole["email"], "name": whole["name"], "id": 1}] + ) + people.decrypt_rows([part]) + + assert opened == [ + {"email": "a@example.com", "name": "Ann", "id": 1}, + {"email": "b@example.com", "people_id": 2}, + ] + + +def test_decrypt_rows_refuses_a_value_stored_for_another_column(): + people = Cipher(Person) + [row] = people.encrypt_rows([{"email": "a@example.com"}]) + with pytest.raises(DecryptError) as raised: + people.decrypt_rows([{"name": row["email"]}]) + assert raised.value.code == "decrypt_foreign_row" + assert raised.value.field == "name" + + +def test_decrypt_rows_of_100_rows_with_one_field_make_one_key_request(): + people = Cipher(Person) + rows = people.encrypt_rows([{"email": f"user{i}@example.com"} for i in range(100)]) + before = _engine.request_count() + + opened = people.decrypt_rows(rows) + + assert _engine.request_count() - before == 1 + assert opened[99] == {"email": "user99@example.com"} From 3c6cc6303675f0560bb42175916b2a232b761adc Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 12:41:06 +0900 Subject: [PATCH 08/29] feat(python): configure_encryption for the SQLAlchemy ORM With the SDK alone, an app calls encrypt before each insert and decrypt after each select. configure_encryption(engine, classes) moves both into SQLAlchemy hooks, so the app reads and writes plaintext attributes. A column type would encrypt one value at a time, with one key request each. The hooks work in batches instead. before_execute encrypts the parameters of each INSERT or UPDATE with one encrypt_rows call. do_orm_execute decrypts the objects that the load and refresh events recorded, with one decrypt_rows call for each class, and sets them with set_committed_value so that they are not saved again. A statement with an encrypted value inside its SQL is refused, because the hook can change only the parameters, and the value would be stored as plaintext. Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 36 ++++ .../python/python/stackencrypt/sqlalchemy.py | 200 ++++++++++++++++++ languages/python/tests/test_sqlalchemy.py | 150 +++++++++++++ .../python/tests/test_sqlalchemy_live.py | 172 +++++++++++++++ 4 files changed, 558 insertions(+) create mode 100644 languages/python/python/stackencrypt/sqlalchemy.py create mode 100644 languages/python/tests/test_sqlalchemy.py create mode 100644 languages/python/tests/test_sqlalchemy_live.py diff --git a/languages/python/README.md b/languages/python/README.md index ab58ba5cf..089e1f0a9 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -138,6 +138,42 @@ The values are checked as `encrypt` checks them. A row that is not a mapping rai `cipher.decrypt_rows` is the reverse. It takes dicts of stored values that contain some of the declared fields, and returns dicts with the plaintext. Other keys are kept unchanged. It follows the same rules, and raises the same errors as `decrypt`. A row that is not a mapping raises `DecryptError` with the code `decrypt_row_shape`. So do rows that contain different declared fields. +## Use it with the SQLAlchemy ORM + +`stackencrypt.sqlalchemy.configure_encryption` makes the ORM encrypt and decrypt for you. Call it once, after you create the engine. Then use sessions as usual. The app reads and writes plaintext attributes, and the tables store EQL values. + +```text +from stackencrypt.sqlalchemy import configure_encryption + +engine = create_engine(DATABASE_URL.replace("postgresql://", "postgresql+psycopg://", 1)) +configure_encryption(engine, [Customer, Order]) + +with Session(engine) as session: + session.add(Customer(email="a@example.com", name="Ann", plan="free")) + session.commit() # encrypted on the way in + customer = session.scalars(select(Customer)).one() + customer.email # "a@example.com" +``` + +- **Writes:** each INSERT or UPDATE on a listed table is encrypted with one batch of key requests for all its rows. +- **Reads:** the objects a query loads are decrypted with one batch for each class. This includes relationship loads, and attributes read again after a commit. +- Only encrypted fields are changed. A passthrough field is stored as SQLAlchemy writes it. + +Each class must be declared with `stackencrypt.model` and mapped by SQLAlchemy. `configure_encryption` refuses a class with `DeclarationError` in these cases: + +- `declaration_not_mapped`: SQLAlchemy does not map the class. +- `declaration_table_mismatch`: the table in `stackencrypt.model` is not the `__tablename__`. +- `declaration_column_name`: an encrypted field's column has another name than the field, as with `mapped_column("email_address")`. +- `declaration_already_configured`: the class was configured before in this process. + +A statement that puts an encrypted value inside its SQL, such as `insert(Customer).values(email=...)`, raises `DeclarationError` with `declaration_inline_value`. The value is not stored. Pass the values as parameters instead, as `session.add` and `session.execute(insert(Customer), rows)` do. + +These are not handled yet: + +- A Core query, such as `conn.execute(text("SELECT ..."))`, returns EQL values, not plaintext. Use `cipher.decrypt_rows` on its rows. +- A search by an encrypted field, such as `where(Customer.email == x)`, is not supported yet. Build the term with `cipher.eq` and compare it in the SQL, as in "Search by equality". +- A query result is read in full before it is returned, so that its objects can be decrypted in one batch. A query that streams its rows does not stream. + ## Declare a SQLAlchemy relationship A SQLAlchemy relationship is not a column. Every annotated field needs a marker, so write a relationship in one of two ways: diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py new file mode 100644 index 000000000..cdd107ba9 --- /dev/null +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -0,0 +1,200 @@ +"""Encryption for SQLAlchemy ORM models, with no SDK call in the app's code:: + + from stackencrypt.sqlalchemy import configure_encryption + + engine = create_engine(DATABASE_URL) + configure_encryption(engine, [Customer, Order]) + +After that, a flush encrypts the declared fields of the listed classes, and a +query decrypts them. The app reads and writes plaintext attributes. + +- Writes: a ``before_execute`` hook on the engine. For each INSERT or UPDATE + on a listed table, it encrypts the parameters of all the statement's rows + with one ``Cipher.encrypt_rows`` call. +- Reads: the ``load`` and ``refresh`` events of each listed class record the + objects a query builds. A ``do_orm_execute`` hook then decrypts them, with + one ``Cipher.decrypt_rows`` call for each class and set of fields. The + plaintext is set with ``set_committed_value``, so the objects are not + marked as changed. + +Only the sealed fields go through the hooks. A passthrough field is stored +as SQLAlchemy writes it. + +This module imports SQLAlchemy. The rest of the package does not. +""" + +from typing import Any, Dict, FrozenSet, List, Sequence, Tuple, Type + +from sqlalchemy import event, inspect +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import set_committed_value +from sqlalchemy.sql.dml import Insert, Update + +from stackencrypt._cipher import Cipher +from stackencrypt._errors import DeclarationError +from stackencrypt._markers import Passthrough + +__all__ = ["configure_encryption"] + +# The objects a load built or refreshed, and which attributes, waiting for +# the do_orm_execute hook. Kept in each session's `info`. +_PENDING = "stackencrypt_pending" + +# Every class configured in this process, with its cipher and sealed fields. +_CONFIGURED: Dict[type, Tuple[Cipher, FrozenSet[str]]] = {} + + +def configure_encryption(engine: Engine, classes: Sequence[type]) -> None: + """Encrypt and decrypt the declared fields of ``classes`` when the ORM + writes them through ``engine`` and reads them in any session. + + Each class must be declared with ``stackencrypt.model`` and mapped by + SQLAlchemy. Its declared table must be its ``__tablename__``, and each + sealed field's column must have the field's name. A class can be + configured once in a process. + + Every class is checked first. A class that fails raises + ``DeclarationError``, and none of the classes is configured. The check + makes no request. + """ + checked = [_check(cls) for cls in classes] + for cls, _, _ in checked: + if cls in _CONFIGURED: + raise DeclarationError( + "declaration_already_configured", + f"{cls.__qualname__} is configured for encryption already. " + "Configure each class once, with every engine it writes through.", + ) + if len({cls for cls, _, _ in checked}) != len(checked): + raise DeclarationError( + "declaration_already_configured", + "the list names a class more than once.", + ) + tables: Dict[str, Tuple[Cipher, FrozenSet[str]]] = {} + for cls, cipher, sealed in checked: + _CONFIGURED[cls] = (cipher, sealed) + tables[cipher.declaration.table] = (cipher, sealed) + event.listen(cls, "load", _remember_load) + event.listen(cls, "refresh", _remember_refresh) + event.listen(engine, "before_execute", _encrypter(tables), retval=True) + if not event.contains(Session, "do_orm_execute", _decrypt_loaded): + event.listen(Session, "do_orm_execute", _decrypt_loaded) + + +def _check(cls: type) -> Tuple[type, Cipher, FrozenSet[str]]: + cipher = Cipher(cls) # refuses a class that is not declared + mapper = inspect(cls, raiseerr=False) if isinstance(cls, type) else None + if mapper is None or not hasattr(mapper, "columns"): + raise DeclarationError( + "declaration_not_mapped", + f"{cls.__qualname__} is not mapped by SQLAlchemy. configure_encryption " + "takes ORM classes. Use Cipher directly for other classes.", + ) + table = cipher.declaration.table + mapped = getattr(mapper.local_table, "name", None) + if mapped != table: + raise DeclarationError( + "declaration_table_mismatch", + f"{cls.__qualname__} is declared for table {table!r}, but SQLAlchemy " + f"maps it to {mapped!r}. Use the same name in stackencrypt.model and " + "__tablename__.", + ) + sealed = frozenset( + field.name for field in cipher.declaration.fields if field.marker is not Passthrough + ) + for name in sorted(sealed): + column = mapper.columns.get(name) + if column is None or column.name != name or column.key != name: + found = None if column is None else column.name + raise DeclarationError( + "declaration_column_name", + f"field {name!r} of {cls.__qualname__} is stored in column {found!r}. " + "configure_encryption needs each encrypted field's column to have " + "the field's name.", + field=name, + ) + return cls, cipher, sealed + + +def _encrypter(tables: Dict[str, Tuple[Cipher, FrozenSet[str]]]) -> Any: + def encrypt_parameters(conn, clause, multiparams, params, execution_options): # type: ignore[no-untyped-def] + if not isinstance(clause, (Insert, Update)): + return clause, multiparams, params + configured = tables.get(getattr(clause.table, "name", None)) + if configured is None: + return clause, multiparams, params + cipher, sealed = configured + rows = [dict(row) for row in multiparams] if multiparams else [dict(params or {})] + _refuse_inline_values(conn, clause, sealed, rows) + names = sealed.intersection(rows[0]) + if not names: + return clause, multiparams, params + encrypted = cipher.encrypt_rows([{name: row[name] for name in names} for row in rows]) + for row, values in zip(rows, encrypted): + row.update(values) + if multiparams: + return clause, rows, params + return clause, multiparams, rows[0] + + return encrypt_parameters + + +def _refuse_inline_values(conn, clause, sealed: FrozenSet[str], rows: List[Dict[str, Any]]) -> None: # type: ignore[no-untyped-def] + """Refuse a statement that carries a sealed value in its SQL, such as + ``insert(table).values(email=...)``: the hook can only change the + parameters passed beside the statement, so the value would be stored + as plaintext.""" + inline = clause.compile(dialect=conn.dialect).params + for name in sorted(sealed): + if inline.get(name) is not None and not any(name in row for row in rows): + raise DeclarationError( + "declaration_inline_value", + f"the statement writes {name!r} of table {clause.table.name!r} as a value " + "inside the SQL, so it cannot be encrypted. Pass the values as " + "parameters, as session.add and session.execute(insert(...), rows) do.", + field=name, + ) + + +def _remember_load(target: object, context: object) -> None: + _remember(target, None) + + +def _remember_refresh(target: object, context: object, attrs: Any) -> None: + _remember(target, attrs) + + +def _remember(target: object, attrs: Any) -> None: + session = Session.object_session(target) + if session is not None: + names = None if attrs is None else frozenset(attrs) + session.info.setdefault(_PENDING, []).append((target, names)) + + +def _decrypt_loaded(state: Any) -> Any: + if not state.is_select: + return None + # Freezing reads every row, so every load and refresh event has fired. + result = state.invoke_statement().freeze() + pending = state.session.info.pop(_PENDING, []) + groups: Dict[Tuple[type, FrozenSet[str]], List[object]] = {} + for target, names in pending: + cipher, sealed = _CONFIGURED[type(target)] + loaded = target.__dict__ + fields = frozenset( + name + for name in (sealed if names is None else sealed.intersection(names)) + if loaded.get(name) is not None + ) + if fields: + groups.setdefault((type(target), fields), []).append(target) + for (cls, fields), targets in groups.items(): + cipher, _ = _CONFIGURED[cls] + opened = cipher.decrypt_rows( + [{name: target.__dict__[name] for name in fields} for target in targets] + ) + for target, values in zip(targets, opened): + for name, value in values.items(): + set_committed_value(target, name, value) + return result() diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py new file mode 100644 index 000000000..23bf388eb --- /dev/null +++ b/languages/python/tests/test_sqlalchemy.py @@ -0,0 +1,150 @@ +"""``stackencrypt.sqlalchemy.configure_encryption`` before any query: the +class checks. + +These tests need fake credentials only. They use an SQLite engine that no +test connects to. The hooks run against PostgreSQL in +``test_sqlalchemy_live.py``. +""" + +import dataclasses +from typing import Annotated + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + +import stackencrypt +from stackencrypt import DeclarationError, Ignore, Passthrough, TextEq +from stackencrypt.sqlalchemy import configure_encryption + +pytestmark = pytest.mark.usefixtures("credentials") + + +class Base(DeclarativeBase): + pass + + +@stackencrypt.model("customers") +class Customer(Base): + __tablename__ = "customers" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + plan: Mapped[Annotated[str, Passthrough]] + + +@pytest.fixture +def engine(): + return create_engine("sqlite://") + + +def refused(engine, classes): + with pytest.raises(DeclarationError) as raised: + configure_encryption(engine, classes) + return raised.value + + +def test_a_class_sqlalchemy_does_not_map_is_refused(engine): + @stackencrypt.model("contacts") + @dataclasses.dataclass + class Contact: + email: Annotated[str, TextEq] + + error = refused(engine, [Contact]) + assert error.code == "declaration_not_mapped" + assert "Contact" in error.message + + +def test_a_class_not_declared_is_refused(engine): + class Plain(Base): + __tablename__ = "plain" + id: Mapped[int] = mapped_column(primary_key=True) + + error = refused(engine, [Plain]) + assert error.code == "declaration_not_decorated" + + +def test_a_declared_table_that_is_not_the_mapped_table_is_refused(engine): + @stackencrypt.model("people") + class Person(Base): + __tablename__ = "persons" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + + error = refused(engine, [Person]) + assert error.code == "declaration_table_mismatch" + assert "people" in error.message + assert "persons" in error.message + + +def test_a_field_whose_column_has_another_name_is_refused(engine): + @stackencrypt.model("accounts") + class Account(Base): + __tablename__ = "accounts" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] = mapped_column("email_address") + + error = refused(engine, [Account]) + assert error.code == "declaration_column_name" + assert error.field == "email" + assert "email_address" in error.message + + +def test_a_class_configured_twice_is_refused(engine): + @stackencrypt.model("members") + class Member(Base): + __tablename__ = "members" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + + configure_encryption(engine, [Member]) + error = refused(create_engine("sqlite://"), [Member]) + assert error.code == "declaration_already_configured" + + +def test_a_refused_list_configures_none_of_its_classes(engine): + @stackencrypt.model("guests") + class Guest(Base): + __tablename__ = "guests" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + + @stackencrypt.model("visitors") + class Visitor(Base): + __tablename__ = "visits" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + + refused(engine, [Guest, Visitor]) + # Guest was not configured by the refused call, so it can be now. + configure_encryption(engine, [Guest]) + + +def test_a_declared_mapped_class_is_accepted_with_no_request(engine, untouched): + configure_encryption(engine, [Customer]) + + +def test_a_sealed_value_inside_the_sql_is_refused_and_not_stored(untouched): + from sqlalchemy import insert, select, update + + @stackencrypt.model("leads") + class Lead(Base): + __tablename__ = "leads" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + + engine = create_engine("sqlite://") + Lead.__table__.create(engine) + configure_encryption(engine, [Lead]) + + for statement in [ + insert(Lead).values(email="secret@example.com"), + update(Lead).where(Lead.id == 1).values(email="secret@example.com"), + ]: + with engine.begin() as conn, pytest.raises(DeclarationError) as raised: + conn.execute(statement) + assert raised.value.code == "declaration_inline_value" + assert raised.value.field == "email" + assert "secret@example.com" not in str(raised.value) + + with engine.connect() as conn: + assert conn.execute(select(Lead.__table__)).all() == [] diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py new file mode 100644 index 000000000..f9709f217 --- /dev/null +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -0,0 +1,172 @@ +"""``configure_encryption`` against ZeroKMS and a PostgreSQL with the EQL v4 +bundle: the app writes and reads plaintext attributes, and the tables hold +EQL values. + +These tests need the CipherStash credentials and ``EQL_TEST_DATABASE_URL``, +as ``test_encrypt_live.py`` does. Each test makes its own tables in a schema +of its own, and drops the schema afterwards. +""" + +import uuid +from typing import Annotated + +import pytest +from _live import DATABASE_URL, LIVE, needs_database +from sqlalchemy import ForeignKey, create_engine, select, text +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship +from sqlalchemy.types import UserDefinedType + +import stackencrypt +from stackencrypt import Ignore, Passthrough, Text, TextEq, _engine +from stackencrypt.sqlalchemy import configure_encryption + +pytestmark = [*LIVE, needs_database] + + +class Eql(UserDefinedType): + """An EQL column type, named by its SQL name.""" + + cache_ok = True + + def __init__(self, sql_name: str) -> None: + self.sql_name = sql_name + + def get_col_spec(self, **kwargs: object) -> str: + return self.sql_name + + +class Base(DeclarativeBase): + pass + + +@stackencrypt.model("customers") +class Customer(Base): + __tablename__ = "customers" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] = mapped_column(Eql("public.eql_v4_text_eq")) + name: Mapped[Annotated[str, Text]] = mapped_column(Eql("public.eql_v4_text")) + plan: Mapped[Annotated[str, Passthrough]] + orders = relationship("Order", back_populates="customer") # a list of Order + + +@stackencrypt.model("orders") +class Order(Base): + __tablename__ = "orders" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + customer_id: Mapped[Annotated[int, Ignore]] = mapped_column(ForeignKey("customers.id")) + note: Mapped[Annotated[str, Text]] = mapped_column(Eql("public.eql_v4_text")) + customer: Mapped[Annotated["Customer", Ignore]] = relationship(back_populates="orders") + + +@pytest.fixture(scope="module") +def engine(): + schema = f"sa_{uuid.uuid4().hex[:12]}" + url = DATABASE_URL.replace("postgresql://", "postgresql+psycopg://", 1) + # The EQL types are in `public` and the functions in `eql_v4`, so both + # stay on the search path after the test schema. + engine = create_engine( + url, + connect_args={"options": f"-c search_path={schema},public"}, + hide_parameters=True, + ) + with engine.begin() as conn: + conn.execute(text(f"CREATE SCHEMA {schema}")) + Base.metadata.create_all(engine) + configure_encryption(engine, [Customer, Order]) + yield engine + with engine.begin() as conn: + conn.execute(text(f"DROP SCHEMA {schema} CASCADE")) + engine.dispose() + + +@pytest.fixture(autouse=True) +def empty(engine): + with engine.begin() as conn: + conn.execute(text("TRUNCATE orders, customers RESTART IDENTITY")) + + +def stored(engine, sql): + with engine.connect() as conn: + return conn.execute(text(sql)).all() + + +def add_ann(engine): + with Session(engine) as session: + ann = Customer(email="ann@example.com", name="Ann", plan="free") + ann.orders = [Order(note="first order"), Order(note="second order")] + session.add(ann) + session.commit() + + +def test_an_added_object_is_stored_encrypted_and_keeps_its_plaintext(engine): + with Session(engine, expire_on_commit=False) as session: + ann = Customer(email="ann@example.com", name="Ann", plan="free") + session.add(ann) + session.commit() + + assert (ann.email, ann.name, ann.plan) == ("ann@example.com", "Ann", "free") + assert not session.dirty + [(email, name, plan)] = stored(engine, "SELECT email::text, name::text, plan FROM customers") + assert "ann@example.com" not in email + assert "Ann" not in name + assert '"c"' in email + assert plan == "free" + + +def test_a_query_decrypts_the_objects_it_loads(engine): + add_ann(engine) + with Session(engine) as session: + [ann] = session.scalars(select(Customer)).all() + + assert (ann.email, ann.name, ann.plan) == ("ann@example.com", "Ann", "free") + assert not session.dirty + + +def test_an_update_of_one_field_is_stored_encrypted(engine): + add_ann(engine) + with Session(engine) as session: + ann = session.scalars(select(Customer)).one() + ann.email = "ann@new.example.com" + session.commit() + [(email,)] = stored(engine, "SELECT email::text FROM customers") + assert "new.example.com" not in email + with Session(engine) as session: + ann = session.scalars(select(Customer)).one() + assert (ann.email, ann.name) == ("ann@new.example.com", "Ann") + + +def test_a_relationship_load_decrypts_the_related_objects(engine): + add_ann(engine) + with Session(engine) as session: + ann = session.scalars(select(Customer)).one() + assert sorted(order.note for order in ann.orders) == ["first order", "second order"] + + +def test_an_attribute_read_after_commit_is_decrypted(engine): + add_ann(engine) + with Session(engine) as session: + ann = session.scalars(select(Customer)).one() + ann.plan = "pro" + session.commit() # expires every attribute + + assert ann.email == "ann@example.com" + assert not session.dirty + + +def test_100_objects_make_one_key_request_to_insert_and_one_to_load(engine): + add_ann(engine) # warms up the client: the first call may load the keyset + with Session(engine) as session: + session.add_all( + [Customer(email=f"user{i}@example.com", name=f"User {i}", plan="free") for i in range(100)] + ) + before = _engine.request_count() + session.commit() + inserted = _engine.request_count() - before + with Session(engine) as session: + before = _engine.request_count() + customers = session.scalars(select(Customer).where(Customer.plan == "free")).all() + loaded = _engine.request_count() - before + + # 100 objects of 2 sealed fields: 200 values, one request of up to 500 keys. + assert (inserted, loaded) == (1, 1) + assert sorted(c.email for c in customers)[0] == "ann@example.com" From c899f61bde31a906f4c85621382531dd966229f4 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 13:26:27 +0900 Subject: [PATCH 09/29] feat(python): search encrypted columns through the SQLAlchemy ORM An app had to build each query term with cipher.eq and write the cast in the SQL itself. eql_type_map gives every encrypted field an EqlType column, read from its marker, so a model names no SQL type. The type's ==, <, <=, >, and >= make the term with the Cipher method of the same meaning and cast it to the marker's query type. Every other operator that would send a value, such as like, in_, and !=, raises QueryError when the query is built. The value would reach the database as plaintext, and could never match an EQL value. configure_encryption now refuses an encrypted column whose type is not its marker's EQL type. Without the type map, SQLAlchemy would make it an ordinary text column, with no error. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 30 +++- .../python/python/stackencrypt/sqlalchemy.py | 141 +++++++++++++++++- languages/python/tests/test_sqlalchemy.py | 104 ++++++++++++- .../python/tests/test_sqlalchemy_live.py | 69 ++++++--- 4 files changed, 307 insertions(+), 37 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 089e1f0a9..a401fea6c 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -143,27 +143,47 @@ The values are checked as `encrypt` checks them. A row that is not a mapping rai `stackencrypt.sqlalchemy.configure_encryption` makes the ORM encrypt and decrypt for you. Call it once, after you create the engine. Then use sessions as usual. The app reads and writes plaintext attributes, and the tables store EQL values. ```text -from stackencrypt.sqlalchemy import configure_encryption +from stackencrypt.sqlalchemy import configure_encryption, eql_type_map + + +class Base(DeclarativeBase): + type_annotation_map = eql_type_map + + +@stackencrypt.model("customers") +class Customer(Base): + __tablename__ = "customers" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + level: Mapped[Annotated[int, IntegerOrd]] + engine = create_engine(DATABASE_URL.replace("postgresql://", "postgresql+psycopg://", 1)) -configure_encryption(engine, [Customer, Order]) +configure_encryption(engine, [Customer]) with Session(engine) as session: - session.add(Customer(email="a@example.com", name="Ann", plan="free")) + session.add(Customer(email="a@example.com", level=3)) session.commit() # encrypted on the way in - customer = session.scalars(select(Customer)).one() + customer = session.scalars(select(Customer).where(Customer.email == "a@example.com")).one() customer.email # "a@example.com" ``` +`eql_type_map` gives each encrypted field its EQL column type, such as `public.eql_v4_text_eq` for `TextEq`. So a column needs no type of its own. `Base.metadata.create_all(engine)` creates the EQL columns. + - **Writes:** each INSERT or UPDATE on a listed table is encrypted with one batch of key requests for all its rows. - **Reads:** the objects a query loads are decrypted with one batch for each class. This includes relationship loads, and attributes read again after a commit. - Only encrypted fields are changed. A passthrough field is stored as SQLAlchemy writes it. +- **Search:** `==`, `<`, `<=`, `>`, and `>=` on an encrypted field compare it with a query term, as `cipher.eq` and `cipher.gte` make. The field's marker must support the operator. For example, `Customer.level >= 3` works, and `Customer.email > x` raises `QueryError` with `query_unsupported`. +- **Other operators** that would send a value, such as `like`, `in_`, `!=`, and `between`, raise `QueryError` with `query_unsupported` when you build the query. The value would reach the database as plaintext, and could never match. `is_(None)`, `is_not(None)`, `asc()`, and `desc()` send no value, so they work. +- A comparison on a class that is not configured raises `QueryError` with `query_not_configured`. + Each class must be declared with `stackencrypt.model` and mapped by SQLAlchemy. `configure_encryption` refuses a class with `DeclarationError` in these cases: - `declaration_not_mapped`: SQLAlchemy does not map the class. - `declaration_table_mismatch`: the table in `stackencrypt.model` is not the `__tablename__`. - `declaration_column_name`: an encrypted field's column has another name than the field, as with `mapped_column("email_address")`. +- `declaration_column_type`: an encrypted field's column is not the EQL type of its marker. Set `type_annotation_map = eql_type_map` on the base. - `declaration_already_configured`: the class was configured before in this process. A statement that puts an encrypted value inside its SQL, such as `insert(Customer).values(email=...)`, raises `DeclarationError` with `declaration_inline_value`. The value is not stored. Pass the values as parameters instead, as `session.add` and `session.execute(insert(Customer), rows)` do. @@ -171,7 +191,7 @@ A statement that puts an encrypted value inside its SQL, such as `insert(Custome These are not handled yet: - A Core query, such as `conn.execute(text("SELECT ..."))`, returns EQL values, not plaintext. Use `cipher.decrypt_rows` on its rows. -- A search by an encrypted field, such as `where(Customer.email == x)`, is not supported yet. Build the term with `cipher.eq` and compare it in the SQL, as in "Search by equality". +- A match search, `matches`, is not supported through the ORM yet. Use `cipher.matches`, as in "Search by match". - A query result is read in full before it is returned, so that its objects can be decrypted in one batch. A query that streams its rows does not stream. ## Declare a SQLAlchemy relationship diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index cdd107ba9..d5726763d 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -20,22 +20,107 @@ Only the sealed fields go through the hooks. A passthrough field is stored as SQLAlchemy writes it. +- Search: the column type that ``eql_type_map`` gives each encrypted field + turns ``==``, ``<``, ``<=``, ``>`` and ``>=`` into a comparison with a query + term, made by ``Cipher.eq``, ``lt``, ``lte``, ``gt`` or ``gte``. Any other + operator that would send a value raises ``QueryError`` when the query is + built, because the value would reach the database as plaintext. + This module imports SQLAlchemy. The rest of the package does not. """ -from typing import Any, Dict, FrozenSet, List, Sequence, Tuple, Type +from typing import Annotated, Any, Callable, Dict, FrozenSet, List, Sequence, Tuple, Type -from sqlalchemy import event, inspect +from sqlalchemy import String, Table, cast, event, inspect, literal from sqlalchemy.engine import Engine from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import set_committed_value +from sqlalchemy.sql import operators from sqlalchemy.sql.dml import Insert, Update +from sqlalchemy.types import UserDefinedType +from stackencrypt import _eql from stackencrypt._cipher import Cipher -from stackencrypt._errors import DeclarationError -from stackencrypt._markers import Passthrough +from stackencrypt._declaration import _KIND_TYPES +from stackencrypt._errors import DeclarationError, QueryError +from stackencrypt._markers import Marker, Passthrough + +__all__ = ["EqlType", "configure_encryption", "eql_type_map"] + + +class EqlType(UserDefinedType): # type: ignore[type-arg] + """The column type of an encrypted field: its marker's EQL type, such as + ``public.eql_v4_text_eq`` for ``TextEq``. ``eql_type_map`` gives it to + each field, so a model does not name it.""" + + cache_ok = True + + def __init__(self, marker: Type[Marker]) -> None: + self.marker = marker + + def get_col_spec(self, **kwargs: Any) -> str: + return self.marker.sql_domain + + def __repr__(self) -> str: + return f"EqlType({self.marker.__name__})" + + class comparator_factory(UserDefinedType.Comparator): # type: ignore[type-arg] + def operate(self, op: Any, *other: Any, **kwargs: Any) -> Any: + if op in _NO_VALUE or (op is operators.eq and other == (None,)): + return super().operate(op, *other, **kwargs) + method = _COMPARISONS.get(op) + if method is None or len(other) != 1: + raise _refused(self.expr, op) + return super().operate(op, _term(self.expr, method, other[0]), **kwargs) + + def reverse_operate(self, op: Any, other: Any, **kwargs: Any) -> Any: + raise _refused(self.expr, op) + + +class _QueryType(UserDefinedType): # type: ignore[type-arg] + """The type a query term is cast to, such as ``eql_v4.query_text_eq``.""" + + cache_ok = True + + def __init__(self, sql_name: str) -> None: + self.sql_name = sql_name + + def get_col_spec(self, **kwargs: Any) -> str: + return self.sql_name -__all__ = ["configure_encryption"] + +# For a DeclarativeBase's `type_annotation_map`: every marker the decorator +# accepts, under the hint a model writes for it, such as +# Annotated[str, TextEq]. +eql_type_map: Dict[Any, EqlType] = { + Annotated[_KIND_TYPES[marker.plaintext], marker]: EqlType(marker) # type: ignore[valid-type] + for marker in _eql.TYPES + if marker.producible and marker.plaintext in _KIND_TYPES +} + +# The comparisons a query term answers, and the Cipher method that makes it. +_COMPARISONS: Dict[Any, str] = { + operators.eq: "eq", + operators.lt: "lt", + operators.le: "lte", + operators.gt: "gt", + operators.ge: "gte", +} + +# The operators that send no value to the database. +_NO_VALUE = frozenset( + { + operators.is_, + operators.is_not, + operators.asc_op, + operators.desc_op, + operators.nulls_first_op, + operators.nulls_last_op, + } +) + +# The configured tables, for the comparisons: a Table to its cipher. +_TABLES: Dict[Table, Cipher] = {} # The objects a load built or refreshed, and which attributes, waiting for # the do_orm_execute hook. Kept in each session's `info`. @@ -74,6 +159,7 @@ def configure_encryption(engine: Engine, classes: Sequence[type]) -> None: tables: Dict[str, Tuple[Cipher, FrozenSet[str]]] = {} for cls, cipher, sealed in checked: _CONFIGURED[cls] = (cipher, sealed) + _TABLES[inspect(cls).local_table] = cipher tables[cipher.declaration.table] = (cipher, sealed) event.listen(cls, "load", _remember_load) event.listen(cls, "refresh", _remember_refresh) @@ -114,6 +200,20 @@ def _check(cls: type) -> Tuple[type, Cipher, FrozenSet[str]]: "the field's name.", field=name, ) + for field in cipher.declaration.fields: + if field.marker is Passthrough: + continue + column_type = mapper.columns[field.name].type + if not (isinstance(column_type, EqlType) and column_type.marker is field.marker): + raise DeclarationError( + "declaration_column_type", + f"field {field.name!r} of {cls.__qualname__} is declared " + f"{field.marker.__name__}, so its column must be " + f"{field.marker.sql_domain}, but SQLAlchemy maps it as {column_type!r}. " + "Set `type_annotation_map = eql_type_map` on the declarative base, " + "and give the column no type of its own.", + field=field.name, + ) return cls, cipher, sealed @@ -198,3 +298,34 @@ def _decrypt_loaded(state: Any) -> Any: for name, value in values.items(): set_committed_value(target, name, value) return result() + + +def _term(column: Any, method: str, value: object) -> Any: + """``value`` as the query term that ``method`` makes for ``column``, + cast to the column marker's query type.""" + cipher = _TABLES.get(getattr(column, "table", None)) + if cipher is None: + raise QueryError( + "query_not_configured", + f"column {_column_name(column)} is encrypted, but its class is not configured. " + "Call configure_encryption with the class before you build a query on it.", + ) + term = getattr(cipher, method)(column.key, value) + return cast(literal(term, String()), _QueryType(column.type.marker.query_sql_domain)) + + +def _refused(column: Any, op: Callable[..., Any]) -> QueryError: + name = getattr(op, "__name__", repr(op)).removesuffix("_op") + return QueryError( + "query_unsupported", + f"{name} is not supported on the encrypted column {_column_name(column)}. " + "It would send the value to the database as plaintext. Use ==, <, <=, >, or >= " + "where the column's marker supports it.", + field=getattr(column, "key", None), + reason=name, + ) + + +def _column_name(column: Any) -> str: + table = getattr(getattr(column, "table", None), "name", "?") + return f"{table}.{getattr(column, 'name', '?')}" diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index 23bf388eb..48adc73a4 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -1,5 +1,5 @@ -"""``stackencrypt.sqlalchemy.configure_encryption`` before any query: the -class checks. +"""``stackencrypt.sqlalchemy`` before any request: the type map, the class +checks of ``configure_encryption``, and the operators a query refuses. These tests need fake credentials only. They use an SQLite engine that no test connects to. The hooks run against PostgreSQL in @@ -10,18 +10,19 @@ class checks. from typing import Annotated import pytest -from sqlalchemy import create_engine +from sqlalchemy import String, create_engine from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column import stackencrypt from stackencrypt import DeclarationError, Ignore, Passthrough, TextEq -from stackencrypt.sqlalchemy import configure_encryption +from stackencrypt import IntegerOrd, QueryError, Text, _eql +from stackencrypt.sqlalchemy import EqlType, configure_encryption, eql_type_map pytestmark = pytest.mark.usefixtures("credentials") class Base(DeclarativeBase): - pass + type_annotation_map = eql_type_map @stackencrypt.model("customers") @@ -133,7 +134,9 @@ class Lead(Base): email: Mapped[Annotated[str, TextEq]] engine = create_engine("sqlite://") - Lead.__table__.create(engine) + # SQLite has no EQL types, and the refusal comes before the database. + with engine.begin() as conn: + conn.exec_driver_sql("CREATE TABLE leads (id INTEGER PRIMARY KEY, email TEXT)") configure_encryption(engine, [Lead]) for statement in [ @@ -148,3 +151,92 @@ class Lead(Base): with engine.connect() as conn: assert conn.execute(select(Lead.__table__)).all() == [] + + +# The type map. + + +def test_the_type_map_gives_every_declarable_marker_its_eql_type(): + declarable = [m for m in _eql.TYPES if m.producible and m.plaintext != "json"] + assert declarable + for marker in declarable: + [column_type] = [t for hint, t in eql_type_map.items() if hint.__metadata__ == (marker,)] + assert isinstance(column_type, EqlType) + assert column_type.marker is marker + assert column_type.get_col_spec() == marker.sql_domain + + +def test_a_column_of_another_type_is_refused(engine): + @stackencrypt.model("tickets") + class Ticket(Base): + __tablename__ = "tickets" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] = mapped_column(String) + + error = refused(engine, [Ticket]) + assert error.code == "declaration_column_type" + assert error.field == "email" + assert "eql_v4_text_eq" in error.message + + +# The operators a query refuses, before any request. + + +@stackencrypt.model("scores") +class Score(Base): + __tablename__ = "scores" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + level: Mapped[Annotated[int, IntegerOrd]] + note: Mapped[Annotated[str, Text]] + + +@pytest.fixture(scope="module") +def scores(): + configure_encryption(create_engine("sqlite://"), [Score]) + + +@pytest.mark.parametrize( + "build", + [ + lambda: Score.note.like("%secret%"), + lambda: Score.note.contains("secret"), + lambda: Score.note.in_(["secret"]), + lambda: Score.note != "secret", + lambda: Score.level.between(1, 9), + lambda: Score.level + 1, + ], + ids=["like", "contains", "in", "not-equal", "between", "add"], +) +def test_an_operator_that_would_send_the_value_is_refused(scores, untouched, build): + with pytest.raises(QueryError) as raised: + build() + assert raised.value.code == "query_unsupported" + assert "secret" not in str(raised.value) + + +@pytest.mark.parametrize( + "build", + [ + lambda: Score.note.is_(None), + lambda: Score.note.is_not(None), + lambda: Score.note == None, # noqa: E711 + lambda: Score.level.asc(), + lambda: Score.level.desc(), + ], + ids=["is-null", "is-not-null", "equals-none", "asc", "desc"], +) +def test_an_operator_that_sends_no_value_is_allowed_with_no_request(scores, untouched, build): + build() + + +def test_a_comparison_on_a_class_not_configured_is_refused(untouched): + @stackencrypt.model("drafts") + class Draft(Base): + __tablename__ = "drafts" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + note: Mapped[Annotated[str, Text]] + + with pytest.raises(QueryError) as raised: + Draft.note == "secret" # noqa: B015 + assert raised.value.code == "query_not_configured" + assert "secret" not in str(raised.value) diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index f9709f217..873ca57f9 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -14,37 +14,24 @@ from _live import DATABASE_URL, LIVE, needs_database from sqlalchemy import ForeignKey, create_engine, select, text from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship -from sqlalchemy.types import UserDefinedType import stackencrypt -from stackencrypt import Ignore, Passthrough, Text, TextEq, _engine -from stackencrypt.sqlalchemy import configure_encryption +from stackencrypt import Ignore, IntegerOrd, Passthrough, QueryError, Text, TextEq, _engine +from stackencrypt.sqlalchemy import configure_encryption, eql_type_map pytestmark = [*LIVE, needs_database] -class Eql(UserDefinedType): - """An EQL column type, named by its SQL name.""" - - cache_ok = True - - def __init__(self, sql_name: str) -> None: - self.sql_name = sql_name - - def get_col_spec(self, **kwargs: object) -> str: - return self.sql_name - - class Base(DeclarativeBase): - pass + type_annotation_map = eql_type_map @stackencrypt.model("customers") class Customer(Base): __tablename__ = "customers" id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) - email: Mapped[Annotated[str, TextEq]] = mapped_column(Eql("public.eql_v4_text_eq")) - name: Mapped[Annotated[str, Text]] = mapped_column(Eql("public.eql_v4_text")) + email: Mapped[Annotated[str, TextEq]] + name: Mapped[Annotated[str, Text]] plan: Mapped[Annotated[str, Passthrough]] orders = relationship("Order", back_populates="customer") # a list of Order @@ -54,10 +41,17 @@ class Order(Base): __tablename__ = "orders" id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) customer_id: Mapped[Annotated[int, Ignore]] = mapped_column(ForeignKey("customers.id")) - note: Mapped[Annotated[str, Text]] = mapped_column(Eql("public.eql_v4_text")) + note: Mapped[Annotated[str, Text]] customer: Mapped[Annotated["Customer", Ignore]] = relationship(back_populates="orders") +@stackencrypt.model("players") +class Player(Base): + __tablename__ = "players" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + level: Mapped[Annotated[int, IntegerOrd]] + + @pytest.fixture(scope="module") def engine(): schema = f"sa_{uuid.uuid4().hex[:12]}" @@ -72,7 +66,7 @@ def engine(): with engine.begin() as conn: conn.execute(text(f"CREATE SCHEMA {schema}")) Base.metadata.create_all(engine) - configure_encryption(engine, [Customer, Order]) + configure_encryption(engine, [Customer, Order, Player]) yield engine with engine.begin() as conn: conn.execute(text(f"DROP SCHEMA {schema} CASCADE")) @@ -82,7 +76,7 @@ def engine(): @pytest.fixture(autouse=True) def empty(engine): with engine.begin() as conn: - conn.execute(text("TRUNCATE orders, customers RESTART IDENTITY")) + conn.execute(text("TRUNCATE orders, customers, players RESTART IDENTITY")) def stored(engine, sql): @@ -170,3 +164,36 @@ def test_100_objects_make_one_key_request_to_insert_and_one_to_load(engine): # 100 objects of 2 sealed fields: 200 values, one request of up to 500 keys. assert (inserted, loaded) == (1, 1) assert sorted(c.email for c in customers)[0] == "ann@example.com" + + +def test_an_equality_search_finds_the_one_matching_row(engine): + add_ann(engine) + with Session(engine) as session: + session.add(Customer(email="bob@example.com", name="Bob", plan="free")) + session.commit() + with Session(engine) as session: + found = session.scalars(select(Customer).where(Customer.email == "bob@example.com")).all() + + assert [(c.email, c.name) for c in found] == [("bob@example.com", "Bob")] + + +def test_order_searches_find_the_rows_on_each_side(engine): + with Session(engine) as session: + session.add_all([Player(level=level) for level in (1, 3, 5, 7)]) + session.commit() + with Session(engine) as session: + high = session.scalars(select(Player).where(Player.level >= 5)).all() + low = session.scalars(select(Player).where(Player.level < 3)).all() + middle = session.scalars( + select(Player).where(Player.level > 1, Player.level <= 5).order_by(Player.id) + ).all() + + assert sorted(p.level for p in high) == [5, 7] + assert [p.level for p in low] == [1] + assert [p.level for p in middle] == [3, 5] + + +def test_an_order_search_on_an_equality_field_is_unsupported(engine): + with pytest.raises(QueryError) as raised: + select(Customer).where(Customer.email > "a@example.com") + assert raised.value.code == "query_unsupported" From feb54db694b1411205b8be2516bcb4267a0e5b28 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 15:02:55 +0900 Subject: [PATCH 10/29] feat(python): Alembic migrations for encrypted columns and their indexes An app that moves to encrypted columns needs migrations that create the EQL columns and the database indexes a search uses. Alembic's autogenerate could not do either alone. A mapped class now gets a database index for each EQL index of each encrypted field, such as hash (eql_v4.eq_term(email)) for TextEq. It is added when the class is mapped, not when a column joins a table: Alembic's add_column creates the indexes of a temporary table, so an index added there was created twice. Two hooks for env.py. render_item writes EqlType into a migration with its imports. process_revision_directives writes a change of an existing column to an EQL type as an ALTER with USING col::jsonb::, which PostgreSQL needs. The cast changes an empty column and refuses one with plaintext rows, which keeps them. It also drops the type change Alembic finds between an EQL type and itself: it compares eql_v4_text_eq from the database with public.eql_v4_text_eq from the model as text. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- languages/python/README.md | 37 +++ languages/python/pyproject.toml | 20 +- .../python/python/stackencrypt/sqlalchemy.py | 163 +++++++++- .../tests/test_sqlalchemy_migrations.py | 286 ++++++++++++++++++ .../tests/test_sqlalchemy_migrations_live.py | 258 ++++++++++++++++ languages/python/uv.lock | 232 +++++++++++++- 7 files changed, 990 insertions(+), 8 deletions(-) create mode 100644 languages/python/tests/test_sqlalchemy_migrations.py create mode 100644 languages/python/tests/test_sqlalchemy_migrations_live.py diff --git a/AGENTS.md b/AGENTS.md index 1fac4da02..3094761b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The package has no runtime dependency, so nothing else is locked. The middle three are for the declaration tests. psycopg is for the live tests. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `languages/python/examples/customer-directory`: An internal FastAPI example of the Python SDK, written twice: with psycopg (`app.py`) and with SQLAlchemy (`app_sqlalchemy.py`). It has its own `pyproject.toml` and `uv.lock`, and installs `stackencrypt` from this checkout. Its tests need real credentials, and CI does not run them. diff --git a/languages/python/README.md b/languages/python/README.md index a401fea6c..b84e1aba7 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -188,6 +188,43 @@ Each class must be declared with `stackencrypt.model` and mapped by SQLAlchemy. A statement that puts an encrypted value inside its SQL, such as `insert(Customer).values(email=...)`, raises `DeclarationError` with `declaration_inline_value`. The value is not stored. Pass the values as parameters instead, as `session.add` and `session.execute(insert(Customer), rows)` do. +### Migrations with Alembic + +A class mapped with `eql_type_map` gets a database index for each search its fields support. For example, `email: Mapped[Annotated[str, TextEq]]` gets `ix_customers_email_eq`, on `hash (eql_v4.eq_term(email))`. Without that index, a search still returns the right rows, but PostgreSQL reads every row. `create_all` and Alembic's `autogenerate` both create these indexes. + +| Marker index | Database index | +|---|---| +| `eq` | `hash (eql_v4.eq_term(col))` | +| `ope` | `btree (eql_v4.ord_term(col))` | +| `ore_block` | `btree (eql_v4.ord_term_ore(col))` | +| `match` | `gin (eql_v4.match_term(col))` | + +An `ore_block` index needs an operator class that EQL's install creates. If the role that installed EQL could not create operator classes, the index still builds. But PostgreSQL never uses it, and gives no error. + +Give Alembic the two hooks in your `env.py`: + +```text +from stackencrypt.sqlalchemy import process_revision_directives, render_item + +context.configure( + connection=connection, + target_metadata=target_metadata, + render_item=render_item, + process_revision_directives=process_revision_directives, +) +``` + +- `render_item` writes an encrypted column's type into the migration as `stackencrypt.sqlalchemy.EqlType(stackencrypt.TextEq)`, and adds the imports it needs. +- `process_revision_directives` changes two kinds of step that Alembic writes for an EQL column. + - A change of an existing column to an EQL type, such as from `text`, becomes `op.execute` of an `ALTER` with `USING email::jsonb::public.eql_v4_text_eq`. PostgreSQL refuses the change without it. The cast changes a column with no rows. A column with rows that are not EQL values fails the migration, and keeps its rows. + - A type change from an EQL type to the same EQL type is left out. Alembic would otherwise write one for every EQL column in each new migration. + +These are not handled by the hooks yet: + +- Encrypting data that a column already holds. The migration fails instead. +- The reverse migration of a change to an EQL type. Its `downgrade` is Alembic's `alter_column`, as Alembic writes it. It has not been tested. +- Installing EQL. Install the bundle before the first migration, as in "Before you start". + These are not handled yet: - A Core query, such as `conn.execute(text("SELECT ..."))`, returns EQL values, not plaintext. Use `cipher.decrypt_rows` on its rows. diff --git a/languages/python/pyproject.toml b/languages/python/pyproject.toml index bebdda457..191850a11 100644 --- a/languages/python/pyproject.toml +++ b/languages/python/pyproject.toml @@ -12,7 +12,8 @@ classifiers = ["Private :: Do Not Upload"] [project.optional-dependencies] # attrs, Pydantic and SQLAlchemy only for the tests that declare a model of -# each kind; the package itself imports none of them. They have lower bounds +# each kind. The package imports none of them, except `stackencrypt.sqlalchemy`, +# which imports SQLAlchemy when an app imports it. They have lower bounds # and no upper bounds on purpose. They are the libraries users build their # classes with, so the tests meet each new release soon after users do. A new # release that fails a test is a finding about users' classes. uv.lock fixes @@ -22,7 +23,17 @@ classifiers = ["Private :: Do Not Upload"] # # psycopg for the live tests, which insert an encrypted row (#1389). The # binary extra ships libpq, so the tests need no system Postgres client. -test = ["pytest>=8", "attrs>=23", "pydantic>=2", "sqlalchemy>=2", "psycopg[binary]>=3.1"] +# +# Alembic for the tests of the migrations it generates from models that use +# `stackencrypt.sqlalchemy`. +test = [ + "pytest>=8", + "attrs>=23", + "pydantic>=2", + "sqlalchemy>=2", + "psycopg[binary]>=3.1", + "alembic>=1.13", +] [tool.maturin] # The pure-Python package is the distribution; the Rust extension is its @@ -42,5 +53,8 @@ testpaths = ["tests"] pythonpath = ["tests"] # `live` marks the tests that need credentials and a database. They skip # without them; `mise run test:live` runs only them and fails on a skip. -markers = ["live: needs CipherStash credentials, and some a PostgreSQL with EQL v4"] +markers = [ + "live: needs CipherStash credentials, and some a PostgreSQL with EQL v4", + "models: the models module of an Alembic test project, in test_sqlalchemy_migrations_live.py", +] addopts = "--strict-markers" diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index d5726763d..06d470255 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -20,6 +20,13 @@ Only the sealed fields go through the hooks. A passthrough field is stored as SQLAlchemy writes it. +- Indexes: a mapped class's table gets a database index for each EQL index + of each encrypted column, such as ``hash (eql_v4.eq_term(email))`` for ``TextEq``. + So ``create_all`` and Alembic's ``autogenerate`` create them. +- Migrations: two hooks for ``context.configure`` in Alembic's ``env.py``. + ``render_item`` writes an ``EqlType`` into a migration file with its + imports. ``process_revision_directives`` writes a change of an existing + column to an EQL type as an ALTER with a USING cast. - Search: the column type that ``eql_type_map`` gives each encrypted field turns ``==``, ``<``, ``<=``, ``>`` and ``>=`` into a comparison with a query term, made by ``Cipher.eq``, ``lt``, ``lte``, ``gt`` or ``gte``. Any other @@ -31,9 +38,9 @@ from typing import Annotated, Any, Callable, Dict, FrozenSet, List, Sequence, Tuple, Type -from sqlalchemy import String, Table, cast, event, inspect, literal +from sqlalchemy import Index, String, Table, cast, event, func, inspect, literal from sqlalchemy.engine import Engine -from sqlalchemy.orm import Session +from sqlalchemy.orm import Mapper, Session from sqlalchemy.orm.attributes import set_committed_value from sqlalchemy.sql import operators from sqlalchemy.sql.dml import Insert, Update @@ -45,7 +52,13 @@ from stackencrypt._errors import DeclarationError, QueryError from stackencrypt._markers import Marker, Passthrough -__all__ = ["EqlType", "configure_encryption", "eql_type_map"] +__all__ = [ + "EqlType", + "configure_encryption", + "eql_type_map", + "process_revision_directives", + "render_item", +] class EqlType(UserDefinedType): # type: ignore[type-arg] @@ -119,6 +132,150 @@ def get_col_spec(self, **kwargs: Any) -> str: } ) +# The database index for each kind of EQL index: the EQL function the index +# is on, and the index method. +_INDEXES: Dict[str, Tuple[str, str]] = { + "eq": ("eq_term", "hash"), + "ope": ("ord_term", "btree"), + "ore_block": ("ord_term_ore", "btree"), + "match": ("match_term", "gin"), +} + + +@event.listens_for(Mapper, "instrument_class") +def _add_indexes(mapper: Any, cls: type) -> None: + """Give a mapped class's table a database index for each EQL index of + each ``EqlType`` column, named ``ix___``. + + It runs when SQLAlchemy maps the class, not when a column joins a table: + Alembic's ``add_column`` puts the new column on a temporary table and + creates every index of that table, so an index added there would be + created twice.""" + table = mapper.local_table + if not isinstance(table, Table): + return + for column in table.columns: + if not isinstance(column.type, EqlType): + continue + for kind in column.type.marker.indexes: + if kind not in _INDEXES: + continue + function, method = _INDEXES[kind] + name = f"ix_{table.name}_{column.name}_{kind}" + if any(index.name == name for index in table.indexes): + continue + Index(name, getattr(func.eql_v4, function)(column), postgresql_using=method) + + +def render_item(type_: str, obj: Any, autogen_context: Any) -> Any: + """For Alembic's ``context.configure(render_item=render_item)`` in + ``env.py``. Writes an ``EqlType`` column type into a migration as + ``stackencrypt.sqlalchemy.EqlType(stackencrypt.TextEq)``, and adds the + imports it needs. Returns ``False`` for anything else, so that Alembic + writes it as usual.""" + if type_ == "type" and isinstance(obj, EqlType): + autogen_context.imports.add("import stackencrypt") + autogen_context.imports.add("import stackencrypt.sqlalchemy") + return f"stackencrypt.sqlalchemy.EqlType(stackencrypt.{obj.marker.__name__})" + return False + + +def process_revision_directives(context: Any, revision: Any, directives: List[Any]) -> None: + """For Alembic's ``context.configure(process_revision_directives=...)`` + in ``env.py``. Rewrites each upgrade step that changes an existing + column to an EQL type. + + Alembic writes that change as ``op.alter_column(..., type_=EqlType(...))``, + and PostgreSQL refuses it: there is no cast from a column's old type to + an EQL type. The step becomes ``op.execute`` of an ALTER with + ``USING ::jsonb::``. That changes the type of a column + with no rows. A column with rows that are not EQL values fails the + migration, and keeps its rows. Any other change to the same column, such + as to ``nullable``, is kept as Alembic wrote it. + + The downgrade steps are not changed. + """ + from alembic.operations import ops + + preparer = context.dialect.identifier_preparer + for script in directives: + for container, rewrite in ( + (script.upgrade_ops, lambda step: _rewrite_type_change(ops, preparer, step)), + (script.downgrade_ops, lambda step: [step]), + ): + kept = [] + for table_ops in container.ops: + if isinstance(table_ops, ops.ModifyTableOps): + table_ops.ops = [ + step + for operation in table_ops.ops + for unchanged in _drop_same_eql_type(ops, operation) + for step in rewrite(unchanged) + ] + if not table_ops.ops: + continue + kept.append(table_ops) + container.ops = kept + + +def _drop_same_eql_type(ops: Any, operation: Any) -> List[Any]: + """Alembic compares a column's type as text: the database gives an EQL + column's type as ``eql_v4_text_eq``, and ``EqlType`` as + ``public.eql_v4_text_eq``. So it finds a change where there is none. + Drop that type change, and the step if it changes nothing else.""" + if not isinstance(operation, ops.AlterColumnOp) or operation.modify_type is None: + return [operation] + if not _same_eql_type(operation.existing_type, operation.modify_type): + return [operation] + operation.modify_type = None + return [operation] if operation.has_changes() else [] + + +def _same_eql_type(a: Any, b: Any) -> bool: + for eql, other in ((a, b), (b, a)): + if isinstance(eql, EqlType): + name = eql.marker.sql_domain.rsplit(".", 1)[-1] + if isinstance(other, EqlType): + return other.marker is eql.marker + return getattr(other, "name", None) == name and getattr(other, "schema", None) in ( + None, + "public", + ) + return False + + +def _rewrite_type_change(ops: Any, preparer: Any, operation: Any) -> List[Any]: + if not ( + isinstance(operation, ops.AlterColumnOp) + and isinstance(operation.modify_type, EqlType) + and not _is_eql(operation.existing_type) + ): + return [operation] + table = preparer.quote(operation.table_name) + if operation.schema: + table = f"{preparer.quote_schema(operation.schema)}.{table}" + column = preparer.quote(operation.column_name) + domain = operation.modify_type.marker.sql_domain + steps = [ + ops.ExecuteSQLOp( + f"ALTER TABLE {table} ALTER COLUMN {column} TYPE {domain} " + f"USING {column}::jsonb::{domain}" + ) + ] + operation.existing_type = operation.modify_type + operation.modify_type = None + if operation.has_changes(): + steps.append(operation) + return steps + + +def _is_eql(column_type: Any) -> bool: + """An EQL type, from the model or as the database gives it.""" + if isinstance(column_type, EqlType): + return True + return str(getattr(column_type, "name", "")).startswith("eql_v4_") + + # The configured tables, for the comparisons: a Table to its cipher. _TABLES: Dict[Table, Cipher] = {} diff --git a/languages/python/tests/test_sqlalchemy_migrations.py b/languages/python/tests/test_sqlalchemy_migrations.py new file mode 100644 index 000000000..f78a766ed --- /dev/null +++ b/languages/python/tests/test_sqlalchemy_migrations.py @@ -0,0 +1,286 @@ +"""The database indexes and the Alembic migration code that +``stackencrypt.sqlalchemy`` gives an encrypted column, with no database. + +``test_sqlalchemy_live.py`` runs a generated migration against PostgreSQL. +""" + +import datetime +from typing import Annotated + +import pytest +from alembic.autogenerate import render_python_code +from alembic.operations import ops +from alembic.autogenerate.api import AutogenContext +from alembic.runtime.migration import MigrationContext +from sqlalchemy import Column, Integer, MetaData, String, Table, Text +from sqlalchemy.dialects import postgresql +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from sqlalchemy.schema import CreateIndex + +import stackencrypt +from stackencrypt import Ignore, IntegerOrd, Passthrough, TextEq, TextSearch, _eql +from stackencrypt.sqlalchemy import ( + EqlType, + eql_type_map, + process_revision_directives, + render_item, +) + +# The index each kind of EQL index needs, as PostgreSQL prints it. +INDEX_SQL = { + "eq": "USING hash (eql_v4.eq_term({col}))", + "ope": "USING btree (eql_v4.ord_term({col}))", + "ore_block": "USING btree (eql_v4.ord_term_ore({col}))", + "match": "USING gin (eql_v4.match_term({col}))", +} + + +class Base(DeclarativeBase): + type_annotation_map = eql_type_map + + +@stackencrypt.model("people") +class Person(Base): + __tablename__ = "people" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + bio: Mapped[Annotated[str, TextSearch]] + level: Mapped[Annotated[int, IntegerOrd]] + plan: Mapped[Annotated[str, Passthrough]] + + +def index_sql(index): + return " ".join(str(CreateIndex(index).compile(dialect=postgresql.dialect())).split()) + + +@pytest.mark.parametrize( + "marker", + [m for m in _eql.TYPES if m.producible and m.plaintext in ("string", "int32", "date")], + ids=lambda m: m.__name__, +) +def test_each_eql_index_of_a_marker_gets_its_database_index(marker): + python_type = {"string": str, "int32": int, "date": datetime.date}[marker.plaintext] + cls = type( + f"T{marker.__name__}", + (Base,), + { + "__tablename__": f"t_{marker.__name__.lower()}", + "__annotations__": { + "id": Mapped[int], + "v": Mapped[Annotated[python_type, marker]], + }, + "id": mapped_column(primary_key=True), + }, + ) + table = cls.__table__.name + + made = {index.name: index_sql(index) for index in cls.__table__.indexes} + + assert made == { + f"ix_{table}_v_{kind}": f"CREATE INDEX ix_{table}_v_{kind} ON {table} " + + INDEX_SQL[kind].format(col="v") + for kind in marker.indexes + } + + +def test_a_table_that_is_not_mapped_gets_no_index(): + # As Alembic's add_column builds one: its indexes would be created twice. + table = Table("t", MetaData(), Column("id", Integer, primary_key=True), Column("v", EqlType(TextEq))) + + assert table.indexes == set() + + +def test_a_model_gets_one_index_for_each_eql_index_of_each_encrypted_field(): + assert sorted(index.name for index in Person.__table__.indexes) == [ + "ix_people_bio_eq", + "ix_people_bio_match", + "ix_people_bio_ope", + "ix_people_email_eq", + "ix_people_level_ope", + ] + + +def migration_code(*operations): + context = MigrationContext.configure(dialect_name="postgresql") + return render_python_code( + ops.UpgradeOps(ops=list(operations)), render_item=render_item, migration_context=context + ) + + +def test_a_generated_migration_names_the_eql_type_by_its_module(): + code = migration_code(ops.CreateTableOp.from_table(Person.__table__)) + + assert "stackencrypt.sqlalchemy.EqlType(stackencrypt.TextEq)" in code + assert "stackencrypt.sqlalchemy.EqlType(stackencrypt.IntegerOrd)" in code + + +def test_render_item_adds_the_imports_the_type_needs(): + autogen = AutogenContext(MigrationContext.configure(dialect_name="postgresql")) + + render_item("type", EqlType(TextEq), autogen) + + assert {"import stackencrypt", "import stackencrypt.sqlalchemy"} <= autogen.imports + + +def test_render_item_leaves_other_items_to_alembic(): + autogen = AutogenContext(MigrationContext.configure(dialect_name="postgresql")) + + assert render_item("type", Integer(), autogen) is False + assert autogen.imports == set() + + +def test_a_generated_migration_creates_the_indexes(): + index = next(i for i in Person.__table__.indexes if i.name == "ix_people_email_eq") + + code = migration_code(ops.CreateIndexOp.from_index(index)) + + assert "op.create_index('ix_people_email_eq', 'people'" in code + assert "eql_v4.eq_term(email)" in code + assert "postgresql_using='hash'" in code + + +def test_the_generated_code_runs_with_those_imports(): + code = migration_code(ops.CreateTableOp.from_table(Person.__table__)) + calls = [] + namespace = {"op": type("Op", (), {"create_table": staticmethod(lambda *a, **k: calls.append(a))})} + exec("import sqlalchemy as sa\nimport stackencrypt\nimport stackencrypt.sqlalchemy\n", namespace) + + exec("def upgrade():\n" + code + "\nupgrade()\n", namespace) + + [(name, *columns)] = calls + assert name == "people" + email = next(c for c in columns if getattr(c, "name", None) == "email") + assert email.type.marker is TextEq + + +# process_revision_directives: a type change to an EQL type. + + +def script_with(*operations): + return ops.MigrationScript( + rev_id="1", + upgrade_ops=ops.UpgradeOps(ops=[ops.ModifyTableOps("accounts", list(operations))]), + downgrade_ops=ops.DowngradeOps(ops=[]), + ) + + +def rewrite(*operations): + script = script_with(*operations) + context = MigrationContext.configure(dialect_name="postgresql") + process_revision_directives(context, ("head",), [script]) + return script.upgrade_ops.ops[0].ops + + +def test_a_type_change_to_an_eql_type_becomes_an_alter_with_a_using_cast(): + [statement] = rewrite( + ops.AlterColumnOp("accounts", "email", existing_type=Text(), modify_type=EqlType(TextEq)) + ) + + assert isinstance(statement, ops.ExecuteSQLOp) + assert statement.sqltext == ( + "ALTER TABLE accounts ALTER COLUMN email TYPE public.eql_v4_text_eq " + "USING email::jsonb::public.eql_v4_text_eq" + ) + + +def test_the_table_schema_and_quoted_names_are_kept(): + script = ops.MigrationScript( + rev_id="1", + upgrade_ops=ops.UpgradeOps( + ops=[ + ops.ModifyTableOps( + "Accounts", + [ + ops.AlterColumnOp( + "Accounts", + "Email", + schema="app", + existing_type=Text(), + modify_type=EqlType(TextEq), + ) + ], + schema="app", + ) + ] + ), + downgrade_ops=ops.DowngradeOps(ops=[]), + ) + process_revision_directives(MigrationContext.configure(dialect_name="postgresql"), ("head",), [script]) + + [statement] = script.upgrade_ops.ops[0].ops + assert statement.sqltext == ( + 'ALTER TABLE app."Accounts" ALTER COLUMN "Email" TYPE public.eql_v4_text_eq ' + 'USING "Email"::jsonb::public.eql_v4_text_eq' + ) + + +def test_other_changes_of_the_same_column_are_kept(): + statement, alter = rewrite( + ops.AlterColumnOp( + "accounts", + "email", + existing_type=Text(), + modify_type=EqlType(TextEq), + existing_nullable=True, + modify_nullable=False, + ) + ) + + assert isinstance(statement, ops.ExecuteSQLOp) + assert alter.modify_type is None + assert alter.modify_nullable is False + + +def test_a_type_change_to_another_type_is_left_to_alembic(): + alter = ops.AlterColumnOp("accounts", "plan", existing_type=Text(), modify_type=String()) + + assert rewrite(alter) == [alter] + + +def test_the_rewritten_migration_renders_as_op_execute(): + script = script_with( + ops.AlterColumnOp("accounts", "email", existing_type=Text(), modify_type=EqlType(TextEq)) + ) + process_revision_directives(MigrationContext.configure(dialect_name="postgresql"), ("head",), [script]) + + code = render_python_code(script.upgrade_ops, render_item=render_item) + + assert "op.execute('ALTER TABLE accounts ALTER COLUMN email TYPE public.eql_v4_text_eq" in code + assert "alter_column" not in code + + +def test_a_type_change_between_the_same_eql_type_is_dropped_in_both_directions(): + # As the database gives the type of an EQL column back to Alembic. + reflected = postgresql.DOMAIN("eql_v4_text_eq", JSONB(), create_type=False) + script = ops.MigrationScript( + rev_id="1", + upgrade_ops=ops.UpgradeOps( + ops=[ + ops.ModifyTableOps( + "accounts", + [ops.AlterColumnOp("accounts", "email", existing_type=reflected, modify_type=EqlType(TextEq))], + ) + ] + ), + downgrade_ops=ops.DowngradeOps( + ops=[ + ops.ModifyTableOps( + "accounts", + [ops.AlterColumnOp("accounts", "email", existing_type=EqlType(TextEq), modify_type=reflected)], + ) + ] + ), + ) + + process_revision_directives(MigrationContext.configure(dialect_name="postgresql"), ("head",), [script]) + + assert script.upgrade_ops.is_empty() + assert script.downgrade_ops.is_empty() + + +def test_a_change_to_another_eql_type_is_kept(): + reflected = postgresql.DOMAIN("eql_v4_text", JSONB(), create_type=False) + alter = ops.AlterColumnOp("accounts", "email", existing_type=reflected, modify_type=EqlType(TextEq)) + + assert rewrite(alter) == [alter] diff --git a/languages/python/tests/test_sqlalchemy_migrations_live.py b/languages/python/tests/test_sqlalchemy_migrations_live.py new file mode 100644 index 000000000..9154673f5 --- /dev/null +++ b/languages/python/tests/test_sqlalchemy_migrations_live.py @@ -0,0 +1,258 @@ +"""An Alembic migration of an existing table to encrypted columns, against +PostgreSQL with the EQL v4 bundle: ``alembic revision --autogenerate`` and +``alembic upgrade``, through an ``env.py`` that uses the two hooks of +``stackencrypt.sqlalchemy``. + +These tests need ``EQL_TEST_DATABASE_URL``, and the CipherStash credentials +for the searches. Each test makes its own schema, and drops it afterwards. +""" + +import pathlib +import sys +import textwrap +import uuid +from urllib.parse import quote + +import pytest +from _live import DATABASE_URL, LIVE, needs_database +from sqlalchemy import create_engine, text + +pytestmark = [*LIVE, needs_database] + +alembic_command = pytest.importorskip("alembic.command") +from alembic.config import Config # noqa: E402 + +MODELS = ''' +from typing import Annotated + +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + +import stackencrypt +from stackencrypt import Ignore, IntegerOrd, Passthrough, TextEq, TextMatch +from stackencrypt.sqlalchemy import eql_type_map + + +class Base(DeclarativeBase): + type_annotation_map = eql_type_map + + +@stackencrypt.model("accounts") +class Account(Base): + __tablename__ = "accounts" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] # was text: changed in place + level: Mapped[Annotated[int, IntegerOrd]] # new + bio: Mapped[Annotated[str, TextMatch]] # new + plan: Mapped[Annotated[str, Passthrough]] # unchanged; `note` is removed +''' + +# Only `email` changes, so the USING cast is the first step that meets a row. +EMAIL_ONLY = ''' +from typing import Annotated + +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + +import stackencrypt +from stackencrypt import Ignore, Passthrough, TextEq +from stackencrypt.sqlalchemy import eql_type_map + + +class Base(DeclarativeBase): + type_annotation_map = eql_type_map + + +@stackencrypt.model("accounts") +class Account(Base): + __tablename__ = "accounts" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + note: Mapped[Annotated[str, Passthrough]] = mapped_column(nullable=True) + plan: Mapped[Annotated[str, Passthrough]] +''' + +ENV = ''' +from alembic import context +from sqlalchemy import create_engine + +from models import Base +from stackencrypt.sqlalchemy import process_revision_directives, render_item + +engine = create_engine(context.config.get_main_option("sqlalchemy.url")) +with engine.connect() as connection: + context.configure( + connection=connection, + target_metadata=Base.metadata, + render_item=render_item, + process_revision_directives=process_revision_directives, + ) + with context.begin_transaction(): + context.run_migrations() +''' + +SCRIPT = '''"""${message}""" +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} +''' + + +@pytest.fixture +def project(tmp_path, monkeypatch, request): + """An Alembic project in ``tmp_path`` with the models above, on a schema + of its own that holds the existing table. A test can name other models + with ``@pytest.mark.models(...)``.""" + marker = request.node.get_closest_marker("models") + models = marker.args[0] if marker else MODELS + schema = f"mig_{uuid.uuid4().hex[:12]}" + url = DATABASE_URL.replace("postgresql://", "postgresql+psycopg://", 1) + url += ("&" if "?" in url else "?") + "options=" + quote(f"-csearch_path={schema},public") + engine = create_engine(url) + with engine.begin() as conn: + conn.execute(text(f"CREATE SCHEMA {schema}")) + conn.execute( + text( + "CREATE TABLE accounts (id serial PRIMARY KEY, email text NOT NULL, " + "note text, plan varchar NOT NULL)" + ) + ) + (tmp_path / "versions").mkdir() + (tmp_path / "models.py").write_text(textwrap.dedent(models)) + (tmp_path / "env.py").write_text(textwrap.dedent(ENV)) + (tmp_path / "script.py.mako").write_text(SCRIPT) + monkeypatch.syspath_prepend(str(tmp_path)) + # env.py imports `models`: each test needs its own file, not the one an + # earlier test imported. + monkeypatch.delitem(sys.modules, "models", raising=False) + config = Config() + config.set_main_option("script_location", str(tmp_path)) + config.set_main_option("sqlalchemy.url", url.replace("%", "%%")) + yield config, engine + with engine.begin() as conn: + conn.execute(text(f"DROP SCHEMA {schema} CASCADE")) + engine.dispose() + + +def generate(config, message): + script = alembic_command.revision(config, message=message, autogenerate=True) + return pathlib.Path(script.path).read_text() + + +def body(migration, function): + """The lines of ``function`` in a migration file, comments left out.""" + lines = migration.split(f"def {function}():\n", 1)[1].split("\n\n\ndef ", 1)[0] + return [line.strip() for line in lines.splitlines() if line.strip() and not line.strip().startswith("#")] + + +def test_autogenerate_writes_the_columns_the_indexes_and_the_using_cast(project): + config, _ = project + + migration = generate(config, "encrypt accounts") + + assert "import stackencrypt.sqlalchemy" in migration + assert ( + "ALTER TABLE accounts ALTER COLUMN email TYPE public.eql_v4_text_eq " + "USING email::jsonb::public.eql_v4_text_eq" + ) in migration + assert "stackencrypt.sqlalchemy.EqlType(stackencrypt.IntegerOrd)" in migration + assert "stackencrypt.sqlalchemy.EqlType(stackencrypt.TextMatch)" in migration + assert "op.drop_column('accounts', 'note')" in migration + for index in ("ix_accounts_email_eq", "ix_accounts_level_ope", "ix_accounts_bio_match"): + assert f"op.create_index('{index}'" in migration + + +def test_the_upgrade_makes_the_columns_and_indexes_and_a_second_autogenerate_finds_nothing(project): + config, engine = project + generate(config, "encrypt accounts") + + alembic_command.upgrade(config, "head") + + with engine.connect() as conn: + columns = dict( + conn.execute( + text( + "SELECT attname, format_type(atttypid, atttypmod) FROM pg_attribute " + "WHERE attrelid = 'accounts'::regclass AND attnum > 0 AND NOT attisdropped" + ) + ).all() + ) + indexes = dict( + conn.execute( + text("SELECT indexname, indexdef FROM pg_indexes WHERE tablename = 'accounts'") + ).all() + ) + assert columns == { + "id": "integer", + "email": "eql_v4_text_eq", + "level": "eql_v4_integer_ord", + "bio": "eql_v4_text_match", + "plan": "character varying", + } + assert "USING hash (eql_v4.eq_term(email))" in indexes["ix_accounts_email_eq"] + assert "USING btree (eql_v4.ord_term(level))" in indexes["ix_accounts_level_ope"] + assert "USING gin (eql_v4.match_term(bio))" in indexes["ix_accounts_bio_match"] + + second = generate(config, "nothing to do") + assert body(second, "upgrade") == ["pass"] + assert body(second, "downgrade") == ["pass"] + + +@pytest.mark.models(EMAIL_ONLY) +def test_the_upgrade_refuses_a_column_with_plaintext_rows_and_keeps_them(project): + config, engine = project + with engine.begin() as conn: + conn.execute(text("INSERT INTO accounts (email, plan) VALUES ('a@example.com', 'free')")) + migration = generate(config, "encrypt email") + assert body(migration, "upgrade")[0].startswith("op.execute('ALTER TABLE accounts ALTER COLUMN email") + + with pytest.raises(Exception) as raised: + alembic_command.upgrade(config, "head") + + assert "invalid input syntax for type json" in str(raised.value) + with engine.connect() as conn: + assert conn.execute(text("SELECT email, plan FROM accounts")).all() == [("a@example.com", "free")] + + +def test_postgresql_uses_each_index_for_its_search(project): + config, engine = project + generate(config, "encrypt accounts") + alembic_command.upgrade(config, "head") + import models + + from stackencrypt import Cipher + + accounts = Cipher(models.Account) + searches = { + "ix_accounts_email_eq": ( + "email = CAST(:term AS eql_v4.query_text_eq)", + accounts.eq("email", "a@example.com"), + ), + "ix_accounts_level_ope": ( + "level >= CAST(:term AS eql_v4.query_integer_ord)", + accounts.gte("level", 3), + ), + "ix_accounts_bio_match": ( + "bio @@ CAST(:term AS eql_v4.query_text_match)", + accounts.matches("bio", "hello").term, + ), + } + with engine.begin() as conn: + # With no rows, PostgreSQL would read the table rather than an index. + conn.execute(text("SET LOCAL enable_seqscan = off")) + for index, (where, term) in searches.items(): + plan = conn.execute( + text(f"EXPLAIN SELECT id FROM accounts WHERE {where}"), {"term": term} + ).scalars().all() + assert any(index in line for line in plan), (index, plan) diff --git a/languages/python/uv.lock b/languages/python/uv.lock index ecac46d56..e00021c0e 100644 --- a/languages/python/uv.lock +++ b/languages/python/uv.lock @@ -7,6 +7,44 @@ resolution-markers = [ "python_full_version < '3.10'", ] +[[package]] +name = "alembic" +version = "1.16.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "mako", version = "1.3.12", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "sqlalchemy", version = "2.0.57", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "typing-extensions", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/ca/4dc52902cf3491892d464f5265a81e9dff094692c8a049a3ed6a05fe7ee8/alembic-1.16.5.tar.gz", hash = "sha256:a88bb7f6e513bd4301ecf4c7f2206fe93f9913f9b48dac3b78babde2d6fe765e", size = 1969868, upload-time = "2025-08-27T18:02:05.668Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/4a/4c61d4c84cfd9befb6fa08a702535b27b21fff08c946bc2f6139decbf7f7/alembic-1.16.5-py3-none-any.whl", hash = "sha256:e845dfe090c5ffa7b92593ae6687c5cb1a101e91fa53868497dbd79847f9dbe3", size = 247355, upload-time = "2025-08-27T18:02:07.37Z" }, +] + +[[package]] +name = "alembic" +version = "1.20.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "mako", version = "1.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sqlalchemy", version = "2.0.57", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "sqlalchemy", version = "2.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/1c/00909e0fbebad2121d916fdcab044cae988146680e9ee4387ba25fef8747/alembic-1.20.1.tar.gz", hash = "sha256:83490def1f723413db5096591f7609ac52544805be93849170088ed04f5a2e0b", size = 2103576, upload-time = "2026-10-22T03:24:35.93Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/3c/8740f5b4e1bd57333e4f84265b9b6363da4f7cf70f5622dd1e2fde7a0bd5/alembic-1.20.1-py3-none-any.whl", hash = "sha256:9970712cfb81ea26c8658d378aaa5b8f6ff9146e5bb111d70497c164016cf598", size = 270355, upload-time = "2026-10-22T03:24:37.667Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -55,7 +93,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -233,6 +271,195 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, ] +[[package]] +name = "mako" +version = "1.3.12" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "markupsafe", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/62/791b31e69ae182791ec67f04850f2f062716bbd205483d63a215f3e062d3/mako-1.3.12.tar.gz", hash = "sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a", size = 400219, upload-time = "2026-04-28T19:01:08.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, +] + +[[package]] +name = "mako" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "markupsafe", marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/09/e07c4b5579a79f4b16f8d4f29f6c54514ac787c4ad506b8c4f28a0e6b0bf/mako-1.4.3.tar.gz", hash = "sha256:cd6537fe88d5fec315c55c2f8529bc4ce7a9a352ad7db3eeaa6a66e2dd4ec37a", size = 412799, upload-time = "2026-09-22T20:54:31.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/a0/053d6af3e8f871e0073b4a36732d9e65be77a72e5434c31b94f6af78a6bb/mako-1.4.3-py3-none-any.whl", hash = "sha256:723296007c870bfd6b3f0c3230dba7198096e5269297ebf5e4eff9e7ffa39d4f", size = 80164, upload-time = "2026-09-22T20:54:33.128Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/aa/9a65962e364bf19745f6bec7bde398fb1f5ca53ad2e734258e6198cc32ba/markupsafe-3.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889", size = 11521, upload-time = "2026-10-02T23:04:20.141Z" }, + { url = "https://files.pythonhosted.org/packages/14/36/927999a34b7d1def6957de89153d327fedc4030061940b5a468584e6c5a8/markupsafe-3.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2", size = 12066, upload-time = "2026-10-02T23:04:21.275Z" }, + { url = "https://files.pythonhosted.org/packages/17/54/69e7b0db9bd687bfd83451eed5666384d67cfed3be062bfc59a06a15474e/markupsafe-3.0.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a", size = 22081, upload-time = "2026-10-02T23:04:22.221Z" }, + { url = "https://files.pythonhosted.org/packages/bd/14/f6f5c97903f7d2db76bbfaced31509a47d360a103b3aaf4849f6536591ec/markupsafe-3.0.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc", size = 26079, upload-time = "2026-10-02T23:04:23.346Z" }, + { url = "https://files.pythonhosted.org/packages/d0/04/c3cc9b75f94f8b54d7e503c44cebd4b4a115ec1d6f1b996b999807daba99/markupsafe-3.0.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8", size = 22611, upload-time = "2026-10-02T23:04:24.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/26/c4708ed3b0f08e8e6d7cbce3314ac130cb5352d1f62951b6fef7878f2b1e/markupsafe-3.0.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9", size = 20712, upload-time = "2026-10-02T23:04:25.386Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b9/f3894d6aae3d7a52c9363317f4baf2fcadc052163d6dbc871266e32639ed/markupsafe-3.0.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a", size = 20742, upload-time = "2026-10-02T23:04:26.403Z" }, + { url = "https://files.pythonhosted.org/packages/59/7c/8e248ddbfe286ab6bddb462bdac0851cbb1510d2cbbb815a415fcb0511af/markupsafe-3.0.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36", size = 21560, upload-time = "2026-10-02T23:04:27.522Z" }, + { url = "https://files.pythonhosted.org/packages/df/26/2353fef7d4bcf2b18e16bad81979fcecff2915156ca6882447917205b8e2/markupsafe-3.0.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be", size = 20973, upload-time = "2026-10-02T23:04:28.573Z" }, + { url = "https://files.pythonhosted.org/packages/67/6f/a9561d98d9a6ee3494b0b970a1c766e58bab128cc84841d56ec009456dbd/markupsafe-3.0.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa", size = 22036, upload-time = "2026-10-02T23:04:29.568Z" }, + { url = "https://files.pythonhosted.org/packages/55/83/6217df9192eca95af3ff0cad955c9854a93fa43b288cf7411ae24713eb52/markupsafe-3.0.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9", size = 20314, upload-time = "2026-10-02T23:04:30.881Z" }, + { url = "https://files.pythonhosted.org/packages/35/7c/9cd8081dae70e17fdab3558121fa618d459c7950c8abd8fc10fc024486d7/markupsafe-3.0.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a", size = 20780, upload-time = "2026-10-02T23:04:31.975Z" }, + { url = "https://files.pythonhosted.org/packages/44/37/c5f2f45f4d8f0c24e4af7b9dc711640c888044cbf5f7e2ba2c0bb74cbb55/markupsafe-3.0.4-cp310-cp310-win32.whl", hash = "sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278", size = 14096, upload-time = "2026-10-02T23:04:33.129Z" }, + { url = "https://files.pythonhosted.org/packages/50/50/394a1c61c9b9972cb4f76875c332af7109bd39bdad09d3a0ec5327a339dc/markupsafe-3.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7", size = 14329, upload-time = "2026-10-02T23:04:34.168Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9a/86d03d2f32ba41a57baa124a7bcb76d0a6d39b1622e9bd4d550074e8b61a/markupsafe-3.0.4-cp310-cp310-win_arm64.whl", hash = "sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf", size = 14196, upload-time = "2026-10-02T23:04:35.201Z" }, + { url = "https://files.pythonhosted.org/packages/32/55/18dbb4778b30ada5ce071608503cc3edc9e14e13d868c17a6d178fc30f7a/markupsafe-3.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346", size = 11520, upload-time = "2026-10-02T23:04:36.299Z" }, + { url = "https://files.pythonhosted.org/packages/6c/14/0b05f79b4733e264a18d08fe08fa1df7347630ff32a6cb82180d9dccec55/markupsafe-3.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91", size = 12063, upload-time = "2026-10-02T23:04:37.291Z" }, + { url = "https://files.pythonhosted.org/packages/ca/3a/63ba10b6c1463216b3e4df669a9f0e5a3b0c3071557d2e8229e3968c79fb/markupsafe-3.0.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef", size = 24321, upload-time = "2026-10-02T23:04:38.262Z" }, + { url = "https://files.pythonhosted.org/packages/1a/2e/5f015261b76ad633d187ef6f388b413aedd64a8773c4df59e530a0be5525/markupsafe-3.0.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169", size = 28845, upload-time = "2026-10-02T23:04:39.295Z" }, + { url = "https://files.pythonhosted.org/packages/33/cf/26e594b26be40c2f1fec63ccf8a8b99d0335a5b2cbe84835c7d82a994375/markupsafe-3.0.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb", size = 24701, upload-time = "2026-10-02T23:04:40.348Z" }, + { url = "https://files.pythonhosted.org/packages/81/a5/a513b76c139a3915b43404324e55c0b7979ae4f0d39eb6f075b0282e90a8/markupsafe-3.0.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808", size = 22961, upload-time = "2026-10-02T23:04:41.372Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/4c66192c100b4542bcbe392ae06696b670f66927be3ac38a213234778ff9/markupsafe-3.0.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692", size = 21920, upload-time = "2026-10-02T23:04:42.429Z" }, + { url = "https://files.pythonhosted.org/packages/cb/17/ac3662678bfbad649893117ada2ba44dc30bf56884e84f13154a792b10f1/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d", size = 23726, upload-time = "2026-10-02T23:04:43.531Z" }, + { url = "https://files.pythonhosted.org/packages/f7/af/fe47cee339180a69ebca3c57fb3483d0f5cbd1e8337d1871fb1d9c1aebee/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21", size = 22539, upload-time = "2026-10-02T23:04:44.618Z" }, + { url = "https://files.pythonhosted.org/packages/12/32/d55440ba140442800e02d799c9cb5ab597bf6ebdb1177b5ea39a11f797bd/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707", size = 23963, upload-time = "2026-10-02T23:04:45.656Z" }, + { url = "https://files.pythonhosted.org/packages/50/9d/9c86042cb364c2ad4c971e6d1247929effd25f714cd7ee11b05e6316445b/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e", size = 21506, upload-time = "2026-10-02T23:04:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/75/ef/5b824f03ba40c3b3652b6272d083d2fc4fcdd440de519a3a39ba2c3e7262/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7", size = 22955, upload-time = "2026-10-02T23:04:47.698Z" }, + { url = "https://files.pythonhosted.org/packages/1e/e8/44cfcb5ea40e5e43cec7793ef90704ed0c475280839076c4345757eb8e59/markupsafe-3.0.4-cp311-cp311-win32.whl", hash = "sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5", size = 14094, upload-time = "2026-10-02T23:04:48.788Z" }, + { url = "https://files.pythonhosted.org/packages/91/89/f2b509f7bf79352e40117824c1070dbeafd4df67031d3fa98165a3134228/markupsafe-3.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3", size = 14328, upload-time = "2026-10-02T23:04:49.801Z" }, + { url = "https://files.pythonhosted.org/packages/2b/5a/ccf22672a0f64dc682306e288f0dabcb06c7a201f3bb6e6cbf86d9e8ad03/markupsafe-3.0.4-cp311-cp311-win_arm64.whl", hash = "sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e", size = 14188, upload-time = "2026-10-02T23:04:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/81/09/4c59d56b8461ae8eb0d8ba34bb25b7e618547044679d58a82ef9b2479fc1/markupsafe-3.0.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6", size = 11658, upload-time = "2026-10-02T23:04:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f0/d6613774d86fbf6d145751d43c59875e47a6f9f17daee0aef173bd36d90e/markupsafe-3.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f", size = 12050, upload-time = "2026-10-02T23:04:52.931Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f2/8f18e0b806eb13c1f8d07d917a720831ead54253a6dec011fbc78098a6f8/markupsafe-3.0.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b", size = 24363, upload-time = "2026-10-02T23:04:53.895Z" }, + { url = "https://files.pythonhosted.org/packages/60/ce/fa07dbe8a5675558fa36dea033e19995bc783de2dec5f540ccb9030b06aa/markupsafe-3.0.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df", size = 28525, upload-time = "2026-10-02T23:04:54.905Z" }, + { url = "https://files.pythonhosted.org/packages/85/40/be87c01f3868ec217f8a2015089d71c22c8c5a75324822e5ed1cdd87210d/markupsafe-3.0.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c", size = 24733, upload-time = "2026-10-02T23:04:56.229Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/aeedb5140afa41fc74c225e9184ab96723a6e873b6ee1c9fede7283456d8/markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581", size = 22985, upload-time = "2026-10-02T23:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fc/e91352bb08c6a59da3ef0909d457bf95a5f5908fbf151b30a06d9dbcfbb4/markupsafe-3.0.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77", size = 22001, upload-time = "2026-10-02T23:04:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f8/bffee5e7d2a3deb59748a797650a48af7e672025cf641a79344a771ad106/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c", size = 23793, upload-time = "2026-10-02T23:04:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/ed/59/b853d6628ecb4d658e1d637224846d5e9bb4adf4f8df97f3be9f29dce2ec/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749", size = 22640, upload-time = "2026-10-02T23:05:00.768Z" }, + { url = "https://files.pythonhosted.org/packages/09/b2/1506df394f0f075797c418d0301498f49e43be194e3ffcb49e6fe6ccf022/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed", size = 24074, upload-time = "2026-10-02T23:05:01.813Z" }, + { url = "https://files.pythonhosted.org/packages/c7/81/5ed69cda630ac69ef60d06c09ba5a7f84ff66a2e28cf986fd5614ab3c6e6/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786", size = 21563, upload-time = "2026-10-02T23:05:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fe/fb1e79be0fea60aa32602ebefc9c35a82bb42b4df157285ab7dfec12341a/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e", size = 23048, upload-time = "2026-10-02T23:05:04.479Z" }, + { url = "https://files.pythonhosted.org/packages/c8/52/7632a53360671a9b750cdbabaf9cdd89f18b42248b8e4cb42c0b0296e459/markupsafe-3.0.4-cp312-cp312-win32.whl", hash = "sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237", size = 14108, upload-time = "2026-10-02T23:05:05.513Z" }, + { url = "https://files.pythonhosted.org/packages/3f/bf/62495e180b7000aaf30000fff849e933f74264638057176cf46852500adc/markupsafe-3.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7", size = 14303, upload-time = "2026-10-02T23:05:06.538Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/4c24208776a65878d656996945aacfbfe010d3720d1a98fc0eb8491fc03b/markupsafe-3.0.4-cp312-cp312-win_arm64.whl", hash = "sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9", size = 14174, upload-time = "2026-10-02T23:05:07.617Z" }, + { url = "https://files.pythonhosted.org/packages/6d/18/4bc5ba32499e87bb2b0ef5b3a9bb9c00a131fa961ddf0be548cb550f548b/markupsafe-3.0.4-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1", size = 12786, upload-time = "2026-10-02T23:05:08.709Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6f/17f0c099bf25f3e31e63cc19244d9f6af861a9a4ab778c203997903cfdd0/markupsafe-3.0.4-cp313-cp313-android_24_x86_64.whl", hash = "sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1", size = 12510, upload-time = "2026-10-02T23:05:09.93Z" }, + { url = "https://files.pythonhosted.org/packages/11/af/1a141081b905036ee904ec4bd945e1f70b4e1b32d33c4e59e8cf1d58b247/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96", size = 11583, upload-time = "2026-10-02T23:05:10.884Z" }, + { url = "https://files.pythonhosted.org/packages/e7/0a/a89385ae590232622a03e091805cff12f24fabe6c11e0e8bae096cece81c/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148", size = 11801, upload-time = "2026-10-02T23:05:11.913Z" }, + { url = "https://files.pythonhosted.org/packages/ed/85/ea548dc013962eb73653124bc595635fbf9e0fa41d1f181a967ccb784dfb/markupsafe-3.0.4-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e", size = 11443, upload-time = "2026-10-02T23:05:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/cc/72/15f2e5ec9cf2eb00d5cdfe968d94e4156a7bd7303832c3f3b2c403a36839/markupsafe-3.0.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248", size = 11665, upload-time = "2026-10-02T23:05:13.829Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e0/4030bea613677e333c8a2c901fd405055f657f9d06acba5b7357984b6ef7/markupsafe-3.0.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72", size = 12061, upload-time = "2026-10-02T23:05:14.807Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a5/28b76a7449eb702966b88bef599e2360b411fbb3afeee8fe560939be06ec/markupsafe-3.0.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2", size = 24395, upload-time = "2026-10-02T23:05:15.909Z" }, + { url = "https://files.pythonhosted.org/packages/07/6c/21232811afc3a063b5e934b1ae2efda52f46154ec382f585149c020e61fe/markupsafe-3.0.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85", size = 27183, upload-time = "2026-10-02T23:05:16.976Z" }, + { url = "https://files.pythonhosted.org/packages/14/38/6ccdfa5b59049cb36fb80cbc80aee9cf1fc9bb77d1335ad435f2070b08cf/markupsafe-3.0.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde", size = 24772, upload-time = "2026-10-02T23:05:18.209Z" }, + { url = "https://files.pythonhosted.org/packages/63/e0/cec6865dfe88cb48fedd4b20aed6af5158e41092adcbf3e028bcc6ec2108/markupsafe-3.0.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6", size = 23017, upload-time = "2026-10-02T23:05:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/ee/76/6ed4940bb7648a9aac457c14f870cfdd5105f139a0fb1f29cd61fafa47d1/markupsafe-3.0.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f", size = 22036, upload-time = "2026-10-02T23:05:20.352Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4f/ed476226d4fe46a09090a36025bf319296810028df55eb12f1253b540f3a/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39", size = 23786, upload-time = "2026-10-02T23:05:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/9a/35/66ff30450e35ef5fba9ebc930c9411747e537fd9447b65e44f5007e2b84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee", size = 22672, upload-time = "2026-10-02T23:05:22.922Z" }, + { url = "https://files.pythonhosted.org/packages/32/0b/72f45ce4b4efcbca4b80cf1b06703eff0be8d37e82abb78f66c85a7ead1e/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2", size = 24107, upload-time = "2026-10-02T23:05:24.175Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/71776e5fdcba04614b384cc102e8a4198208579d896fd1394cb7cb9aa900/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46", size = 21615, upload-time = "2026-10-02T23:05:25.215Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/801ce02a02e7aee0f784b1ec7843026178f6adeb9c93ac67eb1992a9a84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17", size = 23076, upload-time = "2026-10-02T23:05:26.423Z" }, + { url = "https://files.pythonhosted.org/packages/4a/85/c43776625428f3bb4a61e8633940400e3efe6409e3c6f5bff26de5e45618/markupsafe-3.0.4-cp313-cp313-win32.whl", hash = "sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0", size = 14118, upload-time = "2026-10-02T23:05:27.716Z" }, + { url = "https://files.pythonhosted.org/packages/6f/36/163da64de88a13db79214ef75fa041be7fa13bdb42261cf5b7484de14bfb/markupsafe-3.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5", size = 14313, upload-time = "2026-10-02T23:05:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a8/9b662783ffaa1149221432a923cee562f78b9cbbb8baa3df9b3753e63e1e/markupsafe-3.0.4-cp313-cp313-win_arm64.whl", hash = "sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc", size = 14189, upload-time = "2026-10-02T23:05:29.917Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c3/a944f3b0df22bd129e96915b9f4e98d2eeca6516687d7618304a966c3c74/markupsafe-3.0.4-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed", size = 12787, upload-time = "2026-10-02T23:05:30.971Z" }, + { url = "https://files.pythonhosted.org/packages/d4/d6/a44863f69d88b6c7e27889108f70d47aed259edf89d5df3c5fca1eac87d6/markupsafe-3.0.4-cp314-cp314-android_24_x86_64.whl", hash = "sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59", size = 12525, upload-time = "2026-10-02T23:05:32.263Z" }, + { url = "https://files.pythonhosted.org/packages/17/8f/168ba80e532dd6a93f96f8f706f1ad41d7990b6e1aeedc1cc0d211a33497/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453", size = 11589, upload-time = "2026-10-02T23:05:33.251Z" }, + { url = "https://files.pythonhosted.org/packages/32/b3/aa2c95a574d3af39403a469b295886eb9b6d448da568cbebb5a2cbfdc2e5/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b", size = 11801, upload-time = "2026-10-02T23:05:34.315Z" }, + { url = "https://files.pythonhosted.org/packages/60/d0/34b810107d83840e768bf485de795893ebbae35b26ab061b487adfa0a692/markupsafe-3.0.4-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6", size = 11447, upload-time = "2026-10-02T23:05:35.302Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ab/2f8488f0f817a39fca068d2b17daf446bf5cdb3eae28c3720af534d873b4/markupsafe-3.0.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634", size = 11722, upload-time = "2026-10-02T23:05:36.363Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/e2d117b048d47282ade906fbfd92814cbee5647afc13fda88a3406039372/markupsafe-3.0.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f", size = 12061, upload-time = "2026-10-02T23:05:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a8/73a81135e85ba66217f5af7facb03bbb386807e1a729ab64532e4c802652/markupsafe-3.0.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9", size = 24434, upload-time = "2026-10-02T23:05:38.407Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ca/fa9216dd01efee2dfdacafe7df32b4d0170fbac694b0c258a193d6e53999/markupsafe-3.0.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f", size = 27183, upload-time = "2026-10-02T23:05:39.581Z" }, + { url = "https://files.pythonhosted.org/packages/fa/4e/a469509e538d37af51103b17b073126973f2b1cbf197ff32c7ddf025cfe5/markupsafe-3.0.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c", size = 24813, upload-time = "2026-10-02T23:05:40.671Z" }, + { url = "https://files.pythonhosted.org/packages/8f/db/d7282caf7ab03af44d5d6fdbaa019b35c7d7f1c90588b839c07cba640d6a/markupsafe-3.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300", size = 23049, upload-time = "2026-10-02T23:05:41.864Z" }, + { url = "https://files.pythonhosted.org/packages/30/f3/b6a425206e6964efda6acee544d0eb01d1501784d0b8e2dcc74986f33b17/markupsafe-3.0.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0", size = 22088, upload-time = "2026-10-02T23:05:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8a/84d3582fc1f0d5bd466cdf2eebf175e172158a6e70701aacec1de1b35430/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977", size = 23846, upload-time = "2026-10-02T23:05:44.098Z" }, + { url = "https://files.pythonhosted.org/packages/1c/65/db101cce51b7ba4864ac491a9859d297dd1adf0e55b103fee9db9c47c527/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7", size = 22674, upload-time = "2026-10-02T23:05:45.23Z" }, + { url = "https://files.pythonhosted.org/packages/e0/49/ddee9813d71db0c7a5c9d97c832125e6758a0c844777f1cf076569bb0e22/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17", size = 24104, upload-time = "2026-10-02T23:05:46.398Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0e/7d8518d726726870a2399d69fd30d0fa36c5e57a2132c336b58d7c491073/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c", size = 21637, upload-time = "2026-10-02T23:05:47.48Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b0/b505e8a361ba557dbf3b3aa7331ea39b00d2022a26e925ff8463b9714bb3/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4", size = 23081, upload-time = "2026-10-02T23:05:48.611Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ea/9cc3cea873f980c75cbdb6f4277ce30ee955de38be0b3d02f14c108e0698/markupsafe-3.0.4-cp314-cp314-win32.whl", hash = "sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c", size = 14272, upload-time = "2026-10-02T23:05:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/80/f0/5792ff768a410f93ee3f84fc19345295ffc352d2c936b424cb37e514714c/markupsafe-3.0.4-cp314-cp314-win_amd64.whl", hash = "sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe", size = 14501, upload-time = "2026-10-02T23:05:50.788Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cf/3d074a8edffcc6899355232ff2543ae8d929733239596423b7db79698bc9/markupsafe-3.0.4-cp314-cp314-win_arm64.whl", hash = "sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a", size = 14364, upload-time = "2026-10-02T23:05:51.857Z" }, + { url = "https://files.pythonhosted.org/packages/d9/31/87ce42159aae2163cf3bbbd0c44bc87780510eecab1ea3859099aed95dcb/markupsafe-3.0.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2", size = 11759, upload-time = "2026-10-02T23:05:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/5f/53/b047207eeb7752e960aca3eb1df5fb7eefa7dd4c62ac49bb156456c8a702/markupsafe-3.0.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977", size = 12096, upload-time = "2026-10-02T23:05:54.066Z" }, + { url = "https://files.pythonhosted.org/packages/ee/51/4326c88a13c7b755657d44b4bb986f8c3d9843ecba7e22d98661d87f9a57/markupsafe-3.0.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289", size = 25699, upload-time = "2026-10-02T23:05:55.15Z" }, + { url = "https://files.pythonhosted.org/packages/f2/bb/990581b7474bfcf2cf34bed6ba5ea23bd87adb9d671213d68e88620e7a6b/markupsafe-3.0.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe", size = 27804, upload-time = "2026-10-02T23:05:56.29Z" }, + { url = "https://files.pythonhosted.org/packages/6b/89/89491878c28e8291f5aa2fffe2c2d57230d10ae366d55dd810b840513d78/markupsafe-3.0.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a", size = 25912, upload-time = "2026-10-02T23:05:57.416Z" }, + { url = "https://files.pythonhosted.org/packages/30/77/680998b54efdea06fc114565cd739b6d059f826a0279219b218dfa750d29/markupsafe-3.0.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733", size = 23623, upload-time = "2026-10-02T23:05:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/ae/75/2709f5ac5de9467b40b10e2bb8f89cc63dfb74582e09aa734b1124a217de/markupsafe-3.0.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34", size = 23301, upload-time = "2026-10-02T23:05:59.94Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c8/39eadc6c5b14c9c7679bfb98f4d4c6a97863b5beb91839aca4d2d6e16e55/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978", size = 24950, upload-time = "2026-10-02T23:06:01.289Z" }, + { url = "https://files.pythonhosted.org/packages/1a/5e/01037f8a43e8ccb0bffb4fbdc5212db05bf080fdd7286cd392332d58128a/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc", size = 23330, upload-time = "2026-10-02T23:06:02.441Z" }, + { url = "https://files.pythonhosted.org/packages/d4/f4/23e83ce0596bb0cbe670502d31df8f757bbd01a392aa486fa3b40d1ed399/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc", size = 25239, upload-time = "2026-10-02T23:06:03.579Z" }, + { url = "https://files.pythonhosted.org/packages/88/5b/3708897368073cc683d524750474f41a77d2986152c380dcc55b20fdf340/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932", size = 22767, upload-time = "2026-10-02T23:06:04.699Z" }, + { url = "https://files.pythonhosted.org/packages/c6/61/ebda1307864b409e6b3115757a3d4a09cca46cfb6cc65191b5de226b424b/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6", size = 23668, upload-time = "2026-10-02T23:06:05.9Z" }, + { url = "https://files.pythonhosted.org/packages/09/15/98075cceac3b5ba0dbb8e4762a847be967d2befc349a2cf2d0ac77f62c9d/markupsafe-3.0.4-cp314-cp314t-win32.whl", hash = "sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691", size = 14301, upload-time = "2026-10-02T23:06:07.109Z" }, + { url = "https://files.pythonhosted.org/packages/0b/a3/768b560fcc4156685cb563d922b217810cfa7bc135773367f62f1f9d2078/markupsafe-3.0.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464", size = 14526, upload-time = "2026-10-02T23:06:08.276Z" }, + { url = "https://files.pythonhosted.org/packages/93/63/da554b4c97a6b0ea3229ca7fe8cbfb620be81613d517f482e85958550537/markupsafe-3.0.4-cp314-cp314t-win_arm64.whl", hash = "sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c", size = 14421, upload-time = "2026-10-02T23:06:09.402Z" }, + { url = "https://files.pythonhosted.org/packages/a9/30/54d11c8ca027114898cab97421fb39e4ffd9ddf47cdbc44df2ec76722da9/markupsafe-3.0.4-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65", size = 12791, upload-time = "2026-10-02T23:06:10.485Z" }, + { url = "https://files.pythonhosted.org/packages/10/6d/97c913e253a14bd3cd0e15a5c56d13203b823fa7ee32498342896a072dc4/markupsafe-3.0.4-cp315-cp315-android_24_x86_64.whl", hash = "sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163", size = 12527, upload-time = "2026-10-02T23:06:11.834Z" }, + { url = "https://files.pythonhosted.org/packages/26/f9/b86d032042a4d597d9e1997f0e5f63a3eedaf11258e0a05760b0a0a826ea/markupsafe-3.0.4-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92", size = 11602, upload-time = "2026-10-02T23:06:13.122Z" }, + { url = "https://files.pythonhosted.org/packages/f2/dc/73c14c1eedf0ac5fa3292ba43435e6c49d2c2050f33cebde541f8f4807f1/markupsafe-3.0.4-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a", size = 11817, upload-time = "2026-10-02T23:06:14.227Z" }, + { url = "https://files.pythonhosted.org/packages/8f/69/2c2fcaa5fcee22d72c7819c0d536fd181c74a688e6143845419579cd2863/markupsafe-3.0.4-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429", size = 11446, upload-time = "2026-10-02T23:06:15.574Z" }, + { url = "https://files.pythonhosted.org/packages/88/54/9e5ec76c62e6e2834d5a93623018c943e8b3bb41d663e3fd4c03303b9b85/markupsafe-3.0.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8", size = 11719, upload-time = "2026-10-02T23:06:16.701Z" }, + { url = "https://files.pythonhosted.org/packages/96/24/3ec292b44064c16229e064d770b2625bd8ea941aa61f44905a9fa44942c0/markupsafe-3.0.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97", size = 12075, upload-time = "2026-10-02T23:06:17.855Z" }, + { url = "https://files.pythonhosted.org/packages/aa/85/b64fdb1f304848518742136983c24e96d967bfb59a0ea160e92736901ab0/markupsafe-3.0.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b", size = 24656, upload-time = "2026-10-02T23:06:18.963Z" }, + { url = "https://files.pythonhosted.org/packages/9c/18/23997d4c65b355da6390d61cd56e0ab3befd6ba8dda25cb40c602bd0fa6b/markupsafe-3.0.4-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9", size = 27737, upload-time = "2026-10-02T23:06:20.117Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/35998dead3c6af88c38265a56e58100211f036234ab88eb2283fd4cbce44/markupsafe-3.0.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653", size = 25003, upload-time = "2026-10-02T23:06:21.284Z" }, + { url = "https://files.pythonhosted.org/packages/82/96/ef49135ce260db4ca4a12b119ed468449cd248db6b1468e2112b546d7a2e/markupsafe-3.0.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369", size = 23111, upload-time = "2026-10-02T23:06:22.524Z" }, + { url = "https://files.pythonhosted.org/packages/50/7d/83126e338bd88c17a220668235368ad719fd4638e426739858cbb8508f77/markupsafe-3.0.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19", size = 22403, upload-time = "2026-10-02T23:06:23.785Z" }, + { url = "https://files.pythonhosted.org/packages/83/dd/daf7e420de23c8206c365204e7b85e1251d8e19d34196a56336f316e5ed2/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e", size = 24083, upload-time = "2026-10-02T23:06:25.037Z" }, + { url = "https://files.pythonhosted.org/packages/19/3c/11eecdc06bc44ad5570350085b572ebf049e8f9a38d1ece6d76640b739cd/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811", size = 22931, upload-time = "2026-10-02T23:06:26.328Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9e/ac0fd77f2a726e56ecc3ca0235d095feace1358d1b822406c2a2ef26a4dc/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea", size = 24299, upload-time = "2026-10-02T23:06:27.742Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/c6bd842ad58ff5b3bc76eeed7e9a42a6f11adc5d090ec697b72c9672731e/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916", size = 21948, upload-time = "2026-10-02T23:06:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/a3/46/82f586711fed61e86faa1ee1bc317d68cd45a10c8bdbe3f7d1fdf9026ad8/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741", size = 23150, upload-time = "2026-10-02T23:06:30.583Z" }, + { url = "https://files.pythonhosted.org/packages/19/2d/2dfdce99318abbfa26925195fbc17db188c46a1ec6457be121b6f9cfeb42/markupsafe-3.0.4-cp315-cp315-win32.whl", hash = "sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b", size = 14269, upload-time = "2026-10-02T23:06:31.949Z" }, + { url = "https://files.pythonhosted.org/packages/5b/ec/6000fd82e8791e58fcd0456ec20f098957e2b03d5ed02eb73241a577c0ba/markupsafe-3.0.4-cp315-cp315-win_amd64.whl", hash = "sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214", size = 14496, upload-time = "2026-10-02T23:06:33.258Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/e73bd5016421d5d6e2fb6de7dd609f9de020942ac8c626526bd8c6eeaf82/markupsafe-3.0.4-cp315-cp315-win_arm64.whl", hash = "sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67", size = 14361, upload-time = "2026-10-02T23:06:34.539Z" }, + { url = "https://files.pythonhosted.org/packages/90/df/cb8c3dc98d313a951df2f8968f44e4cb5643df6d3cab749a530ce2f7d972/markupsafe-3.0.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad", size = 11760, upload-time = "2026-10-02T23:06:35.807Z" }, + { url = "https://files.pythonhosted.org/packages/d6/bb/4af9b3ca0753d654ac75f9531d5bd741bb77ca6e696f36807c475ffc099a/markupsafe-3.0.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99", size = 12108, upload-time = "2026-10-02T23:06:37.089Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d4/b56429313aee5fd59b079c3df5615299959e25e7113eb6d8caadbdd7d38a/markupsafe-3.0.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002", size = 25762, upload-time = "2026-10-02T23:06:38.419Z" }, + { url = "https://files.pythonhosted.org/packages/65/f5/34c181e891aa4f7d59c918584672e0c5eb7fffe76c1387d1246008bf4081/markupsafe-3.0.4-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e", size = 28400, upload-time = "2026-10-02T23:06:39.819Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b5/ad14694fd0ac9a5ce30bc6498f2999378f418583dd1679cca5a1b512957e/markupsafe-3.0.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c", size = 25946, upload-time = "2026-10-02T23:06:41.381Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a8/26b606445387d0ceb1eb1f21840094b84e4e3c3c3983d80d10b89823b490/markupsafe-3.0.4-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8", size = 23634, upload-time = "2026-10-02T23:06:42.748Z" }, + { url = "https://files.pythonhosted.org/packages/39/a2/b8814de672f1f0094d498bf646f2fec9d6356b503d28ef500b71c5095377/markupsafe-3.0.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe", size = 23488, upload-time = "2026-10-02T23:06:44.176Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/287223376fb73335a3cc5d6eb22c6ab01358cf33945a9c39c06b9dac3f4b/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2", size = 25037, upload-time = "2026-10-02T23:06:45.646Z" }, + { url = "https://files.pythonhosted.org/packages/f9/29/4df8355e313426d19e62ba33e0253c009ca12a0894ee77d67fa67255361c/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38", size = 23543, upload-time = "2026-10-02T23:06:47.264Z" }, + { url = "https://files.pythonhosted.org/packages/71/e5/8377731e8495668dcc768f645e717df18318c841edaf023a99395f6da9b4/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494", size = 25282, upload-time = "2026-10-02T23:06:48.795Z" }, + { url = "https://files.pythonhosted.org/packages/ed/5f/373456e37ceb1478d657d6fe769cbe0a39f0a8dfc1548eeb19c471eefdd9/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d", size = 22993, upload-time = "2026-10-02T23:06:50.31Z" }, + { url = "https://files.pythonhosted.org/packages/d7/93/2cbd5628435afb6f541bbaced4bce0c2edac4b09a142e6e928b8b0da9858/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894", size = 23712, upload-time = "2026-10-02T23:06:51.759Z" }, + { url = "https://files.pythonhosted.org/packages/81/99/157e10966b033b363aeda5263e82596ee232a0b1d082fdbf90aa417ff083/markupsafe-3.0.4-cp315-cp315t-win32.whl", hash = "sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78", size = 14300, upload-time = "2026-10-02T23:06:53.241Z" }, + { url = "https://files.pythonhosted.org/packages/33/05/55884815414c9706a23deca150b72c25a62109e65b0b6ce232077802c719/markupsafe-3.0.4-cp315-cp315t-win_amd64.whl", hash = "sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c", size = 14522, upload-time = "2026-10-02T23:06:54.729Z" }, + { url = "https://files.pythonhosted.org/packages/92/f9/ecbde7149e95b8a0f18e16d5d747f7dc06049d5da2e4f77f6f5e4a1f46a8/markupsafe-3.0.4-cp315-cp315t-win_arm64.whl", hash = "sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba", size = 14417, upload-time = "2026-10-02T23:06:56.246Z" }, + { url = "https://files.pythonhosted.org/packages/2d/2e/a974ab7076b0272a20f3b01af857c5c78a5c7304125ac39d9eacad3fb1d8/markupsafe-3.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd", size = 11515, upload-time = "2026-10-02T23:06:57.721Z" }, + { url = "https://files.pythonhosted.org/packages/91/e5/1f4a9b31bc89d08150aa1211d8187d69e240decc09aa92575087f12c5ded/markupsafe-3.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868", size = 12060, upload-time = "2026-10-02T23:06:59.149Z" }, + { url = "https://files.pythonhosted.org/packages/98/fe/dbcabd91abdb382f9fbd2ae1b02354c727d90bbdcce79db98cfb687d70b0/markupsafe-3.0.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac", size = 21956, upload-time = "2026-10-02T23:07:00.573Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b9/b39a16a1fc3111cce4a6d74ae19b696757503300f410abfc75f5fc9043be/markupsafe-3.0.4-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2", size = 25915, upload-time = "2026-10-02T23:07:02.007Z" }, + { url = "https://files.pythonhosted.org/packages/ee/35/53eef45f90496d9b5ff3c2cc2b1f66a92f7cc1361a80ce6723ace426b375/markupsafe-3.0.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98", size = 22464, upload-time = "2026-10-02T23:07:03.394Z" }, + { url = "https://files.pythonhosted.org/packages/55/a2/72769102a02abdb32faf9f10abbd4d9ba80b50949b2fe668c27e2432fa82/markupsafe-3.0.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0", size = 20583, upload-time = "2026-10-02T23:07:05.055Z" }, + { url = "https://files.pythonhosted.org/packages/54/a0/6b37c580383117881616e5fa13ccfcd27b711c99d17c59247830772677c6/markupsafe-3.0.4-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295", size = 20618, upload-time = "2026-10-02T23:07:06.521Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ff/e878b3a76c9f4e6d050c6709cc36d4e5a24a817c7cb2dd2a4230348677b4/markupsafe-3.0.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16", size = 21420, upload-time = "2026-10-02T23:07:08.332Z" }, + { url = "https://files.pythonhosted.org/packages/77/0e/1b9d5c3197aa7f2a993d476f4ea3fd677656c1be0f8f87b5cdf3e18c88cc/markupsafe-3.0.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97", size = 20826, upload-time = "2026-10-02T23:07:10.093Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ea/906a72b984b8b928d596ee7f57a2051dd2cec94ca9be383fb52c9ad19418/markupsafe-3.0.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768", size = 21876, upload-time = "2026-10-02T23:07:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3b/8099c4f4e2793ca067bdc5670e2898fedeb3f49e6d17e7810fa094aad54e/markupsafe-3.0.4-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3", size = 20167, upload-time = "2026-10-02T23:07:13.338Z" }, + { url = "https://files.pythonhosted.org/packages/28/4b/085aa6f3d1572d8284a6670f66edda0cee772860f3bdc1431cceff5163e7/markupsafe-3.0.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8", size = 20621, upload-time = "2026-10-02T23:07:14.941Z" }, + { url = "https://files.pythonhosted.org/packages/e2/24/34eba90224606cf2b5327914e80c1cda8718b9dc88f7331d1b32331de71a/markupsafe-3.0.4-cp39-cp39-win32.whl", hash = "sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624", size = 14091, upload-time = "2026-10-02T23:07:16.399Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e9/7ae8c8656b913431eabb4e8b30aea311aa7a68217b56609ec20306732b60/markupsafe-3.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b", size = 14324, upload-time = "2026-10-02T23:07:18.982Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7d/87ff065a4d14d8de1f462f851d9cdf903b6e1fc067d0ef3674414da444c0/markupsafe-3.0.4-cp39-cp39-win_arm64.whl", hash = "sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378", size = 14190, upload-time = "2026-10-02T23:07:20.567Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -985,6 +1212,8 @@ source = { editable = "." } [package.optional-dependencies] test = [ + { name = "alembic", version = "1.16.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "alembic", version = "1.20.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "attrs" }, { name = "psycopg", version = "3.2.13", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version < '3.10'" }, { name = "psycopg", version = "3.3.8", source = { registry = "https://pypi.org/simple" }, extra = ["binary"], marker = "python_full_version >= '3.10'" }, @@ -998,6 +1227,7 @@ test = [ [package.metadata] requires-dist = [ + { name = "alembic", marker = "extra == 'test'", specifier = ">=1.13" }, { name = "attrs", marker = "extra == 'test'", specifier = ">=23" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'test'", specifier = ">=3.1" }, { name = "pydantic", marker = "extra == 'test'", specifier = ">=2" }, From 9965ad6e4c699e9107545442b93366eb08a0f901 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 15:12:39 +0900 Subject: [PATCH 11/29] refactor(python): the SQLAlchemy example uses configure_encryption The SQLAlchemy version of the customer directory called encrypt before each insert and decrypt after each select, and built the search term by hand. With configure_encryption, its endpoints use Session as any SQLAlchemy app does. It is the nearest copy of putting the layer into an existing app. The hooks check only the encrypted fields. cipher.encrypt also refused a created_at with no time zone, so the request model now uses Pydantic's AwareDatetime to keep that refusal. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .../examples/customer-directory/README.md | 10 ++- .../customer-directory/app_sqlalchemy.py | 68 ++++++++----------- .../customer-directory/tests/test_app.py | 27 +++++--- .../examples/customer-directory/uv.lock | 1 + 4 files changed, 54 insertions(+), 52 deletions(-) diff --git a/languages/python/examples/customer-directory/README.md b/languages/python/examples/customer-directory/README.md index 7e937cc60..3ddc40741 100644 --- a/languages/python/examples/customer-directory/README.md +++ b/languages/python/examples/customer-directory/README.md @@ -52,7 +52,15 @@ The search is a `POST` so that the email stays out of URLs and access logs. mise run dev:sqlalchemy # http://127.0.0.1:8001 ``` -A SQLAlchemy object contains plaintext, and the columns must receive ciphertext. So the app calls `encrypt` before an `INSERT` and `decrypt` after a `SELECT`. It does not use `Session`. +The app calls `configure_encryption(engine, [Customer])` once, at start-up. After that, the endpoints use `Session` with no SDK calls: + +- `session.add` and `session.commit` encrypt `email` and `name`. +- `select(Customer).where(Customer.email == search.email)` searches by the equality term. +- A query decrypts the customers it loads. + +The base class sets `type_annotation_map = eql_type_map`, so the model names no SQL types. `schema.sql` still creates the table, not SQLAlchemy. + +The hooks check only the encrypted fields. So `CustomerBody` declares `created_at` as Pydantic's `AwareDatetime`. That type refuses a time with no time zone. ## Run the tests diff --git a/languages/python/examples/customer-directory/app_sqlalchemy.py b/languages/python/examples/customer-directory/app_sqlalchemy.py index 32385331f..ef3387c70 100644 --- a/languages/python/examples/customer-directory/app_sqlalchemy.py +++ b/languages/python/examples/customer-directory/app_sqlalchemy.py @@ -2,9 +2,9 @@ It uses the table of `app.py`, so the two apps share rows. -A SQLAlchemy object contains plaintext, and the columns must receive -ciphertext. So the app calls `encrypt` before an INSERT and `decrypt` after a -SELECT. It does not use `Session`. +`configure_encryption` makes the ORM encrypt and decrypt: a flush encrypts +the encrypted fields, and a query decrypts the objects it loads. So the +endpoints use `Session` as any SQLAlchemy app does, with no SDK calls. """ import os @@ -16,11 +16,11 @@ from fastapi import FastAPI, HTTPException, Query, Request from fastapi.responses import JSONResponse from psycopg.errors import UniqueViolation -from pydantic import BaseModel, ConfigDict, Field +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column -from sqlalchemy.types import UserDefinedType +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column from stackencrypt import Ignore, Passthrough, Text, TextEq +from stackencrypt.sqlalchemy import configure_encryption, eql_type_map from errors import install_error_handlers, log_error, problem @@ -32,20 +32,9 @@ ) -class EqlType(UserDefinedType): - """An EQL column type. The tests fail with `sa.Text`, `sa.String`, or `sa.JSON`.""" - - cache_ok = True - - def __init__(self, sql_name: str) -> None: - self.sql_name = sql_name - - def get_col_spec(self, **kwargs) -> str: - return self.sql_name - - class Base(DeclarativeBase): - pass + # Gives each encrypted field its EQL column type, from its marker. + type_annotation_map = eql_type_map @stackencrypt.model("customers") @@ -55,14 +44,18 @@ class Customer(Base): id: Mapped[Annotated[int, Ignore]] = mapped_column( sa.BigInteger, sa.Identity(always=True), primary_key=True ) - email: Mapped[Annotated[str, TextEq]] = mapped_column(EqlType("public.eql_v4_text_eq")) - name: Mapped[Annotated[str, Text]] = mapped_column(EqlType("public.eql_v4_text")) + email: Mapped[Annotated[str, TextEq]] + name: Mapped[Annotated[str, Text]] plan: Mapped[Annotated[str, Passthrough]] created_at: Mapped[Annotated[datetime, Passthrough]] = mapped_column( sa.DateTime(timezone=True) ) +# Once for the process, before the first request. +configure_encryption(engine, [Customer]) + + def utc_now() -> datetime: return datetime.now(timezone.utc) @@ -76,7 +69,9 @@ class CustomerBody(BaseModel): email: str name: str plan: str - created_at: datetime = Field(default_factory=utc_now) + # Aware: the hooks check only the encrypted fields, so a passthrough + # datetime with no time zone is refused here. + created_at: AwareDatetime = Field(default_factory=utc_now) class Search(BaseModel): @@ -85,12 +80,6 @@ class Search(BaseModel): email: str -# `id` is left out: the model marks it Ignore, and `decrypt` does not want it. -COLUMNS = (Customer.email, Customer.name, Customer.plan, Customer.created_at) - -# One Cipher for the process: making one checks the model and the credentials. -cipher = stackencrypt.Cipher(Customer) - app = FastAPI(title="Customer directory (SQLAlchemy)") install_error_handlers(app) @@ -105,28 +94,25 @@ def integrity_error(request: Request, error: IntegrityError) -> JSONResponse: @app.post("/customers", status_code=201, response_model=CustomerBody) def add_customer(body: CustomerBody) -> CustomerBody: - rows = cipher.encrypt([Customer(**body.model_dump())]) - with engine.begin() as conn: - conn.execute(sa.insert(Customer.__table__), rows) + with Session(engine) as session: + session.add(Customer(**body.model_dump())) + session.commit() return body @app.post("/customers/search", response_model=CustomerBody) def find_customer(search: Search) -> Customer: # The email is in the body so that it stays out of URLs and access logs. - term = sa.literal(cipher.eq("email", search.email), sa.Text) - query = sa.select(*COLUMNS).where( - Customer.email == sa.cast(term, EqlType("eql_v4.query_text_eq")) - ) - with engine.connect() as conn: - found: Optional[sa.RowMapping] = conn.execute(query).mappings().first() + query = sa.select(Customer).where(Customer.email == search.email) + with Session(engine) as session: + found: Optional[Customer] = session.scalars(query).first() if found is None: raise HTTPException(status_code=404, detail="No customer has this email.") - return cipher.decrypt([found])[0] + return found @app.get("/customers", response_model=List[CustomerBody]) def list_customers(limit: int = Query(default=100, ge=1, le=500)) -> List[Customer]: - query = sa.select(*COLUMNS).order_by(Customer.id).limit(limit) - with engine.connect() as conn: - return cipher.decrypt(conn.execute(query).mappings().all()) + query = sa.select(Customer).order_by(Customer.id).limit(limit) + with Session(engine) as session: + return list(session.scalars(query).all()) diff --git a/languages/python/examples/customer-directory/tests/test_app.py b/languages/python/examples/customer-directory/tests/test_app.py index 2606989d8..4f38ff8c7 100644 --- a/languages/python/examples/customer-directory/tests/test_app.py +++ b/languages/python/examples/customer-directory/tests/test_app.py @@ -11,9 +11,9 @@ import psycopg import pytest +import stackencrypt from fastapi.testclient import TestClient from pydantic import ValidationError -from sqlalchemy.exc import DataError from sqlalchemy.orm import Session import app as psycopg_app @@ -141,7 +141,12 @@ def test_an_unexpected_error_gives_a_fixed_500(client, module, monkeypatch, capl def fail(*args, **kwargs): raise RuntimeError(f"contains {EMAIL}") - monkeypatch.setattr(module.cipher, "encrypt", fail) + # The psycopg app calls encrypt. The SQLAlchemy app's flush calls + # encrypt_rows, on the Cipher that configure_encryption made. + if module is psycopg_app: + monkeypatch.setattr(module.cipher, "encrypt", fail) + else: + monkeypatch.setattr(stackencrypt.Cipher, "encrypt_rows", fail) with caplog.at_level(logging.INFO): response = client.post("/customers", json=customer()) @@ -179,17 +184,19 @@ def test_each_app_reads_what_the_other_wrote(): assert (from_first.status_code, from_first.json()["name"]) == (200, NAME) -def test_a_session_cannot_store_a_plaintext_object(): +def test_a_session_stores_an_object_encrypted(): empty_the_table() - plaintext = sqlalchemy_app.Customer( + customer_object = sqlalchemy_app.Customer( email=EMAIL, name=NAME, plan="free", created_at=datetime.now(timezone.utc) ) - with pytest.raises(DataError) as caught: - with Session(sqlalchemy_app.engine) as session: - session.add(plaintext) - session.commit() + with Session(sqlalchemy_app.engine, expire_on_commit=False) as session: + session.add(customer_object) + session.commit() - assert EMAIL not in str(caught.value) + # The object keeps its plaintext; the table holds EQL values. + assert (customer_object.email, customer_object.name) == (EMAIL, NAME) with psycopg.connect(DATABASE_URL) as conn: - assert conn.execute("SELECT count(*) FROM customers").fetchone() == (0,) + [(email, name)] = conn.execute("SELECT email::text, name::text FROM customers").fetchall() + assert EMAIL not in email + assert NAME not in name diff --git a/languages/python/examples/customer-directory/uv.lock b/languages/python/examples/customer-directory/uv.lock index 203ec9906..df3d2c36b 100644 --- a/languages/python/examples/customer-directory/uv.lock +++ b/languages/python/examples/customer-directory/uv.lock @@ -1195,6 +1195,7 @@ source = { directory = "../../" } [package.metadata] requires-dist = [ + { name = "alembic", marker = "extra == 'test'", specifier = ">=1.13" }, { name = "attrs", marker = "extra == 'test'", specifier = ">=23" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'test'", specifier = ">=3.1" }, { name = "pydantic", marker = "extra == 'test'", specifier = ">=2" }, From 022f67f681f5a33a011601def7be5361c1e7144f Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 15:32:08 +0900 Subject: [PATCH 12/29] feat(python): !=, in_ and passthrough checks in the SQLAlchemy layer The write hook sent only encrypted fields to encrypt_rows, so a passthrough value was not checked: a datetime with no time zone reached PostgreSQL, where cipher.encrypt refuses it. Every declared field now goes through encrypt_rows, which checks a passthrough value and returns it unchanged. The example no longer needs Pydantic's AwareDatetime. != becomes NOT (col = term), and in_ one equality term for each value, joined with OR. Neither sends a value as plaintext. in_ with a subquery is still refused. render_item wrote stackencrypt.Json, a name the package does not export. A marker that is not exported is written from stackencrypt._eql. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 5 ++- .../examples/customer-directory/README.md | 2 - .../customer-directory/app_sqlalchemy.py | 6 +-- .../python/python/stackencrypt/sqlalchemy.py | 33 +++++++++++--- languages/python/tests/test_sqlalchemy.py | 44 ++++++++++++++++--- .../python/tests/test_sqlalchemy_live.py | 28 ++++++++++++ .../tests/test_sqlalchemy_migrations.py | 12 +++++ 7 files changed, 110 insertions(+), 20 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index b84e1aba7..c2541738a 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -172,10 +172,11 @@ with Session(engine) as session: - **Writes:** each INSERT or UPDATE on a listed table is encrypted with one batch of key requests for all its rows. - **Reads:** the objects a query loads are decrypted with one batch for each class. This includes relationship loads, and attributes read again after a commit. -- Only encrypted fields are changed. A passthrough field is stored as SQLAlchemy writes it. +- Only encrypted fields are changed. A passthrough field is checked as `encrypt` checks it, then stored unchanged. For example, a `datetime` with no time zone raises `InvalidValueError` with `value_naive_datetime`. - **Search:** `==`, `<`, `<=`, `>`, and `>=` on an encrypted field compare it with a query term, as `cipher.eq` and `cipher.gte` make. The field's marker must support the operator. For example, `Customer.level >= 3` works, and `Customer.email > x` raises `QueryError` with `query_unsupported`. -- **Other operators** that would send a value, such as `like`, `in_`, `!=`, and `between`, raise `QueryError` with `query_unsupported` when you build the query. The value would reach the database as plaintext, and could never match. `is_(None)`, `is_not(None)`, `asc()`, and `desc()` send no value, so they work. +- `!=`, `in_([...])`, and `not_in([...])` work where `==` does. `!=` becomes `NOT (col = term)`. `in_` becomes one equality for each value, joined with `OR`. +- **Other operators** that would send a value, such as `like`, `contains`, `between`, and `in_` with a subquery, raise `QueryError` with `query_unsupported` when you build the query. The value would reach the database as plaintext, and could never match. `is_(None)`, `is_not(None)`, `asc()`, and `desc()` send no value, so they work. - A comparison on a class that is not configured raises `QueryError` with `query_not_configured`. Each class must be declared with `stackencrypt.model` and mapped by SQLAlchemy. `configure_encryption` refuses a class with `DeclarationError` in these cases: diff --git a/languages/python/examples/customer-directory/README.md b/languages/python/examples/customer-directory/README.md index 3ddc40741..9ca24b395 100644 --- a/languages/python/examples/customer-directory/README.md +++ b/languages/python/examples/customer-directory/README.md @@ -60,8 +60,6 @@ The app calls `configure_encryption(engine, [Customer])` once, at start-up. Afte The base class sets `type_annotation_map = eql_type_map`, so the model names no SQL types. `schema.sql` still creates the table, not SQLAlchemy. -The hooks check only the encrypted fields. So `CustomerBody` declares `created_at` as Pydantic's `AwareDatetime`. That type refuses a time with no time zone. - ## Run the tests The tests use the real ZeroKMS service and the example's PostgreSQL. They run against both apps. They empty the `customers` table. CI does not run them. diff --git a/languages/python/examples/customer-directory/app_sqlalchemy.py b/languages/python/examples/customer-directory/app_sqlalchemy.py index ef3387c70..f7decba4e 100644 --- a/languages/python/examples/customer-directory/app_sqlalchemy.py +++ b/languages/python/examples/customer-directory/app_sqlalchemy.py @@ -16,7 +16,7 @@ from fastapi import FastAPI, HTTPException, Query, Request from fastapi.responses import JSONResponse from psycopg.errors import UniqueViolation -from pydantic import AwareDatetime, BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column from stackencrypt import Ignore, Passthrough, Text, TextEq @@ -69,9 +69,7 @@ class CustomerBody(BaseModel): email: str name: str plan: str - # Aware: the hooks check only the encrypted fields, so a passthrough - # datetime with no time zone is refused here. - created_at: AwareDatetime = Field(default_factory=utc_now) + created_at: datetime = Field(default_factory=utc_now) class Search(BaseModel): diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 06d470255..25925f9b1 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -17,8 +17,9 @@ plaintext is set with ``set_committed_value``, so the objects are not marked as changed. -Only the sealed fields go through the hooks. A passthrough field is stored -as SQLAlchemy writes it. +On a write, every declared field goes through ``encrypt_rows``, so a +passthrough value is checked as ``encrypt`` checks it, and stored unchanged. +On a read, only the encrypted fields are decrypted. - Indexes: a mapped class's table gets a database index for each EQL index of each encrypted column, such as ``hash (eql_v4.eq_term(email))`` for ``TextEq``. @@ -38,7 +39,7 @@ from typing import Annotated, Any, Callable, Dict, FrozenSet, List, Sequence, Tuple, Type -from sqlalchemy import Index, String, Table, cast, event, func, inspect, literal +from sqlalchemy import Index, String, Table, cast, event, false, func, inspect, literal, not_, or_ from sqlalchemy.engine import Engine from sqlalchemy.orm import Mapper, Session from sqlalchemy.orm.attributes import set_committed_value @@ -46,6 +47,7 @@ from sqlalchemy.sql.dml import Insert, Update from sqlalchemy.types import UserDefinedType +import stackencrypt from stackencrypt import _eql from stackencrypt._cipher import Cipher from stackencrypt._declaration import _KIND_TYPES @@ -79,8 +81,18 @@ def __repr__(self) -> str: class comparator_factory(UserDefinedType.Comparator): # type: ignore[type-arg] def operate(self, op: Any, *other: Any, **kwargs: Any) -> Any: - if op in _NO_VALUE or (op is operators.eq and other == (None,)): + if op in _NO_VALUE or (op in (operators.eq, operators.ne) and other == (None,)): return super().operate(op, *other, **kwargs) + if op is operators.ne and len(other) == 1: + # NOT (col = term): a NULL column matches neither, as with !=. + return not_(self.operate(operators.eq, other[0])) + if op in (operators.in_op, operators.not_in_op) and len(other) == 1: + values = other[0] + if not isinstance(values, (list, tuple, set, frozenset)): + raise _refused(self.expr, op) + # One equality term for each value. An empty list matches no row. + found = or_(false(), *[self.operate(operators.eq, value) for value in values]) + return found if op is operators.in_op else not_(found) method = _COMPARISONS.get(op) if method is None or len(other) != 1: raise _refused(self.expr, op) @@ -176,7 +188,10 @@ def render_item(type_: str, obj: Any, autogen_context: Any) -> Any: if type_ == "type" and isinstance(obj, EqlType): autogen_context.imports.add("import stackencrypt") autogen_context.imports.add("import stackencrypt.sqlalchemy") - return f"stackencrypt.sqlalchemy.EqlType(stackencrypt.{obj.marker.__name__})" + name = obj.marker.__name__ + # A marker the package does not export, such as Json, is in _eql. + module = "stackencrypt" if hasattr(stackencrypt, name) else "stackencrypt._eql" + return f"stackencrypt.sqlalchemy.EqlType({module}.{name})" return False @@ -384,7 +399,9 @@ def encrypt_parameters(conn, clause, multiparams, params, execution_options): # cipher, sealed = configured rows = [dict(row) for row in multiparams] if multiparams else [dict(params or {})] _refuse_inline_values(conn, clause, sealed, rows) - names = sealed.intersection(rows[0]) + # Passthrough fields too: encrypt_rows checks them as encrypt does, + # and returns them unchanged. + names = _declared(cipher).intersection(rows[0]) if not names: return clause, multiparams, params encrypted = cipher.encrypt_rows([{name: row[name] for name in names} for row in rows]) @@ -397,6 +414,10 @@ def encrypt_parameters(conn, clause, multiparams, params, execution_options): # return encrypt_parameters +def _declared(cipher: Cipher) -> FrozenSet[str]: + return frozenset(field.name for field in cipher.declaration.fields) + + def _refuse_inline_values(conn, clause, sealed: FrozenSet[str], rows: List[Dict[str, Any]]) -> None: # type: ignore[no-untyped-def] """Refuse a statement that carries a sealed value in its SQL, such as ``insert(table).values(email=...)``: the hook can only change the diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index 48adc73a4..4468cd2ac 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -10,11 +10,11 @@ from typing import Annotated import pytest -from sqlalchemy import String, create_engine +from sqlalchemy import String, create_engine, select from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column import stackencrypt -from stackencrypt import DeclarationError, Ignore, Passthrough, TextEq +from stackencrypt import DeclarationError, Ignore, InvalidValueError, Passthrough, TextEq from stackencrypt import IntegerOrd, QueryError, Text, _eql from stackencrypt.sqlalchemy import EqlType, configure_encryption, eql_type_map @@ -200,12 +200,11 @@ def scores(): [ lambda: Score.note.like("%secret%"), lambda: Score.note.contains("secret"), - lambda: Score.note.in_(["secret"]), - lambda: Score.note != "secret", + lambda: Score.note.in_(select(Score.note).scalar_subquery()), lambda: Score.level.between(1, 9), lambda: Score.level + 1, ], - ids=["like", "contains", "in", "not-equal", "between", "add"], + ids=["like", "contains", "in-a-subquery", "between", "add"], ) def test_an_operator_that_would_send_the_value_is_refused(scores, untouched, build): with pytest.raises(QueryError) as raised: @@ -220,10 +219,13 @@ def test_an_operator_that_would_send_the_value_is_refused(scores, untouched, bui lambda: Score.note.is_(None), lambda: Score.note.is_not(None), lambda: Score.note == None, # noqa: E711 + lambda: Score.note != None, # noqa: E711 + lambda: Score.note.in_([]), + lambda: Score.note.not_in([]), lambda: Score.level.asc(), lambda: Score.level.desc(), ], - ids=["is-null", "is-not-null", "equals-none", "asc", "desc"], + ids=["is-null", "is-not-null", "equals-none", "not-equals-none", "in-nothing", "not-in-nothing", "asc", "desc"], ) def test_an_operator_that_sends_no_value_is_allowed_with_no_request(scores, untouched, build): build() @@ -240,3 +242,33 @@ class Draft(Base): Draft.note == "secret" # noqa: B015 assert raised.value.code == "query_not_configured" assert "secret" not in str(raised.value) + + +# Passthrough values on writes. + + +def test_a_passthrough_value_is_checked_on_a_flush(untouched): + import datetime + + from sqlalchemy.orm import Session + + @stackencrypt.model("trips") + class Visit(Base): + __tablename__ = "trips" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + at: Mapped[Annotated[datetime.datetime, Passthrough]] + + engine = create_engine("sqlite://") + with engine.begin() as conn: + conn.exec_driver_sql("CREATE TABLE trips (id INTEGER PRIMARY KEY, email TEXT, at TEXT)") + configure_encryption(engine, [Visit]) + + with Session(engine) as session, pytest.raises(InvalidValueError) as raised: + session.add(Visit(email="secret@example.com", at=datetime.datetime(2026, 1, 1))) + session.commit() + + assert raised.value.code == "value_naive_datetime" + assert raised.value.field == "at" + with engine.connect() as conn: + assert conn.exec_driver_sql("SELECT count(*) FROM trips").scalar() == 0 diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index 873ca57f9..a7d823744 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -197,3 +197,31 @@ def test_an_order_search_on_an_equality_field_is_unsupported(engine): with pytest.raises(QueryError) as raised: select(Customer).where(Customer.email > "a@example.com") assert raised.value.code == "query_unsupported" + + +def test_not_equal_and_in_find_the_right_rows(engine): + with Session(engine) as session: + session.add_all( + [Customer(email=f"{name}@example.com", name=name, plan="free") for name in ("ann", "bob", "cy")] + ) + session.commit() + with Session(engine) as session: + def names(*where): + return sorted(c.name for c in session.scalars(select(Customer).where(*where))) + + assert names(Customer.email != "bob@example.com") == ["ann", "cy"] + assert names(Customer.email.in_(["ann@example.com", "cy@example.com"])) == ["ann", "cy"] + assert names(Customer.email.not_in(["ann@example.com"])) == ["bob", "cy"] + assert names(Customer.email.in_([])) == [] + + +def test_an_update_of_a_passthrough_field_only_makes_no_key_request(engine): + add_ann(engine) # warms up the client + with Session(engine) as session: + ann = session.scalars(select(Customer)).one() + ann.plan = "pro" + before = _engine.request_count() + session.commit() + assert _engine.request_count() - before == 0 + with Session(engine) as session: + assert session.scalars(select(Customer)).one().plan == "pro" diff --git a/languages/python/tests/test_sqlalchemy_migrations.py b/languages/python/tests/test_sqlalchemy_migrations.py index f78a766ed..b7d7beb9e 100644 --- a/languages/python/tests/test_sqlalchemy_migrations.py +++ b/languages/python/tests/test_sqlalchemy_migrations.py @@ -123,6 +123,18 @@ def test_render_item_adds_the_imports_the_type_needs(): assert {"import stackencrypt", "import stackencrypt.sqlalchemy"} <= autogen.imports +def test_render_item_names_a_marker_the_package_does_not_export_by_its_module(): + # Json is in _eql only: no model can declare it yet. + autogen = AutogenContext(MigrationContext.configure(dialect_name="postgresql")) + + rendered = render_item("type", EqlType(_eql.Json), autogen) + + assert rendered == "stackencrypt.sqlalchemy.EqlType(stackencrypt._eql.Json)" + namespace = {} + exec("import stackencrypt\nimport stackencrypt.sqlalchemy\nt = " + rendered, namespace) + assert namespace["t"].marker is _eql.Json + + def test_render_item_leaves_other_items_to_alembic(): autogen = AutogenContext(MigrationContext.configure(dialect_name="postgresql")) From 6185df77847cda71d97bbc0d414f716c7cab67b8 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 15:41:46 +0900 Subject: [PATCH 13/29] feat(python): a declared index replaces the SDK's; refuse unique EQL columns The SDK added a hash index on eq_term(email) even when the model, like the example, already had a unique btree index on the same term. Alembic would then create both. Now an index the model declares on the same term of the same column replaces the SDK's index for that term. A unique constraint on an encrypted column compares the stored EQL values. Each one has new random ciphertext, so the constraint never finds a duplicate. configure_encryption refuses it, and points to a unique index on eq_term instead. The example declares the unique index of schema.sql, and gives plan the text type, so Alembic finds no index or plan difference with the table. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 9 ++++ .../customer-directory/app_sqlalchemy.py | 8 +++- .../python/python/stackencrypt/sqlalchemy.py | 48 ++++++++++++++++++- languages/python/tests/test_sqlalchemy.py | 13 +++++ .../tests/test_sqlalchemy_migrations.py | 33 ++++++++++++- 5 files changed, 107 insertions(+), 4 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index c2541738a..44df9655c 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -185,6 +185,7 @@ Each class must be declared with `stackencrypt.model` and mapped by SQLAlchemy. - `declaration_table_mismatch`: the table in `stackencrypt.model` is not the `__tablename__`. - `declaration_column_name`: an encrypted field's column has another name than the field, as with `mapped_column("email_address")`. - `declaration_column_type`: an encrypted field's column is not the EQL type of its marker. Set `type_annotation_map = eql_type_map` on the base. +- `declaration_unique_column`: an encrypted column has `unique=True` or a unique constraint. Each EQL value contains new random ciphertext, so the constraint would never find a duplicate. Use a unique index on `eql_v4.eq_term(col)`, as in "Migrations with Alembic". - `declaration_already_configured`: the class was configured before in this process. A statement that puts an encrypted value inside its SQL, such as `insert(Customer).values(email=...)`, raises `DeclarationError` with `declaration_inline_value`. The value is not stored. Pass the values as parameters instead, as `session.add` and `session.execute(insert(Customer), rows)` do. @@ -200,6 +201,14 @@ A class mapped with `eql_type_map` gets a database index for each search its fie | `ore_block` | `btree (eql_v4.ord_term_ore(col))` | | `match` | `gin (eql_v4.match_term(col))` | +If the model declares its own index on the same term of the same column, the SDK adds none for that term. Declare it in `__table_args__`. For example, a unique email needs a unique btree index, because PostgreSQL cannot make a hash index unique: + +```text +__table_args__ = ( + Index("customers_email_unique", text("eql_v4.eq_term(email)"), unique=True), +) +``` + An `ore_block` index needs an operator class that EQL's install creates. If the role that installed EQL could not create operator classes, the index still builds. But PostgreSQL never uses it, and gives no error. Give Alembic the two hooks in your `env.py`: diff --git a/languages/python/examples/customer-directory/app_sqlalchemy.py b/languages/python/examples/customer-directory/app_sqlalchemy.py index f7decba4e..92992db93 100644 --- a/languages/python/examples/customer-directory/app_sqlalchemy.py +++ b/languages/python/examples/customer-directory/app_sqlalchemy.py @@ -40,13 +40,19 @@ class Base(DeclarativeBase): @stackencrypt.model("customers") class Customer(Base): __tablename__ = "customers" + # As schema.sql makes it. A btree, because PostgreSQL cannot make a hash + # index unique. It also serves the email search, so the SDK adds no + # index of its own on this term. + __table_args__ = ( + sa.Index("customers_email_unique", sa.text("eql_v4.eq_term(email)"), unique=True), + ) id: Mapped[Annotated[int, Ignore]] = mapped_column( sa.BigInteger, sa.Identity(always=True), primary_key=True ) email: Mapped[Annotated[str, TextEq]] name: Mapped[Annotated[str, Text]] - plan: Mapped[Annotated[str, Passthrough]] + plan: Mapped[Annotated[str, Passthrough]] = mapped_column(sa.Text) created_at: Mapped[Annotated[datetime, Passthrough]] = mapped_column( sa.DateTime(timezone=True) ) diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 25925f9b1..5677e8104 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -39,7 +39,21 @@ from typing import Annotated, Any, Callable, Dict, FrozenSet, List, Sequence, Tuple, Type -from sqlalchemy import Index, String, Table, cast, event, false, func, inspect, literal, not_, or_ +from sqlalchemy import ( + Index, + String, + Table, + UniqueConstraint, + cast, + event, + false, + func, + inspect, + literal, + not_, + or_, +) +from sqlalchemy.dialects import postgresql from sqlalchemy.engine import Engine from sqlalchemy.orm import Mapper, Session from sqlalchemy.orm.attributes import set_committed_value @@ -166,6 +180,7 @@ def _add_indexes(mapper: Any, cls: type) -> None: table = mapper.local_table if not isinstance(table, Table): return + declared = {_index_term(expression) for index in table.indexes for expression in index.expressions} for column in table.columns: if not isinstance(column.type, EqlType): continue @@ -176,9 +191,23 @@ def _add_indexes(mapper: Any, cls: type) -> None: name = f"ix_{table.name}_{column.name}_{kind}" if any(index.name == name for index in table.indexes): continue + # The model declares an index on this term itself, such as a + # unique btree on eq_term: that one serves the search. + if f"eql_v4.{function}({column.name})" in declared: + continue Index(name, getattr(func.eql_v4, function)(column), postgresql_using=method) +def _index_term(expression: Any) -> str: + """An index expression as SQL, with no spaces or quotes, such as + ``eql_v4.eq_term(email)``.""" + try: + sql = str(expression.compile(dialect=postgresql.dialect())) + except Exception: # an expression only its own dialect compiles + return "" + return "".join(sql.split()).replace('"', "").lower() + + def render_item(type_: str, obj: Any, autogen_context: Any) -> Any: """For Alembic's ``context.configure(render_item=render_item)`` in ``env.py``. Writes an ``EqlType`` column type into a migration as @@ -375,7 +404,22 @@ def _check(cls: type) -> Tuple[type, Cipher, FrozenSet[str]]: for field in cipher.declaration.fields: if field.marker is Passthrough: continue - column_type = mapper.columns[field.name].type + column = mapper.columns[field.name] + unique = column.unique or any( + isinstance(constraint, UniqueConstraint) and column in constraint.columns + for constraint in mapper.local_table.constraints + ) + if unique: + raise DeclarationError( + "declaration_unique_column", + f"field {field.name!r} of {cls.__qualname__} is an encrypted column with a " + "unique constraint. Each EQL value contains new random ciphertext, so the " + "constraint never finds a duplicate. Use a unique index on its equality " + f"term instead: Index(..., text(\"eql_v4.eq_term({field.name})\"), unique=True) " + "in __table_args__.", + field=field.name, + ) + column_type = column.type if not (isinstance(column_type, EqlType) and column_type.marker is field.marker): raise DeclarationError( "declaration_column_type", diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index 4468cd2ac..2380797c1 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -90,6 +90,19 @@ class Account(Base): assert "email_address" in error.message +def test_a_unique_encrypted_column_is_refused(engine): + @stackencrypt.model("logins") + class Login(Base): + __tablename__ = "logins" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] = mapped_column(unique=True) + + error = refused(engine, [Login]) + assert error.code == "declaration_unique_column" + assert error.field == "email" + assert "eql_v4.eq_term(email)" in error.message + + def test_a_class_configured_twice_is_refused(engine): @stackencrypt.model("members") class Member(Base): diff --git a/languages/python/tests/test_sqlalchemy_migrations.py b/languages/python/tests/test_sqlalchemy_migrations.py index b7d7beb9e..68543ee42 100644 --- a/languages/python/tests/test_sqlalchemy_migrations.py +++ b/languages/python/tests/test_sqlalchemy_migrations.py @@ -12,7 +12,7 @@ from alembic.operations import ops from alembic.autogenerate.api import AutogenContext from alembic.runtime.migration import MigrationContext -from sqlalchemy import Column, Integer, MetaData, String, Table, Text +from sqlalchemy import Column, Index, Integer, MetaData, String, Table, Text, text from sqlalchemy.dialects import postgresql from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -84,6 +84,37 @@ def test_each_eql_index_of_a_marker_gets_its_database_index(marker): } +def test_an_index_the_model_declares_on_the_same_term_replaces_the_sdk_index(): + @stackencrypt.model("members") + class Member(Base): + __tablename__ = "members" + # Unique, so a btree: PostgreSQL cannot make a hash index unique. + __table_args__ = (Index("members_email_unique", text("eql_v4.eq_term(email)"), unique=True),) + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextSearch]] + + assert sorted(index.name for index in Member.__table__.indexes) == [ + "ix_members_email_match", + "ix_members_email_ope", + "members_email_unique", + ] + + +def test_an_index_on_another_column_does_not_replace_the_sdk_index(): + @stackencrypt.model("guests") + class Guest(Base): + __tablename__ = "guests" + __table_args__ = (Index("guests_name_eq", text("eql_v4.eq_term(name)")),) + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + name: Mapped[Annotated[str, TextEq]] + + assert sorted(index.name for index in Guest.__table__.indexes) == [ + "guests_name_eq", + "ix_guests_email_eq", + ] + + def test_a_table_that_is_not_mapped_gets_no_index(): # As Alembic's add_column builds one: its indexes would be created twice. table = Table("t", MetaData(), Column("id", Integer, primary_key=True), Column("v", EqlType(TextEq))) From b6bb749019f78cc2b93ea22ba8b53c5f745ca9b3 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 15:42:17 +0900 Subject: [PATCH 14/29] docs(python): record the SQLAlchemy layer's design in ADR-0004 The layer made several choices that apps will depend on: batch hooks instead of a column type, an explicit list of classes, refusing anything that would store or send plaintext, column types and indexes read from the markers, and two Alembic hooks. ADR-0004 records each one with the option it was chosen over, and what was run to check it. ADR-0002 left open whether the SQLAlchemy layer names a field by the class attribute. It does, and ADR-0002 now points to ADR-0004. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- ...002-a-query-names-its-field-by-a-string.md | 4 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 143 ++++++++++++++++++ 3 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md diff --git a/AGENTS.md b/AGENTS.md index 3094761b0..5e52f8ed8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and the query methods against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. `languages/python/uv.lock` locks the `test` extra: pytest, attrs, Pydantic, SQLAlchemy, psycopg, and Alembic. The package has no runtime dependency, so nothing else is locked. attrs, Pydantic, and SQLAlchemy are for the declaration tests, and SQLAlchemy also for `stackencrypt.sqlalchemy`. psycopg is for the live tests. Alembic is for the tests of the migrations that `stackencrypt.sqlalchemy`'s two Alembic hooks write. `mise run build` installs them with `uv sync --locked`, so a lock that does not match `pyproject.toml` fails the build. After a change to the `test` extra, run `uv lock` in `languages/python`. None has an upper bound. Dependabot's `uv` entry proposes each new minor or patch release as a lock update. maturin, the build backend, is not locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, a query names its field by a string; ADR-0003, one method for each query operator; ADR-0004, the SQLAlchemy layer encrypts in batch hooks), the spec in #1341. `languages/python/docs/scorecard.md` marks the eight shared principles of `docs/sdk-design-principles.md` against the code. The README's SQL and Python blocks run in order in the `live` job, through `tests/test_readme_live.py`, so a change to a README step needs that test to pass. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `languages/python/examples/customer-directory`: An internal FastAPI example of the Python SDK, written twice: with psycopg (`app.py`) and with SQLAlchemy (`app_sqlalchemy.py`). It has its own `pyproject.toml` and `uv.lock`, and installs `stackencrypt` from this checkout. Its tests need real credentials, and CI does not run them. diff --git a/languages/python/docs/adr/0002-a-query-names-its-field-by-a-string.md b/languages/python/docs/adr/0002-a-query-names-its-field-by-a-string.md index 91e9a66a8..d2d750623 100644 --- a/languages/python/docs/adr/0002-a-query-names-its-field-by-a-string.md +++ b/languages/python/docs/adr/0002-a-query-names-its-field-by-a-string.md @@ -3,7 +3,7 @@ status: accepted date: 2026-11-02 ticket: "#1391, the first ticket that adds `query`, as the spec (#1341) says" extends: ADR-0001 -amended_by: ADR-0003 +amended_by: ADR-0003, ADR-0004 --- # A query names its field by a string @@ -50,6 +50,8 @@ A, for now. B stays open for the later SQLAlchemy layer. A SQLAlchemy model already has a column attribute for each field, so B works there without core changing the class. The layer reads the attribute's name and calls core with the string. The spec in #1341 keeps that layer out of scope for now. +> **Answered by ADR-0004.** On the SQLAlchemy ORM, a query names the field by the class attribute, as in `select(Customer).where(Customer.email == x)`. The column type reads the attribute's name, and calls the `Cipher` with it. The `Cipher` methods still take a string. + A does not block B. A later version can accept a string or an attribute in the same position. A caller who passes a string sees no change. An operator can come later as a keyword argument, such as `op=`. A caller who passes none still gets equality. diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md new file mode 100644 index 000000000..576b5332d --- /dev/null +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -0,0 +1,143 @@ +--- +status: accepted +date: 2026-11-11 +ticket: "none yet" +extends: ADR-0002, ADR-0003 +--- + +# The SQLAlchemy layer encrypts in batch hooks + +`stackencrypt.sqlalchemy` lets an app use the SQLAlchemy ORM with encrypted fields, and make no SDK call of its own: + +```python +class Base(DeclarativeBase): + type_annotation_map = eql_type_map + +configure_encryption(engine, [Customer]) + +with Session(engine) as session: + session.add(Customer(email="a@example.com", level=3)) + session.commit() + found = session.scalars(select(Customer).where(Customer.email == "a@example.com")).one() +``` + +This Architecture Decision Record (ADR) records the decisions that shape the layer. Each one had a real choice. Most would be costly to change once apps depend on them. + +## Background + +Before the layer, an app called `cipher.encrypt` before each insert, and `cipher.decrypt` after each select. It also built each query term with `cipher.eq`, and wrote the cast in the SQL itself. `examples/customer-directory/app_sqlalchemy.py` did this until the layer replaced it. + +Each encrypted value needs its own data key from ZeroKMS, CipherStash's key service. The engine fetches the keys for one call in bulk: up to 500 keys in each request. Two live tests show this: `test_1001_one_field_models_make_three_key_requests` and `test_1001_rows_open_with_three_key_requests` in `tests/test_encrypt_live.py`. + +## Decision 1: batch hooks, not a column type + +### The options + +**A. A column type.** A SQLAlchemy `TypeDecorator` encrypts each value as SQLAlchemy writes it, and decrypts each value as it reads a row. This is the usual SQLAlchemy way to change a column's values. + +**B. Hooks that work on a whole statement or result (chosen).** + +- Writes: a `before_execute` hook on the engine. For each `INSERT` or `UPDATE` on a configured table, it encrypts the parameters of every row with one `Cipher.encrypt_rows` call. +- Reads: the `load` and `refresh` events of each class record the objects a query builds. A `do_orm_execute` hook then decrypts them, with one `Cipher.decrypt_rows` call for each class and set of fields. It sets the plaintext with `set_committed_value`, so the objects are not marked as changed. + +### Why B + +SQLAlchemy passes a column type one value at a time. So 100 rows would make 100 key requests. With B, they make one. The live test `test_100_objects_make_one_key_request_to_insert_and_one_to_load` in `tests/test_sqlalchemy_live.py` shows this. + +### What B needed + +- `Cipher.encrypt_rows` and `Cipher.decrypt_rows`. They take dicts that contain only some of the declared fields. An `UPDATE` has only the changed columns, plus keys such as the primary key for the `WHERE`. +- The engine refuses a row that leaves out a field of the plan. So these methods build a smaller plan for each set of fields. A value from the smaller plan opens and searches under the full plan. The live test `test_a_row_updated_with_some_fields_decrypts_and_searches_with_the_full_model` shows this. +- Every row in one call must contain the same declared fields. Then every value is checked before the one batch of key requests. SQLAlchemy sends one set of columns for each statement, so the hook meets this rule. + +## Decision 2: one setup call with an explicit list of classes + +`configure_encryption(engine, [Customer, Order])` takes the engine and the classes to encrypt. It does not find declared classes by itself, such as from the `Base` registry. An explicit list shows which tables are encrypted. A class imported late would otherwise be missed, with no error. + +It checks each class first, and raises `DeclarationError` before any query: + +| Code | The class has | +|---|---| +| `declaration_not_mapped` | no SQLAlchemy mapping | +| `declaration_table_mismatch` | a `__tablename__` that is not the table in `stackencrypt.model` | +| `declaration_column_name` | an encrypted field whose column has another name | +| `declaration_column_type` | an encrypted field whose column is not its marker's EQL type | +| `declaration_unique_column` | an encrypted field with a unique constraint | +| `declaration_already_configured` | a configuration from an earlier call | + +The hooks use field names, and SQLAlchemy uses column names in statement parameters. So a column with another name, such as `mapped_column("email_address")`, is refused for now. + +## Decision 3: refuse what would store or send plaintext + +The hooks can change only the parameters passed beside a statement. Three cases would put plaintext where the hooks cannot reach it. Each one raises an error before anything is sent. + +- **A value inside the SQL**, such as `insert(Customer).values(email=...)`. It raises `DeclarationError` with `declaration_inline_value`. +- **An operator that would send a value as plaintext**, such as `like`, `contains`, `between`, or `in_` with a subquery. It raises `QueryError` with `query_unsupported` when the app builds the query. The value could also never match an EQL value. +- **A unique constraint on an encrypted column.** Each EQL value contains new random ciphertext, so the constraint never finds a duplicate. A unique index on `eql_v4.eq_term(col)` works instead. + +The allowed operators are: + +- `==`, `<`, `<=`, `>`, and `>=`. Each one sends a query term made by the `Cipher` method of the same meaning. +- `!=`, as `NOT (col = term)`. +- `in_` and `not_in` with a list, as one equality term for each value, joined with `OR`. +- `is_(None)`, `is_not(None)`, `asc()`, and `desc()`. These send no value. + +The comparisons answer ADR-0002's open question. On the ORM, the field is named by the class attribute, such as `Customer.email`. The column type reads the attribute's name and calls the `Cipher` with it. + +## Decision 4: the column type comes from the marker + +`eql_type_map` is for SQLAlchemy's `type_annotation_map`. It maps each hint, such as `Annotated[str, TextEq]`, to an `EqlType` with that marker. The SQL type is the marker's `sql_domain`. So a model names no SQL type, and the type cannot disagree with the marker. + +The other option was an explicit type in each column, such as `mapped_column(Encrypted(TextEq))`. That repeats the marker in the hint. + +## Decision 5: the database indexes come from the markers + +Each marker lists its EQL indexes, such as `eq` for `TextEq`. When SQLAlchemy maps a class, the SDK adds one database index for each: + +| EQL index | Database index | +|---|---| +| `eq` | `hash (eql_v4.eq_term(col))` | +| `ope` | `btree (eql_v4.ord_term(col))` | +| `ore_block` | `btree (eql_v4.ord_term_ore(col))` | +| `match` | `gin (eql_v4.match_term(col))` | + +- The indexes are added when a class is mapped, not when a column joins a table. Alembic's `add_column` puts the new column on a temporary table, and creates that table's indexes. So an index added there was created twice. +- If the model declares its own index on the same term of the same column, the SDK adds none for that term. The customer directory example needs this. Its unique btree index on `eq_term(email)` also serves the search. + +The other option was to declare each index by hand. An index that is missing, or on the wrong function, makes a search slow with no error. + +## Decision 6: two Alembic hooks for env.py + +- `render_item` writes `stackencrypt.sqlalchemy.EqlType(stackencrypt.TextEq)` into a migration, with its imports. +- `process_revision_directives` changes two kinds of step: + - A change of an existing column to an EQL type becomes `op.execute` of an `ALTER` with `USING col::jsonb::`. PostgreSQL refuses the change without it, even with no rows. The cast changes an empty column, and refuses a column with plaintext rows. + - A type change between an EQL type and itself is dropped. Alembic compares types as text: `eql_v4_text_eq` from the database, and `public.eql_v4_text_eq` from the model. + +The other option was to leave the `USING` clause to the app, as a hand edit in each migration. + +## Decision 7: writes check passthrough values + +The write hook sends every declared field to `encrypt_rows`, not only the encrypted ones. So a passthrough value is checked as `encrypt` checks it, and stored unchanged. For example, a `datetime` with no time zone raises `value_naive_datetime`. An update of passthrough fields alone makes no key request. + +Reads do not check passthrough values, as `decrypt_rows` reads only the encrypted fields. + +## What was run + +- `tests/test_sqlalchemy.py` and `tests/test_sqlalchemy_migrations.py` run without credentials. They cover the checks, the refused operators, the indexes, and the Alembic rendering. +- `tests/test_sqlalchemy_live.py` runs the hooks against ZeroKMS and PostgreSQL. It covers writes, reads, relationship loads, reads after a commit, search, and the request counts. +- `tests/test_sqlalchemy_migrations_live.py` runs `alembic revision --autogenerate` and `alembic upgrade` on an existing table. A second `autogenerate` writes nothing. With `enable_seqscan` off, `EXPLAIN` shows each index in use. + +## Consequences + +- A query result is read in full before it is returned, so that its objects can be decrypted in one batch. A query that streams its rows does not stream. +- A Core query, such as `conn.execute(text(...))`, is not decrypted. It returns EQL values. +- The layer imports SQLAlchemy. The rest of the package still does not. SQLAlchemy is not a runtime dependency. +- `do_orm_execute` is registered on every session in the process. It acts only on objects of configured classes. + +## Open + +- `between`, as two comparisons. The engine has no range term for EQL types (ADR-0003, amendment). +- `matches` through the ORM. ADR-0003 says the layer should hide `target_limit` and `verify`. +- Core queries. +- The reverse migration of a change to an EQL type. +- Installing EQL from a migration. From 403617d10cad75a5ec961e0b097952975d73432a Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 15:49:04 +0900 Subject: [PATCH 15/29] feat(python): decrypt the encrypted columns of a session query The read hook decrypted only the objects a query loads. A session query of columns, such as select(Customer.email, Customer.plan), returned EQL values, though it is ordinary ORM code. The hook now finds the result columns that are encrypted fields, through attributes and labels, and decrypts them with one decrypt_rows call for each class and set of fields. A query on a connection is still left to the app, which calls cipher.decrypt_rows: a connection has no hook that can change a result. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 4 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 4 +- .../python/python/stackencrypt/sqlalchemy.py | 49 ++++++++++++++++++- .../python/tests/test_sqlalchemy_live.py | 36 ++++++++++++++ 4 files changed, 88 insertions(+), 5 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 44df9655c..68a67a40c 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -171,7 +171,7 @@ with Session(engine) as session: `eql_type_map` gives each encrypted field its EQL column type, such as `public.eql_v4_text_eq` for `TextEq`. So a column needs no type of its own. `Base.metadata.create_all(engine)` creates the EQL columns. - **Writes:** each INSERT or UPDATE on a listed table is encrypted with one batch of key requests for all its rows. -- **Reads:** the objects a query loads are decrypted with one batch for each class. This includes relationship loads, and attributes read again after a commit. +- **Reads:** the objects a query loads are decrypted with one batch for each class. This includes relationship loads, and attributes read again after a commit. A session query of columns, such as `session.execute(select(Customer.email, Customer.plan))`, is decrypted too. - Only encrypted fields are changed. A passthrough field is checked as `encrypt` checks it, then stored unchanged. For example, a `datetime` with no time zone raises `InvalidValueError` with `value_naive_datetime`. - **Search:** `==`, `<`, `<=`, `>`, and `>=` on an encrypted field compare it with a query term, as `cipher.eq` and `cipher.gte` make. The field's marker must support the operator. For example, `Customer.level >= 3` works, and `Customer.email > x` raises `QueryError` with `query_unsupported`. @@ -237,7 +237,7 @@ These are not handled by the hooks yet: These are not handled yet: -- A Core query, such as `conn.execute(text("SELECT ..."))`, returns EQL values, not plaintext. Use `cipher.decrypt_rows` on its rows. +- A query run on a connection, not a session, returns EQL values, not plaintext. Examples are `conn.execute(select(...))` and `conn.execute(text("SELECT ..."))`. Use `cipher.decrypt_rows` on its rows. A connection has no hook that can change a result. - A match search, `matches`, is not supported through the ORM yet. Use `cipher.matches`, as in "Search by match". - A query result is read in full before it is returned, so that its objects can be decrypted in one batch. A query that streams its rows does not stream. diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index 576b5332d..cf0a70fe1 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -39,6 +39,7 @@ Each encrypted value needs its own data key from ZeroKMS, CipherStash's key serv - Writes: a `before_execute` hook on the engine. For each `INSERT` or `UPDATE` on a configured table, it encrypts the parameters of every row with one `Cipher.encrypt_rows` call. - Reads: the `load` and `refresh` events of each class record the objects a query builds. A `do_orm_execute` hook then decrypts them, with one `Cipher.decrypt_rows` call for each class and set of fields. It sets the plaintext with `set_committed_value`, so the objects are not marked as changed. +- Reads of columns: the same hook decrypts the result columns that are encrypted fields, as in `select(Customer.email, Customer.plan)`. It makes one `decrypt_rows` call for each class and set of fields. ### Why B @@ -130,7 +131,7 @@ Reads do not check passthrough values, as `decrypt_rows` reads only the encrypte ## Consequences - A query result is read in full before it is returned, so that its objects can be decrypted in one batch. A query that streams its rows does not stream. -- A Core query, such as `conn.execute(text(...))`, is not decrypted. It returns EQL values. +- A query run on a connection, not a session, is not decrypted. It returns EQL values, and the app calls `cipher.decrypt_rows`. A connection has no hook that can change a result. A session query of columns is decrypted. - The layer imports SQLAlchemy. The rest of the package still does not. SQLAlchemy is not a runtime dependency. - `do_orm_execute` is registered on every session in the process. It acts only on objects of configured classes. @@ -138,6 +139,5 @@ Reads do not check passthrough values, as `decrypt_rows` reads only the encrypte - `between`, as two comparisons. The engine has no range term for EQL types (ADR-0003, amendment). - `matches` through the ORM. ADR-0003 says the layer should hide `target_limit` and `verify`. -- Core queries. - The reverse migration of a change to an EQL type. - Installing EQL from a migration. diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 5677e8104..5824d011c 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -519,7 +519,54 @@ def _decrypt_loaded(state: Any) -> Any: for target, values in zip(targets, opened): for name, value in values.items(): set_committed_value(target, name, value) - return result() + return _decrypt_columns(state.statement, result)() + + +def _decrypt_columns(statement: Any, result: Any) -> Any: + """Decrypt the result columns that are encrypted fields, such as + ``email`` in ``select(Customer.email, Customer.plan)``: with one + ``decrypt_rows`` call for each class and set of fields. A column of a + whole object, as in ``select(Customer)``, is left to the load events.""" + descriptions = getattr(statement, "column_descriptions", None) + if not descriptions: + return result + # Each result column that is an encrypted field: its position, by cipher. + positions: Dict[Cipher, Dict[int, str]] = {} + for at, description in enumerate(descriptions): + column = _column_of(description.get("expr")) + if column is None or not isinstance(column.type, EqlType): + continue + cipher = _TABLES.get(getattr(column, "table", None)) + if cipher is not None: + positions.setdefault(cipher, {})[at] = column.key + if not positions: + return result + rows = [list(row) for row in result.data] + for cipher, fields in positions.items(): + # Rows grouped by which of these fields are not NULL. + groups: Dict[FrozenSet[int], List[List[Any]]] = {} + for row in rows: + present = frozenset(at for at in fields if row[at] is not None) + if present: + groups.setdefault(present, []).append(row) + for present, members in groups.items(): + opened = cipher.decrypt_rows( + [{fields[at]: row[at] for at in present} for row in members] + ) + for row, values in zip(members, opened): + for at in present: + row[at] = values[fields[at]] + return result.with_new_rows([tuple(row) for row in rows]) + + +def _column_of(expression: Any) -> Any: + """The table column behind a result column: through an ORM attribute's + clause element, and through a label.""" + if hasattr(expression, "__clause_element__"): + expression = expression.__clause_element__() + while hasattr(expression, "element") and not isinstance(getattr(expression, "table", None), Table): + expression = expression.element + return expression if isinstance(getattr(expression, "table", None), Table) else None def _term(column: Any, method: str, value: object) -> Any: diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index a7d823744..23c2636e3 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -225,3 +225,39 @@ def test_an_update_of_a_passthrough_field_only_makes_no_key_request(engine): assert _engine.request_count() - before == 0 with Session(engine) as session: assert session.scalars(select(Customer)).one().plan == "pro" + + +def test_a_query_of_columns_decrypts_the_encrypted_ones(engine): + add_ann(engine) + with Session(engine) as session: + rows = session.execute(select(Customer.email, Customer.name, Customer.plan)).all() + labelled = session.execute(select(Customer.email.label("address"))).all() + both = session.execute(select(Customer, Customer.name)).all() + + assert [tuple(row) for row in rows] == [("ann@example.com", "Ann", "free")] + assert [row.address for row in labelled] == ["ann@example.com"] + [(ann, name)] = both + assert (ann.email, name) == ("ann@example.com", "Ann") + + +def test_100_rows_of_columns_make_one_key_request(engine): + with Session(engine) as session: + session.add_all( + [Customer(email=f"user{i}@example.com", name=f"User {i}", plan="free") for i in range(100)] + ) + session.commit() + with Session(engine) as session: + before = _engine.request_count() + rows = session.execute(select(Customer.email, Customer.name)).all() + assert _engine.request_count() - before == 1 + assert sorted(row.email for row in rows)[0] == "user0@example.com" + + +def test_a_core_query_on_a_connection_returns_eql_values(engine): + add_ann(engine) + with engine.connect() as conn: + [email] = conn.execute(select(Customer.__table__.c.email)).scalars().all() + # Left to the app: cipher.decrypt_rows opens it. + assert "ann@example.com" not in str(email) + [opened] = stackencrypt.Cipher(Customer).decrypt_rows([{"email": email}]) + assert opened == {"email": "ann@example.com"} From 53a2f1a0bdcd487b44b36e7ba4e05e443fc87be7 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 16:21:04 +0900 Subject: [PATCH 16/29] feat(python): match search through the SQLAlchemy ORM A match index is a Bloom filter, so the database can return rows that do not hold the text. ADR-0003 says the SQLAlchemy layer should hide this. .match(text) on a field with a match index is col @@ term, and the session hook runs verify on the rows after the query and removes those that fail. Removing rows after the query is right only in a session's select, with the match as a condition of the WHERE joined with AND, and whole objects of the class in the result. Anything else, such as or_, not_, a count, an update, or a query on a connection, raises QueryError before it is sent: it would return, count, or change rows that do not hold the text. A LIMIT is applied before the rows are removed, so a page can be short. ADR-0004 records the choice and the two other options. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 4 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 37 ++++- .../python/python/stackencrypt/sqlalchemy.py | 137 +++++++++++++++++- .../python/tests/test_sqlalchemy_live.py | 106 +++++++++++++- 4 files changed, 270 insertions(+), 14 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 68a67a40c..42019b942 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -176,6 +176,7 @@ with Session(engine) as session: - **Search:** `==`, `<`, `<=`, `>`, and `>=` on an encrypted field compare it with a query term, as `cipher.eq` and `cipher.gte` make. The field's marker must support the operator. For example, `Customer.level >= 3` works, and `Customer.email > x` raises `QueryError` with `query_unsupported`. - `!=`, `in_([...])`, and `not_in([...])` work where `==` does. `!=` becomes `NOT (col = term)`. `in_` becomes one equality for each value, joined with `OR`. +- `.match(text)` on a field whose marker has a match index, such as `TextMatch`, is a match search: `col @@ term`. A match index can return rows that do not hold the text. So after the query, the SDK runs `verify` on each row and removes the rows that fail. The app gets only rows that hold every 3-character piece of the text. - **Other operators** that would send a value, such as `like`, `contains`, `between`, and `in_` with a subquery, raise `QueryError` with `query_unsupported` when you build the query. The value would reach the database as plaintext, and could never match. `is_(None)`, `is_not(None)`, `asc()`, and `desc()` send no value, so they work. - A comparison on a class that is not configured raises `QueryError` with `query_not_configured`. @@ -238,7 +239,8 @@ These are not handled by the hooks yet: These are not handled yet: - A query run on a connection, not a session, returns EQL values, not plaintext. Examples are `conn.execute(select(...))` and `conn.execute(text("SELECT ..."))`. Use `cipher.decrypt_rows` on its rows. A connection has no hook that can change a result. -- A match search, `matches`, is not supported through the ORM yet. Use `cipher.matches`, as in "Search by match". +- `.match()` works only in a `select` run through a session. The match must be a condition of the `WHERE`, joined with `AND` if there are others. The result must include whole objects of the class, such as `select(Profile)`. Any other use raises `QueryError` with `query_unsupported` before the query is sent. Examples are `or_(...)`, `not_(...)`, `select(Profile.plan)`, `func.count()`, `update`, `delete`, and a query on a connection. Each one would return, count, or change rows that do not hold the text. +- With `.match()`, a `LIMIT` can return fewer rows than it names, even when more rows match. The database applies the limit before the SDK removes the rows that fail `verify`. - A query result is read in full before it is returned, so that its objects can be decrypted in one batch. A query that streams its rows does not stream. ## Declare a SQLAlchemy relationship diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index cf0a70fe1..0b7c7b3b3 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -122,10 +122,43 @@ The write hook sends every declared field to `encrypt_rows`, not only the encryp Reads do not check passthrough values, as `decrypt_rows` reads only the encrypted fields. +## Decision 8: a match search removes its false matches, with limits + +`.match(text)` on a field with a match index is `col @@ term`. A match index is a Bloom filter, so the database can return rows that do not hold the text. ADR-0003 says the layer should hide this. So after the query, the session hook runs `MatchQuery.verify` on each row, and removes the rows that fail. + +Removing rows after the query is right only in some places. So `.match()` works only when all three of these are true: + +1. The statement is a `select` run through a session. +2. The match is a condition of the `WHERE`, joined with `AND` if there are others. +3. The result includes whole objects of the class, so the column is loaded. + +Anything else raises `QueryError` with `query_unsupported` before the query is sent: + +| Use | Why it is refused | +|---|---| +| In `or_(...)` | A row kept by the other condition would be removed. | +| In `not_(...)` | The rows to keep are the false matches, which the hook cannot find. | +| `select(Profile.plan)` | The matched column is not in the result, so `verify` cannot run. | +| `func.count()` | The count includes the false matches. | +| `update` or `delete` | The false matches would be changed in the database. | +| A query on a connection | No hook can remove rows from its result. | + +### A LIMIT can return fewer rows + +The database applies a `LIMIT` before the hook removes rows. So `.limit(10)` can return fewer than 10 rows, even when more rows match. + +| Option | Cost | +|---|---| +| **A. Return fewer rows (chosen)** | The page can be short. | +| B. Raise the limit with `target_limit` | An extra query for a histogram of the stored filters, on each search. Without it, there is no finite limit. | +| C. Query again with an `OFFSET` | Several queries for one search, and rows can change between them. | + +A never returns a row that does not hold the text. B or C can come later, if an app needs exact page sizes. + ## What was run - `tests/test_sqlalchemy.py` and `tests/test_sqlalchemy_migrations.py` run without credentials. They cover the checks, the refused operators, the indexes, and the Alembic rendering. -- `tests/test_sqlalchemy_live.py` runs the hooks against ZeroKMS and PostgreSQL. It covers writes, reads, relationship loads, reads after a commit, search, and the request counts. +- `tests/test_sqlalchemy_live.py` runs the hooks against ZeroKMS and PostgreSQL. It covers writes, reads, relationship loads, reads after a commit, search, match search, and the request counts. A false match of the Bloom filter cannot be made on purpose. So the tests that show rows being removed replace `verify` with a function that refuses one row. - `tests/test_sqlalchemy_migrations_live.py` runs `alembic revision --autogenerate` and `alembic upgrade` on an existing table. A second `autogenerate` writes nothing. With `enable_seqscan` off, `EXPLAIN` shows each index in use. ## Consequences @@ -138,6 +171,6 @@ Reads do not check passthrough values, as `decrypt_rows` reads only the encrypte ## Open - `between`, as two comparisons. The engine has no range term for EQL types (ADR-0003, amendment). -- `matches` through the ORM. ADR-0003 says the layer should hide `target_limit` and `verify`. +- Exact page sizes for `.match()` with a `LIMIT`: option B or C of Decision 8. - The reverse migration of a change to an EQL type. - Installing EQL from a migration. diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 5824d011c..d2daec9a5 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -28,6 +28,11 @@ ``render_item`` writes an ``EqlType`` into a migration file with its imports. ``process_revision_directives`` writes a change of an existing column to an EQL type as an ALTER with a USING cast. +- Match: ``.match(text)`` on a field with a match index is ``col @@ term``. + The session hook runs ``verify`` on the rows and removes those that fail. + It works only in a session's select, as a condition of the WHERE joined + with AND, with whole objects of the class in the result. Anything else + raises ``QueryError``. A LIMIT can return fewer rows than it names. - Search: the column type that ``eql_type_map`` gives each encrypted field turns ``==``, ``<``, ``<=``, ``>`` and ``>=`` into a comparison with a query term, made by ``Cipher.eq``, ``lt``, ``lte``, ``gt`` or ``gte``. Any other @@ -54,10 +59,11 @@ or_, ) from sqlalchemy.dialects import postgresql -from sqlalchemy.engine import Engine +from sqlalchemy.engine import Engine, Row from sqlalchemy.orm import Mapper, Session from sqlalchemy.orm.attributes import set_committed_value -from sqlalchemy.sql import operators +from sqlalchemy.sql import operators, visitors +from sqlalchemy.sql.elements import BooleanClauseList from sqlalchemy.sql.dml import Insert, Update from sqlalchemy.types import UserDefinedType @@ -107,6 +113,16 @@ def operate(self, op: Any, *other: Any, **kwargs: Any) -> Any: # One equality term for each value. An empty list matches no row. found = or_(false(), *[self.operate(operators.eq, value) for value in values]) return found if op is operators.in_op else not_(found) + if op is operators.match_op and len(other) == 1: + query = _cipher_of(self.expr).matches(self.expr.key, other[0]) + term = cast( + literal(query.term, String()), + _QueryType(self.expr.type.marker.query_sql_domain), + ) + found = super().operate(_MATCHES, term) + # The session hook finds the search by this annotation, to + # check where it is and to verify the rows. + return found._annotate({_MATCH: (self.expr, query)}) method = _COMPARISONS.get(op) if method is None or len(other) != 1: raise _refused(self.expr, op) @@ -146,6 +162,14 @@ def get_col_spec(self, **kwargs: Any) -> str: operators.ge: "gte", } +# A match search: `col @@ term`, and the annotation that marks it. +_MATCHES = operators.custom_op("@@", is_comparison=True) +_MATCH = "stackencrypt_match" + +# The execution option the session hook sets on a statement whose match +# searches it checked and verifies. +_VERIFIED = "stackencrypt_match_verified" + # The operators that send no value to the database. _NO_VALUE = frozenset( { @@ -435,6 +459,11 @@ def _check(cls: type) -> Tuple[type, Cipher, FrozenSet[str]]: def _encrypter(tables: Dict[str, Tuple[Cipher, FrozenSet[str]]]) -> Any: def encrypt_parameters(conn, clause, multiparams, params, execution_options): # type: ignore[no-untyped-def] + if not execution_options.get(_VERIFIED): + matches = _matches_in(clause) + if matches: + column, _ = matches[0]._annotations[_MATCH] + raise _match_refused(column, "is not run through a session.") if not isinstance(clause, (Insert, Update)): return clause, multiparams, params configured = tables.get(getattr(clause.table, "name", None)) @@ -495,10 +524,16 @@ def _remember(target: object, attrs: Any) -> None: def _decrypt_loaded(state: Any) -> Any: + matches = _matches_in(state.statement) + if matches and not state.is_select: + column, _ = matches[0]._annotations[_MATCH] + raise _match_refused(column, "is in an UPDATE or a DELETE, so it would change rows that do not hold the text.") if not state.is_select: return None + searches = _checked_matches(state.statement, matches) # Freezing reads every row, so every load and refresh event has fired. - result = state.invoke_statement().freeze() + options = {_VERIFIED: True} if searches else {} + result = state.invoke_statement(execution_options=options).freeze() pending = state.session.info.pop(_PENDING, []) groups: Dict[Tuple[type, FrozenSet[str]], List[object]] = {} for target, names in pending: @@ -519,7 +554,70 @@ def _decrypt_loaded(state: Any) -> Any: for target, values in zip(targets, opened): for name, value in values.items(): set_committed_value(target, name, value) - return _decrypt_columns(state.statement, result)() + result = _decrypt_columns(state.statement, result) + if searches: + result = _verify(searches, result) + return result() + + +def _checked_matches(statement: Any, matches: List[Any]) -> List[Tuple[int, str, Any]]: + """For each match search: the result position of its class's objects, + the field, and the ``MatchQuery``. Raises ``QueryError`` for a search + whose rows cannot be verified.""" + if not matches: + return [] + # The conditions as the app passed them to where(), with each and_() + # opened. `whereclause` would join copies of them. + pending = list(getattr(statement, "_where_criteria", ()) or [statement.whereclause]) + top = set() + while pending: + condition = pending.pop() + if isinstance(condition, BooleanClauseList) and condition.operator is operators.and_: + pending.extend(condition.clauses) + else: + top.add(id(condition)) + descriptions = statement.column_descriptions + searches = [] + for match in matches: + column, query = match._annotations[_MATCH] + if id(match) not in top: + raise _match_refused(column, "is not a condition of the WHERE joined with AND.") + cls = next((c for c, (cipher, _) in _CONFIGURED.items() if cipher is _cipher_of(column)), None) + at = next( + ( + i + for i, description in enumerate(descriptions) + if description.get("type") is cls and description.get("expr") is cls + ), + None, + ) + if at is None: + raise _match_refused(column, "has no whole objects of its class in the result.") + searches.append((at, column.key, query)) + return searches + + +def _item(row: Any, at: int) -> Any: + """Item ``at`` of a frozen row. A result of one object, as from + ``select(Customer)``, keeps the object itself, not a one-item row.""" + return row[at] if isinstance(row, (tuple, Row)) else row + + +def _verify(searches: List[Tuple[int, str, Any]], result: Any) -> Any: + """Keep the rows whose objects hold the text of every match search.""" + rows = list(result.data) + keep = [True] * len(rows) + for at, field, query in searches: + values = [getattr(_item(row, at), field) for row in rows] + present = [i for i, value in enumerate(values) if value is not None] + for i in set(range(len(rows))) - set(present): + keep[i] = False + for i, ok in zip(present, query.verify([values[i] for i in present])): + keep[i] = keep[i] and ok + # with_new_rows takes one tuple for each row, whatever the shape it keeps. + return result.with_new_rows( + [row if isinstance(row, (tuple, Row)) else (row,) for row, kept in zip(rows, keep) if kept] + ) def _decrypt_columns(statement: Any, result: Any) -> Any: @@ -541,7 +639,7 @@ def _decrypt_columns(statement: Any, result: Any) -> Any: positions.setdefault(cipher, {})[at] = column.key if not positions: return result - rows = [list(row) for row in result.data] + rows = [list(row) if isinstance(row, (tuple, Row)) else [row] for row in result.data] for cipher, fields in positions.items(): # Rows grouped by which of these fields are not NULL. groups: Dict[FrozenSet[int], List[List[Any]]] = {} @@ -572,6 +670,11 @@ def _column_of(expression: Any) -> Any: def _term(column: Any, method: str, value: object) -> Any: """``value`` as the query term that ``method`` makes for ``column``, cast to the column marker's query type.""" + term = getattr(_cipher_of(column), method)(column.key, value) + return cast(literal(term, String()), _QueryType(column.type.marker.query_sql_domain)) + + +def _cipher_of(column: Any) -> Cipher: cipher = _TABLES.get(getattr(column, "table", None)) if cipher is None: raise QueryError( @@ -579,8 +682,28 @@ def _term(column: Any, method: str, value: object) -> Any: f"column {_column_name(column)} is encrypted, but its class is not configured. " "Call configure_encryption with the class before you build a query on it.", ) - term = getattr(cipher, method)(column.key, value) - return cast(literal(term, String()), _QueryType(column.type.marker.query_sql_domain)) + return cipher + + +def _match_refused(column: Any, why: str) -> QueryError: + return QueryError( + "query_unsupported", + f"a match search on {_column_name(column)} {why} A match index can return rows " + "that do not hold the text, and the SDK removes them after the query. It can do " + "that only in a select run through a session, with the match as a condition of " + "the WHERE, joined with AND, and with whole objects of the class in the result.", + field=getattr(column, "key", None), + reason="match", + ) + + +def _matches_in(statement: Any) -> List[Any]: + """Every match search in ``statement``, as the annotated expressions.""" + return [ + element + for element in visitors.iterate(statement) + if _MATCH in getattr(element, "_annotations", {}) + ] def _refused(column: Any, op: Callable[..., Any]) -> QueryError: diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index 23c2636e3..f38a98545 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -12,11 +12,11 @@ import pytest from _live import DATABASE_URL, LIVE, needs_database -from sqlalchemy import ForeignKey, create_engine, select, text +from sqlalchemy import ForeignKey, create_engine, delete, func, not_, or_, select, text, update from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship import stackencrypt -from stackencrypt import Ignore, IntegerOrd, Passthrough, QueryError, Text, TextEq, _engine +from stackencrypt import Ignore, IntegerOrd, MatchQuery, Passthrough, QueryError, Text, TextEq, TextMatch, _engine from stackencrypt.sqlalchemy import configure_encryption, eql_type_map pytestmark = [*LIVE, needs_database] @@ -45,6 +45,14 @@ class Order(Base): customer: Mapped[Annotated["Customer", Ignore]] = relationship(back_populates="orders") +@stackencrypt.model("profiles") +class Profile(Base): + __tablename__ = "profiles" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + bio: Mapped[Annotated[str, TextMatch]] + plan: Mapped[Annotated[str, Passthrough]] + + @stackencrypt.model("players") class Player(Base): __tablename__ = "players" @@ -66,7 +74,7 @@ def engine(): with engine.begin() as conn: conn.execute(text(f"CREATE SCHEMA {schema}")) Base.metadata.create_all(engine) - configure_encryption(engine, [Customer, Order, Player]) + configure_encryption(engine, [Customer, Order, Player, Profile]) yield engine with engine.begin() as conn: conn.execute(text(f"DROP SCHEMA {schema} CASCADE")) @@ -76,7 +84,7 @@ def engine(): @pytest.fixture(autouse=True) def empty(engine): with engine.begin() as conn: - conn.execute(text("TRUNCATE orders, customers, players RESTART IDENTITY")) + conn.execute(text("TRUNCATE orders, customers, players, profiles RESTART IDENTITY")) def stored(engine, sql): @@ -261,3 +269,93 @@ def test_a_core_query_on_a_connection_returns_eql_values(engine): assert "ann@example.com" not in str(email) [opened] = stackencrypt.Cipher(Customer).decrypt_rows([{"email": email}]) assert opened == {"email": "ann@example.com"} + + +# .match() on a TextMatch field. + +BIOS = ["Alice Smith", "Natalie Wood", "Bob Jones"] + + +def add_profiles(engine): + with Session(engine) as session: + session.add_all([Profile(bio=bio, plan="free") for bio in BIOS]) + session.commit() + + +def test_a_match_search_returns_the_rows_that_hold_the_text(engine): + add_profiles(engine) + with Session(engine) as session: + found = session.scalars(select(Profile).where(Profile.bio.match("ali"))).all() + + assert sorted(p.bio for p in found) == ["Alice Smith", "Natalie Wood"] + + +def test_rows_that_fail_verify_are_dropped(engine, monkeypatch): + add_profiles(engine) + verify = MatchQuery.verify + # A false match of the Bloom filter cannot be made on purpose. So + # verify refuses one row that the database returns. + monkeypatch.setattr( + MatchQuery, "verify", lambda q, values: [v != "Natalie Wood" and ok for v, ok in zip(values, verify(q, values))] + ) + with Session(engine) as session: + found = session.scalars( + select(Profile).where(Profile.bio.match("ali"), Profile.plan == "free") + ).all() + + assert [p.bio for p in found] == ["Alice Smith"] + + +def test_a_limit_can_return_fewer_rows_than_match(engine, monkeypatch): + add_profiles(engine) + verify = MatchQuery.verify + monkeypatch.setattr( + MatchQuery, "verify", lambda q, values: [v != "Alice Smith" and ok for v, ok in zip(values, verify(q, values))] + ) + with Session(engine) as session: + found = session.scalars( + select(Profile).where(Profile.bio.match("ali")).order_by(Profile.id).limit(1) + ).all() + + # The database's one row was Alice, which verify refused. Natalie also + # matches, but was past the limit. + assert found == [] + + +@pytest.mark.parametrize( + "statement", + [ + lambda: select(Profile).where(or_(Profile.bio.match("ali"), Profile.plan == "pro")), + lambda: select(Profile).where(not_(Profile.bio.match("ali"))), + lambda: select(Profile.plan).where(Profile.bio.match("ali")), + lambda: select(func.count()).select_from(Profile).where(Profile.bio.match("ali")), + lambda: update(Profile).where(Profile.bio.match("ali")).values(plan="pro"), + lambda: delete(Profile).where(Profile.bio.match("ali")), + ], + ids=["in-an-or", "in-a-not", "object-not-selected", "count", "update", "delete"], +) +def test_a_match_that_rows_cannot_be_checked_for_is_refused(engine, statement): + add_profiles(engine) + with Session(engine) as session, pytest.raises(QueryError) as raised: + session.execute(statement()) + assert raised.value.code == "query_unsupported" + assert "A match index can return rows" in raised.value.message + assert "ali" not in str(raised.value) + with Session(engine) as session: + assert sorted((p.bio, p.plan) for p in session.scalars(select(Profile))) == sorted( + (bio, "free") for bio in BIOS + ) + + +def test_a_match_on_a_connection_is_refused(engine): + add_profiles(engine) + with engine.connect() as conn, pytest.raises(QueryError) as raised: + conn.execute(select(Profile.__table__).where(Profile.bio.match("ali"))) + assert raised.value.code == "query_unsupported" + assert "is not run through a session" in raised.value.message + + +def test_a_match_on_a_field_with_no_match_index_is_unsupported(engine): + with pytest.raises(QueryError) as raised: + Customer.email.match("ali") + assert raised.value.code == "query_unsupported" From 5c691fab76b50d07066d4830a5a70c1feec87d0d Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 16:30:46 +0900 Subject: [PATCH 17/29] docs(python): scorecard G6 covers the SQLAlchemy layer G6 said the SQLAlchemy layer was a later, separate design. It now exists, and it is a second way to encrypt and decrypt. The entry goes through the principle's four tests with the evidence for each: the need, the shared engine calls, the guarantees kept, and the owners. It also names the one difference, one decrypt_rows call for each class instead of one for the whole result, and the untested echo=True log. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/docs/scorecard.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/languages/python/docs/scorecard.md b/languages/python/docs/scorecard.md index c574f040f..fdd5a4bab 100644 --- a/languages/python/docs/scorecard.md +++ b/languages/python/docs/scorecard.md @@ -63,7 +63,13 @@ Each principle is marked: - There is one way to declare a model: `stackencrypt.model`. Every class library goes through it. `tests/_declaration_cases.py` declares one model with each of dataclasses, attrs, Pydantic, `NamedTuple`, and SQLAlchemy. - No test shows that each refusal fires for every one of the five libraries. -- There is one package, `stackencrypt`. The SQLAlchemy layer is a later, separate design (spec #1341, "Out of Scope"). +- There is one package, `stackencrypt`. +- The SQLAlchemy layer, `stackencrypt.sqlalchemy`, is a second way to encrypt and decrypt. ADR-0004 records its design. It passes the four tests: + 1. **A need the first way cannot meet.** An ORM app uses `Session` and its own objects. With `cipher.encrypt` alone, `examples/customer-directory/app_sqlalchemy.py` could not use `Session` at all. + 2. **The same output, through the same code.** The hooks call `encrypt_rows` and `decrypt_rows`, which run the same `_engine.encrypt` and `_engine.decrypt`. Search terms come from the same `Cipher` methods. `test_a_row_updated_with_some_fields_decrypts_and_searches_with_the_full_model` shows that a value from a smaller plan opens and searches under the full plan. The example's `test_each_app_reads_what_the_other_wrote` shows the two ways read each other's rows. + 3. **The same guarantees, at the same stage.** The declaration is the same decorator. Every value, passthrough values included, is checked before any key request. Errors contain no value. A match search runs `verify` for the app. One difference: the read hook makes one `decrypt_rows` call for each class and set of fields, not one call for the whole result. If a later call fails, the query raises, and the app gets no result. + 4. **Someone owns it.** It has the README section "Use it with the SQLAlchemy ORM", ADR-0004, and four test files. `mise run test` and the `live` job of `tests-python.yml` run them. The owners of this repository, `@cipherstash/developers` in `.github/CODEOWNERS`, own it with the rest of the Python SDK. +- No test checks what SQLAlchemy logs with `echo=True`. The write hook runs before that log, so the log should show EQL values. ## G7. State the design, and run its claims: held From 360d9f733e5148673f1e9133c2a549f4d207a77f Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 17:03:58 +0900 Subject: [PATCH 18/29] feat(python): a field declared Optional stores None as SQL NULL The decorator refused Optional, so a nullable column could only be declared with a wrong type hint, and encrypt still refused None. A row with a missing value could not be stored. Most columns of an app being ported are nullable. Annotated[Optional[T], Marker] and T | None are now accepted. The plan gives the field "null": "sql", so the engine stores nothing for None and the column holds SQL NULL; decrypt reads it back as None. A query term for None is refused, and points to IS NULL. eql_type_map maps the Optional hints too, so SQLAlchemy makes the column nullable. "sql" shows which rows have no value. "sealed" would hide that, but only for fixed-width types with an equality index alone; it can come later. The ADR-0001 amendment records why. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 21 +++- languages/python/crates/core/src/decrypt.rs | 116 ++++++++++++++++++ languages/python/crates/core/src/encrypt.rs | 23 +++- languages/python/crates/core/src/plan.rs | 16 ++- languages/python/crates/core/src/query.rs | 3 + languages/python/crates/core/src/values.rs | 5 + languages/python/crates/engine/src/decrypt.rs | 17 +++ languages/python/crates/engine/src/encrypt.rs | 24 +++- .../python/crates/engine/src/heap_hygiene.rs | 4 + languages/python/crates/engine/src/lib.rs | 10 +- languages/python/crates/engine/src/query.rs | 16 +++ ...on-is-checked-when-the-class-is-defined.md | 28 +++++ .../python/python/stackencrypt/_cipher.py | 1 + .../python/stackencrypt/_declaration.py | 38 +++++- .../python/python/stackencrypt/_engine.pyi | 4 +- .../python/python/stackencrypt/sqlalchemy.py | 11 +- languages/python/tests/_declaration_cases.py | 27 +++- languages/python/tests/test_encrypt_live.py | 39 +++++- languages/python/tests/test_nullable.py | 79 ++++++++++++ languages/python/tests/test_sqlalchemy.py | 28 ++++- .../python/tests/test_sqlalchemy_live.py | 50 +++++++- 21 files changed, 528 insertions(+), 32 deletions(-) create mode 100644 languages/python/tests/test_nullable.py diff --git a/languages/python/README.md b/languages/python/README.md index 42019b942..017964cac 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -87,6 +87,25 @@ A passthrough `datetime` field needs a `timestamptz` column. psycopg reads a `ti `decrypt` checks every row before it sends anything. A value stored under another table or column raises `DecryptError` with the code `decrypt_foreign_row`. A passthrough value of another type than the field declares raises `InvalidValueError`. When any row fails, `decrypt` returns no instances. +## A field that can be missing + +Declare a field that can be `None` with `Optional` inside the `Annotated`: + +```text +city: Annotated[Optional[str], TextEq] +visits: Annotated[Optional[int], Passthrough] +``` + +`str | None` works too, from Python 3.10. Put `Optional` inside the `Annotated`, not around it. `Optional[Annotated[str, TextEq]]` raises `DeclarationError` with `declaration_marker_not_outermost`. + +`encrypt` gives `None` for a missing value, and the column stores SQL `NULL`. `decrypt` reads SQL `NULL` back as `None`. So a column for an `Optional` field must allow `NULL`. + +A missing value is not encrypted. Anyone who can read the table can see which rows have no value in that column. They cannot see the values that are there. + +A query term cannot be made for `None`. `cipher.eq("city", None)` raises `InvalidValueError` with `value_none`. Search for missing values with `WHERE city IS NULL`. + +A field without `Optional` still refuses `None`, with `value_none`. + ## Search by equality `cipher.eq` takes a field's name and a value, and returns a query term as a `str`. Cast the term to the field's query type in the SQL: @@ -168,7 +187,7 @@ with Session(engine) as session: customer.email # "a@example.com" ``` -`eql_type_map` gives each encrypted field its EQL column type, such as `public.eql_v4_text_eq` for `TextEq`. So a column needs no type of its own. `Base.metadata.create_all(engine)` creates the EQL columns. +`eql_type_map` gives each encrypted field its EQL column type, such as `public.eql_v4_text_eq` for `TextEq`. So a column needs no type of its own. `Base.metadata.create_all(engine)` creates the EQL columns. A field declared `Mapped[Annotated[Optional[str], TextEq]]` gets the same type, and SQLAlchemy makes its column nullable. - **Writes:** each INSERT or UPDATE on a listed table is encrypted with one batch of key requests for all its rows. - **Reads:** the objects a query loads are decrypted with one batch for each class. This includes relationship loads, and attributes read again after a commit. A session query of columns, such as `session.execute(select(Customer.email, Customer.plan))`, is decrypted too. diff --git a/languages/python/crates/core/src/decrypt.rs b/languages/python/crates/core/src/decrypt.rs index cf7931c28..bb0e0a611 100644 --- a/languages/python/crates/core/src/decrypt.rs +++ b/languages/python/crates/core/src/decrypt.rs @@ -459,6 +459,7 @@ pub(crate) mod tests { Field { name: name.to_owned(), eql: Some((eql.to_owned(), kind)), + nullable: false, } } @@ -466,6 +467,7 @@ pub(crate) mod tests { Field { name: name.to_owned(), eql: None, + nullable: false, } } @@ -533,6 +535,7 @@ pub(crate) mod tests { row.passthrough(field, Value::Int(*i)).unwrap() } Column::Passthrough(other) => panic!("not an age: {other:?}"), + Column::Null => panic!("no field here is nullable"), } } rows.push(row); @@ -717,4 +720,117 @@ pub(crate) mod tests { assert_eq!(refused.code, "decrypt_kind_mismatch", "{refused:?}"); assert_eq!(refused.field.as_deref(), Some("email")); } + + /// `people` with a nullable sealed field and a nullable passthrough. + fn people() -> ModelPlan { + ModelPlan::new( + "people", + vec![ + Field { + name: "city".to_owned(), + eql: Some(("TextEq".to_owned(), Kind::String)), + nullable: true, + }, + Field { + name: "email".to_owned(), + eql: Some(("TextEq".to_owned(), Kind::String)), + nullable: false, + }, + Field { + name: "note".to_owned(), + eql: None, + nullable: true, + }, + ], + ) + .unwrap() + } + + fn people_rows<'a>(rows: &[(Value<'a>, &'a str, Value<'a>)]) -> ValueFrame { + ValueFrame::of(Source( + rows.iter() + .map(|(city, email, note)| { + vec![ + ("city", Some(Kind::String), *city), + ("email", Some(Kind::String), Value::Text(email)), + ("note", None, *note), + ] + }) + .collect(), + )) + .unwrap() + } + + #[test] + fn a_null_in_a_nullable_field_is_stored_as_sql_null_and_opens_as_null() { + let cipher = cipher(); + let plan = people(); + let source = people_rows(&[ + (Value::Null, "a@example.com", Value::Null), + (Value::Text("Leeds"), "b@example.com", Value::Text("hi")), + ]); + + let stored = run(encrypt_with(&cipher.default_keyset(), &plan, source, 2)).unwrap(); + + assert!(matches!(stored[0][0], Column::Null)); + assert!(matches!( + stored[0][2], + Column::Null | Column::Passthrough(Scalar::Null) + )); + assert!(matches!(stored[1][0], Column::Eql(_))); + + let mut rows = Rows::default(); + for columns in &stored { + let mut row = Row::new(); + for (field, column) in plan.fields().iter().zip(columns) { + match column { + Column::Eql(json) => row.eql(&plan, field, json.as_bytes()).unwrap(), + Column::Passthrough(Scalar::Text(text)) => { + row.passthrough(field, Value::Text(text)).unwrap() + } + // A passthrough is always in the row, as a null here. + Column::Passthrough(Scalar::Null) => { + row.passthrough(field, Value::Null).unwrap() + } + // A nullable EQL field's SQL NULL: no entry. + Column::Null if field.eql.is_some() => {} + Column::Null => row.passthrough(field, Value::Null).unwrap(), + Column::Passthrough(other) => panic!("not a note: {other:?}"), + } + } + rows.push(row); + } + let opened = run(decrypt_with(&cipher, &plan, rows)).unwrap(); + + assert_eq!( + opened, + vec![ + vec![ + Scalar::Null, + Scalar::Text("a@example.com".to_owned()), + Scalar::Null + ], + vec![ + Scalar::Text("Leeds".to_owned()), + Scalar::Text("b@example.com".to_owned()), + Scalar::Text("hi".to_owned()), + ], + ] + ); + } + + #[test] + fn a_null_in_a_field_that_is_not_nullable_is_refused() { + let cipher = cipher(); + let source = ValueFrame::of(Source(vec![vec![ + ("city", Some(Kind::String), Value::Text("Leeds")), + ("email", Some(Kind::String), Value::Null), + ("note", None, Value::Text("hi")), + ]])) + .unwrap(); + + let refused = run(encrypt_with(&cipher.default_keyset(), &people(), source, 1)); + + assert!(refused.is_err(), "{refused:?}"); + } } diff --git a/languages/python/crates/core/src/encrypt.rs b/languages/python/crates/core/src/encrypt.rs index 7c51b70e0..863d1d440 100644 --- a/languages/python/crates/core/src/encrypt.rs +++ b/languages/python/crates/core/src/encrypt.rs @@ -30,6 +30,9 @@ pub struct Field { pub name: String, /// The EQL type and its kind, or `None` for a passthrough. pub eql: Option<(String, Kind)>, + /// Whether the field can be missing: Python's `Optional`. A missing + /// value is written as a null, and comes back as [`Column::Null`]. + pub nullable: bool, } /// A model's plan, built once per `Cipher` and read on every call. @@ -53,7 +56,11 @@ impl ModelPlan { .eql .as_ref() .map_or(plan::Column::Passthrough, |(name, _)| { - plan::Column::Eql(name) + if field.nullable { + plan::Column::NullableEql(name) + } else { + plan::Column::Eql(name) + } }); (field.name.as_str(), column) }) @@ -90,6 +97,8 @@ pub enum Column { Eql(String), /// The echoed value of a passthrough field. Passthrough(Scalar), + /// A nullable field's missing value: SQL `NULL`, Python's `None`. + Null, } /// Encrypt `rows` models, written as one frame, under `plan`. Returns one @@ -152,10 +161,14 @@ fn read_row(plan: &ModelPlan, row: StackCipherText) -> Result, Refus }; let mut columns = Vec::with_capacity(plan.fields.len()); for field in &plan.fields { - let (_, outputs) = entries - .iter() - .find(|(name, _)| name == &field.name) - .ok_or_else(|| engine_fault("a row lacks a declared field"))?; + let Some((_, outputs)) = entries.iter().find(|(name, _)| name == &field.name) else { + // A nullable EQL field's null produces no entry. + if field.nullable { + columns.push(Column::Null); + continue; + } + return Err(engine_fault("a row lacks a declared field")); + }; let CipherText::Map(outputs) = outputs else { return Err(engine_fault("a field is not a map of outputs")); }; diff --git a/languages/python/crates/core/src/plan.rs b/languages/python/crates/core/src/plan.rs index 6687af6a5..83b0de40d 100644 --- a/languages/python/crates/core/src/plan.rs +++ b/languages/python/crates/core/src/plan.rs @@ -18,6 +18,10 @@ use crate::targets; pub enum Column<'a> { /// A field sealed as the EQL type of this name, such as `TextEq`. Eql(&'a str), + /// As [`Column::Eql`], for a field that can be missing. The engine + /// stores nothing for a null (`"null": "sql"`), so the column holds SQL + /// `NULL`, and opens a missing value as a null. + NullableEql(&'a str), /// A field stored as it is. Passthrough, } @@ -90,10 +94,20 @@ struct Spec<'a> { impl Serialize for Spec<'_> { fn serialize(&self, serializer: S) -> Result { - let mut spec = serializer.serialize_struct("Spec", 2)?; + // The frame needs each container's exact length. + let len = if matches!(self.column, Column::NullableEql(_)) { + 3 + } else { + 2 + }; + let mut spec = serializer.serialize_struct("Spec", len)?; spec.serialize_field("context", &self.context)?; match self.column { Column::Eql(name) => spec.serialize_field("target", name)?, + Column::NullableEql(name) => { + spec.serialize_field("target", name)?; + spec.serialize_field("null", "sql")?; + } Column::Passthrough => spec.serialize_field("outputs", &["passthrough"])?, } spec.end() diff --git a/languages/python/crates/core/src/query.rs b/languages/python/crates/core/src/query.rs index a563a6666..dee96490b 100644 --- a/languages/python/crates/core/src/query.rs +++ b/languages/python/crates/core/src/query.rs @@ -243,6 +243,7 @@ mod tests { vec![Field { name: "level".to_owned(), eql: Some(("IntegerOrd".to_owned(), Kind::Int32)), + nullable: false, }], ) .unwrap() @@ -294,6 +295,7 @@ mod tests { vec![Field { name: "notes".to_owned(), eql: Some(("TextMatch".to_owned(), Kind::String)), + nullable: false, }], ) .unwrap() @@ -409,6 +411,7 @@ mod tests { vec![Field { name: "rank".to_owned(), eql: Some((eql.to_owned(), kind)), + nullable: false, }], ) .unwrap(); diff --git a/languages/python/crates/core/src/values.rs b/languages/python/crates/core/src/values.rs index c00bc8f39..84e91c45b 100644 --- a/languages/python/crates/core/src/values.rs +++ b/languages/python/crates/core/src/values.rs @@ -81,6 +81,10 @@ impl Kind { /// A host value, already read by exact type, as the engine takes it. #[derive(Clone, Copy, Debug)] pub enum Value<'a> { + /// A missing value: Python's `None`, for a field declared `Optional`. + /// The frame carries it as a null. A nullable EQL field stores nothing + /// for it, so its column holds SQL `NULL` (`"null": "sql"`). + Null, Text(&'a str), /// Any Python `int` that fits; one that does not is refused by the /// extension as out of range before it gets here. @@ -162,6 +166,7 @@ pub fn scalar( width(i).map(Physical::Int).ok_or(Misfit::OutOfRange) }; Ok(match (kind, value) { + (_, Value::Null) => (None, Physical::Null), (Some(Kind::String), Value::Text(s)) | (None, Value::Text(s)) => (None, Physical::Text(s)), (Some(Kind::Int16), Value::Int(i)) => { (None, int(i, |i| i16::try_from(i).ok().map(Int::I16))?) diff --git a/languages/python/crates/engine/src/decrypt.rs b/languages/python/crates/engine/src/decrypt.rs index 843b5f7f3..0ab1fde14 100644 --- a/languages/python/crates/engine/src/decrypt.rs +++ b/languages/python/crates/engine/src/decrypt.rs @@ -106,6 +106,23 @@ fn stored_row(plan: &Plan, row: &Bound<'_, PyAny>, at: usize) -> PyResult { }, ) }; + // A field declared Optional, stored as SQL NULL: an EQL field has no + // entry, which opens as a null; a passthrough is a null. + if field.nullable && value.is_none() { + if field.eql.is_none() { + stored + .passthrough(field, stackencrypt_core::values::Value::Null) + .map_err(|misfit| { + in_row(Refusal { + code: misfit.code().to_owned(), + message: format!("field {:?}: a null does not fit", field.name), + field: Some(field.name.clone()), + reason: Some("passthrough".to_owned()), + }) + })?; + } + continue; + } match &field.eql { Some(_) => { let text = eql_text(field, &value)?.map_err(in_row)?; diff --git a/languages/python/crates/engine/src/encrypt.rs b/languages/python/crates/engine/src/encrypt.rs index c6d3516f7..e174bcd7d 100644 --- a/languages/python/crates/engine/src/encrypt.rs +++ b/languages/python/crates/engine/src/encrypt.rs @@ -22,8 +22,14 @@ use crate::errors::raise; use crate::values::{read, to_python, PyKind, Read}; /// One plan field as the decorator hands it over: `(name, eql_type, kind, -/// value_type)`. -type FieldSpec<'py> = (String, Option, Option, Bound<'py, PyType>); +/// value_type, nullable)`. +type FieldSpec<'py> = ( + String, + Option, + Option, + Bound<'py, PyType>, + bool, +); /// A model's plan, built once by `Cipher` and passed to every call. #[pyclass(frozen, module = "stackencrypt._engine")] @@ -44,7 +50,7 @@ pub struct Plan { pub fn build_plan(py: Python<'_>, table: &str, fields: Vec>) -> PyResult { let mut kinds = Vec::with_capacity(fields.len()); let mut specs = Vec::with_capacity(fields.len()); - for (name, eql_type, kind, value_type) in fields { + for (name, eql_type, kind, value_type, nullable) in fields { let eql = match (eql_type, kind) { (Some(eql_type), Some(kind)) => { let kind = Kind::parse(&kind).ok_or_else(|| { @@ -73,7 +79,11 @@ pub fn build_plan(py: Python<'_>, table: &str, fields: Vec>) -> Py )) } }; - specs.push(Field { name, eql }); + specs.push(Field { + name, + eql, + nullable, + }); } let plan = ModelPlan::new(table, specs).map_err(|refused| raise(py, refused))?; Ok(Plan { plan, kinds }) @@ -121,6 +131,7 @@ pub fn encrypt<'py>( let value = match column { Column::Eql(json) => PyString::new(py, json).into_any().unbind(), // A passthrough value came from Python, so it fits. + Column::Null => py.None(), Column::Passthrough(scalar) => { to_python(py, scalar)?.map_err(|unfit| PyValueError::new_err(unfit.cause()))? } @@ -191,6 +202,11 @@ pub(crate) fn field_value<'a>( field: &Field, kind: PyKind, ) -> PyResult, Refusal>> { + // A field declared Optional: None is a missing value. The engine stores + // nothing for it, and the column holds SQL NULL. + if field.nullable && value.is_none() { + return Ok(Ok(Read::Other(values::Value::Null))); + } let engine_kind = field.eql.as_ref().map(|(_, kind)| *kind); let reason = engine_kind.map_or("passthrough", Kind::name); let named = |refused: Refusal| Refusal { diff --git a/languages/python/crates/engine/src/heap_hygiene.rs b/languages/python/crates/engine/src/heap_hygiene.rs index a95ef9cf5..1d84b7285 100644 --- a/languages/python/crates/engine/src/heap_hygiene.rs +++ b/languages/python/crates/engine/src/heap_hygiene.rs @@ -209,18 +209,21 @@ fn plan(py: Python<'_>) -> Plan { Some("TextEq".into()), Some("string".into()), text(), + false, ), ( "note".into(), Some("Text".into()), Some("string".into()), text(), + false, ), ( "amount".into(), Some("NumericEq".into()), Some("decimal".into()), decimal, + false, ), ], ) @@ -397,6 +400,7 @@ fn verify_frees_no_copy_of_the_plaintext_unwiped() { Some("TextMatch".into()), Some("string".into()), py.get_type::(), + false, )], ) .unwrap(); diff --git a/languages/python/crates/engine/src/lib.rs b/languages/python/crates/engine/src/lib.rs index 38f839607..494c59953 100644 --- a/languages/python/crates/engine/src/lib.rs +++ b/languages/python/crates/engine/src/lib.rs @@ -45,11 +45,15 @@ fn build_info() -> (&'static str, &'static str) { /// accepts the plan, or its refusal as `(code, message, field, reason)`. The /// Python side raises it as a `DeclarationError`. #[pyfunction] -fn check_plan(table: &str, fields: Vec<(String, Option)>) -> Option { +fn check_plan(table: &str, fields: Vec<(String, Option, bool)>) -> Option { let fields: Vec<(&str, Column<'_>)> = fields .iter() - .map(|(name, eql_type)| { - let column = eql_type.as_deref().map_or(Column::Passthrough, Column::Eql); + .map(|(name, eql_type, nullable)| { + let column = match (eql_type.as_deref(), nullable) { + (Some(eql_type), true) => Column::NullableEql(eql_type), + (Some(eql_type), false) => Column::Eql(eql_type), + (None, _) => Column::Passthrough, + }; (name.as_str(), column) }) .collect(); diff --git a/languages/python/crates/engine/src/query.rs b/languages/python/crates/engine/src/query.rs index 05ae86ff2..42568f48d 100644 --- a/languages/python/crates/engine/src/query.rs +++ b/languages/python/crates/engine/src/query.rs @@ -44,6 +44,10 @@ pub fn query( let Some((_, engine_kind)) = declared.eql else { return Err(raise(py, not_sealed(field))); }; + // A nullable field reads None as a missing value, which has no term. + if declared.nullable && value.is_none() { + return Err(raise(py, no_term_for_none(field, engine_kind.name()))); + } let read = field_value(value, declared, kind)?.map_err(|refused| raise(py, refused))?; let value = read.value(); py.detach(|| core::query(&plan.plan, field, op, engine_kind, value)) @@ -117,6 +121,18 @@ pub fn target_limit( .map_err(|refused| raise(py, refused)) } +fn no_term_for_none(field: &str, kind: &str) -> Refusal { + Refusal { + code: "value_none".to_owned(), + message: format!( + "field {field:?}: is None. A missing value is stored as SQL NULL, with no \ + term. Search for it with `IS NULL`." + ), + field: Some(field.to_owned()), + reason: Some(kind.to_owned()), + } +} + fn as_text<'r>(read: &'r Read<'_>) -> Option<&'r str> { match read.value() { stackencrypt_core::values::Value::Text(text) => Some(text), diff --git a/languages/python/docs/adr/0001-a-declaration-is-checked-when-the-class-is-defined.md b/languages/python/docs/adr/0001-a-declaration-is-checked-when-the-class-is-defined.md index 4ee1b91a0..3bdd90a3b 100644 --- a/languages/python/docs/adr/0001-a-declaration-is-checked-when-the-class-is-defined.md +++ b/languages/python/docs/adr/0001-a-declaration-is-checked-when-the-class-is-defined.md @@ -4,6 +4,7 @@ date: 2026-10-16 accepted: 2026-10-27, by the review on PR #1284; the status edit rode with the first implementation PR, as the spec (#1341) says extends: ADR-0007, ADR-0008 (packages/stack-encrypt/docs/adr) ticket: the Python SDK discovery ticket, backlog item 1 (its 15 October 2026 amendment) +amended_by: "Amendment, 2026-11-11: a field can be Optional" --- # A declaration is checked when Python defines the class, by construction @@ -503,3 +504,30 @@ file and a hand-written one reports the class's module. shapes) and 1.3 (typing the encrypted output). This ADR takes `Annotated` as the input they compare against, which is decision 5's default. + +## Amendment, 2026-11-11: a field can be Optional + +The decorator now accepts `Optional[T]` inside the `Annotated`, as in `Annotated[Optional[str], TextEq]`. It also accepts `T | None`. Before this, it refused them with `declaration_type_mismatch`. + +An app that ports a schema has many nullable columns. Without `Optional`, it could only write `Annotated[str, TextEq]` on a nullable column. That type hint was wrong, and `encrypt` still refused `None` with `value_none`. So a row with a missing value could not be stored at all. + +A field declared `Optional` can be missing: + +- `encrypt` writes `None` as a missing value. The plan gives the field `"null": "sql"`, so the engine stores nothing for it, and the column holds SQL `NULL`. +- `decrypt` reads SQL `NULL` back as `None`. +- A passthrough `None` is stored as a null. +- A query term cannot be made for `None`. It raises `value_none`, and the message points to `IS NULL`. + +A field without `Optional` still refuses `None`, so the check still runs when the class statement runs and before any request. + +### Why `"sql"` and not `"sealed"` + +The engine has two ways to store a missing value (`packages/stack-encrypt/src/dynamic/record.rs`, `NullMode`): + +- `"sql"` stores nothing. The column shows which rows have no value. Any EQL type can use it. +- `"sealed"` encrypts the missing value like a value, so the column does not show which rows have none. It works only for a fixed-width type, such as a number or a date, with an equality index alone. + +`Optional` means `"sql"`. It is what a nullable column means in a normal table, and it works for every marker, text included. An app that stores a placeholder, such as an empty string, in place of `None` shows the same thing: with an equality index, every placeholder row has the same term. `"sealed"` can come later, as an option a fixed-width field asks for. + +The Go SDK declares a nullable field as a pointer, with a `null=sql` or `null=sealed` tag, and refuses a pointer with no tag (`languages/golang/encrypt/gensupport/declaration.go`). Python needs no tag for the common case, because `Optional` already says the field can be missing. + diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index 417c800c2..000f6069b 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -409,6 +409,7 @@ def _build_plan(declaration: Declaration, fields: Sequence[DeclaredField]) -> ob None if field.marker is Passthrough else field.marker.name, None if field.marker is Passthrough else field.marker.plaintext, field.value_type, + field.nullable, ) for field in fields ], diff --git a/languages/python/python/stackencrypt/_declaration.py b/languages/python/python/stackencrypt/_declaration.py index 91f34ca13..f8e1a4c0d 100644 --- a/languages/python/python/stackencrypt/_declaration.py +++ b/languages/python/python/stackencrypt/_declaration.py @@ -32,6 +32,7 @@ class around ``Annotated`` is read by structure, so ``Mapped[Annotated[str, import dataclasses import inspect +import types import sys from dataclasses import dataclass from datetime import date, datetime @@ -47,6 +48,7 @@ class around ``Annotated`` is read by structure, so ``Mapped[Annotated[str, Tuple, Type, TypeVar, + Union, get_args, get_origin, get_type_hints, @@ -58,6 +60,12 @@ class around ``Annotated`` is read by structure, so ``Mapped[Annotated[str, T = TypeVar("T", bound=type) +# The origins of a union: typing.Union, and types.UnionType for `T | None` +# from Python 3.10. +_UNIONS = tuple( + origin for origin in (Union, getattr(types, "UnionType", None)) if origin is not None +) + # The one attribute the decorator stores, in the class's own __dict__. DECLARATION_ATTRIBUTE = "__stash_declaration__" @@ -99,6 +107,8 @@ class DeclaredField: name: str marker: Type[Marker] value_type: type = dataclasses.field(default=object, repr=False, compare=False) + # Declared Optional: None is a missing value, stored as SQL NULL. + nullable: bool = False @dataclass(frozen=True) @@ -108,6 +118,7 @@ class _ReadField: name: str marker: Type[Marker] value_type: Any + nullable: bool = False @dataclass(frozen=True) @@ -177,7 +188,9 @@ def decorate(cls: T) -> T: declaration = Declaration( table=table, fields=tuple( - DeclaredField(name=f.name, marker=f.marker, value_type=f.value_type) + DeclaredField( + name=f.name, marker=f.marker, value_type=f.value_type, nullable=f.nullable + ) for f in planned ), hints=hints, @@ -297,7 +310,10 @@ def _read_fields(cls: type, hints: Dict[str, Any]) -> List[_ReadField]: read = [] for name in names: marker, value_type = _marker_of(name, hints.get(name), from_hints=from_hints) - read.append(_ReadField(name=name, marker=marker, value_type=value_type)) + value_type, nullable = _without_none(value_type) + read.append( + _ReadField(name=name, marker=marker, value_type=value_type, nullable=nullable) + ) return read @@ -405,6 +421,18 @@ def _is_wrapper(hint: Any) -> bool: ) +def _without_none(value_type: Any) -> Tuple[Any, bool]: + """``Optional[T]`` as ``(T, True)``; any other type as ``(type, False)``. + ``Optional[T]`` is ``Union[T, None]``, and ``T | None`` from Python 3.10. + A union of more than one type besides ``None`` is left as it is, and + the type check refuses it.""" + if get_origin(value_type) in _UNIONS: + others = [arg for arg in get_args(value_type) if arg is not type(None)] + if len(others) == 1 and len(get_args(value_type)) == 2: + return others[0], True + return value_type, False + + def _marker_of(name: str, hint: Any, *, from_hints: bool) -> Tuple[Type[Marker], Any]: """The one marker of a field and its value type, or a ``DeclarationError``. @@ -606,7 +634,11 @@ def _check_plan(table: str, planned: List[_ReadField]) -> None: refused = _engine.check_plan( table, [ - (field.name, None if field.marker is Passthrough else field.marker.name) + ( + field.name, + None if field.marker is Passthrough else field.marker.name, + field.nullable, + ) for field in planned ], ) diff --git a/languages/python/python/stackencrypt/_engine.pyi b/languages/python/python/stackencrypt/_engine.pyi index 70c2e8f89..3dd5efc59 100644 --- a/languages/python/python/stackencrypt/_engine.pyi +++ b/languages/python/python/stackencrypt/_engine.pyi @@ -6,7 +6,7 @@ def build_info() -> tuple[str, str]: """``(version, engine_commit)`` as recorded at build time.""" def check_plan( - table: str, fields: List[Tuple[str, Optional[str]]] + table: str, fields: List[Tuple[str, Optional[str], bool]] ) -> Optional[Tuple[str, str, Optional[str], Optional[str]]]: """The engine's check of a model's plan: ``None``, or its refusal as ``(code, message, field, reason)``.""" @@ -20,7 +20,7 @@ class Plan: """A model's plan, built once by ``Cipher``. Opaque.""" def build_plan( - table: str, fields: List[Tuple[str, Optional[str], Optional[str], type]] + table: str, fields: List[Tuple[str, Optional[str], Optional[str], type, bool]] ) -> Plan: """The plan for a model stored in ``table``: ``(name, eql_type, kind, value_type)`` per field, ``eql_type`` and ``kind`` ``None`` for a diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index d2daec9a5..7ea76cb44 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -42,7 +42,7 @@ This module imports SQLAlchemy. The rest of the package does not. """ -from typing import Annotated, Any, Callable, Dict, FrozenSet, List, Sequence, Tuple, Type +from typing import Annotated, Any, Callable, Dict, FrozenSet, List, Optional, Sequence, Tuple, Type from sqlalchemy import ( Index, @@ -147,10 +147,17 @@ def get_col_spec(self, **kwargs: Any) -> str: # For a DeclarativeBase's `type_annotation_map`: every marker the decorator # accepts, under the hint a model writes for it, such as # Annotated[str, TextEq]. +# Annotated[str, TextEq], and Annotated[Optional[str], TextEq] for a field +# that can be missing: SQLAlchemy looks up the whole hint, and makes the +# column nullable for the Optional one. eql_type_map: Dict[Any, EqlType] = { - Annotated[_KIND_TYPES[marker.plaintext], marker]: EqlType(marker) # type: ignore[valid-type] + hint: EqlType(marker) for marker in _eql.TYPES if marker.producible and marker.plaintext in _KIND_TYPES + for hint in ( + Annotated[_KIND_TYPES[marker.plaintext], marker], # type: ignore[valid-type] + Annotated[Optional[_KIND_TYPES[marker.plaintext]], marker], # type: ignore[valid-type] + ) } # The comparisons a query term answers, and the Cipher method that makes it. diff --git a/languages/python/tests/_declaration_cases.py b/languages/python/tests/_declaration_cases.py index a5912b1aa..fc2359383 100644 --- a/languages/python/tests/_declaration_cases.py +++ b/languages/python/tests/_declaration_cases.py @@ -10,8 +10,9 @@ import datetime import decimal import pickle +import sys import uuid -from typing import Annotated, ClassVar, Final, List, NamedTuple, Optional +from typing import Annotated, ClassVar, Final, List, NamedTuple, Optional, Union import attrs import pydantic @@ -449,7 +450,9 @@ class User: @pytest.mark.parametrize( "value_type, marker", [ - (Optional[str], TextEq), + (Optional[int], TextEq), + (Union[str, int], TextEq), + (Optional[Union[str, int]], TextEq), (bool, IntegerEq), (datetime.datetime, DateEq), (int, DoubleEq), @@ -462,6 +465,26 @@ def test_a_value_type_must_be_exactly_the_kinds(value_type, marker): assert error.code == "declaration_type_mismatch" +def test_an_optional_field_is_nullable(): + @stackencrypt.model("users") + @dataclasses.dataclass + class User: + email: Annotated[str, TextEq] + city: Annotated[Optional[str], TextEq] + age: Annotated[Optional[int], Passthrough] + + fields = {f.name: (f.value_type, f.nullable) for f in declaration_of(User).fields} + assert fields == {"email": (str, False), "city": (str, True), "age": (int, True)} + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason="`T | None` is Python 3.10 and later") +def test_a_union_with_none_is_nullable_too(): + User = dataclasses.make_dataclass("User", [("city", Annotated[eval("str | None"), TextEq])]) + + [field] = declaration_of(stackencrypt.model("users")(User)).fields + assert (field.value_type, field.nullable) == (str, True) + + @pytest.mark.parametrize( "value_type", [str, int, float, bool, bytes, datetime.date, datetime.datetime, decimal.Decimal], diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py index cab201374..8cf8442c1 100644 --- a/languages/python/tests/test_encrypt_live.py +++ b/languages/python/tests/test_encrypt_live.py @@ -14,7 +14,7 @@ import json import math import struct -from typing import Annotated, List, NamedTuple +from typing import Annotated, List, NamedTuple, Optional import attrs import pydantic @@ -606,3 +606,40 @@ def test_decrypt_rows_of_100_rows_with_one_field_make_one_key_request(): assert _engine.request_count() - before == 1 assert opened[99] == {"email": "user99@example.com"} + + +@stackencrypt.model("places") +@dataclasses.dataclass +class Place: + name: Annotated[str, TextEq] + city: Annotated[Optional[str], TextEq] + visits: Annotated[Optional[int], Passthrough] + + +@needs_database +def test_none_in_an_optional_field_is_stored_as_null_and_decrypts_as_none(): + psycopg = pytest.importorskip("psycopg") + from psycopg.rows import dict_row + + places = Cipher(Place) + rows = places.encrypt([Place("a", None, None), Place("b", "Leeds", 3)]) + + assert rows[0]["city"] is None and rows[0]["visits"] is None + with psycopg.connect(DATABASE_URL) as conn, conn.cursor(row_factory=dict_row) as cur: + cur.execute( + "CREATE TEMPORARY TABLE places (name public.eql_v4_text_eq NOT NULL, " + "city public.eql_v4_text_eq, visits int)" + ) + for row in rows: + cur.execute("INSERT INTO places VALUES (%(name)s, %(city)s, %(visits)s)", row) + cur.execute("SELECT count(*) AS n FROM places WHERE city IS NULL") + assert cur.fetchone()["n"] == 1 + cur.execute( + "SELECT * FROM places WHERE city = %s::eql_v4.query_text_eq", (places.eq("city", "Leeds"),) + ) + leeds = places.decrypt(cur.fetchall()) + cur.execute("SELECT * FROM places ORDER BY visits NULLS FIRST") + both = places.decrypt(cur.fetchall()) + + assert [dataclasses.astuple(p) for p in leeds] == [("b", "Leeds", 3)] + assert [dataclasses.astuple(p) for p in both] == [("a", None, None), ("b", "Leeds", 3)] diff --git a/languages/python/tests/test_nullable.py b/languages/python/tests/test_nullable.py new file mode 100644 index 000000000..6554f239b --- /dev/null +++ b/languages/python/tests/test_nullable.py @@ -0,0 +1,79 @@ +"""A field declared ``Optional``, before any request: ``None`` passes the +checks of ``encrypt``, ``decrypt`` and the ``*_rows`` methods, and a query +term refuses it. + +These tests need fake credentials only. A call that passes every check +fails building the engine client, with no request. The round trip is in +``test_encrypt_live.py``. +""" + +import dataclasses +import json +from typing import Annotated, Optional + +import pytest + +import stackencrypt +from stackencrypt import ( + Cipher, + DecryptError, + InvalidValueError, + Passthrough, + StashError, + TextEq, +) + +pytestmark = pytest.mark.usefixtures("credentials") + + +@stackencrypt.model("places") +@dataclasses.dataclass +class Place: + name: Annotated[str, TextEq] + city: Annotated[Optional[str], TextEq] + note: Annotated[Optional[str], Passthrough] + + +STORED = json.dumps({"i": {"t": "places", "c": "name"}, "v": 2}) + + +def reaches_the_engine(call): + # Refused later by the engine for want of a client key, not by the + # value checks: they passed. + with pytest.raises(StashError) as raised: + call() + assert not isinstance(raised.value, (InvalidValueError, DecryptError)) + + +def test_none_in_an_optional_field_passes_the_encrypt_checks(): + reaches_the_engine(lambda: Cipher(Place).encrypt([Place(name="a", city=None, note=None)])) + + +def test_none_in_a_field_that_is_not_optional_is_still_refused(untouched): + with pytest.raises(InvalidValueError) as raised: + Cipher(Place).encrypt([Place(name=None, city="Leeds", note=None)]) + assert raised.value.code == "value_none" + assert raised.value.field == "name" + + +def test_none_in_an_optional_field_passes_the_encrypt_rows_checks(): + reaches_the_engine(lambda: Cipher(Place).encrypt_rows([{"city": None, "note": None}])) + + +def test_a_null_column_of_an_optional_field_passes_the_decrypt_checks(): + reaches_the_engine(lambda: Cipher(Place).decrypt([{"name": STORED, "city": None, "note": None}])) + + +def test_a_null_column_of_a_field_that_is_not_optional_is_still_refused(untouched): + with pytest.raises(DecryptError) as raised: + Cipher(Place).decrypt([{"name": None, "city": None, "note": None}]) + assert raised.value.code == "decrypt_malformed" + assert raised.value.field == "name" + + +def test_a_query_term_for_none_is_refused_and_points_to_is_null(untouched): + with pytest.raises(InvalidValueError) as raised: + Cipher(Place).eq("city", None) + assert raised.value.code == "value_none" + assert raised.value.field == "city" + assert "IS NULL" in raised.value.message diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index 2380797c1..2b594e0db 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -7,7 +7,7 @@ """ import dataclasses -from typing import Annotated +from typing import Annotated, Optional import pytest from sqlalchemy import String, create_engine, select @@ -173,10 +173,28 @@ def test_the_type_map_gives_every_declarable_marker_its_eql_type(): declarable = [m for m in _eql.TYPES if m.producible and m.plaintext != "json"] assert declarable for marker in declarable: - [column_type] = [t for hint, t in eql_type_map.items() if hint.__metadata__ == (marker,)] - assert isinstance(column_type, EqlType) - assert column_type.marker is marker - assert column_type.get_col_spec() == marker.sql_domain + # Two hints for each marker: the value type, and Optional of it. + types = [t for hint, t in eql_type_map.items() if hint.__metadata__ == (marker,)] + assert len(types) == 2 + for column_type in types: + assert isinstance(column_type, EqlType) + assert column_type.marker is marker + assert column_type.get_col_spec() == marker.sql_domain + + +def test_an_optional_field_gets_a_nullable_eql_column(): + @stackencrypt.model("places") + class Place(Base): + __tablename__ = "places" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + name: Mapped[Annotated[str, TextEq]] + city: Mapped[Annotated[Optional[str], TextEq]] + note: Mapped[Annotated[Optional[str], Passthrough]] + + columns = Place.__table__.c + assert (repr(columns.city.type), columns.city.nullable) == ("EqlType(TextEq)", True) + assert columns.name.nullable is False + assert columns.note.nullable is True def test_a_column_of_another_type_is_refused(engine): diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index f38a98545..c4f0bfe3f 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -8,7 +8,7 @@ """ import uuid -from typing import Annotated +from typing import Annotated, Optional import pytest from _live import DATABASE_URL, LIVE, needs_database @@ -53,6 +53,15 @@ class Profile(Base): plan: Mapped[Annotated[str, Passthrough]] +@stackencrypt.model("venues") +class Venue(Base): + __tablename__ = "venues" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + name: Mapped[Annotated[str, TextEq]] + city: Mapped[Annotated[Optional[str], TextEq]] + visits: Mapped[Annotated[Optional[int], Passthrough]] + + @stackencrypt.model("players") class Player(Base): __tablename__ = "players" @@ -74,7 +83,7 @@ def engine(): with engine.begin() as conn: conn.execute(text(f"CREATE SCHEMA {schema}")) Base.metadata.create_all(engine) - configure_encryption(engine, [Customer, Order, Player, Profile]) + configure_encryption(engine, [Customer, Order, Player, Profile, Venue]) yield engine with engine.begin() as conn: conn.execute(text(f"DROP SCHEMA {schema} CASCADE")) @@ -84,7 +93,7 @@ def engine(): @pytest.fixture(autouse=True) def empty(engine): with engine.begin() as conn: - conn.execute(text("TRUNCATE orders, customers, players, profiles RESTART IDENTITY")) + conn.execute(text("TRUNCATE orders, customers, players, profiles, venues RESTART IDENTITY")) def stored(engine, sql): @@ -359,3 +368,38 @@ def test_a_match_on_a_field_with_no_match_index_is_unsupported(engine): with pytest.raises(QueryError) as raised: Customer.email.match("ali") assert raised.value.code == "query_unsupported" + + +# A field declared Optional. + + +def test_an_optional_field_stores_null_reads_none_and_is_found_with_is_null(engine): + with Session(engine) as session: + session.add_all([Venue(name="hall", city=None, visits=None), Venue(name="park", city="Leeds", visits=2)]) + session.commit() + with Session(engine) as session: + missing = session.scalars(select(Venue).where(Venue.city.is_(None))).all() + leeds = session.scalars(select(Venue).where(Venue.city == "Leeds")).all() + cities = session.execute(select(Venue.name, Venue.city).order_by(Venue.id)).all() + + assert [(v.name, v.city, v.visits) for v in missing] == [("hall", None, None)] + assert [(v.name, v.city, v.visits) for v in leeds] == [("park", "Leeds", 2)] + assert [tuple(row) for row in cities] == [("hall", None), ("park", "Leeds")] + + +def test_an_optional_field_can_be_set_to_none_and_back(engine): + with Session(engine) as session: + session.add(Venue(name="hall", city="York", visits=1)) + session.commit() + with Session(engine) as session: + hall = session.scalars(select(Venue)).one() + hall.city = None + session.commit() + assert hall.city is None + with Session(engine) as session: + hall = session.scalars(select(Venue)).one() + assert hall.city is None + hall.city = "Bath" + session.commit() + with Session(engine) as session: + assert session.scalars(select(Venue).where(Venue.city == "Bath")).one().name == "hall" From 496b79313b9dd5b9db60a9a7317157c21e8f979b Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 17:12:37 +0900 Subject: [PATCH 19/29] feat(python): an or_ of match searches through the SQLAlchemy ORM A search box that looks in several text fields is an or_ of match searches. The read hook refused every match inside an or_, so an app had to run one query for each field and join the rows in Python, which stops the database sorting and paging the combined result. A row that passes verify always passes @@, so the database returns every row that any branch would keep. The hook now accepts an or_ whose branches are all match searches, or and_ of them, and keeps a row when any branch passes verify. That gives exactly the rows verify would keep over the whole table. An or_ with any other condition, and a match under not_, are still refused. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 2 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 19 ++- .../python/python/stackencrypt/sqlalchemy.py | 137 +++++++++++++----- .../python/tests/test_sqlalchemy_live.py | 103 ++++++++++++- 4 files changed, 221 insertions(+), 40 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 017964cac..f448f340a 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -258,7 +258,7 @@ These are not handled by the hooks yet: These are not handled yet: - A query run on a connection, not a session, returns EQL values, not plaintext. Examples are `conn.execute(select(...))` and `conn.execute(text("SELECT ..."))`. Use `cipher.decrypt_rows` on its rows. A connection has no hook that can change a result. -- `.match()` works only in a `select` run through a session. The match must be a condition of the `WHERE`, joined with `AND` if there are others. The result must include whole objects of the class, such as `select(Profile)`. Any other use raises `QueryError` with `query_unsupported` before the query is sent. Examples are `or_(...)`, `not_(...)`, `select(Profile.plan)`, `func.count()`, `update`, `delete`, and a query on a connection. Each one would return, count, or change rows that do not hold the text. +- `.match()` works only in a `select` run through a session. The match must be a condition of the `WHERE`, joined with `AND` if there are others. It can also be a branch of an `or_` whose branches are all match searches, or `and_` of them. An example is a search box that looks in several fields: `or_(Activity.name.match(q), Activity.town.match(q), Activity.city.match(q))`. A row is kept if any branch passes `verify`. A branch on a field that is NULL fails for that row. The result must include whole objects of the class, such as `select(Profile)`. Any other use raises `QueryError` with `query_unsupported` before the query is sent. Examples are an `or_` with a condition that is not a match search, `not_(...)`, `select(Profile.plan)`, `func.count()`, `update`, `delete`, and a query on a connection. Each one would return, count, or change rows that do not hold the text. - With `.match()`, a `LIMIT` can return fewer rows than it names, even when more rows match. The database applies the limit before the SDK removes the rows that fail `verify`. - A query result is read in full before it is returned, so that its objects can be decrypted in one batch. A query that streams its rows does not stream. diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index 0b7c7b3b3..8ba3ba727 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -129,20 +129,35 @@ Reads do not check passthrough values, as `decrypt_rows` reads only the encrypte Removing rows after the query is right only in some places. So `.match()` works only when all three of these are true: 1. The statement is a `select` run through a session. -2. The match is a condition of the `WHERE`, joined with `AND` if there are others. +2. The match is a condition of the `WHERE`, joined with `AND` if there are others. Or it is a branch of an `or_` whose branches are all match searches, or `and_` of them (see "An `or_` of match searches" below). 3. The result includes whole objects of the class, so the column is loaded. Anything else raises `QueryError` with `query_unsupported` before the query is sent: | Use | Why it is refused | |---|---| -| In `or_(...)` | A row kept by the other condition would be removed. | +| In an `or_` with a condition that is not a match search | A row kept by the other condition would be removed. | | In `not_(...)` | The rows to keep are the false matches, which the hook cannot find. | | `select(Profile.plan)` | The matched column is not in the result, so `verify` cannot run. | | `func.count()` | The count includes the false matches. | | `update` or `delete` | The false matches would be changed in the database. | | A query on a connection | No hook can remove rows from its result. | +### An `or_` of match searches + +A search box that looks in several text fields is an `or_` of match searches, such as `or_(Activity.name.match(q), Activity.town.match(q))`. The hook accepts an `or_` whose branches are all match searches, or `and_` of them. It keeps a row if any branch passes `verify`. A branch on a field that is NULL fails for that row. + +This gives exactly the rows that `verify` would keep over the whole table. A match search never leaves out a row that `verify` would keep: + +- `verify` keeps a value only if it contains every 3-character token of the search text. +- Each of those tokens sets bits in the value's stored Bloom filter, so `@@` returns the row. + +So the database returns every row that any branch would keep, plus some false matches. `verify` on each branch removes the false matches. + +An `or_` with any other condition is still refused. When the match branch fails `verify`, the hook cannot tell whether the other condition kept the row. A match under `not_` is still refused, inside an `or_` or around one. + +Each branch can add false matches before a `LIMIT`. So with a wide `or_`, a page is short more often. + ### A LIMIT can return fewer rows The database applies a `LIMIT` before the hook removes rows. So `.limit(10)` can return fewer than 10 rows, even when more rows match. diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 7ea76cb44..e121cc5da 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -30,9 +30,10 @@ column to an EQL type as an ALTER with a USING cast. - Match: ``.match(text)`` on a field with a match index is ``col @@ term``. The session hook runs ``verify`` on the rows and removes those that fail. - It works only in a session's select, as a condition of the WHERE joined - with AND, with whole objects of the class in the result. Anything else - raises ``QueryError``. A LIMIT can return fewer rows than it names. + It works only in a session's select, with whole objects of the class in + the result. The match must be a condition of the WHERE joined with AND, + or a branch of an ``or_`` of match searches only. Anything else raises + ``QueryError``. A LIMIT can return fewer rows than it names. - Search: the column type that ``eql_type_map`` gives each encrypted field turns ``==``, ``<``, ``<=``, ``>`` and ``>=`` into a comparison with a query term, made by ``Cipher.eq``, ``lt``, ``lte``, ``gt`` or ``gte``. Any other @@ -63,7 +64,7 @@ from sqlalchemy.orm import Mapper, Session from sqlalchemy.orm.attributes import set_committed_value from sqlalchemy.sql import operators, visitors -from sqlalchemy.sql.elements import BooleanClauseList +from sqlalchemy.sql.elements import BooleanClauseList, Grouping from sqlalchemy.sql.dml import Insert, Update from sqlalchemy.types import UserDefinedType @@ -567,28 +568,31 @@ def _decrypt_loaded(state: Any) -> Any: return result() -def _checked_matches(statement: Any, matches: List[Any]) -> List[Tuple[int, str, Any]]: - """For each match search: the result position of its class's objects, - the field, and the ``MatchQuery``. Raises ``QueryError`` for a search - whose rows cannot be verified.""" +# A match search the hook verifies: the result position of its class's +# objects, the field, and the MatchQuery. +_Search = Tuple[int, str, Any] +# A condition of the WHERE, joined to the others with AND: an OR of +# branches, each an AND of searches. A plain match is one branch of one. +_Condition = List[List[_Search]] + + +def _checked_matches(statement: Any, matches: List[Any]) -> List[_Condition]: + """The match searches of ``statement``, as conditions the hook can + verify. Raises ``QueryError`` for a search whose rows cannot be + verified. + + A search can be verified when it is a condition of the WHERE joined + with AND, or a branch of an ``or_`` whose branches are all match + searches or ``and_`` of them. A row that passes ``verify`` always + passes ``@@``, so the database returns every row such an ``or_`` + keeps, and ``verify`` on each branch removes the rest.""" if not matches: return [] - # The conditions as the app passed them to where(), with each and_() - # opened. `whereclause` would join copies of them. - pending = list(getattr(statement, "_where_criteria", ()) or [statement.whereclause]) - top = set() - while pending: - condition = pending.pop() - if isinstance(condition, BooleanClauseList) and condition.operator is operators.and_: - pending.extend(condition.clauses) - else: - top.add(id(condition)) descriptions = statement.column_descriptions - searches = [] - for match in matches: + placed: Dict[int, _Search] = {} + + def search(match: Any) -> _Search: column, query = match._annotations[_MATCH] - if id(match) not in top: - raise _match_refused(column, "is not a condition of the WHERE joined with AND.") cls = next((c for c, (cipher, _) in _CONFIGURED.items() if cipher is _cipher_of(column)), None) at = next( ( @@ -600,8 +604,59 @@ def _checked_matches(statement: Any, matches: List[Any]) -> List[Tuple[int, str, ) if at is None: raise _match_refused(column, "has no whole objects of its class in the result.") - searches.append((at, column.key, query)) - return searches + placed[id(match)] = (at, column.key, query) + return placed[id(match)] + + conditions: List[_Condition] = [] + # The conditions as the app passed them to where(), with each and_() + # opened. `whereclause` would join copies of them. + for condition in _joined(list(getattr(statement, "_where_criteria", ()) or [statement.whereclause]), operators.and_): + if _is_match(condition): + conditions.append([[search(condition)]]) + continue + branches = _joined([condition], operators.or_) if _is_bool(condition, operators.or_) else None + if branches is None: + continue + ands = [_joined([branch], operators.and_) for branch in branches] + if all(part and all(_is_match(match) for match in part) for part in ands): + conditions.append([[search(match) for match in part] for part in ands]) + for match in matches: + if id(match) not in placed: + column, _ = match._annotations[_MATCH] + raise _match_refused( + column, + "is not a condition of the WHERE joined with AND, or a branch of an or_ " + "of match searches only.", + ) + return conditions + + +def _ungrouped(element: Any) -> Any: + while isinstance(element, Grouping): + element = element.element + return element + + +def _is_bool(element: Any, operator: Any) -> bool: + return isinstance(element, BooleanClauseList) and element.operator is operator + + +def _is_match(element: Any) -> bool: + return _MATCH in getattr(element, "_annotations", {}) + + +def _joined(elements: List[Any], operator: Any) -> List[Any]: + """``elements`` with each group removed, and each clause list of + ``operator`` opened into its clauses, at any depth.""" + out = [] + pending = list(reversed(elements)) + while pending: + element = _ungrouped(pending.pop()) + if _is_bool(element, operator): + pending.extend(reversed(list(element.clauses))) + else: + out.append(element) + return out def _item(row: Any, at: int) -> Any: @@ -610,17 +665,31 @@ def _item(row: Any, at: int) -> Any: return row[at] if isinstance(row, (tuple, Row)) else row -def _verify(searches: List[Tuple[int, str, Any]], result: Any) -> Any: - """Keep the rows whose objects hold the text of every match search.""" +def _verify(conditions: List[_Condition], result: Any) -> Any: + """Keep the rows that pass every condition: for each, at least one + branch whose match searches all pass ``verify``. A field that is NULL + fails its search.""" rows = list(result.data) - keep = [True] * len(rows) - for at, field, query in searches: - values = [getattr(_item(row, at), field) for row in rows] - present = [i for i, value in enumerate(values) if value is not None] - for i in set(range(len(rows))) - set(present): - keep[i] = False - for i, ok in zip(present, query.verify([values[i] for i in present])): - keep[i] = keep[i] and ok + passed: Dict[int, List[bool]] = {} + + def results(search: _Search) -> List[bool]: + if id(search) not in passed: + at, field, query = search + values = [getattr(_item(row, at), field) for row in rows] + present = [i for i, value in enumerate(values) if value is not None] + ok = [False] * len(rows) + for i, verified in zip(present, query.verify([values[i] for i in present])): + ok[i] = verified + passed[id(search)] = ok + return passed[id(search)] + + keep = [ + all( + any(all(results(search)[i] for search in branch) for branch in condition) + for condition in conditions + ) + for i in range(len(rows)) + ] # with_new_rows takes one tuple for each row, whatever the shape it keeps. return result.with_new_rows( [row if isinstance(row, (tuple, Row)) else (row,) for row, kept in zip(rows, keep) if kept] diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index c4f0bfe3f..4b731a8db 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -12,7 +12,7 @@ import pytest from _live import DATABASE_URL, LIVE, needs_database -from sqlalchemy import ForeignKey, create_engine, delete, func, not_, or_, select, text, update +from sqlalchemy import ForeignKey, and_, create_engine, delete, func, not_, or_, select, text, update from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship import stackencrypt @@ -62,6 +62,15 @@ class Venue(Base): visits: Mapped[Annotated[Optional[int], Passthrough]] +@stackencrypt.model("trips") +class Trip(Base): + __tablename__ = "trips" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + name: Mapped[Annotated[str, TextMatch]] + town: Mapped[Annotated[str, TextMatch]] + city: Mapped[Annotated[Optional[str], TextMatch]] + + @stackencrypt.model("players") class Player(Base): __tablename__ = "players" @@ -83,7 +92,7 @@ def engine(): with engine.begin() as conn: conn.execute(text(f"CREATE SCHEMA {schema}")) Base.metadata.create_all(engine) - configure_encryption(engine, [Customer, Order, Player, Profile, Venue]) + configure_encryption(engine, [Customer, Order, Player, Profile, Venue, Trip]) yield engine with engine.begin() as conn: conn.execute(text(f"DROP SCHEMA {schema} CASCADE")) @@ -93,7 +102,7 @@ def engine(): @pytest.fixture(autouse=True) def empty(engine): with engine.begin() as conn: - conn.execute(text("TRUNCATE orders, customers, players, profiles, venues RESTART IDENTITY")) + conn.execute(text("TRUNCATE orders, customers, players, profiles, venues, trips RESTART IDENTITY")) def stored(engine, sql): @@ -403,3 +412,91 @@ def test_an_optional_field_can_be_set_to_none_and_back(engine): session.commit() with Session(engine) as session: assert session.scalars(select(Venue).where(Venue.city == "Bath")).one().name == "hall" + + +# An or_ whose branches are all match searches. + +TRIPS = [ + ("Morning run", "Hove", "Brighton"), + ("Evening ride", "Leeds", None), + ("Hill walk", "Keswick", "Carlisle"), +] + + +def add_trips(engine): + with Session(engine) as session: + session.add_all([Trip(name=n, town=t, city=c) for n, t, c in TRIPS]) + session.commit() + + +def names(engine, *where): + with Session(engine) as session: + return sorted(t.name for t in session.scalars(select(Trip).where(*where))) + + +def any_field(text): + return or_(Trip.name.match(text), Trip.town.match(text), Trip.city.match(text)) + + +def test_an_or_of_matches_returns_the_rows_that_hold_the_text_in_any_field(engine): + add_trips(engine) + + assert names(engine, any_field("run")) == ["Morning run"] + assert names(engine, any_field("leeds")) == ["Evening ride"] + assert names(engine, any_field("brighton")) == ["Morning run"] + assert names(engine, any_field("hove")) == ["Morning run"] + + +def test_a_null_field_fails_its_branch_only(engine): + add_trips(engine) + + # Evening ride has no city; its town still matches. + assert names(engine, or_(Trip.city.match("leeds"), Trip.town.match("leeds"))) == ["Evening ride"] + + +def test_a_branch_can_be_an_and_of_matches(engine): + add_trips(engine) + + where = or_(and_(Trip.name.match("hill"), Trip.town.match("keswick")), Trip.city.match("brighton")) + assert names(engine, where) == ["Hill walk", "Morning run"] + + +def test_a_false_match_in_one_branch_is_removed_and_another_branch_keeps_the_row(engine, monkeypatch): + add_trips(engine) + verify = MatchQuery.verify + # A false match of the Bloom filter cannot be made on purpose. So verify + # refuses Hove as a town: Morning run must then be kept by its name. + monkeypatch.setattr( + MatchQuery, "verify", lambda q, values: [v != "Hove" and ok for v, ok in zip(values, verify(q, values))] + ) + + assert names(engine, or_(Trip.town.match("hove"), Trip.name.match("run"))) == ["Morning run"] + assert names(engine, or_(Trip.town.match("hove"), Trip.name.match("walk"))) == ["Hill walk"] + + +def test_two_or_groups_joined_with_and_keep_the_rows_that_pass_both(engine): + add_trips(engine) + + where = ( + or_(Trip.name.match("run"), Trip.name.match("ride")), + or_(Trip.town.match("leeds"), Trip.city.match("carlisle")), + ) + assert names(engine, *where) == ["Evening ride"] + + +@pytest.mark.parametrize( + "where", + [ + lambda: or_(Trip.name.match("run"), Trip.id == 1), + lambda: or_(not_(Trip.name.match("run")), Trip.town.match("hove")), + lambda: not_(or_(Trip.name.match("run"), Trip.town.match("hove"))), + lambda: or_(and_(Trip.name.match("run"), Trip.id == 1), Trip.town.match("hove")), + ], + ids=["mixed", "not-in-a-branch", "or-in-a-not", "mixed-and-in-a-branch"], +) +def test_an_or_that_is_not_only_matches_is_refused(engine, where): + add_trips(engine) + with Session(engine) as session, pytest.raises(QueryError) as raised: + session.execute(select(Trip).where(where())) + assert raised.value.code == "query_unsupported" + assert "A match index can return rows" in raised.value.message From c555603980a2137eda0463f779ba2aa60e4a34db Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 17:24:49 +0900 Subject: [PATCH 20/29] fix(python): an or_ of matches across an outer join no longer crashes In an outer join, a row with no related object has None in that object's place. With AND alone such a row never came back, because its @@ is NULL. With an or_ of matches, another branch can bring it back, and _verify then read the field from None and raised AttributeError. A missing object now counts as a NULL value: its search fails for that row, and another branch can still keep it. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 2 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 2 +- .../python/python/stackencrypt/sqlalchemy.py | 6 ++-- .../python/tests/test_sqlalchemy_live.py | 31 +++++++++++++++++-- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index f448f340a..fc28b3a1d 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -258,7 +258,7 @@ These are not handled by the hooks yet: These are not handled yet: - A query run on a connection, not a session, returns EQL values, not plaintext. Examples are `conn.execute(select(...))` and `conn.execute(text("SELECT ..."))`. Use `cipher.decrypt_rows` on its rows. A connection has no hook that can change a result. -- `.match()` works only in a `select` run through a session. The match must be a condition of the `WHERE`, joined with `AND` if there are others. It can also be a branch of an `or_` whose branches are all match searches, or `and_` of them. An example is a search box that looks in several fields: `or_(Activity.name.match(q), Activity.town.match(q), Activity.city.match(q))`. A row is kept if any branch passes `verify`. A branch on a field that is NULL fails for that row. The result must include whole objects of the class, such as `select(Profile)`. Any other use raises `QueryError` with `query_unsupported` before the query is sent. Examples are an `or_` with a condition that is not a match search, `not_(...)`, `select(Profile.plan)`, `func.count()`, `update`, `delete`, and a query on a connection. Each one would return, count, or change rows that do not hold the text. +- `.match()` works only in a `select` run through a session. The match must be a condition of the `WHERE`, joined with `AND` if there are others. It can also be a branch of an `or_` whose branches are all match searches, or `and_` of them. An example is a search box that looks in several fields: `or_(Activity.name.match(q), Activity.town.match(q), Activity.city.match(q))`. A row is kept if any branch passes `verify`. A branch fails for a row when its field is NULL, or when an outer join gives no object for its class. The result must include whole objects of the class, such as `select(Profile)`. Any other use raises `QueryError` with `query_unsupported` before the query is sent. Examples are an `or_` with a condition that is not a match search, `not_(...)`, `select(Profile.plan)`, `func.count()`, `update`, `delete`, and a query on a connection. Each one would return, count, or change rows that do not hold the text. - With `.match()`, a `LIMIT` can return fewer rows than it names, even when more rows match. The database applies the limit before the SDK removes the rows that fail `verify`. - A query result is read in full before it is returned, so that its objects can be decrypted in one batch. A query that streams its rows does not stream. diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index 8ba3ba727..5bdca3746 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -145,7 +145,7 @@ Anything else raises `QueryError` with `query_unsupported` before the query is s ### An `or_` of match searches -A search box that looks in several text fields is an `or_` of match searches, such as `or_(Activity.name.match(q), Activity.town.match(q))`. The hook accepts an `or_` whose branches are all match searches, or `and_` of them. It keeps a row if any branch passes `verify`. A branch on a field that is NULL fails for that row. +A search box that looks in several text fields is an `or_` of match searches, such as `or_(Activity.name.match(q), Activity.town.match(q))`. The hook accepts an `or_` whose branches are all match searches, or `and_` of them. It keeps a row if any branch passes `verify`. A branch fails for a row when its field is NULL, or when an outer join gives no object for its class. With `AND` alone, such a row never comes back, because its `@@` is NULL. With `or_`, another branch can bring it back. This gives exactly the rows that `verify` would keep over the whole table. A match search never leaves out a row that `verify` would keep: diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index e121cc5da..1745d8379 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -668,14 +668,16 @@ def _item(row: Any, at: int) -> Any: def _verify(conditions: List[_Condition], result: Any) -> Any: """Keep the rows that pass every condition: for each, at least one branch whose match searches all pass ``verify``. A field that is NULL - fails its search.""" + fails its search, and so does an object an outer join leaves out.""" rows = list(result.data) passed: Dict[int, List[bool]] = {} def results(search: _Search) -> List[bool]: if id(search) not in passed: at, field, query = search - values = [getattr(_item(row, at), field) for row in rows] + # An outer join gives None for a missing object. Its search fails, + # as for a NULL value. + values = [getattr(_item(row, at), field, None) for row in rows] present = [i for i, value in enumerate(values) if value is not None] ok = [False] * len(rows) for i, verified in zip(present, query.verify([values[i] for i in present])): diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index 4b731a8db..84b0d167c 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -71,6 +71,14 @@ class Trip(Base): city: Mapped[Annotated[Optional[str], TextMatch]] +@stackencrypt.model("guides") +class Guide(Base): + __tablename__ = "guides" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + trip_id: Mapped[Annotated[int, Ignore]] = mapped_column(ForeignKey("trips.id")) + bio: Mapped[Annotated[str, TextMatch]] + + @stackencrypt.model("players") class Player(Base): __tablename__ = "players" @@ -92,7 +100,7 @@ def engine(): with engine.begin() as conn: conn.execute(text(f"CREATE SCHEMA {schema}")) Base.metadata.create_all(engine) - configure_encryption(engine, [Customer, Order, Player, Profile, Venue, Trip]) + configure_encryption(engine, [Customer, Order, Player, Profile, Venue, Trip, Guide]) yield engine with engine.begin() as conn: conn.execute(text(f"DROP SCHEMA {schema} CASCADE")) @@ -102,7 +110,7 @@ def engine(): @pytest.fixture(autouse=True) def empty(engine): with engine.begin() as conn: - conn.execute(text("TRUNCATE orders, customers, players, profiles, venues, trips RESTART IDENTITY")) + conn.execute(text("TRUNCATE orders, customers, players, profiles, venues, guides, trips RESTART IDENTITY")) def stored(engine, sql): @@ -500,3 +508,22 @@ def test_an_or_that_is_not_only_matches_is_refused(engine, where): session.execute(select(Trip).where(where())) assert raised.value.code == "query_unsupported" assert "A match index can return rows" in raised.value.message + + +def test_an_or_across_an_outer_join_fails_the_branch_of_a_missing_object(engine): + add_trips(engine) + with Session(engine) as session: + # Only Hill walk (id 3) has a guide. + session.add(Guide(trip_id=3, bio="Mountain leader")) + session.commit() + with Session(engine) as session: + statement = ( + select(Trip, Guide) + .outerjoin(Guide, Guide.trip_id == Trip.id) + .where(or_(Trip.name.match("run"), Guide.bio.match("mountain"))) + .order_by(Trip.id) + ) + found = [(trip.name, guide and guide.bio) for trip, guide in session.execute(statement)] + + # Morning run has no guide: its Guide branch fails, and its name keeps it. + assert found == [("Morning run", None), ("Hill walk", "Mountain leader")] From 69bb47541166f4598afe633da2b31fbaa92602c3 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 17:42:23 +0900 Subject: [PATCH 21/29] fix(python): sort encrypted columns by their order term A bare ORDER BY on an EQL v4 column sorted in a wrong order, with no error. EQL v4 gives its column types comparisons but no default sort order, so PostgreSQL sorted the stored JSON. On 98, 120, 150 and 175 it gave 98, 150, 175, 120. The EQL v4 on next behaves the same. The layer now sorts an encrypted column by eql_v4.ord_term(col), or ord_term_ore for an ore_block marker. That sorts by the plaintext, and the SDK's index on the term serves the sort. asc, desc, and the nulls orderings go through the column type. A bare order_by(col) and sqlalchemy.asc(col) do not, so both hooks rewrite the ORDER BY. A column with no order index cannot be sorted, and raises QueryError. ADR-0004 decision 9 records the EQL gap and the options. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 5 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 28 +++++- .../python/python/stackencrypt/sqlalchemy.py | 87 ++++++++++++++++--- languages/python/tests/test_sqlalchemy.py | 59 +++++++++++++ .../python/tests/test_sqlalchemy_live.py | 81 ++++++++++++++++- 5 files changed, 245 insertions(+), 15 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index fc28b3a1d..219afd6d3 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -196,7 +196,10 @@ with Session(engine) as session: - **Search:** `==`, `<`, `<=`, `>`, and `>=` on an encrypted field compare it with a query term, as `cipher.eq` and `cipher.gte` make. The field's marker must support the operator. For example, `Customer.level >= 3` works, and `Customer.email > x` raises `QueryError` with `query_unsupported`. - `!=`, `in_([...])`, and `not_in([...])` work where `==` does. `!=` becomes `NOT (col = term)`. `in_` becomes one equality for each value, joined with `OR`. - `.match(text)` on a field whose marker has a match index, such as `TextMatch`, is a match search: `col @@ term`. A match index can return rows that do not hold the text. So after the query, the SDK runs `verify` on each row and removes the rows that fail. The app gets only rows that hold every 3-character piece of the text. -- **Other operators** that would send a value, such as `like`, `contains`, `between`, and `in_` with a subquery, raise `QueryError` with `query_unsupported` when you build the query. The value would reach the database as plaintext, and could never match. `is_(None)`, `is_not(None)`, `asc()`, and `desc()` send no value, so they work. +- **Other operators** that would send a value, such as `like`, `contains`, `between`, and `in_` with a subquery, raise `QueryError` with `query_unsupported` when you build the query. The value would reach the database as plaintext, and could never match. `is_(None)` and `is_not(None)` send no value, so they work. +- **Sorting:** `order_by` on an encrypted field sorts by its order term, such as `eql_v4.ord_term(level)`. This covers `.asc()`, `.desc()`, `.nulls_first()`, `.nulls_last()`, `sqlalchemy.asc(col)`, and a bare `order_by(col)`. It works in a session and on a connection. The field's marker must have an order index, such as `IntegerOrd`, `NumericOrd`, or `TextOrd`. Sorting a field with no order index raises `QueryError` with `query_unsupported`. + + The SDK rewrites the sort because of a gap in EQL v4. A bare `ORDER BY level` on an EQL v4 column sorts in a wrong order, with no error. The EQL column types have no sort order of their own, so PostgreSQL sorts the stored JSON. The order term sorts by the plaintext, and the SDK's index on it serves the sort. If you write SQL by hand, sort by `eql_v4.ord_term(col)` too. - A comparison on a class that is not configured raises `QueryError` with `query_not_configured`. Each class must be declared with `stackencrypt.model` and mapped by SQLAlchemy. `configure_encryption` refuses a class with `DeclarationError` in these cases: diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index 5bdca3746..cb5dce8d5 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -81,7 +81,8 @@ The allowed operators are: - `==`, `<`, `<=`, `>`, and `>=`. Each one sends a query term made by the `Cipher` method of the same meaning. - `!=`, as `NOT (col = term)`. - `in_` and `not_in` with a list, as one equality term for each value, joined with `OR`. -- `is_(None)`, `is_not(None)`, `asc()`, and `desc()`. These send no value. +- `is_(None)` and `is_not(None)`. These send no value. +- The sort operators, which sort by the order term (Decision 9). The comparisons answer ADR-0002's open question. On the ORM, the field is named by the class attribute, such as `Customer.email`. The column type reads the attribute's name and calls the `Cipher` with it. @@ -170,6 +171,30 @@ The database applies a `LIMIT` before the hook removes rows. So `.limit(10)` can A never returns a row that does not hold the text. B or C can come later, if an app needs exact page sizes. +## Decision 9: sort by the order term + +A bare `ORDER BY col` on an EQL v4 column sorts in a wrong order, with no error. On values 98, 120, 150, and 175 in an `IntegerOrd` column, it gave 98, 150, 175, 120. A sort with new ciphertext gave another order. + +The cause is in EQL v4. It defines `<` and the other comparisons for its column types. But a sort needs a default sort order for the type, and the EQL domains have none. So PostgreSQL sorts the stored JSON. The same test gave the same wrong order with the EQL v4 on the `next` branch as of 2026-11-11. `main` has no EQL v4. The EQL docs say a bare `ORDER BY col` works, but they were written for EQL v3. + +`ORDER BY eql_v4.ord_term(col)` sorts by the plaintext. The EQL docs also recommend that form for v3, because only that form uses the index (`packages/eql/docs/reference/database-indexes.md`, "The sort-key trap"). + +### The options + +| Option | Result | Cost | +|---|---|---| +| A. Fix it in EQL | A bare sort works for every SDK and for SQL written by hand. | An EQL change and release. | +| **B. Sort by the order term in the layer (chosen)** | It works now, and the sort uses the SDK's index. | The layer works around EQL. Go, TypeScript, and SQL written by hand still get the wrong order. | +| C. A test marked as an expected failure | Nothing changes yet. | Sorts through the layer stay wrong. | + +### What B does + +- `.asc()`, `.desc()`, `.nulls_first()`, and `.nulls_last()` on an encrypted column sort by `eql_v4.ord_term(col)`, or `eql_v4.ord_term_ore(col)` for an `ore_block` marker. +- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for an ORM query, and the engine hook for a query on a connection. +- Sorting a column with no order index raises `QueryError` with `query_unsupported`. Before, it gave the order of the stored JSON, with no error. + +If EQL gains a sort order for its v4 column types, the rewrite can stay. The order term is also the form that uses the index. + ## What was run - `tests/test_sqlalchemy.py` and `tests/test_sqlalchemy_migrations.py` run without credentials. They cover the checks, the refused operators, the indexes, and the Alembic rendering. @@ -189,3 +214,4 @@ A never returns a row that does not hold the text. B or C can come later, if an - Exact page sizes for `.match()` with a `LIMIT`: option B or C of Decision 8. - The reverse migration of a change to an EQL type. - Installing EQL from a migration. +- A sort order for the EQL v4 column types in EQL itself, option A of Decision 9. diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 1745d8379..337e45231 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -34,6 +34,10 @@ the result. The match must be a condition of the WHERE joined with AND, or a branch of an ``or_`` of match searches only. Anything else raises ``QueryError``. A LIMIT can return fewer rows than it names. +- Sorting: ``order_by`` on an encrypted column sorts by its order term, such + as ``eql_v4.ord_term(col)``, in both hooks. A bare ``ORDER BY col`` on an + EQL v4 column sorts the stored JSON, in a wrong order (ADR-0004, decision + 9). A column with no order index cannot be sorted. - Search: the column type that ``eql_type_map`` gives each encrypted field turns ``==``, ``<``, ``<=``, ``>`` and ``>=`` into a comparison with a query term, made by ``Cipher.eq``, ``lt``, ``lte``, ``gt`` or ``gte``. Any other @@ -66,6 +70,8 @@ from sqlalchemy.sql import operators, visitors from sqlalchemy.sql.elements import BooleanClauseList, Grouping from sqlalchemy.sql.dml import Insert, Update +from sqlalchemy.sql.functions import FunctionElement +from sqlalchemy.sql.selectable import Select from sqlalchemy.types import UserDefinedType import stackencrypt @@ -102,6 +108,10 @@ def __repr__(self) -> str: class comparator_factory(UserDefinedType.Comparator): # type: ignore[type-arg] def operate(self, op: Any, *other: Any, **kwargs: Any) -> Any: + if op in _SORTS: + # Sorted by the order term: EQL v4 gives its column types no + # sort order of their own (see _sort_key). + return getattr(_sort_key(self.expr), _SORTS[op])() if op in _NO_VALUE or (op in (operators.eq, operators.ne) and other == (None,)): return super().operate(op, *other, **kwargs) if op is operators.ne and len(other) == 1: @@ -179,16 +189,18 @@ def get_col_spec(self, **kwargs: Any) -> str: _VERIFIED = "stackencrypt_match_verified" # The operators that send no value to the database. -_NO_VALUE = frozenset( - { - operators.is_, - operators.is_not, - operators.asc_op, - operators.desc_op, - operators.nulls_first_op, - operators.nulls_last_op, - } -) +_NO_VALUE = frozenset({operators.is_, operators.is_not}) + +# The sort operators, and the method that applies each to a sort key. +_SORTS = { + operators.asc_op: "asc", + operators.desc_op: "desc", + operators.nulls_first_op: "nulls_first", + operators.nulls_last_op: "nulls_last", +} + +# The order term for each kind of order index. +_ORDER_TERMS = {"ope": "ord_term", "ore_block": "ord_term_ore"} # The database index for each kind of EQL index: the EQL function the index # is on, and the index method. @@ -472,6 +484,11 @@ def encrypt_parameters(conn, clause, multiparams, params, execution_options): # if matches: column, _ = matches[0]._annotations[_MATCH] raise _match_refused(column, "is not run through a session.") + if isinstance(clause, Select): + # An ORM select is rewritten in the session hook. + if clause._propagate_attrs.get("compile_state_plugin") != "orm": + clause = _sorted_by_terms(clause) + return clause, multiparams, params if not isinstance(clause, (Insert, Update)): return clause, multiparams, params configured = tables.get(getattr(clause.table, "name", None)) @@ -539,6 +556,7 @@ def _decrypt_loaded(state: Any) -> Any: if not state.is_select: return None searches = _checked_matches(state.statement, matches) + state.statement = _sorted_by_terms(state.statement) # Freezing reads every row, so every load and refresh event has fired. options = {_VERIFIED: True} if searches else {} result = state.invoke_statement(execution_options=options).freeze() @@ -752,6 +770,55 @@ def _term(column: Any, method: str, value: object) -> Any: return cast(literal(term, String()), _QueryType(column.type.marker.query_sql_domain)) +def _sort_key(column: Any) -> Any: + """What to sort an encrypted column by: its order term, such as + ``eql_v4.ord_term(col)``. Raises ``QueryError`` for a column with no + order index. + + A bare ``ORDER BY col`` sorts EQL v4 values in a wrong order, with no + error: the EQL column types have no sort order of their own, so + PostgreSQL sorts the stored JSON. The order term sorts by the + plaintext, and the index on it serves the sort.""" + function = next( + (_ORDER_TERMS[kind] for kind in column.type.marker.indexes if kind in _ORDER_TERMS), None + ) + if function is None: + raise QueryError( + "query_unsupported", + f"the encrypted column {_column_name(column)} has no order index, so it cannot " + f"be sorted. Its marker, {column.type.marker.__name__}, has the indexes " + f"{list(column.type.marker.indexes)}. Use a marker with an order index, such as " + "IntegerOrd or TextOrd.", + field=getattr(column, "key", None), + reason="order", + ) + return getattr(func.eql_v4, function)(column) + + +def _sorted_by_terms(statement: Any) -> Any: + """``statement`` with each encrypted column in its ORDER BY replaced by + its order term. A bare ``order_by(col)``, or ``sqlalchemy.asc(col)``, + does not go through the column type's operators, so the hooks rewrite + it here.""" + clauses = getattr(statement, "_order_by_clauses", ()) + if not clauses: + return statement + + def replace(element: Any) -> Any: + if isinstance(element, FunctionElement) and getattr(element, "name", None) in _ORDER_TERMS.values(): + return element # already a sort key: not opened again + if isinstance(getattr(element, "type", None), EqlType) and isinstance( + getattr(element, "table", None), Table + ): + return _sort_key(element) + return None + + rewritten = [visitors.replacement_traverse(clause, {}, replace) for clause in clauses] + if all(new is old for new, old in zip(rewritten, clauses)): + return statement + return statement.order_by(None).order_by(*rewritten) + + def _cipher_of(column: Any) -> Cipher: cipher = _TABLES.get(getattr(column, "table", None)) if cipher is None: diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index 2b594e0db..fe2ca0332 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -303,3 +303,62 @@ class Visit(Base): assert raised.value.field == "at" with engine.connect() as conn: assert conn.exec_driver_sql("SELECT count(*) FROM trips").scalar() == 0 + + +# ORDER BY an encrypted column. + + +def order_sql(statement): + from sqlalchemy.dialects import postgresql + + sql = str(statement.compile(dialect=postgresql.dialect())) + return " ".join(sql.split("ORDER BY", 1)[1].split()) + + +def test_asc_and_desc_on_an_order_column_sort_by_its_order_term(scores, untouched): + assert order_sql(select(Score).order_by(Score.level.asc())) == "eql_v4.ord_term(scores.level) ASC" + assert order_sql(select(Score).order_by(Score.level.desc().nulls_last())) == ( + "eql_v4.ord_term(scores.level) DESC NULLS LAST" + ) + + +def test_sorting_a_column_with_no_order_index_is_refused(scores, untouched): + with pytest.raises(QueryError) as raised: + Score.note.asc() + assert raised.value.code == "query_unsupported" + assert "order index" in raised.value.message + + +def test_a_bare_order_by_is_sorted_by_the_order_term_in_a_session(untouched): + from sqlalchemy import asc + from sqlalchemy.orm import Session + + @stackencrypt.model("laps") + class Lap(Base): + __tablename__ = "laps" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + seconds: Mapped[Annotated[int, IntegerOrd]] + note: Mapped[Annotated[str, Text]] + + engine = create_engine("sqlite://") + seen = [] + from sqlalchemy import event + + @event.listens_for(engine, "before_cursor_execute") + def capture(conn, cursor, statement, parameters, context, executemany): + seen.append(statement) + + with engine.begin() as conn: + conn.exec_driver_sql("CREATE TABLE laps (id INTEGER PRIMARY KEY, seconds TEXT, note TEXT)") + configure_encryption(engine, [Lap]) + + with Session(engine) as session: + for statement in (select(Lap).order_by(Lap.seconds), select(Lap).order_by(asc(Lap.seconds))): + try: + session.execute(statement) + except Exception: + pass # SQLite has no eql_v4 functions: only the SQL matters here. + with pytest.raises(QueryError) as raised: + session.execute(select(Lap).order_by(Lap.note)) + assert all("ORDER BY eql_v4.ord_term(laps.seconds)" in sql for sql in seen[-2:]), seen + assert raised.value.code == "query_unsupported" diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index 84b0d167c..d24e6d78d 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -7,6 +7,7 @@ of its own, and drops the schema afterwards. """ +import decimal import uuid from typing import Annotated, Optional @@ -16,7 +17,7 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship import stackencrypt -from stackencrypt import Ignore, IntegerOrd, MatchQuery, Passthrough, QueryError, Text, TextEq, TextMatch, _engine +from stackencrypt import Ignore, IntegerOrd, MatchQuery, NumericOrd, Passthrough, QueryError, Text, TextEq, TextMatch, _engine from stackencrypt.sqlalchemy import configure_encryption, eql_type_map pytestmark = [*LIVE, needs_database] @@ -79,6 +80,15 @@ class Guide(Base): bio: Mapped[Annotated[str, TextMatch]] +@stackencrypt.model("runs") +class Run(Base): + __tablename__ = "runs" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + label: Mapped[Annotated[str, Passthrough]] + average_hr: Mapped[Annotated[Optional[int], IntegerOrd]] + pace: Mapped[Annotated[Optional[decimal.Decimal], NumericOrd]] + + @stackencrypt.model("players") class Player(Base): __tablename__ = "players" @@ -100,7 +110,7 @@ def engine(): with engine.begin() as conn: conn.execute(text(f"CREATE SCHEMA {schema}")) Base.metadata.create_all(engine) - configure_encryption(engine, [Customer, Order, Player, Profile, Venue, Trip, Guide]) + configure_encryption(engine, [Customer, Order, Player, Profile, Venue, Trip, Guide, Run]) yield engine with engine.begin() as conn: conn.execute(text(f"DROP SCHEMA {schema} CASCADE")) @@ -110,7 +120,7 @@ def engine(): @pytest.fixture(autouse=True) def empty(engine): with engine.begin() as conn: - conn.execute(text("TRUNCATE orders, customers, players, profiles, venues, guides, trips RESTART IDENTITY")) + conn.execute(text("TRUNCATE orders, customers, players, profiles, venues, guides, trips, runs RESTART IDENTITY")) def stored(engine, sql): @@ -527,3 +537,68 @@ def test_an_or_across_an_outer_join_fails_the_branch_of_a_missing_object(engine) # Morning run has no guide: its Guide branch fails, and its name keeps it. assert found == [("Morning run", None), ("Hill walk", "Mountain leader")] + + +# ORDER BY an encrypted column with an order index. + +RUNS = [ + ("a", 150, decimal.Decimal("5.30")), + ("b", None, decimal.Decimal("4.75")), + ("c", 120, None), + ("d", 175, decimal.Decimal("6.10")), + ("e", 98, decimal.Decimal("4.80")), +] + + +def labels(engine, *order): + with Session(engine) as session: + return [r.label for r in session.scalars(select(Run).order_by(*order))] + + +@pytest.fixture +def runs(engine): + with Session(engine) as session: + session.add_all([Run(label=l, average_hr=hr, pace=p) for l, hr, p in RUNS]) + session.commit() + + +def test_order_by_an_integer_ord_column(engine, runs): + assert labels(engine, Run.average_hr.asc().nulls_last()) == ["e", "c", "a", "d", "b"] + assert labels(engine, Run.average_hr.desc().nulls_last()) == ["d", "a", "c", "e", "b"] + + +def test_order_by_a_numeric_ord_column(engine, runs): + assert labels(engine, Run.pace.asc().nulls_last()) == ["b", "e", "a", "d", "c"] + assert labels(engine, Run.pace.desc().nulls_last()) == ["d", "a", "e", "b", "c"] + + +def test_nulls_first_puts_the_missing_values_first(engine, runs): + assert labels(engine, Run.average_hr.asc().nulls_first())[0] == "b" + assert labels(engine, Run.pace.desc().nulls_first())[0] == "c" + + +def test_order_by_with_a_limit_pages_in_order(engine, runs): + with Session(engine) as session: + page = session.scalars( + select(Run).order_by(Run.average_hr.desc().nulls_last()).limit(2).offset(1) + ).all() + assert [r.label for r in page] == ["a", "c"] + + + +def test_a_bare_order_by_and_sqlalchemy_asc_sort_by_the_plaintext(engine, runs): + from sqlalchemy import asc, desc + + assert labels(engine, Run.average_hr) == ["e", "c", "a", "d", "b"] + assert labels(engine, asc(Run.pace)) == ["b", "e", "a", "d", "c"] + assert labels(engine, desc(Run.average_hr).nulls_last()) == ["d", "a", "c", "e", "b"] + + +def test_an_order_by_on_a_connection_sorts_by_the_plaintext(engine, runs): + table = Run.__table__ + with engine.connect() as conn: + found = conn.execute(select(table.c.label).order_by(table.c.average_hr.asc().nulls_last())).scalars().all() + bare = conn.execute(select(table.c.label).order_by(table.c.pace)).scalars().all() + + assert found == ["e", "c", "a", "d", "b"] + assert bare == ["b", "e", "a", "d", "c"] From 8e70626e0026b33562d8bcc660c03865e3323cc9 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 17:47:05 +0900 Subject: [PATCH 22/29] feat(python): a SQLAlchemy relationship needs no marker Rule A asked for a marker on every annotated attribute, relationships included. A list relationship could not be marked Ignore, because SQLAlchemy reads the type directly inside Mapped, so an app removed the annotation and lost the type. Its hint also had to resolve when the decorator ran, so the target class had to come first. A relationship is never a column, so leaving it out cannot store plaintext. The decorator now finds the relationships in the class's SQLAlchemy mapper, which lists them when the class statement runs, and leaves them out of the plan without resolving their hints. Every column still needs a marker, as Rule A says. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 24 +++---- ...on-is-checked-when-the-class-is-defined.md | 16 ++++- .../python/stackencrypt/_declaration.py | 67 +++++++++++++++++-- languages/python/tests/_declaration_cases.py | 53 ++++++++++++--- .../python/tests/test_sqlalchemy_live.py | 7 +- 5 files changed, 132 insertions(+), 35 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 219afd6d3..2c87ff072 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -267,15 +267,13 @@ These are not handled yet: ## Declare a SQLAlchemy relationship -A SQLAlchemy relationship is not a column. Every annotated field needs a marker, so write a relationship in one of two ways: - -- **A collection,** such as an author's books: leave the relationship without an annotation. The decorator never reads it. -- **One object,** such as a book's author: mark it `Ignore`. The SDK then leaves it out. +A SQLAlchemy relationship is never a column, so it needs no marker. Write it as SQLAlchemy documents it. The decorator leaves it out of the plan, and does not read its hint: ```python +from typing import List + from sqlalchemy import ForeignKey from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship -from stackencrypt import Ignore class Base(DeclarativeBase): @@ -287,7 +285,7 @@ class Author(Base): __tablename__ = "authors" id: Mapped[Annotated[int, Passthrough]] = mapped_column(primary_key=True) name: Mapped[Annotated[str, TextEq]] - books = relationship("Book", back_populates="author") # a list of Book + books: Mapped[List["Book"]] = relationship(back_populates="author") @stackencrypt.model("books") @@ -295,20 +293,14 @@ class Book(Base): __tablename__ = "books" id: Mapped[Annotated[int, Passthrough]] = mapped_column(primary_key=True) author_id: Mapped[Annotated[int, Passthrough]] = mapped_column(ForeignKey("authors.id")) - author: Mapped[Annotated["Author", Ignore]] = relationship(back_populates="books") + author: Mapped["Author"] = relationship(back_populates="books") ``` -Do not mark a collection `Ignore`. SQLAlchemy reads the type directly inside `Mapped[...]`, and it does not look inside `Annotated`. So `Mapped[Annotated[list["Book"], Ignore]]` does not give a list. The cost of the unannotated form is that a type checker does not know the type of `books`. - -If a book can have no author, write `Optional["Author"]`. Then the relationship must name its target, as `relationship("Author")` does: - -```text -author: Mapped[Annotated[Optional["Author"], Ignore]] = relationship("Author", back_populates="books") -``` +`Author` can name `"Book"` before `Book` is defined. A type checker knows that `books` is a list of `Book`. -With this form, a new `Book` has `author` set to `None` until you set it. A bare `relationship()` fails here, because SQLAlchemy takes `Optional["Author"]` as the target. +The decorator finds the relationships in SQLAlchemy's mapper. So apply `stackencrypt.model` above the class statement of a declarative class, as here, after SQLAlchemy has mapped it. -`Author` must be defined before `Book`, because the decorator evaluates `"Author"` in the hint when `Book`'s class statement runs. +Every column still needs a marker. A column with no marker, such as `body: Mapped[str]`, raises `DeclarationError` with `declaration_no_marker`. So no column is stored as plaintext by mistake. A relationship marked `Ignore` also still works. ## What a printed model shows diff --git a/languages/python/docs/adr/0001-a-declaration-is-checked-when-the-class-is-defined.md b/languages/python/docs/adr/0001-a-declaration-is-checked-when-the-class-is-defined.md index 3bdd90a3b..7a41a59e9 100644 --- a/languages/python/docs/adr/0001-a-declaration-is-checked-when-the-class-is-defined.md +++ b/languages/python/docs/adr/0001-a-declaration-is-checked-when-the-class-is-defined.md @@ -4,7 +4,7 @@ date: 2026-10-16 accepted: 2026-10-27, by the review on PR #1284; the status edit rode with the first implementation PR, as the spec (#1341) says extends: ADR-0007, ADR-0008 (packages/stack-encrypt/docs/adr) ticket: the Python SDK discovery ticket, backlog item 1 (its 15 October 2026 amendment) -amended_by: "Amendment, 2026-11-11: a field can be Optional" +amended_by: "Amendments, 2026-11-11: a field can be Optional; a relationship needs no marker" --- # A declaration is checked when Python defines the class, by construction @@ -531,3 +531,17 @@ The engine has two ways to store a missing value (`packages/stack-encrypt/src/dy The Go SDK declares a nullable field as a pointer, with a `null=sql` or `null=sealed` tag, and refuses a pointer with no tag (`languages/golang/encrypt/gensupport/declaration.go`). Python needs no tag for the common case, because `Optional` already says the field can be missing. +## Amendment, 2026-11-11: a relationship needs no marker + +The decorator now leaves out a SQLAlchemy relationship, such as `books: Mapped[list["Book"]] = relationship()`. It needs no marker, and its hint is not resolved. + +Before this, Rule A applied to every annotated attribute of a SQLAlchemy class. A relationship could not be marked `Ignore` if it was a list: SQLAlchemy reads the type directly inside `Mapped[...]`, not inside `Annotated`. So an app had to remove the annotation, and a type checker no longer knew the type. The hint also had to resolve when the decorator ran, so the target class had to be defined first. Endurain's `User` class has 13 such relationships. + +A relationship is never a column, so leaving it out cannot store plaintext. Rule A still holds for every column: a column with no marker still raises `declaration_no_marker`. So leaving out a column is never a silent choice. + +The decorator finds the relationships in the class's SQLAlchemy mapper, `__mapper__._props`. SQLAlchemy maps a declarative class when its class statement runs, so the mapper lists them before the decorator runs, and before the classes they name exist. `_props` is a private attribute. If it is missing, nothing is left out, and Rule A refuses an unmarked relationship as before. + +### What was not done + +A class-level option, such as `unmarked=Passthrough`, would store every unmarked column as it is. It would save marking many columns `Passthrough` or `Ignore`. But a sensitive column that someone forgot to mark would then be stored as plaintext, with no error. That breaks Rule A, so it was not added. + diff --git a/languages/python/python/stackencrypt/_declaration.py b/languages/python/python/stackencrypt/_declaration.py index f8e1a4c0d..aba90b7d5 100644 --- a/languages/python/python/stackencrypt/_declaration.py +++ b/languages/python/python/stackencrypt/_declaration.py @@ -263,10 +263,62 @@ def _class_changed( def _hints(cls: type) -> Dict[str, Any]: - try: - return get_type_hints(cls, include_extras=True) - except (NameError, TypeError) as error: - raise _hints_unresolved(cls, error) from error + """The class's resolved type hints, except its SQLAlchemy relationships. + + A relationship's hint often names a class defined later, such as + ``Mapped[list["Gear"]]``, so it may not resolve yet. It is never a + column, so it needs no hint here (see ``_relationships``).""" + skip = _relationships(cls) + if not skip: + try: + return get_type_hints(cls, include_extras=True) + except (NameError, TypeError) as error: + raise _hints_unresolved(cls, error) from error + hints: Dict[str, Any] = {} + # As get_type_hints does: each base's own hints in its own namespaces, + # the class's own last, so that they win. + for base in reversed(cls.__mro__): + try: + own = _own_annotations(base) + except (NameError, TypeError) as error: + raise _hints_unresolved(cls, error) from error + module = sys.modules.get(base.__module__) + global_names = dict(vars(module)) if module is not None else {} + for name, hint in own.items(): + if name in skip: + continue + + def probe() -> None: ... + + probe.__annotations__ = {name: hint} + try: + hints.update(get_type_hints(probe, global_names, dict(vars(base)), include_extras=True)) + except (NameError, TypeError) as error: + raise _hints_unresolved(cls, error) from error + return hints + + +def _relationships(cls: type) -> frozenset: + """The names of a SQLAlchemy class's relationships, such as ``books`` in + ``books: Mapped[list["Book"]] = relationship()``. Empty for any other + class. + + A relationship is never a column, so it is not a field of the plan, + and it needs no marker. Rule A still holds for every column. SQLAlchemy + maps a declarative class when its class statement runs, so its mapper + lists the relationships before the decorator runs, and before the + classes they name exist. ``Mapper._props`` is SQLAlchemy's own record + of them; if it is missing, nothing is skipped.""" + orm = sys.modules.get("sqlalchemy.orm") + if orm is None: + return frozenset() + mapper = cls.__dict__.get("__mapper__") + props = getattr(mapper, "_props", None) + if not isinstance(props, dict): + return frozenset() + return frozenset( + name for name, prop in props.items() if isinstance(prop, orm.RelationshipProperty) + ) def _signature(cls: type) -> Optional[inspect.Signature]: @@ -306,7 +358,12 @@ def _read_fields(cls: type, hints: Dict[str, Any]) -> List[_ReadField]: names = _library_field_names(cls) from_hints = names is None if names is None: - names = [name for name, hint in hints.items() if not _is_class_level(hint)] + relationships = _relationships(cls) + names = [ + name + for name, hint in hints.items() + if not _is_class_level(hint) and name not in relationships + ] read = [] for name in names: marker, value_type = _marker_of(name, hints.get(name), from_hints=from_hints) diff --git a/languages/python/tests/_declaration_cases.py b/languages/python/tests/_declaration_cases.py index fc2359383..7a97de2ff 100644 --- a/languages/python/tests/_declaration_cases.py +++ b/languages/python/tests/_declaration_cases.py @@ -17,6 +17,7 @@ import attrs import pydantic import pytest +from sqlalchemy import inspect from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship from sqlalchemy import ForeignKey @@ -308,20 +309,52 @@ class LaterBase(DeclarativeBase): pass -def test_a_relationship_to_a_class_defined_later_is_refused_with_its_field(): - # Author is defined after Book, as a relationship target often is. +def test_a_relationship_to_a_class_defined_later_needs_no_marker(): + # Author is defined after Book, as a relationship target often is. A + # relationship is never a column, so it needs no marker, and its hint + # is not resolved. + @stackencrypt.model("books") + class Book(LaterBase): + __tablename__ = "books" + id: Mapped[Annotated[int, Passthrough]] = mapped_column(primary_key=True) + title: Mapped[Annotated[str, TextEq]] + authors: Mapped[List["Author"]] = relationship(back_populates="book") # noqa: F821 + + class Author(LaterBase): + __tablename__ = "authors" + id: Mapped[int] = mapped_column(primary_key=True) + book_id: Mapped[int] = mapped_column(ForeignKey("books.id")) + book: Mapped["Book"] = relationship(back_populates="authors") + + assert planned(Book) == [("id", Passthrough), ("title", TextEq)] + # SQLAlchemy still reads the hint: the relationship is a list of Author. + assert inspect(Book).relationships["authors"].uselist is True + stackencrypt.Cipher(Book) # the class is as the decorator left it + + +def test_a_relationship_to_one_object_needs_no_marker(): + @stackencrypt.model("comments") + class Comment(Base): + __tablename__ = "comments" + id: Mapped[Annotated[int, Passthrough]] = mapped_column(primary_key=True) + body: Mapped[Annotated[str, TextEq]] + user_id: Mapped[Annotated[int, Passthrough]] = mapped_column(ForeignKey("users.id")) + user: Mapped["User"] = relationship() + + assert planned(Comment) == [("id", Passthrough), ("body", TextEq), ("user_id", Passthrough)] + + +def test_a_sqlalchemy_column_with_no_marker_is_still_refused(): def build(): - @stackencrypt.model("books") - class Book(LaterBase): - __tablename__ = "books" + @stackencrypt.model("notes") + class Note(Base): + __tablename__ = "notes" id: Mapped[Annotated[int, Passthrough]] = mapped_column(primary_key=True) - title: Mapped[Annotated[str, TextEq]] - authors: Mapped[Annotated[List["Author"], Ignore]] = relationship("Author") # noqa: F821 + body: Mapped[str] error = refusal(build) - assert error.code == "declaration_hints_unresolved" - assert error.field == "authors" - assert isinstance(error.__cause__, NameError) + assert error.code == "declaration_no_marker" + assert error.field == "body" def test_a_class_is_declared_once(): diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index d24e6d78d..65d840851 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -9,7 +9,7 @@ import decimal import uuid -from typing import Annotated, Optional +from typing import Annotated, List, Optional import pytest from _live import DATABASE_URL, LIVE, needs_database @@ -34,7 +34,8 @@ class Customer(Base): email: Mapped[Annotated[str, TextEq]] name: Mapped[Annotated[str, Text]] plan: Mapped[Annotated[str, Passthrough]] - orders = relationship("Order", back_populates="customer") # a list of Order + # A relationship needs no marker: it is never a column. + orders: Mapped[List["Order"]] = relationship(back_populates="customer") @stackencrypt.model("orders") @@ -43,7 +44,7 @@ class Order(Base): id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) customer_id: Mapped[Annotated[int, Ignore]] = mapped_column(ForeignKey("customers.id")) note: Mapped[Annotated[str, Text]] - customer: Mapped[Annotated["Customer", Ignore]] = relationship(back_populates="orders") + customer: Mapped["Customer"] = relationship(back_populates="orders") @stackencrypt.model("profiles") From 7073225cd8d0756503a5eb1ff39c59a69b57eef4 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 17:57:02 +0900 Subject: [PATCH 23/29] fix(python): rewrite ORDER BY in an ORM select run on a connection The engine hook rewrote the ORDER BY only for selects without the ORM mark, and left the rest to the session hook. But SQLAlchemy marks a select as ORM whenever it names a mapped class or attribute, even when it runs on a connection, where the session hook does not run. So conn.execute(select(T.level).order_by(T.level)) sorted the stored JSON. The engine hook now rewrites every select. A select from a session reaches it already rewritten, and is left as it is. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- ...-the-sqlalchemy-layer-encrypts-in-batch-hooks.md | 2 +- languages/python/python/stackencrypt/sqlalchemy.py | 10 ++++++---- languages/python/tests/test_sqlalchemy_live.py | 13 +++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index cb5dce8d5..2cef75cb5 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -190,7 +190,7 @@ The cause is in EQL v4. It defines `<` and the other comparisons for its column ### What B does - `.asc()`, `.desc()`, `.nulls_first()`, and `.nulls_last()` on an encrypted column sort by `eql_v4.ord_term(col)`, or `eql_v4.ord_term_ore(col)` for an `ore_block` marker. -- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for an ORM query, and the engine hook for a query on a connection. +- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for a query in a session, and the engine hook for a query on a connection. The engine hook rewrites every `select`. SQLAlchemy marks a select as ORM when it names a mapped class or attribute, as `select(Run.label)` does, even when it runs on a connection. A select from a session reaches the engine hook already rewritten, and is left as it is. - Sorting a column with no order index raises `QueryError` with `query_unsupported`. Before, it gave the order of the stored JSON, with no error. If EQL gains a sort order for its v4 column types, the rewrite can stay. The order term is also the form that uses the index. diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 337e45231..f56a7c4f4 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -485,10 +485,12 @@ def encrypt_parameters(conn, clause, multiparams, params, execution_options): # column, _ = matches[0]._annotations[_MATCH] raise _match_refused(column, "is not run through a session.") if isinstance(clause, Select): - # An ORM select is rewritten in the session hook. - if clause._propagate_attrs.get("compile_state_plugin") != "orm": - clause = _sorted_by_terms(clause) - return clause, multiparams, params + # Every select, ORM-marked or not. SQLAlchemy marks a select as + # ORM when it names a mapped class or attribute, even when it + # runs on a connection, where the session hook does not run. A + # select from a session reaches here already rewritten, and + # comes back unchanged. + return _sorted_by_terms(clause), multiparams, params if not isinstance(clause, (Insert, Update)): return clause, multiparams, params configured = tables.get(getattr(clause.table, "name", None)) diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index 65d840851..0266ca902 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -603,3 +603,16 @@ def test_an_order_by_on_a_connection_sorts_by_the_plaintext(engine, runs): assert found == ["e", "c", "a", "d", "b"] assert bare == ["b", "e", "a", "d", "c"] + + +def test_an_orm_select_on_a_connection_sorts_by_the_plaintext(engine, runs): + # select(Run.label) names a mapped attribute, so SQLAlchemy marks it as + # an ORM statement, even when it runs on a connection. + from sqlalchemy import asc + + with engine.connect() as conn: + bare = conn.execute(select(Run.label).order_by(Run.average_hr)).scalars().all() + function = conn.execute(select(Run.label).order_by(asc(Run.pace))).scalars().all() + + assert bare == ["e", "c", "a", "d", "b"] + assert function == ["b", "e", "a", "d", "c"] From ef420d4723c36b5fe8defea3ad7fae49fb62ce94 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 18:11:49 +0900 Subject: [PATCH 24/29] fix(python): sort by the order term in aliases and subqueries The ORDER BY rewrite took only columns whose table is a Table, and read only the top select. So a bare order_by on an aliased class, or in a subquery, still sorted the stored JSON. It now takes any column of an EQL type, and rewrites every select in the statement, copying it only when an ORDER BY changes. Copying a statement made SQLAlchemy compare columns with ==, which showed two bugs in the column type. Its None check compared a SQL expression for truth and raised TypeError, and it treated another column as a value to encrypt. A comparison with an expression that holds no value, such as another column, now passes through. One that holds a value, such as literal("x"), is refused: the value would reach the database as plaintext. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 3 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 4 +- .../python/python/stackencrypt/sqlalchemy.py | 70 +++++++++++++++---- languages/python/tests/test_sqlalchemy.py | 29 +++++++- .../python/tests/test_sqlalchemy_live.py | 45 ++++++++++++ 5 files changed, 133 insertions(+), 18 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 2c87ff072..529c54018 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -194,10 +194,11 @@ with Session(engine) as session: - Only encrypted fields are changed. A passthrough field is checked as `encrypt` checks it, then stored unchanged. For example, a `datetime` with no time zone raises `InvalidValueError` with `value_naive_datetime`. - **Search:** `==`, `<`, `<=`, `>`, and `>=` on an encrypted field compare it with a query term, as `cipher.eq` and `cipher.gte` make. The field's marker must support the operator. For example, `Customer.level >= 3` works, and `Customer.email > x` raises `QueryError` with `query_unsupported`. +- A comparison with another column, such as `Customer.email == Other.email`, sends no value, so it is passed to the database as it is. A comparison with a SQL expression that holds a Python value, such as `literal("x")` or `func.lower("x")`, raises `QueryError` with `query_unsupported`. The value would reach the database as plaintext. Compare with the Python value itself, such as `Customer.email == "x"`. - `!=`, `in_([...])`, and `not_in([...])` work where `==` does. `!=` becomes `NOT (col = term)`. `in_` becomes one equality for each value, joined with `OR`. - `.match(text)` on a field whose marker has a match index, such as `TextMatch`, is a match search: `col @@ term`. A match index can return rows that do not hold the text. So after the query, the SDK runs `verify` on each row and removes the rows that fail. The app gets only rows that hold every 3-character piece of the text. - **Other operators** that would send a value, such as `like`, `contains`, `between`, and `in_` with a subquery, raise `QueryError` with `query_unsupported` when you build the query. The value would reach the database as plaintext, and could never match. `is_(None)` and `is_not(None)` send no value, so they work. -- **Sorting:** `order_by` on an encrypted field sorts by its order term, such as `eql_v4.ord_term(level)`. This covers `.asc()`, `.desc()`, `.nulls_first()`, `.nulls_last()`, `sqlalchemy.asc(col)`, and a bare `order_by(col)`. It works in a session and on a connection. The field's marker must have an order index, such as `IntegerOrd`, `NumericOrd`, or `TextOrd`. Sorting a field with no order index raises `QueryError` with `query_unsupported`. +- **Sorting:** `order_by` on an encrypted field sorts by its order term, such as `eql_v4.ord_term(level)`. This covers `.asc()`, `.desc()`, `.nulls_first()`, `.nulls_last()`, `sqlalchemy.asc(col)`, and a bare `order_by(col)`. It works in a session and on a connection, on an aliased class, and in a subquery. The field's marker must have an order index, such as `IntegerOrd`, `NumericOrd`, or `TextOrd`. Sorting a field with no order index raises `QueryError` with `query_unsupported`. The SDK rewrites the sort because of a gap in EQL v4. A bare `ORDER BY level` on an EQL v4 column sorts in a wrong order, with no error. The EQL column types have no sort order of their own, so PostgreSQL sorts the stored JSON. The order term sorts by the plaintext, and the SDK's index on it serves the sort. If you write SQL by hand, sort by `eql_v4.ord_term(col)` too. - A comparison on a class that is not configured raises `QueryError` with `query_not_configured`. diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index 2cef75cb5..89e87d452 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -76,6 +76,8 @@ The hooks can change only the parameters passed beside a statement. Three cases - **An operator that would send a value as plaintext**, such as `like`, `contains`, `between`, or `in_` with a subquery. It raises `QueryError` with `query_unsupported` when the app builds the query. The value could also never match an EQL value. - **A unique constraint on an encrypted column.** Each EQL value contains new random ciphertext, so the constraint never finds a duplicate. A unique index on `eql_v4.eq_term(col)` works instead. +A comparison with another SQL expression is allowed when the expression holds no Python value, such as another column. It sends no value. An expression that holds one, such as `literal("x")` or `func.lower("x")`, is refused: the hooks cannot encrypt a value inside an expression. SQLAlchemy itself compares columns this way when it copies a statement. + The allowed operators are: - `==`, `<`, `<=`, `>`, and `>=`. Each one sends a query term made by the `Cipher` method of the same meaning. @@ -190,7 +192,7 @@ The cause is in EQL v4. It defines `<` and the other comparisons for its column ### What B does - `.asc()`, `.desc()`, `.nulls_first()`, and `.nulls_last()` on an encrypted column sort by `eql_v4.ord_term(col)`, or `eql_v4.ord_term_ore(col)` for an `ore_block` marker. -- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for a query in a session, and the engine hook for a query on a connection. The engine hook rewrites every `select`. SQLAlchemy marks a select as ORM when it names a mapped class or attribute, as `select(Run.label)` does, even when it runs on a connection. A select from a session reaches the engine hook already rewritten, and is left as it is. +- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for a query in a session, and the engine hook for a query on a connection. The rewrite covers every `select` in a statement, subqueries included, and a column of a table, an alias, or a subquery. The statement is copied only when an `ORDER BY` changes. The engine hook rewrites every `select`. SQLAlchemy marks a select as ORM when it names a mapped class or attribute, as `select(Run.label)` does, even when it runs on a connection. A select from a session reaches the engine hook already rewritten, and is left as it is. - Sorting a column with no order index raises `QueryError` with `query_unsupported`. Before, it gave the order of the stored JSON, with no error. If EQL gains a sort order for its v4 column types, the rewrite can stay. The order term is also the form that uses the index. diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index f56a7c4f4..b20344beb 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -68,7 +68,13 @@ from sqlalchemy.orm import Mapper, Session from sqlalchemy.orm.attributes import set_committed_value from sqlalchemy.sql import operators, visitors -from sqlalchemy.sql.elements import BooleanClauseList, Grouping +from sqlalchemy.sql.elements import ( + BindParameter, + BooleanClauseList, + ClauseElement, + ColumnClause, + Grouping, +) from sqlalchemy.sql.dml import Insert, Update from sqlalchemy.sql.functions import FunctionElement from sqlalchemy.sql.selectable import Select @@ -112,8 +118,25 @@ def operate(self, op: Any, *other: Any, **kwargs: Any) -> Any: # Sorted by the order term: EQL v4 gives its column types no # sort order of their own (see _sort_key). return getattr(_sort_key(self.expr), _SORTS[op])() - if op in _NO_VALUE or (op in (operators.eq, operators.ne) and other == (None,)): + # `is None`: `other == (None,)` would compare a SQL expression, + # which has no truth value. + if op in _NO_VALUE or ( + op in (operators.eq, operators.ne) and len(other) == 1 and other[0] is None + ): return super().operate(op, *other, **kwargs) + if (op in _COMPARISONS or op is operators.ne) and len(other) == 1: + expression = other[0] + if hasattr(expression, "__clause_element__"): # an ORM attribute + expression = expression.__clause_element__() + if isinstance(expression, ClauseElement): + # A SQL expression that holds a Python value, such as + # literal("x") or func.lower("x"), would send it to the + # database as plaintext. + if any(isinstance(e, BindParameter) for e in visitors.iterate(expression)): + raise _refused(self.expr, op) + # Another column: no value is sent. SQLAlchemy also + # compares columns this way when it copies a statement. + return super().operate(op, expression, **kwargs) if op is operators.ne and len(other) == 1: # NOT (col = term): a NULL column matches neither, as with !=. return not_(self.operate(operators.eq, other[0])) @@ -798,27 +821,44 @@ def _sort_key(column: Any) -> Any: def _sorted_by_terms(statement: Any) -> Any: - """``statement`` with each encrypted column in its ORDER BY replaced by - its order term. A bare ``order_by(col)``, or ``sqlalchemy.asc(col)``, - does not go through the column type's operators, so the hooks rewrite - it here.""" - clauses = getattr(statement, "_order_by_clauses", ()) - if not clauses: + """``statement`` with each encrypted column in an ORDER BY replaced by + its order term, in every select of it, subqueries included. A bare + ``order_by(col)``, or ``sqlalchemy.asc(col)``, does not go through the + column type's operators, so the hooks rewrite it here. The statement is + copied only when an ORDER BY changes.""" + if not any( + _sort_keys(clauses) is not clauses + for clauses in ( + element._order_by_clauses + for element in visitors.iterate(statement) + if isinstance(element, Select) and element._order_by_clauses + ) + ): return statement + def visit_select(select: Any) -> None: + # On the copy that cloned_traverse made, as SQLAlchemy's own + # generative methods change it. + if select._order_by_clauses: + select._order_by_clauses = _sort_keys(select._order_by_clauses) + + return visitors.cloned_traverse(statement, {}, {"select": visit_select}) + + +def _sort_keys(clauses: Any) -> Any: + """``clauses`` with each encrypted column replaced by its order term; the + same object when nothing changes.""" + def replace(element: Any) -> Any: if isinstance(element, FunctionElement) and getattr(element, "name", None) in _ORDER_TERMS.values(): return element # already a sort key: not opened again - if isinstance(getattr(element, "type", None), EqlType) and isinstance( - getattr(element, "table", None), Table - ): + # Any column of an EQL type: of a table, an alias, or a subquery. + if isinstance(element, ColumnClause) and isinstance(element.type, EqlType): return _sort_key(element) return None - rewritten = [visitors.replacement_traverse(clause, {}, replace) for clause in clauses] - if all(new is old for new, old in zip(rewritten, clauses)): - return statement - return statement.order_by(None).order_by(*rewritten) + rewritten = tuple(visitors.replacement_traverse(clause, {}, replace) for clause in clauses) + return clauses if all(new is old for new, old in zip(rewritten, clauses)) else rewritten def _cipher_of(column: Any) -> Cipher: diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index fe2ca0332..786d78f5f 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -10,7 +10,7 @@ from typing import Annotated, Optional import pytest -from sqlalchemy import String, create_engine, select +from sqlalchemy import String, create_engine, func, literal, select from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column import stackencrypt @@ -362,3 +362,30 @@ def capture(conn, cursor, statement, parameters, context, executemany): session.execute(select(Lap).order_by(Lap.note)) assert all("ORDER BY eql_v4.ord_term(laps.seconds)" in sql for sql in seen[-2:]), seen assert raised.value.code == "query_unsupported" + + +# A comparison with a SQL expression, not a Python value. + + +@pytest.mark.parametrize( + "build", + [ + lambda: Score.note == literal("secret"), + lambda: Score.note == func.lower("secret"), + lambda: Score.level > literal(3) + 1, + ], + ids=["literal", "function-of-a-value", "expression-of-a-value"], +) +def test_a_comparison_with_an_expression_that_holds_a_value_is_refused(scores, untouched, build): + with pytest.raises(QueryError) as raised: + build() + assert raised.value.code == "query_unsupported" + assert "secret" not in str(raised.value) + + +def test_a_comparison_with_another_column_sends_no_value(scores, untouched): + from sqlalchemy.dialects import postgresql + + sql = str((Score.note == Score.note).compile(dialect=postgresql.dialect())) + + assert sql == "scores.note = scores.note" diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index 0266ca902..f9936b1e6 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -616,3 +616,48 @@ def test_an_orm_select_on_a_connection_sorts_by_the_plaintext(engine, runs): assert bare == ["e", "c", "a", "d", "b"] assert function == ["b", "e", "a", "d", "c"] + + +@pytest.fixture +def sent(engine): + """The SQL the engine sends while a test runs. A wrong sort gives a + random order, which can be the right one by chance, so a sort test + also checks the SQL.""" + from sqlalchemy import event + + statements = [] + + def record(conn, cursor, statement, *args): + statements.append(statement) + + event.listen(engine, "before_cursor_execute", record) + yield statements + event.remove(engine, "before_cursor_execute", record) + + +def test_a_bare_order_by_on_an_aliased_class_sorts_by_the_plaintext(engine, runs, sent): + from sqlalchemy.orm import aliased + + other = aliased(Run) + found = labels_of(engine, select(other).order_by(other.average_hr)) + + assert found == ["e", "c", "a", "d", "b"] + assert "ORDER BY eql_v4.ord_term(runs_1.average_hr)" in sent[-1], sent[-1] + + +def test_a_bare_order_by_in_a_subquery_sorts_by_the_plaintext(engine, runs, sent): + # The three lowest heart rates, then their labels in the outer query. + lowest = select(Run.label, Run.average_hr).order_by(Run.average_hr).limit(3).subquery() + with Session(engine) as session: + found = session.execute(select(lowest.c.label)).scalars().all() + with engine.connect() as conn: + on_a_connection = conn.execute(select(lowest.c.label)).scalars().all() + + assert sorted(found) == ["a", "c", "e"] + assert sorted(on_a_connection) == ["a", "c", "e"] + assert all("ORDER BY eql_v4.ord_term(runs.average_hr)" in sql for sql in sent[-2:]), sent[-2:] + + +def labels_of(engine, statement): + with Session(engine) as session: + return [r.label for r in session.scalars(statement)] From 951983fec5ba1c5a21f356d830ebe99f985e59b8 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 18:25:56 +0900 Subject: [PATCH 25/29] fix(python): sort a labelled encrypted column by its order term order_by(col.label("x")) and order_by("x") sent ORDER BY x, which sorts by the selected column: the stored JSON, in a wrong order. SQLAlchemy keeps a label reference in the ORDER BY, not the column, so the rewrite did not see it. The rewrite now collects each select's labelled encrypted columns by name, and replaces a label reference, by object or by name, with the order term of the column it names. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 2 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 2 +- .../python/python/stackencrypt/sqlalchemy.py | 50 +++++++++++++++---- languages/python/tests/test_sqlalchemy.py | 37 ++++++++++++++ .../python/tests/test_sqlalchemy_live.py | 13 +++++ 5 files changed, 91 insertions(+), 13 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 529c54018..154e4d0cb 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -198,7 +198,7 @@ with Session(engine) as session: - `!=`, `in_([...])`, and `not_in([...])` work where `==` does. `!=` becomes `NOT (col = term)`. `in_` becomes one equality for each value, joined with `OR`. - `.match(text)` on a field whose marker has a match index, such as `TextMatch`, is a match search: `col @@ term`. A match index can return rows that do not hold the text. So after the query, the SDK runs `verify` on each row and removes the rows that fail. The app gets only rows that hold every 3-character piece of the text. - **Other operators** that would send a value, such as `like`, `contains`, `between`, and `in_` with a subquery, raise `QueryError` with `query_unsupported` when you build the query. The value would reach the database as plaintext, and could never match. `is_(None)` and `is_not(None)` send no value, so they work. -- **Sorting:** `order_by` on an encrypted field sorts by its order term, such as `eql_v4.ord_term(level)`. This covers `.asc()`, `.desc()`, `.nulls_first()`, `.nulls_last()`, `sqlalchemy.asc(col)`, and a bare `order_by(col)`. It works in a session and on a connection, on an aliased class, and in a subquery. The field's marker must have an order index, such as `IntegerOrd`, `NumericOrd`, or `TextOrd`. Sorting a field with no order index raises `QueryError` with `query_unsupported`. +- **Sorting:** `order_by` on an encrypted field sorts by its order term, such as `eql_v4.ord_term(level)`. This covers `.asc()`, `.desc()`, `.nulls_first()`, `.nulls_last()`, `sqlalchemy.asc(col)`, and a bare `order_by(col)`. It works in a session and on a connection, on an aliased class, in a subquery, and for a labelled column, as in `order_by(col.label("x"))` or `order_by("x")`. The field's marker must have an order index, such as `IntegerOrd`, `NumericOrd`, or `TextOrd`. Sorting a field with no order index raises `QueryError` with `query_unsupported`. The SDK rewrites the sort because of a gap in EQL v4. A bare `ORDER BY level` on an EQL v4 column sorts in a wrong order, with no error. The EQL column types have no sort order of their own, so PostgreSQL sorts the stored JSON. The order term sorts by the plaintext, and the SDK's index on it serves the sort. If you write SQL by hand, sort by `eql_v4.ord_term(col)` too. - A comparison on a class that is not configured raises `QueryError` with `query_not_configured`. diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index 89e87d452..6903f7bc2 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -192,7 +192,7 @@ The cause is in EQL v4. It defines `<` and the other comparisons for its column ### What B does - `.asc()`, `.desc()`, `.nulls_first()`, and `.nulls_last()` on an encrypted column sort by `eql_v4.ord_term(col)`, or `eql_v4.ord_term_ore(col)` for an `ore_block` marker. -- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for a query in a session, and the engine hook for a query on a connection. The rewrite covers every `select` in a statement, subqueries included, and a column of a table, an alias, or a subquery. The statement is copied only when an `ORDER BY` changes. The engine hook rewrites every `select`. SQLAlchemy marks a select as ORM when it names a mapped class or attribute, as `select(Run.label)` does, even when it runs on a connection. A select from a session reaches the engine hook already rewritten, and is left as it is. +- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for a query in a session, and the engine hook for a query on a connection. The rewrite covers every `select` in a statement, subqueries included, and a column of a table, an alias, or a subquery. A label in the `ORDER BY`, by object or by name, would sort by the selected column, so it becomes the order term of the column it names. The statement is copied only when an `ORDER BY` changes. The engine hook rewrites every `select`. SQLAlchemy marks a select as ORM when it names a mapped class or attribute, as `select(Run.label)` does, even when it runs on a connection. A select from a session reaches the engine hook already rewritten, and is left as it is. - Sorting a column with no order index raises `QueryError` with `query_unsupported`. Before, it gave the order of the stored JSON, with no error. If EQL gains a sort order for its v4 column types, the rewrite can stay. The order term is also the form that uses the index. diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index b20344beb..2b22ac0ee 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -74,6 +74,9 @@ ClauseElement, ColumnClause, Grouping, + Label, + _label_reference, + _textual_label_reference, ) from sqlalchemy.sql.dml import Insert, Update from sqlalchemy.sql.functions import FunctionElement @@ -827,12 +830,9 @@ def _sorted_by_terms(statement: Any) -> Any: column type's operators, so the hooks rewrite it here. The statement is copied only when an ORDER BY changes.""" if not any( - _sort_keys(clauses) is not clauses - for clauses in ( - element._order_by_clauses - for element in visitors.iterate(statement) - if isinstance(element, Select) and element._order_by_clauses - ) + _select_sort_keys(element) is not element._order_by_clauses + for element in visitors.iterate(statement) + if isinstance(element, Select) and element._order_by_clauses ): return statement @@ -840,20 +840,37 @@ def visit_select(select: Any) -> None: # On the copy that cloned_traverse made, as SQLAlchemy's own # generative methods change it. if select._order_by_clauses: - select._order_by_clauses = _sort_keys(select._order_by_clauses) + select._order_by_clauses = _select_sort_keys(select) return visitors.cloned_traverse(statement, {}, {"select": visit_select}) -def _sort_keys(clauses: Any) -> Any: - """``clauses`` with each encrypted column replaced by its order term; the - same object when nothing changes.""" +def _select_sort_keys(select: Any) -> Any: + """The ORDER BY of ``select`` with each encrypted column replaced by its + order term; the same object when nothing changes. A label in the ORDER + BY, as in ``order_by(col.label("x"))`` or ``order_by("x")``, would sort + by the selected column, so it is replaced by the order term of the + column it names.""" + clauses = select._order_by_clauses + # The select's labelled encrypted columns, by label name. + labelled = { + column.name: column.element + for column in getattr(select, "_raw_columns", ()) + if isinstance(column, Label) and _is_eql_column(column.element) + } def replace(element: Any) -> Any: if isinstance(element, FunctionElement) and getattr(element, "name", None) in _ORDER_TERMS.values(): return element # already a sort key: not opened again + if isinstance(element, (_label_reference, _textual_label_reference)): + named = element.element + column = labelled.get(named) if isinstance(named, str) else _labelled_column(named) + return _sort_key(column) if column is not None else None + column = _labelled_column(element) + if column is not None: + return _sort_key(column) # Any column of an EQL type: of a table, an alias, or a subquery. - if isinstance(element, ColumnClause) and isinstance(element.type, EqlType): + if _is_eql_column(element): return _sort_key(element) return None @@ -861,6 +878,17 @@ def replace(element: Any) -> Any: return clauses if all(new is old for new, old in zip(rewritten, clauses)) else rewritten +def _is_eql_column(element: Any) -> bool: + return isinstance(element, ColumnClause) and isinstance(element.type, EqlType) + + +def _labelled_column(element: Any) -> Any: + """The encrypted column inside a label, or ``None``.""" + if isinstance(element, Label) and _is_eql_column(element.element): + return element.element + return None + + def _cipher_of(column: Any) -> Cipher: cipher = _TABLES.get(getattr(column, "table", None)) if cipher is None: diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index 786d78f5f..68f2ee974 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -389,3 +389,40 @@ def test_a_comparison_with_another_column_sends_no_value(scores, untouched): sql = str((Score.note == Score.note).compile(dialect=postgresql.dialect())) assert sql == "scores.note = scores.note" + + +def test_a_labelled_column_in_order_by_is_sorted_by_the_order_term(untouched): + from sqlalchemy import event + from sqlalchemy.orm import Session + + @stackencrypt.model("splits") + class Split(Base): + __tablename__ = "splits" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + seconds: Mapped[Annotated[int, IntegerOrd]] + + engine = create_engine("sqlite://") + sent = [] + event.listen(engine, "before_cursor_execute", lambda conn, cur, sql, *a: sent.append(sql)) + with engine.begin() as conn: + conn.exec_driver_sql("CREATE TABLE splits (id INTEGER PRIMARY KEY, seconds TEXT)") + configure_encryption(engine, [Split]) + secs = Split.seconds.label("secs") + + with Session(engine) as session: + for statement in ( + select(secs).order_by(secs), + select(secs).order_by("secs"), + select(secs).order_by(secs.desc()), + ): + try: + session.execute(statement) + except Exception: + pass # SQLite has no eql_v4 functions: only the SQL matters here. + + sorts = [sql.split("ORDER BY", 1)[1].strip() for sql in sent[-3:]] + assert sorts == [ + "eql_v4.ord_term(splits.seconds)", + "eql_v4.ord_term(splits.seconds)", + "eql_v4.ord_term(splits.seconds) DESC", + ] diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index f9936b1e6..c28a57173 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -661,3 +661,16 @@ def test_a_bare_order_by_in_a_subquery_sorts_by_the_plaintext(engine, runs, sent def labels_of(engine, statement): with Session(engine) as session: return [r.label for r in session.scalars(statement)] + + +def test_a_labelled_column_in_order_by_sorts_by_the_plaintext(engine, runs, sent): + hr = Run.average_hr.label("hr") + with Session(engine) as session: + by_label = session.execute(select(Run.label, hr).order_by(hr)).all() + by_name = session.execute(select(Run.label, hr).order_by("hr")).all() + + assert [row.label for row in by_label] == ["e", "c", "a", "d", "b"] + assert [row.label for row in by_name] == ["e", "c", "a", "d", "b"] + assert all("ORDER BY eql_v4.ord_term(runs.average_hr)" in sql for sql in sent[-2:]), sent[-2:] + # The selected column is still the decrypted value, under its label. + assert [row.hr for row in by_label] == [98, 120, 150, 175, None] From 21ef41d72ada41118eb57bdfa8fbdda7295f3344 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 18:34:58 +0900 Subject: [PATCH 26/29] fix(python): sort a column named by a string by its order term order_by("level") and desc("level") sent ORDER BY laps.level, which sorts the stored JSON. SQLAlchemy matches a string to a column only when it compiles the statement, so the rewrite did not see a column. The rewrite now looks the name up as SQLAlchemy does: the labels, then the selected columns, then the columns of the FROM tables. A name that belongs to an encrypted column becomes its order term. A name that two FROM tables share, and that is not selected, raises QueryError, because the rewrite cannot tell which column is meant. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 2 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 2 +- .../python/python/stackencrypt/sqlalchemy.py | 45 +++++++++--- languages/python/tests/test_sqlalchemy.py | 71 +++++++++++++++++++ .../python/tests/test_sqlalchemy_live.py | 9 +++ 5 files changed, 117 insertions(+), 12 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 154e4d0cb..268538219 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -198,7 +198,7 @@ with Session(engine) as session: - `!=`, `in_([...])`, and `not_in([...])` work where `==` does. `!=` becomes `NOT (col = term)`. `in_` becomes one equality for each value, joined with `OR`. - `.match(text)` on a field whose marker has a match index, such as `TextMatch`, is a match search: `col @@ term`. A match index can return rows that do not hold the text. So after the query, the SDK runs `verify` on each row and removes the rows that fail. The app gets only rows that hold every 3-character piece of the text. - **Other operators** that would send a value, such as `like`, `contains`, `between`, and `in_` with a subquery, raise `QueryError` with `query_unsupported` when you build the query. The value would reach the database as plaintext, and could never match. `is_(None)` and `is_not(None)` send no value, so they work. -- **Sorting:** `order_by` on an encrypted field sorts by its order term, such as `eql_v4.ord_term(level)`. This covers `.asc()`, `.desc()`, `.nulls_first()`, `.nulls_last()`, `sqlalchemy.asc(col)`, and a bare `order_by(col)`. It works in a session and on a connection, on an aliased class, in a subquery, and for a labelled column, as in `order_by(col.label("x"))` or `order_by("x")`. The field's marker must have an order index, such as `IntegerOrd`, `NumericOrd`, or `TextOrd`. Sorting a field with no order index raises `QueryError` with `query_unsupported`. +- **Sorting:** `order_by` on an encrypted field sorts by its order term, such as `eql_v4.ord_term(level)`. This covers `.asc()`, `.desc()`, `.nulls_first()`, `.nulls_last()`, `sqlalchemy.asc(col)`, and a bare `order_by(col)`. It works in a session and on a connection, on an aliased class, in a subquery, and for a column named by a label or a string, as in `order_by(col.label("x"))`, `order_by("x")`, or `order_by(desc("level"))`. The field's marker must have an order index, such as `IntegerOrd`, `NumericOrd`, or `TextOrd`. Sorting a field with no order index raises `QueryError` with `query_unsupported`. The SDK rewrites the sort because of a gap in EQL v4. A bare `ORDER BY level` on an EQL v4 column sorts in a wrong order, with no error. The EQL column types have no sort order of their own, so PostgreSQL sorts the stored JSON. The order term sorts by the plaintext, and the SDK's index on it serves the sort. If you write SQL by hand, sort by `eql_v4.ord_term(col)` too. - A comparison on a class that is not configured raises `QueryError` with `query_not_configured`. diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index 6903f7bc2..08e1ee8d9 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -192,7 +192,7 @@ The cause is in EQL v4. It defines `<` and the other comparisons for its column ### What B does - `.asc()`, `.desc()`, `.nulls_first()`, and `.nulls_last()` on an encrypted column sort by `eql_v4.ord_term(col)`, or `eql_v4.ord_term_ore(col)` for an `ore_block` marker. -- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for a query in a session, and the engine hook for a query on a connection. The rewrite covers every `select` in a statement, subqueries included, and a column of a table, an alias, or a subquery. A label in the `ORDER BY`, by object or by name, would sort by the selected column, so it becomes the order term of the column it names. The statement is copied only when an `ORDER BY` changes. The engine hook rewrites every `select`. SQLAlchemy marks a select as ORM when it names a mapped class or attribute, as `select(Run.label)` does, even when it runs on a connection. A select from a session reaches the engine hook already rewritten, and is left as it is. +- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for a query in a session, and the engine hook for a query on a connection. The rewrite covers every `select` in a statement, subqueries included, and a column of a table, an alias, or a subquery. A label or a string name in the `ORDER BY`, such as `order_by("level")`, would sort by the selected column. SQLAlchemy matches a string to a column only when it compiles the statement. So the rewrite looks the name up as SQLAlchemy does: among the labels, then the selected columns, then the columns of the `FROM` tables. A name that belongs to an encrypted column becomes the order term of that column. A name that two `FROM` tables share, and that is not selected, raises `QueryError`, because the rewrite cannot tell which column is meant. The statement is copied only when an `ORDER BY` changes. The engine hook rewrites every `select`. SQLAlchemy marks a select as ORM when it names a mapped class or attribute, as `select(Run.label)` does, even when it runs on a connection. A select from a session reaches the engine hook already rewritten, and is left as it is. - Sorting a column with no order index raises `QueryError` with `query_unsupported`. Before, it gave the order of the stored JSON, with no error. If EQL gains a sort order for its v4 column types, the rewrite can stay. The order term is also the form that uses the index. diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 2b22ac0ee..a736b37af 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -847,23 +847,48 @@ def visit_select(select: Any) -> None: def _select_sort_keys(select: Any) -> Any: """The ORDER BY of ``select`` with each encrypted column replaced by its - order term; the same object when nothing changes. A label in the ORDER - BY, as in ``order_by(col.label("x"))`` or ``order_by("x")``, would sort - by the selected column, so it is replaced by the order term of the - column it names.""" + order term; the same object when nothing changes. A label or a name in + the ORDER BY, as in ``order_by(col.label("x"))``, ``order_by("x")`` or + ``order_by(desc("level"))``, would sort by the selected column, so it is + replaced by the order term of the column it names.""" clauses = select._order_by_clauses - # The select's labelled encrypted columns, by label name. - labelled = { - column.name: column.element - for column in getattr(select, "_raw_columns", ()) - if isinstance(column, Label) and _is_eql_column(column.element) - } + # The encrypted columns a string in the ORDER BY can name, as SQLAlchemy + # matches it when it compiles the statement: a label's name first, then + # a selected column's key, then a column of a FROM table. A name that + # two FROM tables share cannot be told apart, so it is refused. + labelled: Dict[str, Any] = {} + shared = set() + for table in select.get_final_froms(): + for column in getattr(table, "c", ()): + if _is_eql_column(column): + if column.key in labelled and labelled[column.key] is not column: + shared.add(column.key) + labelled[column.key] = column + labelled.update( + {key: column for key, column in select.selected_columns.items() if _is_eql_column(column)} + ) + labelled.update( + { + column.name: column.element + for column in getattr(select, "_raw_columns", ()) + if isinstance(column, Label) and _is_eql_column(column.element) + } + ) + selected = set(select.selected_columns.keys()) def replace(element: Any) -> Any: if isinstance(element, FunctionElement) and getattr(element, "name", None) in _ORDER_TERMS.values(): return element # already a sort key: not opened again if isinstance(element, (_label_reference, _textual_label_reference)): named = element.element + if isinstance(named, str) and named in shared and named not in selected: + raise QueryError( + "query_unsupported", + f"order_by({named!r}) names an encrypted column of more than one table " + "in the query, so the SDK cannot sort by its order term. Pass the " + "column itself, such as order_by(Model.field).", + reason="order", + ) column = labelled.get(named) if isinstance(named, str) else _labelled_column(named) return _sort_key(column) if column is not None else None column = _labelled_column(element) diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index 68f2ee974..fe01a6a8b 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -426,3 +426,74 @@ class Split(Base): "eql_v4.ord_term(splits.seconds)", "eql_v4.ord_term(splits.seconds) DESC", ] + + +def test_a_column_named_by_a_string_in_order_by_is_sorted_by_the_order_term(untouched): + from sqlalchemy import desc, event + from sqlalchemy.orm import Session + + @stackencrypt.model("legs") + class Leg(Base): + __tablename__ = "legs" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + seconds: Mapped[Annotated[int, IntegerOrd]] + + engine = create_engine("sqlite://") + sent = [] + event.listen(engine, "before_cursor_execute", lambda conn, cur, sql, *a: sent.append(sql)) + with engine.begin() as conn: + conn.exec_driver_sql("CREATE TABLE legs (id INTEGER PRIMARY KEY, seconds TEXT)") + configure_encryption(engine, [Leg]) + statements = [ + select(Leg.seconds).order_by("seconds"), + select(Leg.seconds).order_by(desc("seconds")), + select(Leg).order_by("seconds"), + select(Leg.__table__).order_by("seconds"), + ] + + with Session(engine) as session: + for statement in statements: + try: + session.execute(statement) + except Exception: + pass # SQLite has no eql_v4 functions: only the SQL matters here. + + sorts = [sql.split("ORDER BY", 1)[1].strip() for sql in sent[-4:]] + assert sorts == [ + "eql_v4.ord_term(legs.seconds)", + "eql_v4.ord_term(legs.seconds) DESC", + "eql_v4.ord_term(legs.seconds)", + "eql_v4.ord_term(legs.seconds)", + ] + + +def test_a_string_naming_an_encrypted_column_that_is_not_selected_is_sorted_or_refused(untouched): + from sqlalchemy import event + from sqlalchemy.orm import Session, aliased + + @stackencrypt.model("stages") + class Stage(Base): + __tablename__ = "stages" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + name: Mapped[Annotated[str, Passthrough]] + seconds: Mapped[Annotated[int, IntegerOrd]] + + engine = create_engine("sqlite://") + sent = [] + event.listen(engine, "before_cursor_execute", lambda conn, cur, sql, *a: sent.append(sql)) + with engine.begin() as conn: + conn.exec_driver_sql("CREATE TABLE stages (id INTEGER PRIMARY KEY, name TEXT, seconds TEXT)") + configure_encryption(engine, [Stage]) + other = aliased(Stage) + + with Session(engine) as session: + try: + session.execute(select(Stage.name).order_by("seconds")) + except Exception: + pass # SQLite has no eql_v4 functions: only the SQL matters here. + with pytest.raises(QueryError) as raised: + session.execute(select(Stage.name, other.name).join(other, other.id == Stage.id).order_by("seconds")) + + assert sent[-1].split("ORDER BY", 1)[1].strip() == "eql_v4.ord_term(stages.seconds)" + assert raised.value.code == "query_unsupported" + assert "seconds" in raised.value.message diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index c28a57173..9d6e62434 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -674,3 +674,12 @@ def test_a_labelled_column_in_order_by_sorts_by_the_plaintext(engine, runs, sent assert all("ORDER BY eql_v4.ord_term(runs.average_hr)" in sql for sql in sent[-2:]), sent[-2:] # The selected column is still the decrypted value, under its label. assert [row.hr for row in by_label] == [98, 120, 150, 175, None] + + +def test_a_column_named_by_a_string_in_order_by_sorts_by_the_plaintext(engine, runs, sent): + from sqlalchemy import desc + + assert labels_of(engine, select(Run).order_by("average_hr")) == ["e", "c", "a", "d", "b"] + assert labels_of(engine, select(Run).order_by(desc("pace").nulls_last())) == ["d", "a", "e", "b", "c"] + assert "ORDER BY eql_v4.ord_term(runs.average_hr)" in sent[-2], sent[-2] + assert "ORDER BY eql_v4.ord_term(runs.pace) DESC NULLS LAST" in sent[-1], sent[-1] From 6770900a87542357db1ccfed727a513bf1d4f4f0 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 18:40:32 +0900 Subject: [PATCH 27/29] fix(python): a plaintext label hides an encrypted column in ORDER BY The name lookup for a string in an ORDER BY kept only encrypted columns. So a plaintext label or column with the name of an encrypted FROM column did not replace it, and the rewrite sorted by the encrypted column instead: select(T.rank.label("level")).order_by("level") sorted by T.level, with no error. The lookup now keeps every candidate, plaintext ones included, in SQLAlchemy's order: FROM columns, then selected columns, then labels. Only a name that resolves to an encrypted column is rewritten. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 2 +- .../python/python/stackencrypt/sqlalchemy.py | 54 ++++++++++--------- languages/python/tests/test_sqlalchemy.py | 31 +++++++++++ .../python/tests/test_sqlalchemy_live.py | 12 ++++- 4 files changed, 71 insertions(+), 28 deletions(-) diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index 08e1ee8d9..c3f10781c 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -192,7 +192,7 @@ The cause is in EQL v4. It defines `<` and the other comparisons for its column ### What B does - `.asc()`, `.desc()`, `.nulls_first()`, and `.nulls_last()` on an encrypted column sort by `eql_v4.ord_term(col)`, or `eql_v4.ord_term_ore(col)` for an `ore_block` marker. -- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for a query in a session, and the engine hook for a query on a connection. The rewrite covers every `select` in a statement, subqueries included, and a column of a table, an alias, or a subquery. A label or a string name in the `ORDER BY`, such as `order_by("level")`, would sort by the selected column. SQLAlchemy matches a string to a column only when it compiles the statement. So the rewrite looks the name up as SQLAlchemy does: among the labels, then the selected columns, then the columns of the `FROM` tables. A name that belongs to an encrypted column becomes the order term of that column. A name that two `FROM` tables share, and that is not selected, raises `QueryError`, because the rewrite cannot tell which column is meant. The statement is copied only when an `ORDER BY` changes. The engine hook rewrites every `select`. SQLAlchemy marks a select as ORM when it names a mapped class or attribute, as `select(Run.label)` does, even when it runs on a connection. A select from a session reaches the engine hook already rewritten, and is left as it is. +- A bare `order_by(col)` and `sqlalchemy.asc(col)` do not go through the column type's operators. So the hooks rewrite the `ORDER BY` of each `select`: the session hook for a query in a session, and the engine hook for a query on a connection. The rewrite covers every `select` in a statement, subqueries included, and a column of a table, an alias, or a subquery. A label or a string name in the `ORDER BY`, such as `order_by("level")`, would sort by the selected column. SQLAlchemy matches a string to a column only when it compiles the statement. So the rewrite looks the name up as SQLAlchemy does: among the labels, then the selected columns, then the columns of the `FROM` tables. Plaintext columns and labels are candidates too, so a plaintext label hides an encrypted column with the same name, as it does in SQLAlchemy. Only a name that resolves to an encrypted column becomes the order term of that column. A name that two `FROM` tables share, and that is not selected, raises `QueryError`, because the rewrite cannot tell which column is meant. The statement is copied only when an `ORDER BY` changes. The engine hook rewrites every `select`. SQLAlchemy marks a select as ORM when it names a mapped class or attribute, as `select(Run.label)` does, even when it runs on a connection. A select from a session reaches the engine hook already rewritten, and is left as it is. - Sorting a column with no order index raises `QueryError` with `query_unsupported`. Before, it gave the order of the stored JSON, with no error. If EQL gains a sort order for its v4 column types, the rewrite can stay. The order term is also the form that uses the index. diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index a736b37af..791aaac90 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -852,45 +852,47 @@ def _select_sort_keys(select: Any) -> Any: ``order_by(desc("level"))``, would sort by the selected column, so it is replaced by the order term of the column it names.""" clauses = select._order_by_clauses - # The encrypted columns a string in the ORDER BY can name, as SQLAlchemy - # matches it when it compiles the statement: a label's name first, then - # a selected column's key, then a column of a FROM table. A name that - # two FROM tables share cannot be told apart, so it is refused. - labelled: Dict[str, Any] = {} - shared = set() + # What a string in the ORDER BY names, as SQLAlchemy matches it when it + # compiles the statement: a label's name first, then a selected + # column's key, then a column of a FROM table. Every candidate is kept, + # plaintext ones included, so that a plaintext label hides an encrypted + # column of the same name. Only a name that resolves to an encrypted + # column is rewritten. A name that two FROM tables share, with an + # encrypted column among them, cannot be told apart, so it is refused. + named: Dict[str, Any] = {} + candidates: Dict[str, List[Any]] = {} for table in select.get_final_froms(): for column in getattr(table, "c", ()): - if _is_eql_column(column): - if column.key in labelled and labelled[column.key] is not column: - shared.add(column.key) - labelled[column.key] = column - labelled.update( - {key: column for key, column in select.selected_columns.items() if _is_eql_column(column)} - ) - labelled.update( - { - column.name: column.element - for column in getattr(select, "_raw_columns", ()) - if isinstance(column, Label) and _is_eql_column(column.element) - } - ) - selected = set(select.selected_columns.keys()) + named[column.key] = column + candidates.setdefault(column.key, []).append(column) + shared = { + name + for name, columns in candidates.items() + if len({id(c) for c in columns}) > 1 and any(_is_eql_column(c) for c in columns) + } + for key, column in select.selected_columns.items(): + named[key] = column + shared.discard(key) + for column in getattr(select, "_raw_columns", ()): + if isinstance(column, Label): + named[column.name] = column.element + shared.discard(column.name) def replace(element: Any) -> Any: if isinstance(element, FunctionElement) and getattr(element, "name", None) in _ORDER_TERMS.values(): return element # already a sort key: not opened again if isinstance(element, (_label_reference, _textual_label_reference)): - named = element.element - if isinstance(named, str) and named in shared and named not in selected: + name = element.element + if isinstance(name, str) and name in shared: raise QueryError( "query_unsupported", - f"order_by({named!r}) names an encrypted column of more than one table " + f"order_by({name!r}) names an encrypted column of more than one table " "in the query, so the SDK cannot sort by its order term. Pass the " "column itself, such as order_by(Model.field).", reason="order", ) - column = labelled.get(named) if isinstance(named, str) else _labelled_column(named) - return _sort_key(column) if column is not None else None + column = named.get(name) if isinstance(name, str) else _labelled_column(name) + return _sort_key(column) if _is_eql_column(column) else None column = _labelled_column(element) if column is not None: return _sort_key(column) diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index fe01a6a8b..28d235daa 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -497,3 +497,34 @@ class Stage(Base): assert sent[-1].split("ORDER BY", 1)[1].strip() == "eql_v4.ord_term(stages.seconds)" assert raised.value.code == "query_unsupported" assert "seconds" in raised.value.message + + +def test_a_plaintext_label_with_an_encrypted_columns_name_is_left_alone(untouched): + from sqlalchemy import event + from sqlalchemy.orm import Session + + @stackencrypt.model("heats") + class Heat(Base): + __tablename__ = "heats" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + rank: Mapped[Annotated[int, Passthrough]] + seconds: Mapped[Annotated[int, IntegerOrd]] + + engine = create_engine("sqlite://") + sent = [] + event.listen(engine, "before_cursor_execute", lambda conn, cur, sql, *a: sent.append(sql)) + with engine.begin() as conn: + conn.exec_driver_sql("CREATE TABLE heats (id INTEGER PRIMARY KEY, rank INTEGER, seconds TEXT)") + configure_encryption(engine, [Heat]) + + with Session(engine) as session: + # The label "seconds" names the plaintext rank, so the sort is by rank. + session.execute(select(Heat.rank.label("seconds")).order_by("seconds")) + try: + session.execute(select(Heat.rank, Heat.seconds.label("rank")).order_by("rank")) + except Exception: + pass # SQLite has no eql_v4 functions: only the SQL matters here. + + assert sent[-2].split("ORDER BY", 1)[1].strip() == "seconds" + # Here the label "rank" names the encrypted column, so the sort is its term. + assert sent[-1].split("ORDER BY", 1)[1].strip() == "eql_v4.ord_term(heats.seconds)" diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index 9d6e62434..8f254a417 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -13,7 +13,7 @@ import pytest from _live import DATABASE_URL, LIVE, needs_database -from sqlalchemy import ForeignKey, and_, create_engine, delete, func, not_, or_, select, text, update +from sqlalchemy import ForeignKey, and_, create_engine, delete, desc, func, not_, or_, select, text, update from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship import stackencrypt @@ -683,3 +683,13 @@ def test_a_column_named_by_a_string_in_order_by_sorts_by_the_plaintext(engine, r assert labels_of(engine, select(Run).order_by(desc("pace").nulls_last())) == ["d", "a", "e", "b", "c"] assert "ORDER BY eql_v4.ord_term(runs.average_hr)" in sent[-2], sent[-2] assert "ORDER BY eql_v4.ord_term(runs.pace) DESC NULLS LAST" in sent[-1], sent[-1] + + +def test_a_plaintext_label_with_an_encrypted_columns_name_sorts_by_the_plaintext(engine, runs, sent): + # The label "average_hr" names the plaintext label column. + renamed = Run.label.label("average_hr") + with Session(engine) as session: + found = session.execute(select(renamed).order_by(desc("average_hr"))).scalars().all() + + assert found == ["e", "d", "c", "b", "a"] + assert "ord_term" not in sent[-1], sent[-1] From 42a3de95de525d6373aa936e93ac09866d4cf4d1 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 11 Nov 2026 19:01:22 +0900 Subject: [PATCH 28/29] feat(python)!: configure_encryption covers every engine in a process configure_encryption(engine, classes) installed the write hook on one engine, and refused a second call for a class. An app that writes through more than one engine, such as an app engine, a test engine, and a worker run twice in one test, had no way to cover them, and a missed engine wrote without encryption. configure_encryption(classes) now installs the write hook once, on SQLAlchemy's Engine class, so every engine runs it, including one made later. A table is found by its Table object, not its name, so another database's table of the same name is not encrypted. A repeat call for a configured class does nothing; it still refuses a changed class, or a second class for the same table. The package is not released, so the signature change breaks no user. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- languages/python/README.md | 9 +- ...qlalchemy-layer-encrypts-in-batch-hooks.md | 23 ++- .../examples/customer-directory/README.md | 2 +- .../customer-directory/app_sqlalchemy.py | 4 +- .../python/python/stackencrypt/sqlalchemy.py | 141 ++++++++++-------- languages/python/tests/test_sqlalchemy.py | 124 ++++++++++++--- .../python/tests/test_sqlalchemy_live.py | 2 +- 7 files changed, 212 insertions(+), 93 deletions(-) diff --git a/languages/python/README.md b/languages/python/README.md index 268538219..f8380dbec 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -159,7 +159,7 @@ The values are checked as `encrypt` checks them. A row that is not a mapping rai ## Use it with the SQLAlchemy ORM -`stackencrypt.sqlalchemy.configure_encryption` makes the ORM encrypt and decrypt for you. Call it once, after you create the engine. Then use sessions as usual. The app reads and writes plaintext attributes, and the tables store EQL values. +`stackencrypt.sqlalchemy.configure_encryption` makes the ORM encrypt and decrypt for you. Call it once, with the classes to encrypt. Then use sessions as usual, with any engine. The app reads and writes plaintext attributes, and the tables store EQL values. ```text from stackencrypt.sqlalchemy import configure_encryption, eql_type_map @@ -178,7 +178,7 @@ class Customer(Base): engine = create_engine(DATABASE_URL.replace("postgresql://", "postgresql+psycopg://", 1)) -configure_encryption(engine, [Customer]) +configure_encryption([Customer]) with Session(engine) as session: session.add(Customer(email="a@example.com", level=3)) @@ -189,7 +189,8 @@ with Session(engine) as session: `eql_type_map` gives each encrypted field its EQL column type, such as `public.eql_v4_text_eq` for `TextEq`. So a column needs no type of its own. `Base.metadata.create_all(engine)` creates the EQL columns. A field declared `Mapped[Annotated[Optional[str], TextEq]]` gets the same type, and SQLAlchemy makes its column nullable. -- **Writes:** each INSERT or UPDATE on a listed table is encrypted with one batch of key requests for all its rows. +- **Writes:** each INSERT or UPDATE on a listed class's table is encrypted with one batch of key requests for all its rows. This works on every engine in the process, including one made after the call, such as a test's. The table is found by its `Table` object, not its name. So a Core insert through `sa.table("customers", ...)`, which is a separate object, is not encrypted. PostgreSQL then refuses its plaintext, because it does not fit the EQL column type. +- A repeat call for a configured class does nothing. It raises `DeclarationError` if the class changed since it was declared, or if another class is configured for the same table. - **Reads:** the objects a query loads are decrypted with one batch for each class. This includes relationship loads, and attributes read again after a commit. A session query of columns, such as `session.execute(select(Customer.email, Customer.plan))`, is decrypted too. - Only encrypted fields are changed. A passthrough field is checked as `encrypt` checks it, then stored unchanged. For example, a `datetime` with no time zone raises `InvalidValueError` with `value_naive_datetime`. @@ -210,7 +211,7 @@ Each class must be declared with `stackencrypt.model` and mapped by SQLAlchemy. - `declaration_column_name`: an encrypted field's column has another name than the field, as with `mapped_column("email_address")`. - `declaration_column_type`: an encrypted field's column is not the EQL type of its marker. Set `type_annotation_map = eql_type_map` on the base. - `declaration_unique_column`: an encrypted column has `unique=True` or a unique constraint. Each EQL value contains new random ciphertext, so the constraint would never find a duplicate. Use a unique index on `eql_v4.eq_term(col)`, as in "Migrations with Alembic". -- `declaration_already_configured`: the class was configured before in this process. +- `declaration_already_configured`: another class is configured for the same table. A statement that puts an encrypted value inside its SQL, such as `insert(Customer).values(email=...)`, raises `DeclarationError` with `declaration_inline_value`. The value is not stored. Pass the values as parameters instead, as `session.add` and `session.execute(insert(Customer), rows)` do. diff --git a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md index c3f10781c..ed5b36eb7 100644 --- a/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md +++ b/languages/python/docs/adr/0004-the-sqlalchemy-layer-encrypts-in-batch-hooks.md @@ -13,7 +13,7 @@ extends: ADR-0002, ADR-0003 class Base(DeclarativeBase): type_annotation_map = eql_type_map -configure_encryption(engine, [Customer]) +configure_encryption([Customer]) with Session(engine) as session: session.add(Customer(email="a@example.com", level=3)) @@ -37,7 +37,7 @@ Each encrypted value needs its own data key from ZeroKMS, CipherStash's key serv **B. Hooks that work on a whole statement or result (chosen).** -- Writes: a `before_execute` hook on the engine. For each `INSERT` or `UPDATE` on a configured table, it encrypts the parameters of every row with one `Cipher.encrypt_rows` call. +- Writes: a `before_execute` hook on SQLAlchemy's `Engine` class, so every engine runs it. For each `INSERT` or `UPDATE` on a configured table, it encrypts the parameters of every row with one `Cipher.encrypt_rows` call. - Reads: the `load` and `refresh` events of each class record the objects a query builds. A `do_orm_execute` hook then decrypts them, with one `Cipher.decrypt_rows` call for each class and set of fields. It sets the plaintext with `set_committed_value`, so the objects are not marked as changed. - Reads of columns: the same hook decrypts the result columns that are encrypted fields, as in `select(Customer.email, Customer.plan)`. It makes one `decrypt_rows` call for each class and set of fields. @@ -53,7 +53,7 @@ SQLAlchemy passes a column type one value at a time. So 100 rows would make 100 ## Decision 2: one setup call with an explicit list of classes -`configure_encryption(engine, [Customer, Order])` takes the engine and the classes to encrypt. It does not find declared classes by itself, such as from the `Base` registry. An explicit list shows which tables are encrypted. A class imported late would otherwise be missed, with no error. +`configure_encryption([Customer, Order])` takes the classes to encrypt. It does not find declared classes by itself, such as from the `Base` registry. An explicit list shows which tables are encrypted. A class imported late would otherwise be missed, with no error. It checks each class first, and raises `DeclarationError` before any query: @@ -64,7 +64,22 @@ It checks each class first, and raises `DeclarationError` before any query: | `declaration_column_name` | an encrypted field whose column has another name | | `declaration_column_type` | an encrypted field whose column is not its marker's EQL type | | `declaration_unique_column` | an encrypted field with a unique constraint | -| `declaration_already_configured` | a configuration from an earlier call | +| `declaration_already_configured` | a table that another class is configured for | + +### Every engine, once in a process + +The first version took one engine: `configure_encryption(engine, classes)`. It installed the write hook on that engine only, and refused a second call for a class. But an app can write through more than one engine in a process: Endurain has its app engine, a test engine, and a worker that one test runs twice. A missed engine would write without encryption. The options were these: + +| Option | Problem | +|---|---| +| A list of engines | Every engine must exist at the call. A test engine made later is missed. | +| A second call adds an engine | Each engine needs a call. A forgotten one writes without encryption. | +| **The hooks on SQLAlchemy's `Engine` class (chosen)** | Every engine runs them. | + +So the call takes only the classes. The hooks are installed once, on `Engine` and on `Session`. Two changes go with it: + +- **A table is found by its `Table` object, not its name.** With every engine hooked, another database with a table of the same name would otherwise have its writes encrypted. A Core insert through `sa.table("users", ...)` uses a separate object, so it is not encrypted. PostgreSQL refuses its plaintext, because it does not fit the EQL column type, so the failure is not silent. +- **A repeat call for a configured class does nothing.** It has nothing left to install. It still raises `DeclarationError` if the class changed since it was declared (`declaration_class_changed`), or if another class is configured for the same table (`declaration_already_configured`). The hooks use field names, and SQLAlchemy uses column names in statement parameters. So a column with another name, such as `mapped_column("email_address")`, is refused for now. diff --git a/languages/python/examples/customer-directory/README.md b/languages/python/examples/customer-directory/README.md index 9ca24b395..b769ab58c 100644 --- a/languages/python/examples/customer-directory/README.md +++ b/languages/python/examples/customer-directory/README.md @@ -52,7 +52,7 @@ The search is a `POST` so that the email stays out of URLs and access logs. mise run dev:sqlalchemy # http://127.0.0.1:8001 ``` -The app calls `configure_encryption(engine, [Customer])` once, at start-up. After that, the endpoints use `Session` with no SDK calls: +The app calls `configure_encryption([Customer])` once, at start-up. After that, the endpoints use `Session` with no SDK calls: - `session.add` and `session.commit` encrypt `email` and `name`. - `select(Customer).where(Customer.email == search.email)` searches by the equality term. diff --git a/languages/python/examples/customer-directory/app_sqlalchemy.py b/languages/python/examples/customer-directory/app_sqlalchemy.py index 92992db93..e0011bfe5 100644 --- a/languages/python/examples/customer-directory/app_sqlalchemy.py +++ b/languages/python/examples/customer-directory/app_sqlalchemy.py @@ -58,8 +58,8 @@ class Customer(Base): ) -# Once for the process, before the first request. -configure_encryption(engine, [Customer]) +# Once for the process, before the first request. Every engine is covered. +configure_encryption([Customer]) def utc_now() -> datetime: diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 791aaac90..26bb37b89 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -2,15 +2,15 @@ from stackencrypt.sqlalchemy import configure_encryption - engine = create_engine(DATABASE_URL) - configure_encryption(engine, [Customer, Order]) + configure_encryption([Customer, Order]) After that, a flush encrypts the declared fields of the listed classes, and a query decrypts them. The app reads and writes plaintext attributes. -- Writes: a ``before_execute`` hook on the engine. For each INSERT or UPDATE - on a listed table, it encrypts the parameters of all the statement's rows - with one ``Cipher.encrypt_rows`` call. +- Writes: a ``before_execute`` hook on SQLAlchemy's ``Engine`` class, so + every engine runs it. For each INSERT or UPDATE on the Table object of a + listed class, it encrypts the parameters of all the statement's rows with + one ``Cipher.encrypt_rows`` call. - Reads: the ``load`` and ``refresh`` events of each listed class record the objects a query builds. A ``do_orm_execute`` hook then decrypts them, with one ``Cipher.decrypt_rows`` call for each class and set of fields. The @@ -393,6 +393,9 @@ def _is_eql(column_type: Any) -> bool: # The configured tables, for the comparisons: a Table to its cipher. _TABLES: Dict[Table, Cipher] = {} +# The configured tables, by their Table object: the class configured for each. +_TABLE_CLASSES: Dict[Table, type] = {} + # The objects a load built or refreshed, and which attributes, waiting for # the do_orm_execute hook. Kept in each session's `info`. _PENDING = "stackencrypt_pending" @@ -401,40 +404,51 @@ def _is_eql(column_type: Any) -> bool: _CONFIGURED: Dict[type, Tuple[Cipher, FrozenSet[str]]] = {} -def configure_encryption(engine: Engine, classes: Sequence[type]) -> None: +def configure_encryption(classes: Sequence[type]) -> None: """Encrypt and decrypt the declared fields of ``classes`` when the ORM - writes them through ``engine`` and reads them in any session. + writes them through any engine, and reads them in any session. Each class must be declared with ``stackencrypt.model`` and mapped by SQLAlchemy. Its declared table must be its ``__tablename__``, and each - sealed field's column must have the field's name. A class can be - configured once in a process. + sealed field's column must have the field's name. + + The hooks are installed once in a process, on SQLAlchemy's ``Engine`` + and ``Session`` classes, so every engine gets them, including one made + later, such as a test's. A write is encrypted when its table is the + ``Table`` object of a configured class. + + A repeat call for a class that is already configured does nothing. + It still raises ``DeclarationError`` when the class changed since it + was declared, or when another class is configured for its table. Every class is checked first. A class that fails raises ``DeclarationError``, and none of the classes is configured. The check makes no request. """ - checked = [_check(cls) for cls in classes] - for cls, _, _ in checked: - if cls in _CONFIGURED: + checked = [_check(cls) for cls in classes] # refuses a changed class too + owners = dict(_TABLE_CLASSES) + new = [] + for cls, cipher, sealed in checked: + table = inspect(cls).local_table + owner = owners.get(table) + if owner is not None and owner is not cls: raise DeclarationError( "declaration_already_configured", - f"{cls.__qualname__} is configured for encryption already. " - "Configure each class once, with every engine it writes through.", + f"{cls.__qualname__} maps table {table.name!r}, and " + f"{owner.__qualname__} is configured for that table already. " + "Configure one class for each table.", ) - if len({cls for cls, _, _ in checked}) != len(checked): - raise DeclarationError( - "declaration_already_configured", - "the list names a class more than once.", - ) - tables: Dict[str, Tuple[Cipher, FrozenSet[str]]] = {} - for cls, cipher, sealed in checked: + if owner is None: + owners[table] = cls + new.append((cls, cipher, sealed, table)) + for cls, cipher, sealed, table in new: _CONFIGURED[cls] = (cipher, sealed) - _TABLES[inspect(cls).local_table] = cipher - tables[cipher.declaration.table] = (cipher, sealed) + _TABLES[table] = cipher + _TABLE_CLASSES[table] = cls event.listen(cls, "load", _remember_load) event.listen(cls, "refresh", _remember_refresh) - event.listen(engine, "before_execute", _encrypter(tables), retval=True) + if not event.contains(Engine, "before_execute", _encrypt_parameters): + event.listen(Engine, "before_execute", _encrypt_parameters, retval=True) if not event.contains(Session, "do_orm_execute", _decrypt_loaded): event.listen(Session, "do_orm_execute", _decrypt_loaded) @@ -503,41 +517,50 @@ def _check(cls: type) -> Tuple[type, Cipher, FrozenSet[str]]: return cls, cipher, sealed -def _encrypter(tables: Dict[str, Tuple[Cipher, FrozenSet[str]]]) -> Any: - def encrypt_parameters(conn, clause, multiparams, params, execution_options): # type: ignore[no-untyped-def] - if not execution_options.get(_VERIFIED): - matches = _matches_in(clause) - if matches: - column, _ = matches[0]._annotations[_MATCH] - raise _match_refused(column, "is not run through a session.") - if isinstance(clause, Select): - # Every select, ORM-marked or not. SQLAlchemy marks a select as - # ORM when it names a mapped class or attribute, even when it - # runs on a connection, where the session hook does not run. A - # select from a session reaches here already rewritten, and - # comes back unchanged. - return _sorted_by_terms(clause), multiparams, params - if not isinstance(clause, (Insert, Update)): - return clause, multiparams, params - configured = tables.get(getattr(clause.table, "name", None)) - if configured is None: - return clause, multiparams, params - cipher, sealed = configured - rows = [dict(row) for row in multiparams] if multiparams else [dict(params or {})] - _refuse_inline_values(conn, clause, sealed, rows) - # Passthrough fields too: encrypt_rows checks them as encrypt does, - # and returns them unchanged. - names = _declared(cipher).intersection(rows[0]) - if not names: - return clause, multiparams, params - encrypted = cipher.encrypt_rows([{name: row[name] for name in names} for row in rows]) - for row, values in zip(rows, encrypted): - row.update(values) - if multiparams: - return clause, rows, params - return clause, multiparams, rows[0] - - return encrypt_parameters +def _encrypt_parameters(conn, clause, multiparams, params, execution_options): # type: ignore[no-untyped-def] + """The write hook, on SQLAlchemy's ``Engine`` class: every engine runs it.""" + if not execution_options.get(_VERIFIED): + matches = _matches_in(clause) + if matches: + column, _ = matches[0]._annotations[_MATCH] + raise _match_refused(column, "is not run through a session.") + if isinstance(clause, Select): + # Every select, ORM-marked or not. SQLAlchemy marks a select as + # ORM when it names a mapped class or attribute, even when it + # runs on a connection, where the session hook does not run. A + # select from a session reaches here already rewritten, and + # comes back unchanged. + return _sorted_by_terms(clause), multiparams, params + if not isinstance(clause, (Insert, Update)): + return clause, multiparams, params + cls = _configured_class(clause.table) + if cls is None: + return clause, multiparams, params + cipher, sealed = _CONFIGURED[cls] + rows = [dict(row) for row in multiparams] if multiparams else [dict(params or {})] + _refuse_inline_values(conn, clause, sealed, rows) + # Passthrough fields too: encrypt_rows checks them as encrypt does, + # and returns them unchanged. + names = _declared(cipher).intersection(rows[0]) + if not names: + return clause, multiparams, params + encrypted = cipher.encrypt_rows([{name: row[name] for name in names} for row in rows]) + for row, values in zip(rows, encrypted): + row.update(values) + if multiparams: + return clause, rows, params + return clause, multiparams, rows[0] + + +def _configured_class(table: Any) -> Any: + """The class configured for ``table``, found by its Table object, not + its name: another database's table of the same name is not encrypted. + An ORM statement can carry an annotated copy of the Table, so the copy + is also looked up as its original.""" + found = _TABLE_CLASSES.get(table) + if found is None and hasattr(table, "_deannotate"): + found = _TABLE_CLASSES.get(table._deannotate()) + return found def _declared(cipher: Cipher) -> FrozenSet[str]: diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index 28d235daa..15c9245b4 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -38,9 +38,9 @@ def engine(): return create_engine("sqlite://") -def refused(engine, classes): +def refused(classes): with pytest.raises(DeclarationError) as raised: - configure_encryption(engine, classes) + configure_encryption(classes) return raised.value @@ -50,7 +50,7 @@ def test_a_class_sqlalchemy_does_not_map_is_refused(engine): class Contact: email: Annotated[str, TextEq] - error = refused(engine, [Contact]) + error = refused([Contact]) assert error.code == "declaration_not_mapped" assert "Contact" in error.message @@ -60,7 +60,7 @@ class Plain(Base): __tablename__ = "plain" id: Mapped[int] = mapped_column(primary_key=True) - error = refused(engine, [Plain]) + error = refused([Plain]) assert error.code == "declaration_not_decorated" @@ -71,7 +71,7 @@ class Person(Base): id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) email: Mapped[Annotated[str, TextEq]] - error = refused(engine, [Person]) + error = refused([Person]) assert error.code == "declaration_table_mismatch" assert "people" in error.message assert "persons" in error.message @@ -84,7 +84,7 @@ class Account(Base): id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) email: Mapped[Annotated[str, TextEq]] = mapped_column("email_address") - error = refused(engine, [Account]) + error = refused([Account]) assert error.code == "declaration_column_name" assert error.field == "email" assert "email_address" in error.message @@ -97,22 +97,102 @@ class Login(Base): id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) email: Mapped[Annotated[str, TextEq]] = mapped_column(unique=True) - error = refused(engine, [Login]) + error = refused([Login]) assert error.code == "declaration_unique_column" assert error.field == "email" assert "eql_v4.eq_term(email)" in error.message -def test_a_class_configured_twice_is_refused(engine): +def test_a_repeat_call_for_a_configured_class_does_nothing(untouched): @stackencrypt.model("members") class Member(Base): __tablename__ = "members" id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) email: Mapped[Annotated[str, TextEq]] - configure_encryption(engine, [Member]) - error = refused(create_engine("sqlite://"), [Member]) + configure_encryption([Member]) + configure_encryption([Member]) + configure_encryption([Member, Member]) + + +def test_a_class_whose_declaration_changed_is_refused_on_a_repeat_call(): + @stackencrypt.model("pilots") + class Pilot(Base): + __tablename__ = "pilots" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + + configure_encryption([Pilot]) + Pilot.__repr__ = lambda self: "Pilot(...)" + + assert refused([Pilot]).code == "declaration_class_changed" + + +def test_another_class_that_maps_a_configured_table_is_refused(): + @stackencrypt.model("crews") + class Crew(Base): + __tablename__ = "crews" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + + class OtherBase(DeclarativeBase): + type_annotation_map = eql_type_map + + @stackencrypt.model("crews") + class CrewAgain(OtherBase): + __table__ = Crew.__table__ + id: Mapped[Annotated[int, Ignore]] + email: Mapped[Annotated[str, TextEq]] + + configure_encryption([Crew]) + error = refused([CrewAgain]) + assert error.code == "declaration_already_configured" + assert "Crew" in error.message + + +def test_an_engine_made_after_the_call_is_encrypted_too(untouched): + import datetime + + from sqlalchemy.orm import Session + + @stackencrypt.model("flights") + class Flight(Base): + __tablename__ = "flights" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + at: Mapped[Annotated[datetime.datetime, Passthrough]] + + configure_encryption([Flight]) + later = create_engine("sqlite://") + with later.begin() as conn: + conn.exec_driver_sql("CREATE TABLE flights (id INTEGER PRIMARY KEY, email TEXT, at TEXT)") + + # The write hook runs on the new engine: it checks the passthrough value. + with Session(later) as session, pytest.raises(InvalidValueError) as raised: + session.add(Flight(email="secret@example.com", at=datetime.datetime(2026, 1, 1))) + session.commit() + assert raised.value.code == "value_naive_datetime" + + +def test_a_table_of_the_same_name_that_is_not_configured_is_left_alone(untouched): + from sqlalchemy import Column, Integer, MetaData, Table, insert + + @stackencrypt.model("tickets_seen") + class Seen(Base): + __tablename__ = "tickets_seen" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + email: Mapped[Annotated[str, TextEq]] + + configure_encryption([Seen]) + # Another database's table, with the same name, and no encryption. + other = Table("tickets_seen", MetaData(), Column("id", Integer, primary_key=True), Column("email", String)) + engine = create_engine("sqlite://") + other.create(engine) + + with engine.begin() as conn: + conn.execute(insert(other), [{"email": "plain@example.com"}]) + assert conn.execute(other.select()).one().email == "plain@example.com" def test_a_refused_list_configures_none_of_its_classes(engine): @@ -128,13 +208,13 @@ class Visitor(Base): id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) email: Mapped[Annotated[str, TextEq]] - refused(engine, [Guest, Visitor]) + refused([Guest, Visitor]) # Guest was not configured by the refused call, so it can be now. - configure_encryption(engine, [Guest]) + configure_encryption([Guest]) def test_a_declared_mapped_class_is_accepted_with_no_request(engine, untouched): - configure_encryption(engine, [Customer]) + configure_encryption([Customer]) def test_a_sealed_value_inside_the_sql_is_refused_and_not_stored(untouched): @@ -150,7 +230,7 @@ class Lead(Base): # SQLite has no EQL types, and the refusal comes before the database. with engine.begin() as conn: conn.exec_driver_sql("CREATE TABLE leads (id INTEGER PRIMARY KEY, email TEXT)") - configure_encryption(engine, [Lead]) + configure_encryption([Lead]) for statement in [ insert(Lead).values(email="secret@example.com"), @@ -204,7 +284,7 @@ class Ticket(Base): id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) email: Mapped[Annotated[str, TextEq]] = mapped_column(String) - error = refused(engine, [Ticket]) + error = refused([Ticket]) assert error.code == "declaration_column_type" assert error.field == "email" assert "eql_v4_text_eq" in error.message @@ -223,7 +303,7 @@ class Score(Base): @pytest.fixture(scope="module") def scores(): - configure_encryption(create_engine("sqlite://"), [Score]) + configure_encryption([Score]) @pytest.mark.parametrize( @@ -293,7 +373,7 @@ class Visit(Base): engine = create_engine("sqlite://") with engine.begin() as conn: conn.exec_driver_sql("CREATE TABLE trips (id INTEGER PRIMARY KEY, email TEXT, at TEXT)") - configure_encryption(engine, [Visit]) + configure_encryption([Visit]) with Session(engine) as session, pytest.raises(InvalidValueError) as raised: session.add(Visit(email="secret@example.com", at=datetime.datetime(2026, 1, 1))) @@ -350,7 +430,7 @@ def capture(conn, cursor, statement, parameters, context, executemany): with engine.begin() as conn: conn.exec_driver_sql("CREATE TABLE laps (id INTEGER PRIMARY KEY, seconds TEXT, note TEXT)") - configure_encryption(engine, [Lap]) + configure_encryption([Lap]) with Session(engine) as session: for statement in (select(Lap).order_by(Lap.seconds), select(Lap).order_by(asc(Lap.seconds))): @@ -406,7 +486,7 @@ class Split(Base): event.listen(engine, "before_cursor_execute", lambda conn, cur, sql, *a: sent.append(sql)) with engine.begin() as conn: conn.exec_driver_sql("CREATE TABLE splits (id INTEGER PRIMARY KEY, seconds TEXT)") - configure_encryption(engine, [Split]) + configure_encryption([Split]) secs = Split.seconds.label("secs") with Session(engine) as session: @@ -443,7 +523,7 @@ class Leg(Base): event.listen(engine, "before_cursor_execute", lambda conn, cur, sql, *a: sent.append(sql)) with engine.begin() as conn: conn.exec_driver_sql("CREATE TABLE legs (id INTEGER PRIMARY KEY, seconds TEXT)") - configure_encryption(engine, [Leg]) + configure_encryption([Leg]) statements = [ select(Leg.seconds).order_by("seconds"), select(Leg.seconds).order_by(desc("seconds")), @@ -483,7 +563,7 @@ class Stage(Base): event.listen(engine, "before_cursor_execute", lambda conn, cur, sql, *a: sent.append(sql)) with engine.begin() as conn: conn.exec_driver_sql("CREATE TABLE stages (id INTEGER PRIMARY KEY, name TEXT, seconds TEXT)") - configure_encryption(engine, [Stage]) + configure_encryption([Stage]) other = aliased(Stage) with Session(engine) as session: @@ -515,7 +595,7 @@ class Heat(Base): event.listen(engine, "before_cursor_execute", lambda conn, cur, sql, *a: sent.append(sql)) with engine.begin() as conn: conn.exec_driver_sql("CREATE TABLE heats (id INTEGER PRIMARY KEY, rank INTEGER, seconds TEXT)") - configure_encryption(engine, [Heat]) + configure_encryption([Heat]) with Session(engine) as session: # The label "seconds" names the plaintext rank, so the sort is by rank. diff --git a/languages/python/tests/test_sqlalchemy_live.py b/languages/python/tests/test_sqlalchemy_live.py index 8f254a417..a65dbd0d2 100644 --- a/languages/python/tests/test_sqlalchemy_live.py +++ b/languages/python/tests/test_sqlalchemy_live.py @@ -111,7 +111,7 @@ def engine(): with engine.begin() as conn: conn.execute(text(f"CREATE SCHEMA {schema}")) Base.metadata.create_all(engine) - configure_encryption(engine, [Customer, Order, Player, Profile, Venue, Trip, Guide, Run]) + configure_encryption([Customer, Order, Player, Profile, Venue, Trip, Guide, Run]) yield engine with engine.begin() as conn: conn.execute(text(f"DROP SCHEMA {schema} CASCADE")) From 63327b0d545535456d19d7e54744e371e5a28bb2 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Thu, 12 Nov 2026 11:43:48 +0900 Subject: [PATCH 29/29] fix(python): refuse an encrypted column inside a UniqueConstraint configure_encryption checked each table constraint with `column in constraint.columns`. A column collection accepts only a string key with `in`. So SQLAlchemy raised ArgumentError for a class with an encrypted field and any UniqueConstraint on its table. That includes the one that unique=True makes for a plaintext column. The check now uses contains_column. An encrypted column in a composite UniqueConstraint is now refused with declaration_unique_column, the same as a column declared unique=True. A unique plaintext column beside an encrypted one is accepted. Refs #1341 Co-Authored-By: Claude Opus 5.5 --- .../python/python/stackencrypt/sqlalchemy.py | 4 ++- languages/python/tests/test_sqlalchemy.py | 27 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/languages/python/python/stackencrypt/sqlalchemy.py b/languages/python/python/stackencrypt/sqlalchemy.py index 26bb37b89..2072acc64 100644 --- a/languages/python/python/stackencrypt/sqlalchemy.py +++ b/languages/python/python/stackencrypt/sqlalchemy.py @@ -490,7 +490,9 @@ def _check(cls: type) -> Tuple[type, Cipher, FrozenSet[str]]: continue column = mapper.columns[field.name] unique = column.unique or any( - isinstance(constraint, UniqueConstraint) and column in constraint.columns + # contains_column, not `in`: a column collection takes only a + # string key with `in`, and raises ArgumentError for a Column. + isinstance(constraint, UniqueConstraint) and constraint.columns.contains_column(column) for constraint in mapper.local_table.constraints ) if unique: diff --git a/languages/python/tests/test_sqlalchemy.py b/languages/python/tests/test_sqlalchemy.py index 15c9245b4..3f3bc010e 100644 --- a/languages/python/tests/test_sqlalchemy.py +++ b/languages/python/tests/test_sqlalchemy.py @@ -10,7 +10,7 @@ from typing import Annotated, Optional import pytest -from sqlalchemy import String, create_engine, func, literal, select +from sqlalchemy import String, UniqueConstraint, create_engine, func, literal, select from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column import stackencrypt @@ -103,6 +103,31 @@ class Login(Base): assert "eql_v4.eq_term(email)" in error.message +def test_an_encrypted_column_in_a_unique_constraint_is_refused(engine): + @stackencrypt.model("seats") + class Seat(Base): + __tablename__ = "seats" + __table_args__ = (UniqueConstraint("venue", "holder"),) + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + venue: Mapped[Annotated[int, Ignore]] + holder: Mapped[Annotated[str, TextEq]] + + error = refused([Seat]) + assert error.code == "declaration_unique_column" + assert error.field == "holder" + + +def test_a_unique_plaintext_column_beside_an_encrypted_one_is_accepted(untouched): + @stackencrypt.model("passes") + class Pass(Base): + __tablename__ = "passes" + id: Mapped[Annotated[int, Ignore]] = mapped_column(primary_key=True) + code: Mapped[Annotated[str, Ignore]] = mapped_column(unique=True) + holder: Mapped[Annotated[str, TextEq]] + + configure_encryption([Pass]) + + def test_a_repeat_call_for_a_configured_class_does_nothing(untouched): @stackencrypt.model("members") class Member(Base):