Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 41 additions & 22 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,24 @@ jobs:
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Check out the repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
- name: Install just
uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
with:
just-version: "1.55.0" # Keep in sync with minimum-version in justfile
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
- name: Install uv
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
cache-suffix: lint
- name: Check the lockfile is up to date
run: uv lock --check
- run: just lint
- run: uv build
- name: Lint and check formatting
run: just lint
- name: Build the package
run: uv build
- name: Check generated models match the spec
run: |
just codegen
Expand All @@ -50,14 +55,16 @@ jobs:
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Check out the repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: zizmorcore/zizmor-action@45d7276c9ca790aabd705e31ca99c6185c3e76dd # v0.8.1
- name: Audit the workflows with zizmor
uses: zizmorcore/zizmor-action@45d7276c9ca790aabd705e31ca99c6185c3e76dd # v0.8.1
with:
advanced-security: false

test:
unit-tests-and-type-checks:
name: Unit tests and type checks (Python ${{ matrix.python }}${{ matrix.lowest_direct && ', lowest direct dependencies' || '' }})
runs-on: ubuntu-latest
permissions:
Expand All @@ -70,21 +77,26 @@ jobs:
lowest_direct: true
- python: "3.14"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Check out the repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
- name: Install just
uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
with:
just-version: "1.55.0" # Keep in sync with minimum-version in justfile
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
- name: Install uv
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
cache-suffix: tests-py${{ matrix.python }}${{ matrix.lowest_direct && '-lowest-direct' || '' }}
- run: just test --python="$PYTHON" ${LOWEST_DIRECT:+--lowest-direct} ${UPGRADE:+--upgrade}
- name: Run the unit tests
run: just test --python="$PYTHON" ${LOWEST_DIRECT:+--lowest-direct} ${UPGRADE:+--upgrade}
env:
PYTHON: ${{ matrix.python }}
LOWEST_DIRECT: ${{ matrix.lowest_direct && 'true' || '' }}
UPGRADE: ${{ (github.event_name == 'schedule' || inputs.upgrade) && 'true' || '' }}
- run: just typecheck --python="$PYTHON" ${LOWEST_DIRECT:+--lowest-direct}
- name: Run the type checks
run: just typecheck --python="$PYTHON" ${LOWEST_DIRECT:+--lowest-direct}
if: ${{ !cancelled() }}
env:
PYTHON: ${{ matrix.python }}
Expand All @@ -96,10 +108,12 @@ jobs:
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Check out the repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Check out the PDC service repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
repository: PhilanthropyDataCommons/service
Expand All @@ -110,36 +124,41 @@ jobs:
working-directory: service
run: docker compose -f compose-ci.yml up -d --wait
timeout-minutes: 5
- uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
- name: Install just
uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
with:
just-version: "1.55.0" # Keep in sync with minimum-version in justfile
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
- name: Install uv
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
cache-suffix: integration
- run: just integration ${UPGRADE:+--upgrade}
- name: Run the integration tests
run: just integration ${UPGRADE:+--upgrade}
timeout-minutes: 5
env:
UPGRADE: ${{ (github.event_name == 'schedule' || inputs.upgrade) && 'true' || '' }}

all-green:
name: All green
if: always()
needs: [lint, zizmor, test, integration]
needs: [lint, zizmor, unit-tests-and-type-checks, integration]
runs-on: ubuntu-latest
permissions: {}
steps:
- uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # v1.3.0
- name: Check that every required job passed
uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # v1.3.0
with:
jobs: ${{ toJSON(needs) }}

notify:
name: Notify failed build
needs: [lint, zizmor, test, integration]
needs: [lint, zizmor, unit-tests-and-type-checks, integration]
if: failure() && (github.event_name == 'push' || github.event_name == 'schedule')
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- uses: drivendataorg/failed-build-issue-action@940568ce50eeef3f31a920614cefcc6102d2f930 # v1.3.0
- name: Open an issue for the failed build
uses: drivendataorg/failed-build-issue-action@940568ce50eeef3f31a920614cefcc6102d2f930 # v1.3.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ coverage.xml
htmlcov/
.pytest_cache/
.ruff_cache/
.mypy_cache/
.DS_Store
codegen/openapi.preprocessed.json
local/
5 changes: 3 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ A framework-free Python client for the Philanthropy Data Commons (PDC) API. The
- `src/pdc_api_client/_generated.py` is generated from `src/pdc_api_client/openapi.json` by `just codegen`. Never edit it by hand, and import models from `pdc_api_client.models`.
- Generated code owns field shapes. `models.py` owns client concerns: bundle `pdc_endpoint` and `default_page_size` ClassVars, aliases, and write-only request models. `permission_grants.py` models the grant union by hand.
- After `just update-api-spec`, record the spec's `info.version` and the download date in `CHANGELOG.md`; PDC changes the spec without bumping that version.
- When the spec disagrees with what PDC returns, add a rule to `codegen/preprocess.py` with a one-line docstring naming the mismatch, then regenerate. Delete a rule once PDC fixes the cause. CI fails if `_generated.py` drifts from the spec.
- When the spec disagrees with what PDC returns, add a rule to `codegen/preprocess.py` with a one-line comment naming the mismatch, then regenerate. Delete a rule once PDC fixes the cause. CI fails if `_generated.py` drifts from the spec.

## API conventions

- `session` is the first positional argument; every other argument is keyword-only. The exception is the bundle type in `get_bundle_page` and `pdc_api_client.testing.add_page_response`, which names the endpoint.
- Domain errors end in `Error`.
- Errors end in `Error`, except the base class `PDCException`. A lookup's not-found error subclasses `PDCNotFoundError`.

## Commands

Expand All @@ -34,5 +34,6 @@ A framework-free Python client for the Philanthropy Data Commons (PDC) API. The
- Readability, simplicity, and elegance, in that order.
- Write the test before the code. Assert transformations and branch behavior, not that a mock returned what it was given.
- Every client function gets an integration test against PDC as well as unit tests, unless PDC's CI stack can't support it.
- Tests live in `Test<Subject>` classes named for the function or class under test, with methods named `test_<behavior>` in the present tense, such as `TestGetProposal.test_raises_proposal_not_found_on_404`. Integration tests that cover a workflow across several functions may group by area instead, such as `TestProposals`.
- No docstrings, and comments only for a non-obvious why such as a PDC quirk, in one line.
- Lower bounds only in `pyproject.toml`. `uv.lock` pins the versions that development and CI use. CI also tests the lowest allowed versions on the oldest supported Python, and a weekly run tests the newest versions released at least a week ago, so raise a bound when you use something newer, and fix breakage from new releases rather than pinning.
16 changes: 16 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,22 @@ This project follows [Semantic Versioning](https://semver.org). Until 1.0, a min

PDC does not bump its spec version for every API change, so each entry names the spec version and the date the spec was downloaded.

## Unreleased

Breaking changes that make names consistent before a wider release:

- `get_basefields` is now `get_base_fields`, `create_changemaker_proposal` is `create_proposal`, `create_changemaker_proposal_version` is `create_proposal_version`, and `register_file_upload` is `create_file`.
- `get_application_form` takes `application_form_id` instead of `form_id`.
- `list_permission_grants` filters by `changemaker_id`, `funder_short_code`, and `proposal_id` instead of `changemaker`, `funder`, and `proposal`.
- The permission grant create and update functions take `grant` instead of `spec`.
- `ProposalNotFoundError`, `ChangemakerNotFoundError`, `FunderNotFoundError`, and `ChangemakerFieldValueBatchNotFoundError` now subclass `PDCNotFoundError`, and their `.detail` names the missing resource.
- Seeding's `ensure_funder_for_kc_org`, `ensure_changemaker_for_kc_org`, and `ensure_data_provider_for_kc_org` are now `ensure_*_for_keycloak_org`, with a `keycloak_org` argument.
- Seeding drops its own `link_changemaker_to_proposal`; use the client function of the same name.

Other changes:

- `ChangemakerFieldValueBatchNotFoundError` and `TokenRequestError` can be imported from `pdc_api_client`.

## 0.3.0 (2026-11-18)

- The package is now type checked with mypy (including strict mode), pyrefly, and ty, so its `py.typed` marker is accurate. The testing helpers and seeding modules are fully annotated.
Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ just test --python 3.12 --lowest-direct -- -k token

`just typecheck` takes `--python` and `--lowest-direct` too. CI type checks Python 3.12 with the lowest versions and Python 3.14 with `uv.lock`.

`uv.lock` pins the versions used for development and CI. Update it with `uv lock --upgrade`. CI also runs both suites weekly with `--upgrade`, and opens an issue when that run fails.
`uv.lock` pins the versions used for development and CI. Update it with `uv lock --upgrade`. CI also runs both suites weekly with `--upgrade`, and opens an issue when that run or a push to main fails.

`just pdc up` downloads PDC's [`compose-ci.yml`](https://github.com/PhilanthropyDataCommons/service/blob/main/compose-ci.yml) and Keycloak realm from the service repo's main branch and starts the stack: the PDC API, Keycloak, Postgres, and S3. The integration tests reach Keycloak at `pdc-auth`, so add this line to `/etc/hosts`:

Expand Down
30 changes: 19 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ PDC applies the caller's permissions to every request. The same call can return

There are two ways to read a list.

**Iterate everything.** `get_changemakers`, `get_funders`, `get_proposals`, `get_initiatives`, and the other `get_*` list functions are generators that walk every page for you:
**Iterate everything.** `get_changemakers`, `get_funders`, `get_proposals`, `get_initiatives`, and the other plural `get_*` functions are generators that walk every page for you. The exception is `get_base_fields`, which returns a list:

```python
from pdc_api_client.client import get_changemaker_proposals
Expand All @@ -103,11 +103,11 @@ Bundles for nested endpoints take path parameters, such as `get_bundle_page(sess
The client covers the writes a proposal workflow needs: creating a proposal, adding a version with field values, linking a changemaker to it, and uploading files.

```python
from pdc_api_client.client import create_changemaker_proposal, create_changemaker_proposal_version
from pdc_api_client.client import create_proposal, create_proposal_version
from pdc_api_client.models import NewProposalFieldValue, NewProposalVersion

proposal = create_changemaker_proposal(session, data={"opportunityId": 5, "externalId": "app-2026-001"})
version = create_changemaker_proposal_version(
proposal = create_proposal(session, data={"opportunityId": 5, "externalId": "app-2026-001"})
version = create_proposal_version(
session,
proposal_version=NewProposalVersion(
proposal_id=proposal.id,
Expand All @@ -122,7 +122,7 @@ A changemaker can also write its own profile values. `create_changemaker_source`

`create_initiative` creates an initiative led by a changemaker, and `create_initiative_field_value` adds a value to it from a given source.

Uploading a file takes two steps. `register_file_upload` creates the file record and returns a presigned S3 POST, and `upload_file_to_s3_presigned_url` sends the bytes to it.
Uploading a file takes two steps. `create_file` creates the file record and returns a presigned S3 POST, and `upload_file_to_s3_presigned_url` sends the bytes to it.

## Permission grants

Expand All @@ -134,7 +134,7 @@ from pdc_api_client.permission_grants import ChangemakerPermissionGrant, create_

grant = create_permission_grant(
session,
spec=ChangemakerPermissionGrant(
grant=ChangemakerPermissionGrant(
changemaker_id=7,
grantee_type=PermissionGrantGranteeType.USER_GROUP,
grantee_keycloak_organization_id="1f0c6e2a-7d4b-4c55-9a51-0d8c2f0b6a11",
Expand All @@ -143,14 +143,14 @@ grant = create_permission_grant(
),
)

page = list_permission_grants(session, changemaker=7, verb=PermissionGrantVerb.VIEW)
page = list_permission_grants(session, changemaker_id=7, verb=PermissionGrantVerb.VIEW)
```

The models check that the grantee fields match `grantee_type` before anything is sent. Create, get, update, and delete functions exist for grants and for default permission grants. A default permission grant has no context entity; PDC copies it into a real grant on each new entity of its type as that entity is created.

## Errors

`APISession` raises a subclass of `pdc_api_client.PDCException` for every failed request. Each exception has `.code` (the HTTP status) and `.detail` (the parsed JSON body, or the raw text when the body isn't JSON).
`APISession` raises a subclass of `pdc_api_client.PDCException` for every failed request. Each exception has `.code` (the HTTP status) and `.detail` (the parsed JSON body, or the raw text when the body isn't JSON). Before raising, it retries a GET or HEAD up to three times on a 502, 503, or 504 or a dropped connection.

| Status | Exception |
| ------------- | ----------------------------- |
Expand All @@ -163,7 +163,15 @@ The models check that the grantee fields match `grantee_type` before anything is
| 5xx, timeout, or connection failure | `PDCServiceError` |
| Any other non-2xx | `PDCException` |

A few lookups turn a 404 into an error named for the resource: `get_proposal` and `get_changemaker_proposal` raise `ProposalNotFoundError`, `get_changemaker` raises `ChangemakerNotFoundError`, `get_funder` raises `FunderNotFoundError`, `get_changemaker_field_value_batch` raises `ChangemakerFieldValueBatchNotFoundError`, and `get_application_form_by_name` raises `ApplicationFormNotFoundError`, a `PDCNotFoundError`, when no form has that name. `KeycloakClientCredentialsProvider` raises `TokenRequestError` when Keycloak rejects the grant.
A few lookups raise a subclass of `PDCNotFoundError` named for the resource:

- `get_proposal` and `get_changemaker_proposal` raise `ProposalNotFoundError`.
- `get_changemaker` raises `ChangemakerNotFoundError`.
- `get_funder` raises `FunderNotFoundError`.
- `get_changemaker_field_value_batch` raises `ChangemakerFieldValueBatchNotFoundError`.
- `get_application_form_by_name` raises `ApplicationFormNotFoundError` when no form has that name.

`KeycloakClientCredentialsProvider` raises `TokenRequestError` when Keycloak rejects the grant.

## Models

Expand All @@ -177,8 +185,8 @@ The spec doesn't always match what PDC returns. Some fields it marks required ar

The client covers most of the API but not all of it:

- **No client functions:** `/applicationFormFields`, `/sources`, `/tasks` (bulk uploads and base field copies), `/terminologySets`, and `/users`. Their models are generated, and `session.get`/`session.post` reach any endpoint directly.
- **Writes outside the proposal workflow:** creating funders, changemakers, opportunities, sources, and the like exists only as seeding helpers in `pdc_api_client.seeding`, which aren't meant for production data.
- **No client functions:** `/applicationFormFields`, `/sources` (apart from `create_changemaker_source`), `/tasks` (bulk uploads and base field copies), `/terminologySets`, and `/users`. Their models are generated, and `session.get`/`session.post` reach any endpoint directly.
- **Other writes:** creating funders, changemakers, opportunities, and funder or data-provider sources is possible only through the seeding helpers in `pdc_api_client.seeding`, which aren't meant for production data.
- **Async:** the client is synchronous and built on `requests`.

Issues and pull requests that fill these gaps are welcome.
Expand Down
36 changes: 32 additions & 4 deletions integration_tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import uuid

import pytest
import requests
Expand All @@ -21,8 +22,8 @@
KEYCLOAK_CLIENT_ID = os.environ.get("KEYCLOAK_CLIENT_ID", "pdc-dev-ingest")
KEYCLOAK_CLIENT_SECRET = os.environ.get("KEYCLOAK_CLIENT_SECRET", "password")
PDC_API_TIMEOUT = float(os.environ.get("PDC_API_TIMEOUT", "30"))
KC_ADMIN_USERNAME = os.environ.get("KEYCLOAK_ADMIN_USERNAME", "admin")
KC_ADMIN_PASSWORD = os.environ.get("KEYCLOAK_ADMIN_PASSWORD", "admin")
KEYCLOAK_ADMIN_USERNAME = os.environ.get("KEYCLOAK_ADMIN_USERNAME", "admin")
KEYCLOAK_ADMIN_PASSWORD = os.environ.get("KEYCLOAK_ADMIN_PASSWORD", "admin")


@pytest.fixture(scope="session")
Expand Down Expand Up @@ -55,8 +56,8 @@ def keycloak_admin() -> KeycloakAdmin:
return KeycloakAdmin(
keycloak_url=KEYCLOAK_SERVER_URL,
realm=KEYCLOAK_REALM,
admin_username=KC_ADMIN_USERNAME,
admin_password=KC_ADMIN_PASSWORD,
admin_username=KEYCLOAK_ADMIN_USERNAME,
admin_password=KEYCLOAK_ADMIN_PASSWORD,
)


Expand Down Expand Up @@ -116,3 +117,30 @@ def make(username: str, org_alias: str) -> APISession:
@pytest.fixture(scope="session")
def test_user_password() -> str:
return TEST_USER_PASSWORD


@pytest.fixture
def suffix() -> str:
return uuid.uuid4().hex[:8]


@pytest.fixture
def fresh_keycloak_org(keycloak_admin, suffix):
alias = f"fresh-org-{suffix}"
return keycloak_admin.ensure_organization(
name=f"Fresh Org {suffix}",
alias=alias,
domain=f"{alias}.example.com",
)


@pytest.fixture
def add_user_to_org(keycloak_admin, keycloak_client_id, test_user_password):
keycloak_admin.ensure_direct_grants_enabled(client_id=keycloak_client_id)

def add(username: str, *orgs) -> None:
user = keycloak_admin.ensure_user(username=username, email=username, password=test_user_password)
for org in orgs:
keycloak_admin.ensure_user_in_organization(organization_id=org.id, user_id=user.id)

return add
Loading