Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions .github/workflows/ci-post.yml
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,37 @@ jobs:
- if: matrix.python-version == '3.13'
run: python -m twine check --strict dist/*

network-proof:
name: Live PyPI hashes
needs: [authz, main-ref]
runs-on: ubuntu-24.04
env:
PF_CI_LABELS: '["ubuntu-24.04"]'
PINEFORGE_NETWORK_TEST: "1"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: '3.13'
cache: pip
- run: python -m pip install -e '.[dev]' pytest-xdist
- name: Execute the required live PyPI hash row with one actual worker
run: pytest -n 1 tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi --junitxml=network.xml
- name: Derive the required zero-skip inventory from the actual JUnit
env:
PYTHONPATH: scripts
run: python -m release_chain.main_proofs network --directory . --output network-proof.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: network-proof-${{ github.run_attempt }}
path: |
network.xml
network-proof.json
if-no-files-found: error

docker:
name: Docker runtime
needs: [authz, main-ref]
Expand Down Expand Up @@ -399,7 +430,7 @@ jobs:

ready:
name: release/ready
needs: [authz, main-ref, test, contract, docker, database-e2e, distribution]
needs: [authz, main-ref, test, contract, docker, database-e2e, distribution, network-proof]
runs-on: ubuntu-24.04
timeout-minutes: 15
env:
Expand All @@ -421,16 +452,25 @@ jobs:
RUN_ID: ${{ github.run_id }}
ATTEMPT: ${{ github.run_attempt }}
run: gh run download "$RUN_ID" --name "python-package-distributions-$ATTEMPT" --dir dist
- name: Download this attempt's actual live PyPI report
if: needs.authz.result == 'success'
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
RUN_ID: ${{ github.run_id }}
ATTEMPT: ${{ github.run_attempt }}
run: gh run download "$RUN_ID" --name "network-proof-$ATTEMPT" --dir reports/network
- name: Require the complete job inventory and bind evidence to this head/base/tree
if: needs.authz.result == 'success'
env:
PYTHONPATH: scripts
NEEDS_JSON: >-
{"test":${{ toJSON(needs.test) }},"contract":${{ toJSON(needs.contract) }},
"docker":${{ toJSON(needs.docker) }},"database-e2e":${{ toJSON(needs.database-e2e) }},
"distribution":${{ toJSON(needs.distribution) }}}
"distribution":${{ toJSON(needs.distribution) }},
"network-proof":${{ toJSON(needs.network-proof) }}}
GH_TOKEN: ${{ github.token }}
run: python -m release_chain.evidence aggregate --path dist
run: python -m release_chain.evidence aggregate --path dist --network-proof reports/network
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: needs.authz.result == 'success'
with:
Expand All @@ -439,7 +479,7 @@ jobs:
if-no-files-found: error

check:
needs: [authz, main-ref, workflow-lint, test, docker, database-e2e, contract, distribution, docs-tests, ready]
needs: [authz, main-ref, workflow-lint, test, docker, database-e2e, contract, distribution, docs-tests, network-proof, ready]
if: ${{ always() }}
runs-on: ubuntu-24.04
timeout-minutes: 10
Expand Down
143 changes: 137 additions & 6 deletions .github/workflows/native-feed-post.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ name: Native feed
on:
push:
branches: [main]
paths: ['native/feed/**', '.github/workflows/native-feed.yml', '.github/workflows/native-feed-post.yml']
workflow_dispatch:

permissions:
Expand Down Expand Up @@ -111,12 +110,30 @@ jobs:
run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl"
- name: Configure, build and test
run: |
set -euo pipefail
export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}"
cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \
-DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL"
cmake --build build-feed -j4
ctest --test-dir build-feed --output-on-failure
ctest --test-dir build-feed --show-only=json-v1 > native-registration.json
ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml"
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: '3.13'
- name: Bind the actual registration and JUnit to this leg
env:
PYTHONPATH: scripts
run: python -m release_chain.main_proofs native-leg --leg release-amd64 --directory . --output native-leg.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: native-leg-release-amd64-${{ github.run_attempt }}
path: |
native-registration.json
native-junit.xml
native-leg.json
if-no-files-found: error

release-arm64:
name: Release (ubuntu-24.04-arm)
Expand All @@ -139,12 +156,30 @@ jobs:
run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl"
- name: Configure, build and test
run: |
set -euo pipefail
export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}"
cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \
-DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL"
cmake --build build-feed -j4
ctest --test-dir build-feed --output-on-failure
ctest --test-dir build-feed --show-only=json-v1 > native-registration.json
ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml"
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: '3.13'
- name: Bind the actual registration and JUnit to this leg
env:
PYTHONPATH: scripts
run: python -m release_chain.main_proofs native-leg --leg release-arm64 --directory . --output native-leg.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: native-leg-release-arm64-${{ github.run_attempt }}
path: |
native-registration.json
native-junit.xml
native-leg.json
if-no-files-found: error

release-macos:
name: Release (macos-15)
Expand All @@ -167,12 +202,30 @@ jobs:
run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl"
- name: Configure, build and test
run: |
set -euo pipefail
export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}"
cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \
-DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL"
cmake --build build-feed -j4
ctest --test-dir build-feed --output-on-failure
ctest --test-dir build-feed --show-only=json-v1 > native-registration.json
ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml"
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: '3.13'
- name: Bind the actual registration and JUnit to this leg
env:
PYTHONPATH: scripts
run: python -m release_chain.main_proofs native-leg --leg release-macos --directory . --output native-leg.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: native-leg-release-macos-${{ github.run_attempt }}
path: |
native-registration.json
native-junit.xml
native-leg.json
if-no-files-found: error

asan-ubsan:
name: ASan + UBSan
Expand All @@ -190,12 +243,30 @@ jobs:
ASAN_OPTIONS: detect_leaks=1:abort_on_error=1
UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
run: |
set -euo pipefail
export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \
-DPINEFORGE_FEED_SANITIZER=address,undefined
cmake --build build-feed -j4
ctest --test-dir build-feed --output-on-failure
ctest --test-dir build-feed --show-only=json-v1 > native-registration.json
ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml"
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: '3.13'
- name: Bind the actual registration and JUnit to this leg
env:
PYTHONPATH: scripts
run: python -m release_chain.main_proofs native-leg --leg asan-ubsan --directory . --output native-leg.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: native-leg-asan-ubsan-${{ github.run_attempt }}
path: |
native-registration.json
native-junit.xml
native-leg.json
if-no-files-found: error

tsan:
name: TSan
Expand All @@ -220,10 +291,70 @@ jobs:
env:
TSAN_OPTIONS: halt_on_error=1
run: |
set -euo pipefail
export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \
-DPINEFORGE_FEED_SANITIZER=thread \
-DPINEFORGE_FEED_TSAN_OPTIONS_EXTRA=external_symbolizer_path=/usr/lib/llvm-18/bin/llvm-symbolizer
cmake --build build-feed -j4
ctest --test-dir build-feed --output-on-failure
ctest --test-dir build-feed --show-only=json-v1 > native-registration.json
ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml"
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: '3.13'
- name: Bind the actual registration and JUnit to this leg
env:
PYTHONPATH: scripts
run: python -m release_chain.main_proofs native-leg --leg tsan --directory . --output native-leg.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: native-leg-tsan-${{ github.run_attempt }}
path: |
native-registration.json
native-junit.xml
native-leg.json
if-no-files-found: error

native-proof:
name: Native proof
needs: [authz, main-ref, release-amd64, release-arm64, release-macos, asan-ubsan, tsan]
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: '3.13'
- uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: native-leg-release-amd64-${{ github.run_attempt }}
path: native-reports/release-amd64
- uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: native-leg-release-arm64-${{ github.run_attempt }}
path: native-reports/release-arm64
- uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: native-leg-release-macos-${{ github.run_attempt }}
path: native-reports/release-macos
- uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: native-leg-asan-ubsan-${{ github.run_attempt }}
path: native-reports/asan-ubsan
- uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
with:
name: native-leg-tsan-${{ github.run_attempt }}
path: native-reports/tsan
- name: Require authoritative leg results and reparse their actual reports
env:
PYTHONPATH: scripts
NEEDS_JSON: ${{ toJSON(needs) }}
run: python -m release_chain.main_proofs native-aggregate --directory native-reports --output native-evidence.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: native-evidence-${{ github.run_attempt }}
path: native-evidence.json
if-no-files-found: error
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ on:
required: false
type: boolean
workflow_run:
workflows: [CI]
workflows: [CI, Native feed]
types: [completed]
branches: [main]

Expand Down
22 changes: 11 additions & 11 deletions native/feed/src/serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ constexpr auto keepalive = std::chrono::seconds(5); // the runner's WebSoc
Json number(std::uint64_t value) { return Json::number(std::to_string(value)); }

struct Client {
std::uint64_t next = 0; // the next message index this client receives
std::atomic<std::uint64_t> next{0}; // the next message index this client receives
std::deque<std::shared_ptr<const std::string>> queue;
std::size_t queue_bytes = 0;
bool live = false, slow = false;
Expand Down Expand Up @@ -57,7 +57,7 @@ struct Hub {
// A client that cannot keep up is cut off; it never slows the producer or another client.
if (shared->size() > config.client_queue_bytes - std::min(client->queue_bytes, config.client_queue_bytes)) {
client->slow = true;
log("warn", "client_slow", Json::object({{"next", number(client->next)}, {"queued_bytes", number(client->queue_bytes)}}));
log("warn", "client_slow", Json::object({{"next", number(client->next.load())}, {"queued_bytes", number(client->queue_bytes)}}));
continue;
}
client->queue_bytes += shared->size();
Expand Down Expand Up @@ -178,7 +178,7 @@ int admit(const mg_connection* constant, void* data) {
else if (hub.connected >= hub.config.max_clients) status = 503, reason = "client limit reached (--max-clients)";
else {
auto* client = new Client;
client->next = index;
client->next.store(index);
hub.clients.insert(client);
++hub.connected;
mg_set_user_connection_data(constant, client);
Expand Down Expand Up @@ -219,15 +219,15 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) {
const auto send = [&](const std::string& bytes, int opcode) {
const auto written = mg_websocket_write(connection, opcode, bytes.data(), bytes.size());
if (written <= 0 || (!bytes.empty() && static_cast<std::size_t>(written) != bytes.size())) {
log("warn", "client_write_failed", Json::object({{"next", number(client->next)}}));
log("warn", "client_write_failed", Json::object({{"next", number(client->next.load())}}));
return false;
}
last_send = std::chrono::steady_clock::now();
return true;
};
const auto expired = [&] {
std::lock_guard<std::mutex> guard(hub.mutex);
return client->next < hub.first_retained;
return client->next.load() < hub.first_retained;
};
// The journal is read only while catching up: a client at the head holds no segment open.
std::optional<JournalReader> reader;
Expand All @@ -238,18 +238,18 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) {
std::lock_guard<std::mutex> guard(hub.mutex);
stopping = hub.stopping;
head = hub.published;
if (!stopping && client->next == head) client->live = true;
if (!stopping && client->next.load() == head) client->live = true;
}
if (stopping) { send_close(connection, 1001, "producer stopping"); return; }
if (client->live) break;
if (!reader) {
reader.emplace(hub.journal);
if (!reader->seek(client->next)) {
if (!reader->seek(client->next.load())) {
send_close(connection, 4410, "cursor expired: older than the retained journal");
return;
}
}
while (client->next < head) {
while (client->next.load() < head) {
if (hub.halting) { send_close(connection, 1001, "producer stopping"); return; }
auto line = reader->next();
if (!line) {
Expand All @@ -270,7 +270,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) {
}
}
if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return;
++client->next;
client->next.fetch_add(1);
}
}
reader.reset();
Expand Down Expand Up @@ -299,7 +299,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) {
}
if (line) {
if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return;
++client->next;
client->next.fetch_add(1);
} else if (std::chrono::steady_clock::now() - last_send >= keepalive && !send("", MG_WEBSOCKET_OPCODE_PONG)) return;
}
}
Expand Down Expand Up @@ -344,7 +344,7 @@ void on_connection_close(const mg_connection* connection) {
hub->clients.erase(client);
--hub->connected;
}
const auto next = client->next;
const auto next = client->next.load();
delete client;
mg_set_user_connection_data(connection, nullptr);
try { log("info", "client_disconnected", Json::object({{"next", number(next)}})); } catch (...) {}
Expand Down
Loading
Loading