diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index a556e1cde8..a26eadba65 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -408,17 +408,33 @@ jobs: echo "PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION" >> "$GITHUB_ENV" echo "STORE_DIR=hugegraph-store/apache-hugegraph-store-$VERSION" >> "$GITHUB_ENV" - - name: Run Store ingress, Arthas security and metrics tests + # -Dtest scopes the run to hg-store-node's own tests (including ArthasSecurityTest and + # HgStoreMetricsControllerTest); without it -am would also run the test suites of every + # upstream module. StoredRowIngressTest is listed by name because hg-store-core also has + # tests in org.apache.hugegraph.store.business. + - name: Run store-node tests run: | set -o pipefail RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX") mvn test -pl hugegraph-store/hg-store-node -am \ - -Dtest=StoredRowIngressTest,ArthasSecurityTest,HgStoreMetricsControllerTest \ - -Dsurefire.failIfNoSpecifiedTests=false \ + -Dtest='org/apache/hugegraph/store/node/**/*Test,StoredRowIngressTest' \ + -Dsurefire.failIfNoSpecifiedTests=false -DreuseForks=false \ -Djacoco.sessionId=store-node-test \ -Djacoco.destFile="$PWD/hugegraph-store/hg-store-test/target/store-node-test.exec" 2>&1 | tee "$RUNTIME_LOG" bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs" + # Unit tests for the optional S3 cloud-storage provider plugin. -Dtest scopes the run to this + # module's own provider test (-am would otherwise also run every upstream module's tests); + # the module's *Benchmark / *E2ETest classes stay excluded by its Surefire config. + - name: Run S3 cloud storage provider test + run: | + set -o pipefail + RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX") + mvn test -pl hugegraph-store/hg-store-cloud-s3 -am \ + -Dtest=S3CloudStorageProviderTest -Dsurefire.failIfNoSpecifiedTests=false \ + -DreuseForks=false -ntp 2>&1 | tee "$RUNTIME_LOG" + bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs" + - name: Run common test run: | set -o pipefail diff --git a/docker/cloud-storage/.gitignore b/docker/cloud-storage/.gitignore new file mode 100644 index 0000000000..56a3ee1594 --- /dev/null +++ b/docker/cloud-storage/.gitignore @@ -0,0 +1,5 @@ +.artifacts/ +logs/ +data/ +.generated/ + diff --git a/docker/cloud-storage/README.md b/docker/cloud-storage/README.md new file mode 100644 index 0000000000..0529cc4eaf --- /dev/null +++ b/docker/cloud-storage/README.md @@ -0,0 +1,920 @@ +# Cloud Storage (MinIO + HStore) Local Stack + +This stack runs: + +- MinIO (S3-compatible storage) +- 1 PD node +- 3 Store nodes +- `hg-store-cloud-s3` plugin loaded from local build artifacts + +The goal is to validate that: + +1. RocksDB SST files are mirrored to S3. +2. Recovery metadata (`MANIFEST-*`, `CURRENT`, `OPTIONS-*`) is mirrored alongside SSTs. +3. Together, these uploaded files are enough for cloud recovery after local RocksDB data loss. + +## Prerequisites + +- Docker + Docker Compose +- Java 17+ +- Maven 3.5+ +- `curl` and `jq` (optional, for manual API testing) + +From repo root, build all modules: + +```bash +mvn clean package -DskipTests -Ps3-cloud-storage +``` + +This builds the necessary artifacts for the Docker images. `-Ps3-cloud-storage` is the opt-in +profile that bundles the S3 provider (`hg-store-cloud-s3` + AWS SDK) with the Store node; the +harness image stages the provider jar and its dependencies under `/hugegraph-store/plugins/`. + +## Quick Start + +Detect repo root first, then run the script: + +```bash +export REPO_ROOT="$(git rev-parse --show-toplevel)" +test -d "${REPO_ROOT}/docker/cloud-storage" + +cd "${REPO_ROOT}/docker/cloud-storage" +chmod +x ./scripts/*.sh ./entrypoints/*.sh +./scripts/test-graph-queries-and-sst.sh +``` + +```bash +# (Optional) Keep the stack running after the test for manual inspection +./scripts/test-graph-queries-and-sst.sh --keep-stack +``` + +```bash + +# (Optional) Start infrastructure only (skip data creation + validation tests) +./scripts/test-graph-queries-and-sst.sh --keep-stack --infra-only +``` + +## Test Output Files + +After running `make test` or `./scripts/test-graph-queries-and-sst.sh`, check `.generated/` for: + +- **`FULL-TEST-REPORT.txt`** — Complete test summary +- **`test-report.txt`** — Graph query test results +- **`minio-verification.txt`** — MinIO S3 object listing and SST count +- **`store-cluster.log`** — All store node logs with RocksDB/S3 events +- **`cli-load.log`** — Data load job output from hg-store-cli +- **`load-data.tsv`** — Generated test data file (200k entries) + +## Manual Verification (Core Flow) + +**Prerequisites:** Complete Core Step 1 first and wait for the infrastructure ready message. + +For hands-on validation with manual graph creation and queries, start the cluster using the +automated script with `--keep-stack`, then follow interactive steps. + +**Core goals in this flow:** +1. Create schema and load enough data to generate SST files +2. Flush stores to trigger cloud upload +3. Verify SST + recovery metadata in MinIO UI + +### Step 1: Set Environment and Start Infrastructure (No Auto-Load) + +Use the automated test script to build images and start the stack. For manual-only flows, +prefer `--infra-only` to skip scripted data creation and validation checks while keeping the stack up: + +```bash +export REPO_ROOT="$(git rev-parse --show-toplevel)" +echo "$REPO_ROOT" +# The harness runs its own compose project from a generated file; every `docker compose` +# command in this guide targets that stack through these two variables. +export COMPOSE_PROJECT_NAME=cloud-storage-test +export CF="$REPO_ROOT/docker/cloud-storage/.generated/docker-compose.yml" + +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh --keep-stack --infra-only +``` + +#### Shared Helper: Resolve `HG_NET` Once Per Shell + +Run this once in your current shell after infrastructure is up. Re-run it if you open a new shell. + +```bash +resolve_hg_net() { + local net + net=$(docker inspect cloud-storage-minio \ + --format '{{range $k, $v := .NetworkSettings.Networks}}{{println $k}}{{end}}' \ + 2>/dev/null | head -n1 || true) + + if [[ -z "$net" ]]; then + if docker network inspect cloud-storage-test_hg-net >/dev/null 2>&1; then + net="cloud-storage-test_hg-net" + elif docker network inspect cloud-storage-net >/dev/null 2>&1; then + net="cloud-storage-net" + fi + fi + + [[ -n "$net" ]] || { + echo "No cloud-storage Docker network found. Ensure Core Step 1 completed successfully." >&2 + return 1 + } + + export HG_NET="$net" + echo "Using Docker network: $HG_NET" +} + +resolve_hg_net +``` + +This will: +- Build local artifacts (PD, Store, Store CLI, S3 plugin) +- Build Docker images +- Start MinIO + PD + 3 Store nodes +- Wait for all services to be healthy +- **Skip** initial data load phase +- **Keep the stack running** for manual steps + +The output will indicate when infrastructure is ready. + +### Step 2: Verify Cluster Health + +Verify all services are accessible. This may take a few minutes as the HugeGraph server needs time to initialize: + +```bash +# Wait for services to be healthy (may take 2-3 minutes on first run) +echo "Waiting for services to be healthy..." +for i in {1..60}; do + pd_ok=$(curl -fsS http://127.0.0.1:8620/v1/health >/dev/null 2>&1 && echo "yes" || echo "no") + store0_ok=$(curl -fsS http://127.0.0.1:8520/v1/health >/dev/null 2>&1 && echo "yes" || echo "no") + store1_ok=$(curl -fsS http://127.0.0.1:8521/v1/health >/dev/null 2>&1 && echo "yes" || echo "no") + store2_ok=$(curl -fsS http://127.0.0.1:8522/v1/health >/dev/null 2>&1 && echo "yes" || echo "no") + server_ok=$(curl -fsS http://127.0.0.1:8080/versions >/dev/null 2>&1 && echo "yes" || echo "no") + + if [[ "$pd_ok" == "yes" && "$store0_ok" == "yes" && "$store1_ok" == "yes" && "$store2_ok" == "yes" && "$server_ok" == "yes" ]]; then + break + fi + + echo " Attempt $i/60: PD=$pd_ok Store0=$store0_ok Store1=$store1_ok Store2=$store2_ok Server=$server_ok" + sleep 2 +done + +# Final verification +curl -fsS http://127.0.0.1:8620/v1/health >/dev/null 2>&1 && echo "✓ PD OK" || echo "✗ PD FAILED" +curl -fsS http://127.0.0.1:8520/v1/health >/dev/null 2>&1 && echo "✓ Store0 OK" || echo "✗ Store0 FAILED" +curl -fsS http://127.0.0.1:8521/v1/health >/dev/null 2>&1 && echo "✓ Store1 OK" || echo "✗ Store1 FAILED" +curl -fsS http://127.0.0.1:8522/v1/health >/dev/null 2>&1 && echo "✓ Store2 OK" || echo "✗ Store2 FAILED" +curl -fsS http://127.0.0.1:8080/versions >/dev/null 2>&1 && echo "✓ Server OK" || echo "✗ Server FAILED" +``` + +If all show "✓ OK", proceed to Step 3. If any show "✗ FAILED", check logs: +```bash +docker compose -f "$CF" logs --tail=50 server +``` + +**Note:** The `/versions` endpoint returns a clean 200 OK response without touching a graph, and is the endpoint the compose health checks use. The `/gremlin` endpoint requires query parameters. + +### Step 3: Create Graph Schema + +Schema persists across restarts via rocksdb-cloud. If you see `ExistedException`, schema already exists. + +```bash +create_pk() { + local name="$1" dtype="$2" + local found=$(curl -s --compressed "http://localhost:8080/graphs/hugegraph/schema/propertykeys/$name" \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null) + [[ "$found" == "$name" ]] && { echo " ✓ property key '$name' exists"; return; } + curl -s -X POST http://localhost:8080/graphs/hugegraph/schema/propertykeys \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"$name\",\"data_type\":\"$dtype\",\"cardinality\":\"SINGLE\"}" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(' ✓ created property key:', d.get('property_key',{}).get('name','?'))" +} + +create_vl() { + local name="$1" props="$2" + local found=$(curl -s --compressed "http://localhost:8080/graphs/hugegraph/schema/vertexlabels/$name" \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null) + [[ "$found" == "$name" ]] && { echo " ✓ vertex label '$name' exists"; return; } + curl -s -X POST http://localhost:8080/graphs/hugegraph/schema/vertexlabels \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"$name\",\"id_strategy\":\"AUTOMATIC\",\"properties\":$props}" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(' ✓ created vertex label:', d.get('name','?'))" +} + +create_pk "name" "TEXT" +create_pk "age" "INT" +create_pk "city" "TEXT" + +create_vl "person" '["name","age","city"]' +create_vl "location" '["name"]' + +FOUND_EL=$(curl -s --compressed http://localhost:8080/graphs/hugegraph/schema/edgelabels/lives_in \ + | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null) +if [[ "$FOUND_EL" == "lives_in" ]]; then + echo " ✓ edge label 'lives_in' exists" +else + curl -s -X POST http://localhost:8080/graphs/hugegraph/schema/edgelabels \ + -H 'Content-Type: application/json' \ + -d '{"name":"lives_in","source_label":"person","target_label":"location"}' \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(' ✓ created edge label:', d.get('name','?'))" +fi + +echo "✓ Schema ready" +``` + +### Step 4: Add Vertices and Edges + +Add sufficient data to trigger RocksDB compaction (minimum 150+ vertices): + +```bash +insert_vertex() { + local label="$1" props="$2" + local tmpfile=$(mktemp) + local code=$(curl -s -o "$tmpfile" -w "%{http_code}" -X POST \ + http://localhost:8080/graphs/hugegraph/graph/vertices \ + -H 'Content-Type: application/json' \ + -d "{\"label\":\"$label\",\"properties\":$props}") + local body=$(cat "$tmpfile") + rm -f "$tmpfile" + if [[ "$code" == "201" || "$code" == "200" ]]; then + echo "$body" | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])" + else + echo "ERROR: HTTP $code - $body" >&2 + echo "" + fi +} + +echo "Inserting test vertices (150+)..." +for i in {1..150}; do + age=$((20 + i % 50)) + city=$((i % 5)) + vid=$(insert_vertex "person" "{\"name\":\"person_$i\",\"age\":$age,\"city\":\"city_$city\"}") + if (( i % 50 == 0 )); then + echo " ✓ Inserted $i vertices" + fi +done + +echo "Adding location vertices..." +for i in {0..4}; do + insert_vertex "location" "{\"name\":\"city_$i\"}" >/dev/null +done + +echo "✓ Inserted 150+ test vertices (sufficient for RocksDB compaction)" +``` + +**Why 150+ vertices?** Smaller datasets may not trigger RocksDB compaction. 150+ vertices across 3 store nodes ensures enough write activity to generate SST files. + +Optional quick sanity check before flush (uses Gremlin because the REST `/graph/vertices` listing +is capped at 100 vertices by default): + +```bash +curl -s --compressed -X POST http://localhost:8080/gremlin -H 'Content-Type: application/json' \ + -d '{"gremlin":"g.V().count()","bindings":{},"language":"gremlin-groovy","aliases":{"g":"__g_DEFAULT-hugegraph"}}' \ + | python3 -c "import sys,json; print('Total vertices:', json.load(sys.stdin)['result']['data'][0])" +``` + +### Step 5: Manual Flush + MinIO UI Verification + +Use this step when you want a purely manual verification flow in the UI. + +1. Open MinIO Console: `http://localhost:9001` (default credentials: `minioadmin` / `minioadmin`). +2. Open buckets `hugegraph-store0`, `hugegraph-store1`, and `hugegraph-store2`. +3. In each bucket, note the current SST/metadata state before flush: + - `*.sst` objects + - recovery metadata files (`CURRENT`, `MANIFEST-*`, `OPTIONS-*`) + - latest modified timestamps shown in the object list + +Trigger flush on all store nodes: + +```bash +echo "=== Trigger flush on all stores ===" +for p in 8520 8521 8522; do + echo "Flushing store REST port ${p}..." + curl -fsS "http://127.0.0.1:${p}/test/flush" >/dev/null +done +echo "Flush requests sent" +``` + +Wait for asynchronous cloud upload, then refresh MinIO UI: + +```bash +echo "Waiting for flush upload to reach MinIO..." +sleep 20 +``` + +Re-check the same three buckets in MinIO UI. + +**Success criteria:** +- ✅ You can browse all three buckets in MinIO UI +- ✅ `*.sst` objects exist in each bucket after flush +- ✅ Recovery metadata files (`CURRENT`, `MANIFEST-*`, `OPTIONS-*`) are visible +- ✅ At least one object timestamp advances after flush (new upload activity) + +If the UI does not update after ~20 seconds, wait another 20-40 seconds and refresh again. + +## Advanced Scenarios (Optional) + +### Advanced 1: Strict Script Assertions + +Run this if you want script-based strict assertions in addition to the manual UI checks in Core Step 5. +The commands are wrapped in a guarded subshell so any `exit 1` fails only this block, not your +terminal (even if `set -e` was enabled earlier in the shell). + +```bash + +if ! ( +# Requires the helper from Core Step 1. +type resolve_hg_net >/dev/null 2>&1 || { echo "Run Core Step 1 helper first"; exit 1; } +resolve_hg_net + +echo "=== Assert bucket object + SST counts ===" +docker run --rm --entrypoint /bin/sh --network "$HG_NET" pgsty/mc:RELEASE.2026-10-18T00-00-00Z \ + -c 'if mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + elif mc alias set local http://cloud-storage-minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + else \ + echo "ERROR: cannot reach MinIO at minio:9000 or cloud-storage-minio:9000" >&2; \ + exit 1; \ + fi; \ + fail=0; \ + for b in hugegraph-store0 hugegraph-store1 hugegraph-store2; do \ + total_count=$(mc ls --recursive local/$b 2>/dev/null | wc -l); \ + sst_count=$(mc find local/$b --name "*.sst" 2>/dev/null | wc -l); \ + echo "### $b: total=$total_count sst=$sst_count"; \ + if [ "$total_count" -eq 0 ] || [ "$sst_count" -eq 0 ]; then \ + echo " ERROR: expected total>0 and sst>0" >&2; \ + fail=1; \ + fi; \ + done; \ + exit "$fail"' + +echo "=== Assert metadata set (CURRENT / MANIFEST-* / OPTIONS-*) ===" +docker run --rm --entrypoint /bin/sh --network "$HG_NET" pgsty/mc:RELEASE.2026-10-18T00-00-00Z \ + -c 'if mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + elif mc alias set local http://cloud-storage-minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + else \ + echo "ERROR: cannot reach MinIO at minio:9000 or cloud-storage-minio:9000" >&2; \ + exit 1; \ + fi; \ + fail=0; \ + for b in hugegraph-store0 hugegraph-store1 hugegraph-store2; do \ + current=$(mc find local/$b --name CURRENT 2>/dev/null | wc -l); \ + manifest=$(mc find local/$b --name "MANIFEST-*" 2>/dev/null | wc -l); \ + options=$(mc find local/$b --name "OPTIONS-*" 2>/dev/null | wc -l); \ + sst=$(mc find local/$b --name "*.sst" 2>/dev/null | wc -l); \ + echo "### $b: CURRENT=$current MANIFEST=$manifest OPTIONS=$options SST=$sst"; \ + if [ "$current" -lt 1 ] || [ "$manifest" -lt 1 ] || [ "$options" -lt 1 ] || [ "$sst" -lt 1 ]; then \ + echo " ERROR: missing required recovery files" >&2; \ + fail=1; \ + fi; \ + done; \ + exit "$fail"' + +); then + echo "Advanced 1 assertions failed (see errors above)" +fi +``` + +**Success criteria:** +- ✅ Both assertion commands exit with status `0` +- ✅ All buckets have `total > 0` and `sst > 0` +- ✅ All buckets include `CURRENT`, `MANIFEST-*`, `OPTIONS-*`, and `*.sst` + +**If buckets are still empty after flush:** +1. Check store logs for upload errors: `docker compose -f "$CF" logs store0 | grep -i "s3\|cloud\|error"` +2. Verify cloud storage was initialized: `docker compose -f "$CF" logs store0 | grep -i "Cloud storage provider.*initialized\|S3CloudStorageProvider initialized"` +3. Check that data was written: `docker exec cloud-storage-store0 sh -lc 'find /hugegraph-store/storage -name "*.sst" | wc -l'` + +### Advanced 2: Clear Semantics & No-Orphan Re-hydration + +This step validates what `graph.clear()` guarantees for cloud storage. Clearing a graph runs a +per-key-range delete (`deleteRange`) on the partition's RocksDB instance — it does **not** purge the +whole remote DB prefix. That instance can be shared by multiple graphs, so a whole-prefix purge would +destroy co-tenant graphs' cloud data; instead the cleared range converges in cloud through normal +SST mirroring/compaction. The durable guarantee under test is therefore: **after `clear()` and a store +restart, the graph does not re-hydrate stale data** (vertex count stays `0`). + +Because the prefix is intentionally not purged, the object count under the prefix is expected to remain +non-zero after clear (live `CURRENT`/`MANIFEST-*`/`OPTIONS-*` and not-yet-compacted SSTs) — that is +by design, not a leak. Whole-prefix purge is exercised only by real DB deletion (see Advanced 3). + +> **Warning:** This deletes graph data. Run it after Core Step 5 (and Advanced 1 if you use it), +> then re-run Core Steps 3-5 if you want to repeat the flush/recovery setup. + +#### Manual Flow: Before Clear -> Clear -> After Clear -> Restart Check + +1. Resolve Docker network and detect one graph DB prefix in MinIO. + +```bash +GRAPH_NAME=hugegraph +type resolve_hg_net >/dev/null 2>&1 || { echo "Run Core Step 1 helper first"; exit 1; } +resolve_hg_net + +BUCKET="" +DB_PREFIX="" +for CANDIDATE_BUCKET in hugegraph-store0 hugegraph-store1 hugegraph-store2; do + MATCH=$(docker run --rm --entrypoint /bin/sh --network "$HG_NET" \ + pgsty/mc:RELEASE.2026-10-18T00-00-00Z \ + -c 'if mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + elif mc alias set local http://cloud-storage-minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + else \ + echo "ERROR: cannot reach MinIO at minio:9000 or cloud-storage-minio:9000" >&2; \ + exit 1; \ + fi; \ + mc find local/'"$CANDIDATE_BUCKET"' --name CURRENT 2>/dev/null' \ + | grep "/${GRAPH_NAME}/" | head -n1 || true) + if [[ -n "$MATCH" ]]; then + BUCKET="$CANDIDATE_BUCKET" + DB_PREFIX="${MATCH#local/${CANDIDATE_BUCKET}/}" + DB_PREFIX="${DB_PREFIX%/CURRENT}" + break + fi +done + +echo "Detected DB prefix: bucket=${BUCKET}, prefix=${DB_PREFIX}" +[[ -n "$BUCKET" && -n "$DB_PREFIX" ]] || { + echo "Failed to detect DB prefix for ${GRAPH_NAME}" + exit 1 +} +``` + +2. Observe objects before clear (CLI + optional MinIO UI check). + +```bash +BEFORE_COUNT=$(docker run --rm --entrypoint /bin/sh --network "$HG_NET" \ + pgsty/mc:RELEASE.2026-10-18T00-00-00Z \ + -c 'if mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + elif mc alias set local http://cloud-storage-minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + else \ + echo "ERROR: cannot reach MinIO at minio:9000 or cloud-storage-minio:9000" >&2; \ + exit 1; \ + fi; \ + mc ls --recursive local/'"$BUCKET"'/'"$DB_PREFIX"' 2>/dev/null | wc -l') + +echo "Objects before clear: ${BEFORE_COUNT}" + +docker run --rm --entrypoint /bin/sh --network "$HG_NET" \ + pgsty/mc:RELEASE.2026-10-18T00-00-00Z \ + -c 'if mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + elif mc alias set local http://cloud-storage-minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + else \ + echo "ERROR: cannot reach MinIO at minio:9000 or cloud-storage-minio:9000" >&2; \ + exit 1; \ + fi; \ + mc ls --recursive local/'"$BUCKET"'/'"$DB_PREFIX"' 2>/dev/null' | head -20 +``` + +Optional UI view: open `http://localhost:9001`, go to `${BUCKET}/${DB_PREFIX}`, and note object +names / timestamps. + +3. Run clear action. + +```bash +curl -s -X DELETE "http://localhost:8080/graphs/${GRAPH_NAME}/clear" \ + --get --data-urlencode "confirm_message=I'm sure to delete all data" | head -c 200 +echo +sleep 5 +``` + +4. Observe objects after clear (expected non-zero). + +```bash +AFTER_COUNT=$(docker run --rm --entrypoint /bin/sh --network "$HG_NET" \ + pgsty/mc:RELEASE.2026-10-18T00-00-00Z \ + -c 'if mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + elif mc alias set local http://cloud-storage-minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + else \ + echo "ERROR: cannot reach MinIO at minio:9000 or cloud-storage-minio:9000" >&2; \ + exit 1; \ + fi; \ + mc ls --recursive local/'"$BUCKET"'/'"$DB_PREFIX"' 2>/dev/null | wc -l') + +echo "Objects after clear (expected non-zero): ${AFTER_COUNT}" + +docker run --rm --entrypoint /bin/sh --network "$HG_NET" \ + pgsty/mc:RELEASE.2026-10-18T00-00-00Z \ + -c 'if mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + elif mc alias set local http://cloud-storage-minio:9000 minioadmin minioadmin >/dev/null 2>&1; then \ + :; \ + else \ + echo "ERROR: cannot reach MinIO at minio:9000 or cloud-storage-minio:9000" >&2; \ + exit 1; \ + fi; \ + mc ls --recursive local/'"$BUCKET"'/'"$DB_PREFIX"' 2>/dev/null' | head -20 +``` + +5. Manual no-orphan re-hydration verification. + +```bash +echo "Vertex count immediately after clear (expected 0):" +curl -s --compressed -X POST http://localhost:8080/gremlin -H 'Content-Type: application/json' \ + -d "{\"gremlin\":\"g.V().count()\",\"bindings\":{},\"language\":\"gremlin-groovy\",\"aliases\":{\"g\":\"__g_DEFAULT-${GRAPH_NAME}\"}}" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['result']['data'][0])" + +docker restart cloud-storage-store0 cloud-storage-store1 cloud-storage-store2 +sleep 20 + +echo "Vertex count after store restart (must remain 0):" +curl -s --compressed -X POST http://localhost:8080/gremlin -H 'Content-Type: application/json' \ + -d "{\"gremlin\":\"g.V().count()\",\"bindings\":{},\"language\":\"gremlin-groovy\",\"aliases\":{\"g\":\"__g_DEFAULT-${GRAPH_NAME}\"}}" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['result']['data'][0])" +``` + +If the post-restart count is non-zero, stale data was re-hydrated (unexpected). + +#### Optional Automation + +Run the one-shot helper if you prefer scripted execution of the same checks: + +```bash +bash "$REPO_ROOT/docker/cloud-storage/scripts/test-advanced-clear-semantics.sh" +``` + +Optional overrides: + +```bash +GRAPH_NAME=hugegraph RESTART_WAIT_SECONDS=20 \ + bash "$REPO_ROOT/docker/cloud-storage/scripts/test-advanced-clear-semantics.sh" +``` + +**Success criteria:** +- ✅ `Objects before clear` is non-zero for the detected prefix +- ✅ `Objects after clear` is typically still non-zero (shared prefix is not purged by design) +- ✅ Vertex count immediately after clear is `0` +- ✅ Restart check returns vertex count `0` — cleared data is not re-hydrated from cloud +- ℹ️ Whole-prefix purge is validated by Advanced 3 + +### Advanced 3: DB Delete Callbacks (`onDBDeleteBegin` + `onDBDeleted`) + +This step validates the DB-destroy lifecycle callbacks (not truncate/clear): + +- `onDBDeleteBegin` writes a tombstone object as a **sibling** of the DB prefix (`_DELETED`, never `/_DELETED`); it is removed again once the purge succeeds +- `onDBDeleted` purges the DB prefix from cloud storage + +It uses the **test-only** Store endpoint `/test/raftDelete/{groupId}` to destroy one partition +engine, which triggers `destroyGraphDB(...)` internally. + +> **Important:** This is a **simulation of GraphDB deletion** via an internal test endpoint. +> It is useful for callback behavior verification, but it is not the production graph-delete API path. +> Use Advanced 2 to validate production `clear()` semantics and no-orphan re-hydration. + +> **Warning:** This is destructive and intended only for callback verification. Run it near the +> end of manual testing. After this step, restart from Core Step 1 for a fresh cluster state. + +Run this phase through the main end-to-end harness: + +```bash +bash "$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh" --keep-stack +``` + +What this phase does: +1. picks a partition id and cloud DB prefix, +2. writes a probe object, +3. triggers `/test/raftDelete/{groupId}`, +4. polls store logs for callback markers, +5. verifies the prefix is fully purged. + +**Success criteria:** +- ✅ Logs include `Cloud DB tombstone written` (from `onDBDeleteBegin`); the `tombstone-sibling-key` phase asserts the sibling key `_DELETED` was observed or logged and that `/_DELETED` never exists +- ✅ Logs include `Cloud DB purge completed` (from `onDBDeleted`, may appear up to ~60s later) +- ✅ `Objects after db delete` is `0` for the detected DB prefix + +### Advanced 4: Local Disk-Loss Recovery (Automated) + +This scenario is covered by the main end-to-end harness (`recovery-test` phase): + +```bash +bash "$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh" --keep-stack +``` + +What this phase does: +1. checks baseline vertex count (> 0), +2. triggers `/test/flush` on all stores, +3. verifies every bucket contains recovery metadata (`CURRENT` / `MANIFEST-*` / `OPTIONS-*`) and SSTs, +4. wipes each store's local state-machine data while preserving `raft/` and `snapshot/`, +5. restarts stores and validates recovered vertex count matches baseline, +6. checks logs for restore-consistency errors and stale `.hyd-tmp` leftovers. + +**Success criteria:** +- ✅ Baseline vertex count is non-zero before the wipe +- ✅ Recovery metadata + SSTs exist in all three buckets before the wipe +- ✅ Recovered vertex count after restart equals baseline +- ✅ No `Cloud restore inconsistent` log errors or stale `.hyd-tmp` files + +#### Full disk wipe (all files including `raft/` and `.cloud-store-scope`) + +The automated harness above preserves the Raft state directory. For a true **full-volume wipe** (simulating complete disk failure), the node also loses the `.cloud-store-scope` marker file, which normally caches the cloud key scope across restarts. On restart you will see: + +``` +WARN: Cloud key scope seeded from runtime network address: 'store-store0_8510'. +This scope is only stable while the node's address does not change. +For guaranteed recovery after an IP/hostname change or local disk loss, +set a stable cloud.storage.node-id. +``` + +**This WARN is safe** in this Docker stack because `HG_CLOUD_STORAGE_NODE_ID=store0_8510` is already set in `docker-compose.yml`. The node-id env var and the runtime network address produce the identical sanitised scope string (`store0_8510`), so the node re-scopes correctly to its own remote objects and recovery completes fully. + +The WARN **does indicate a real risk** for deployments where `cloud.storage.node-id` is not set: if the node's hostname or IP changes after a disk wipe (new VM, rescheduled pod), the runtime-address fallback produces a different scope string, the node silently finds zero remote objects, and starts with empty data. **Always set `cloud.storage.node-id`** to a deployment-stable value in production (e.g. a StatefulSet pod ordinal or provisioned node UUID). See [pluggable-cloud-storage-architecture.md — Cloud key scope resolution](../../hugegraph-store/docs/pluggable-cloud-storage-architecture.md) for the full precedence rules. + +## Cleanup (Optional) + +When done with manual verification, stop and clean up. Use the **same project and compose file +the harness created the stack with** — the committed `docker-compose.yml` declares +`name: cloud-storage`, so a `down` against it targets a different project and leaves the +`cloud-storage-test` stack running: + +```bash +export COMPOSE_PROJECT_NAME=cloud-storage-test +CF="$REPO_ROOT/docker/cloud-storage/.generated/docker-compose.yml" +docker compose -f "$CF" down +``` + +To also remove the store data volumes and the generated artifacts: + +```bash +export COMPOSE_PROJECT_NAME=cloud-storage-test +CF="$REPO_ROOT/docker/cloud-storage/.generated/docker-compose.yml" +docker compose -f "$CF" down -v +rm -rf $REPO_ROOT/docker/cloud-storage/.generated/ +``` + +To reset everything including built Docker images (for a fresh start): + +```bash +export COMPOSE_PROJECT_NAME=cloud-storage-test +CF="$REPO_ROOT/docker/cloud-storage/.generated/docker-compose.yml" +docker compose -f "$CF" down +rm -rf $REPO_ROOT/docker/cloud-storage/data/ $REPO_ROOT/docker/cloud-storage/logs/ $REPO_ROOT/docker/cloud-storage/.artifacts/ +docker rmi \ + hugegraph-pd-cloud-storage:local \ + hugegraph-store-cloud-storage:local \ + 2>/dev/null || true +``` + + +## Architecture Notes + +- **Entrypoints**: Store containers inject `cloud.storage.*` config via `SPRING_APPLICATION_JSON` in `entrypoints/store-entrypoint.sh`. +- **Config precedence**: values injected by `SPRING_APPLICATION_JSON` override defaults in `/hugegraph-store/conf/application.yml`. +- **Plugin Loading (test-harness image only)**: `entrypoints/store-entrypoint.sh` starts the Store with `PropertiesLauncher` and `-Dloader.path=/hugegraph-store/plugins` so the S3 provider JAR staged by this harness is discovered via `ServiceLoader`. This is specific to the harness image; the stock Store distribution does not scan a `plugins/` directory (see "Enabling S3 on the stock distribution" in the [Store README](../../hugegraph-store/README.md)) +- **Logging**: Store containers run with `DEBUG` level for: + - `org.apache.hugegraph.store.node.cloud` + - `org.apache.hugegraph.rocksdb.access` +- **Networking**: Containers run on `cloud-storage-net` (static compose) or `cloud-storage-test_hg-net` (default generated test stack) +- **Storage**: Local bind mounts under `data/` and `logs/` for inspection +- **Server backend**: the `server` service runs on the `hstore` backend with PD peers `pd:8686`, configured by `conf/graphs/hugegraph.properties` (mounted over `/hugegraph-server/conf/graphs`). The generated test stack copies the same file, so both stacks use the same graph config; `STORE_REST=store0:8520` lets the server entrypoint wait for partition assignment +- **Bucket layout**: each Store node writes SSTs to a dedicated bucket (`store0 -> hugegraph-store0`, `store1 -> hugegraph-store1`, `store2 -> hugegraph-store2`) +- **Cloud key scope and `HG_CLOUD_STORAGE_NODE_ID`**: Each node scopes its remote objects under a key prefix derived from `cloud.storage.node-id` (env `HG_CLOUD_STORAGE_NODE_ID`). This value is already set per-node in `docker-compose.yml` (e.g. `store0_8510`). After a full disk wipe the local `.cloud-store-scope` marker file is lost; the node falls back to the runtime network address to re-seed the scope and logs a WARN. Recovery succeeds here because the node-id and the network address both resolve to the same string. **In production, always set `cloud.storage.node-id`** to a deployment-stable value so the scope survives both disk wipes and address changes. See the [architecture doc](../../hugegraph-store/docs/pluggable-cloud-storage-architecture.md) for details. + +### Troubleshooting Manual Verification Steps + +The commands below target the harness stack through `COMPOSE_PROJECT_NAME` and `CF` from +[Step 1](#step-1-set-environment-and-start-infrastructure-no-auto-load); set them again in a new shell. + +**Step 2: Server not becoming healthy** +```bash +# Check if server container is running +docker compose -f "$CF" ps server + +# Check server logs +docker compose -f "$CF" logs --tail=100 server + +# Verify port 8080 is accessible and the /versions health endpoint works +curl -v http://127.0.0.1:8080/versions +``` + +**Note:** The server health check uses the `/versions` endpoint (HTTP 200 OK), not `/gremlin` (requires query parameter). + +**Step 3-4: Graph API errors (HTTP errors)** +```bash +# Verify server is healthy +curl -fsS http://127.0.0.1:8080/versions && echo "Server OK" || echo "Server not ready" + +# Check if graph exists +curl -s --compressed http://localhost:8080/graphs/hugegraph/graph/vertices | python3 -c "import sys,json; print(json.load(sys.stdin).keys())" + +# Check server logs for errors +docker compose -f "$CF" logs server | grep -i "error\|exception" | tail -20 +``` + +**Optional partition check shows empty** +```bash +# Data may still be in RocksDB memory, not yet in partitions +# This is normal - proceed to Core Step 5 to flush + +# Or check if data was actually written +docker compose -f "$CF" logs store0 | grep -i "vertex\|edge\|insert" | tail -10 +``` + +**Core Step 5: Flush endpoint fails or stores become unhealthy** +```bash +# Check individual store health +for i in 0 1 2; do + port=$((8520 + i)) + echo "Store$i:" + curl -v http://127.0.0.1:$port/v1/health 2>&1 | grep -E "HTTP/|Connection refused" +done + +# Check store logs for startup errors +docker compose -f "$CF" logs store0 | tail -50 | grep -i "error\|exception" +``` + +**Core Step 5 / Advanced 1: Buckets showing 0 files after flush** + +This is the most common issue. Debug step-by-step: + +1. **Verify stores have data on disk:** + ```bash + docker compose -f "$CF" exec store0 \ + find /hugegraph-store/storage -name "*.sst" | wc -l + ``` + If output is 0, no SST files were created. Go back to Step 4 and ensure data was inserted. + +2. **Verify cloud storage was initialized:** + ```bash + docker compose -f "$CF" logs store0 | \ + grep -i "S3CloudStorageProvider\|cloud storage" | head -5 + ``` + If no output, plugin didn't load. In this harness image, check `/hugegraph-store/plugins/` for `hg-store-cloud-s3.jar`. + +3. **Check for S3 upload errors:** + ```bash + docker compose -f "$CF" logs store0 | \ + grep -i "upload\|s3\|error" | tail -20 + ``` + +4. **Verify MinIO connectivity:** + ```bash + docker compose -f "$CF" exec store0 \ + curl -v http://minio:9000/minio/health/live + ``` + +5. **Manually check MinIO buckets:** + ```bash + type resolve_hg_net >/dev/null 2>&1 || { echo "Run Core Step 1 helper first"; exit 1; } + resolve_hg_net + docker run --rm --entrypoint /bin/sh --network "$HG_NET" pgsty/mc:RELEASE.2026-10-18T00-00-00Z \ + -c 'mc alias set local http://minio:9000 minioadmin minioadmin && mc ls local/' + ``` + +### Troubleshooting: General Infrastructure Issues + +These commands also use `COMPOSE_PROJECT_NAME` and `CF` from +[Step 1](#step-1-set-environment-and-start-infrastructure-no-auto-load). + +### Store crashes with SIGSEGV on ARM64 (libjvm.so) + +If store logs show a fatal JVM crash like: + +```text +SIGSEGV ... libjvm.so ... linux-aarch64 +``` + +this is typically seen on ARM64 hosts with JVM + RocksDB startup. + +**On ARM64 hosts, the script automatically applies safe JVM defaults:** +- Uses Java 17 runtime image (not Java 11) +- Applies conservative JVM flags: `-XX:+UseSerialGC -XX:-UseCompressedOops -XX:-UseCompressedClassPointers` + +Simply run without special flags: + +```bash +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh +``` + +The script detects your host architecture and applies appropriate defaults automatically. + +### Custom overrides on ARM64 + +If you need different JVM flags or runtime images: + +```bash +HG_DOCKER_DEFAULT_PLATFORM=linux/amd64 \ +HG_PD_JAVA_RUNTIME_IMAGE=eclipse-temurin:17-jre \ +HG_STORE_JAVA_RUNTIME_IMAGE=eclipse-temurin:17-jre \ +HG_STORE_JAVA_OPTS="-XX:+UseSerialGC -XX:-UseCompressedOops -XX:-UseCompressedClassPointers" \ +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh +``` + +But in most cases, running without overrides should work: + +```bash +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh +``` + +If old containers are still running, force recreate with fresh env: + +```bash +docker compose -f "$CF" down +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh + +# Verify container runtime JDK actually changed +docker compose -f "$CF" run --rm --entrypoint java store0 -version +``` + +If the crash persists, inspect the generated JVM error file from store logs/data path: + +```bash +docker compose -f "$CF" logs store0 | tail -200 +docker compose -f "$CF" exec store0 ls -lah /hugegraph-store/hs_err_pid*.log +``` + +### No SST files appear in MinIO + +- Short test runs may not trigger compaction; SST deletion especially requires heavier/longer load +- Check store logs for upload errors: + ```bash + docker compose -f "$CF" logs store0 | grep -i "cloud\|s3" + ``` +- Verify MinIO is accessible: + ```bash + docker compose -f "$CF" exec minio mc ls local/ 2>&1 | head -20 + ``` + +### Query tests show FAILED + +- Ensure all store nodes are healthy: check `/v1/health` endpoints +- Manually test: `curl -fsS http://127.0.0.1:8520/v1/partitions | jq` +- Check store logs with: + ```bash + docker compose -f "$CF" logs store0 store1 store2 + ``` + +### Keep stack running for debugging + +```bash +# Run test but keep stack up +KEEP_STACK=1 $REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh +``` + +Then inspect manually: +```bash +docker compose -f "$CF" logs store0 store1 store2 +docker compose -f "$CF" exec store0 ls -la /hugegraph-store/storage +``` + +Stop manually when done: +```bash +docker compose -f "$CF" down +``` + +## Summary of Manual Verification Workflow + +Use this sequence for the end-to-end manual flow: + +1. **Core Step 1:** Set environment and start infrastructure (`--keep-stack --infra-only`) +2. **Core Step 2:** Wait for all services to become healthy +3. **Core Step 3:** Create graph schema +4. **Core Step 4:** Load 150+ vertices +5. **Core Step 5:** Flush stores and verify SST + metadata in MinIO UI +6. **Advanced 1 (optional):** Run strict script assertions +7. **Advanced 2 (optional):** Verify `clear()` semantics and no-orphan re-hydration +8. **Advanced 3 (optional):** Verify DB delete callbacks and remote prefix purge +9. **Advanced 4 (optional):** Simulate local disk loss and validate cloud re-hydration +10. **Cleanup (optional):** Tear down containers/volumes + +- **Note:** For manual Core Step 3+ workflows, always use Core Step 1 with `--infra-only` so + automation starts infrastructure but doesn't mutate graph state. + +For automated validation (including local state wipe + cloud re-hydration checks), run: + +```bash +$REPO_ROOT/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh --keep-stack +``` + + +**Success = SST and recovery metadata appear in all three buckets after Core Step 5** +(or after Advanced 1 strict checks). + +## What Gets Verified + +✅ Graph schema creation works +✅ Vertex/edge insertion works +✅ RocksDB SST file generation via explicit flush/compaction +✅ Cloud storage plugin uploads SST files to MinIO +✅ Multiple buckets receive files consistently +✅ A consistent `CURRENT` + `MANIFEST-*` + `OPTIONS-*` metadata set is mirrored alongside SSTs +✅ `clear()` keeps graph data empty after restart (optional Advanced 2) +✅ `clear()` does not require whole-prefix purge in shared cloud DB layouts (optional Advanced 2) +✅ DB delete callbacks write tombstone and purge remote DB prefix (optional advanced step) +✅ A store recovers all data from cloud after losing its local RocksDB state (automated script) + +## Notes + +- In this test-harness image the S3 provider JAR must be in `/hugegraph-store/plugins/` for `ServiceLoader` discovery (the stock distribution instead bundles it into the node jar via `-Ps3-cloud-storage`) +- Plugin dependency staging filters extra SLF4J binding jars to avoid duplicate binding warnings at runtime +- Cloud settings are read from environment variables at store startup time +- **Minimum data size:** 150+ vertices is recommended to trigger RocksDB SST file generation. Very small datasets may not create any SST files. +- MinIO buckets are pre-created by `minio-init` service: `hugegraph-store0`, `hugegraph-store1`, `hugegraph-store2` + + + diff --git a/docker/cloud-storage/conf/graphs/hugegraph.properties b/docker/cloud-storage/conf/graphs/hugegraph.properties new file mode 100644 index 0000000000..96b0995c50 --- /dev/null +++ b/docker/cloud-storage/conf/graphs/hugegraph.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Graph config for the cloud-storage test stack: HStore backend, placed by the PD service. +# Used by both docker-compose.yml and scripts/test-graph-queries-and-sst.sh. +gremlin.graph=org.apache.hugegraph.HugeFactory +backend=hstore +serializer=binary +store=hugegraph +task.scheduler_type=local +pd.peers=pd:8686 diff --git a/docker/cloud-storage/docker-compose.yml b/docker/cloud-storage/docker-compose.yml new file mode 100644 index 0000000000..89702bf1e8 --- /dev/null +++ b/docker/cloud-storage/docker-compose.yml @@ -0,0 +1,244 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: cloud-storage + +networks: + hg-net: + name: cloud-storage-net + driver: bridge + +volumes: + minio-data: + +services: + minio: + image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z + container_name: cloud-storage-minio + command: server /data --console-address ':9001' + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio-data:/data + networks: [hg-net] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 5s + retries: 20 + + minio-init: + image: pgsty/mc:RELEASE.2026-10-18T00-00-00Z + container_name: cloud-storage-minio-init + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c ' + mc alias set local http://minio:9000 minioadmin minioadmin && + mc mb -p local/hugegraph-store0 || true && + mc mb -p local/hugegraph-store1 || true && + mc mb -p local/hugegraph-store2 || true && + mc anonymous set none local/hugegraph-store0 || true && + mc anonymous set none local/hugegraph-store1 || true && + mc anonymous set none local/hugegraph-store2 || true + ' + networks: [hg-net] + + pd: + build: + context: ../.. + dockerfile: docker/cloud-storage/images/pd.Dockerfile + args: + JAVA_RUNTIME_IMAGE: ${HG_PD_JAVA_RUNTIME_IMAGE:-eclipse-temurin:17-jre-jammy} + image: hugegraph/pd:cloud-storage-local + container_name: cloud-storage-pd + depends_on: + minio-init: + condition: service_completed_successfully + environment: + HG_PD_GRPC_HOST: pd + HG_PD_GRPC_PORT: "8686" + HG_PD_REST_PORT: "8620" + HG_PD_RAFT_ADDRESS: pd:8610 + HG_PD_RAFT_PEERS_LIST: pd:8610 + HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 + HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_INITIAL_STORE_COUNT: "3" + ports: + - "8620:8620" + - "8686:8686" + volumes: + - ./data/pd:/hugegraph-pd/pd_data + - ./logs/pd:/hugegraph-pd/logs + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8620/v1/health >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 30 + start_period: 60s + + store0: + build: + context: ../.. + dockerfile: docker/cloud-storage/images/store.Dockerfile + args: + JAVA_RUNTIME_IMAGE: ${HG_STORE_JAVA_RUNTIME_IMAGE:-eclipse-temurin:17-jre-jammy} + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store0 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store0 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store0:8510 + HG_CLOUD_STORAGE_ENDPOINT: http://minio:9000 + HG_CLOUD_STORAGE_BUCKET: hugegraph-store0 + HG_CLOUD_STORAGE_NODE_ID: store0_8510 + JAVA_OPTS: ${HG_STORE_JAVA_OPTS:--Xms4g -Xmx4g -XX:+UseG1GC} + ports: + - "8500:8500" + - "8510:8510" + - "8520:8520" + volumes: + - ./data/store0:/hugegraph-store/storage + - ./logs/store0:/hugegraph-store/logs + networks: [hg-net] + deploy: + resources: + limits: + memory: 8G + reservations: + memory: 6G + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + + store1: + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store1 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store1 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store1:8510 + HG_CLOUD_STORAGE_ENDPOINT: http://minio:9000 + HG_CLOUD_STORAGE_BUCKET: hugegraph-store1 + HG_CLOUD_STORAGE_NODE_ID: store1_8510 + JAVA_OPTS: ${HG_STORE_JAVA_OPTS:--Xms4g -Xmx4g -XX:+UseG1GC} + ports: + - "8501:8500" + - "8511:8510" + - "8521:8520" + volumes: + - ./data/store1:/hugegraph-store/storage + - ./logs/store1:/hugegraph-store/logs + networks: [hg-net] + deploy: + resources: + limits: + memory: 8G + reservations: + memory: 6G + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + + store2: + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store2 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store2 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store2:8510 + HG_CLOUD_STORAGE_ENDPOINT: http://minio:9000 + HG_CLOUD_STORAGE_BUCKET: hugegraph-store2 + HG_CLOUD_STORAGE_NODE_ID: store2_8510 + JAVA_OPTS: ${HG_STORE_JAVA_OPTS:--Xms4g -Xmx4g -XX:+UseG1GC} + ports: + - "8502:8500" + - "8512:8510" + - "8522:8520" + volumes: + - ./data/store2:/hugegraph-store/storage + - ./logs/store2:/hugegraph-store/logs + networks: [hg-net] + deploy: + resources: + limits: + memory: 8G + reservations: + memory: 6G + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + + server: + image: hugegraph/server:1.7.0 + container_name: cloud-storage-server + depends_on: + store0: + condition: service_healthy + store1: + condition: service_healthy + store2: + condition: service_healthy + environment: + GREMLIN_SERVER_HOST: 0.0.0.0 + GREMLIN_SERVER_PORT: "8182" + # Used by the entrypoint's post-startup partition check (hstore backend) + STORE_REST: store0:8520 + ports: + - "8080:8080" + - "8182:8182" + volumes: + - ./logs/server:/hugegraph-server/logs + # HStore backend + PD peers (pd:8686); same graph config the generated test stack mounts + - ./conf/graphs:/hugegraph-server/conf/graphs + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/versions >/dev/null || exit 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s diff --git a/docker/cloud-storage/entrypoints/pd-entrypoint.sh b/docker/cloud-storage/entrypoints/pd-entrypoint.sh new file mode 100755 index 0000000000..8ad2159d5e --- /dev/null +++ b/docker/cloud-storage/entrypoints/pd-entrypoint.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +: "${HG_PD_GRPC_HOST:=pd}" +: "${HG_PD_GRPC_PORT:=8686}" +: "${HG_PD_REST_PORT:=8620}" +: "${HG_PD_RAFT_ADDRESS:=pd:8610}" +: "${HG_PD_RAFT_PEERS_LIST:=pd:8610}" +: "${HG_PD_DATA_PATH:=/hugegraph-pd/pd_data}" +: "${HG_PD_INITIAL_STORE_LIST:=store0:8500,store1:8500,store2:8500}" +: "${HG_PD_INITIAL_STORE_COUNT:=3}" + +json_escape() { + local s="$1" + s=${s//\\/\\\\} + s=${s//\"/\\\"} + s=${s//$'\n'/} + printf "%s" "$s" +} + +export SPRING_APPLICATION_JSON="$(cat <&2 + exit 2 + fi +} + +json_escape() { + local s="$1" + s=${s//\\/\\\\} + s=${s//\"/\\\"} + s=${s//$'\n'/} + printf "%s" "$s" +} + +is_truthy() { + case "${1:-}" in + 1|[Tt][Rr][Uu][Ee]|[Yy]|[Yy][Ee][Ss]|[Oo][Nn]) return 0 ;; + *) return 1 ;; + esac +} + +require_env "HG_STORE_PD_ADDRESS" +require_env "HG_STORE_GRPC_HOST" +require_env "HG_STORE_RAFT_ADDRESS" + +: "${HG_STORE_GRPC_PORT:=8500}" +: "${HG_STORE_REST_PORT:=8520}" +: "${HG_STORE_DATA_PATH:=/hugegraph-store/storage}" +: "${HG_CLOUD_STORAGE_PROVIDER:=s3}" +: "${HG_CLOUD_STORAGE_ENABLED:=true}" +: "${HG_CLOUD_STORAGE_BUCKET:=hugegraph-store}" +: "${HG_CLOUD_STORAGE_REGION:=us-east-1}" +: "${HG_CLOUD_STORAGE_ENDPOINT:=http://minio:9000}" +: "${HG_CLOUD_STORAGE_ACCESS_KEY:=minioadmin}" +: "${HG_CLOUD_STORAGE_SECRET_KEY:=minioadmin}" +: "${HG_CLOUD_STORAGE_PATH_PREFIX:=hugegraph}" +# Stable per-node identity for the cloud key scope. Set this to a deployment-stable +# value (e.g. the container name) so hydration finds the correct remote objects after a +# disk wipe — without it the scope falls back to the runtime Raft address, which may +# produce a different sanitised string and silently find zero remote files. +: "${HG_CLOUD_STORAGE_NODE_ID:=}" + +# Build the node-id fragment conditionally so a blank value is omitted from the JSON +# rather than written as an empty string (which would override the default behaviour). +if [[ -n "${HG_CLOUD_STORAGE_NODE_ID}" ]]; then + NODE_ID_JSON="\"node-id\": \"$(json_escape "${HG_CLOUD_STORAGE_NODE_ID}")\"," +else + NODE_ID_JSON="" +fi + +export SPRING_APPLICATION_JSON="$(cat <&2 + exit 2 +fi + +# Avoid duplicate/conflicting RocksDB API jars in PropertiesLauncher loader.path. +rm -f /hugegraph-store/plugins/rocksdbjni-*.jar >/dev/null 2>&1 || true + +# RocksDB native callback threads may resolve Java types via AppClassLoader; keep rocksdbjni +# explicitly on the JVM system classpath to avoid NoClassDefFoundError for callback types. +ROCKSDB_JAR="" +if compgen -G "/hugegraph-store/lib/rocksdbjni-*.jar" > /dev/null; then + ROCKSDB_JAR=$(ls -1 /hugegraph-store/lib/rocksdbjni-*.jar | sort | tail -n 1) +elif compgen -G "/hugegraph-store/plugins/rocksdbjni-*.jar" > /dev/null; then + ROCKSDB_JAR=$(ls -1 /hugegraph-store/plugins/rocksdbjni-*.jar | sort | tail -n 1) +fi + +APP_CLASSPATH="$STORE_JAR" +if [[ -n "$ROCKSDB_JAR" ]]; then + APP_CLASSPATH="$ROCKSDB_JAR:$APP_CLASSPATH" + echo "Using explicit rocksdbjni on system classpath: $ROCKSDB_JAR" +else + if is_truthy "$HG_CLOUD_STORAGE_ENABLED"; then + echo "ERROR: no external rocksdbjni jar found while cloud storage is enabled" >&2 + echo "ERROR: refusing to start to avoid callback NoClassDefFoundError at runtime" >&2 + exit 2 + fi + echo "WARNING: no external rocksdbjni jar found; continuing because cloud storage is disabled" >&2 +fi + +exec java ${JAVA_OPTS:-} \ + -Dlog4j.configurationFile=/hugegraph-store/conf/log4j2.xml \ + -Dspring.config.location=/hugegraph-store/conf/application.yml \ + -Dloader.path=/hugegraph-store/plugins \ + -cp "$APP_CLASSPATH" \ + org.springframework.boot.loader.PropertiesLauncher + diff --git a/docker/cloud-storage/images/pd.Dockerfile b/docker/cloud-storage/images/pd.Dockerfile new file mode 100644 index 0000000000..244308986e --- /dev/null +++ b/docker/cloud-storage/images/pd.Dockerfile @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# syntax=docker/dockerfile:1.4 +ARG JAVA_RUNTIME_IMAGE=eclipse-temurin:17-jre-jammy +FROM ${JAVA_RUNTIME_IMAGE} + +WORKDIR /hugegraph-pd +COPY docker/cloud-storage/.artifacts/pd-dist/ /hugegraph-pd/ +COPY docker/cloud-storage/entrypoints/pd-entrypoint.sh /hugegraph-pd/pd-entrypoint.sh + +RUN chmod +x /hugegraph-pd/pd-entrypoint.sh /hugegraph-pd/bin/*.sh + +EXPOSE 8620 8686 8610 +ENTRYPOINT ["/hugegraph-pd/pd-entrypoint.sh"] diff --git a/docker/cloud-storage/images/store.Dockerfile b/docker/cloud-storage/images/store.Dockerfile new file mode 100644 index 0000000000..42fdc6b474 --- /dev/null +++ b/docker/cloud-storage/images/store.Dockerfile @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# syntax=docker/dockerfile:1.4 +ARG JAVA_RUNTIME_IMAGE=eclipse-temurin:17-jre-jammy +FROM ${JAVA_RUNTIME_IMAGE} + +WORKDIR /hugegraph-store +COPY docker/cloud-storage/.artifacts/store-dist/ /hugegraph-store/ +COPY docker/cloud-storage/.artifacts/plugins/ /hugegraph-store/plugins/ +COPY docker/cloud-storage/entrypoints/store-entrypoint.sh /hugegraph-store/store-entrypoint.sh + +RUN chmod +x /hugegraph-store/store-entrypoint.sh /hugegraph-store/bin/*.sh && \ + mkdir -p /hugegraph-store/logs /hugegraph-store/storage /hugegraph-store/plugins + +EXPOSE 8500 8510 8520 +ENTRYPOINT ["/hugegraph-store/store-entrypoint.sh"] diff --git a/docker/cloud-storage/scripts/test-advanced-clear-semantics.sh b/docker/cloud-storage/scripts/test-advanced-clear-semantics.sh new file mode 100755 index 0000000000..adbdbe5040 --- /dev/null +++ b/docker/cloud-storage/scripts/test-advanced-clear-semantics.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prerequisite: create/populate the cloud-storage environment first via +# test-graph-queries-and-sst.sh. + +set -euo pipefail + +GRAPH_NAME="${GRAPH_NAME:-hugegraph}" +MC_IMAGE="${MC_IMAGE:-pgsty/mc:RELEASE.2026-10-18T00-00-00Z}" +MINIO_USER="${MINIO_USER:-minioadmin}" +MINIO_PASS="${MINIO_PASS:-minioadmin}" +RESTART_WAIT_SECONDS="${RESTART_WAIT_SECONDS:-20}" + +BUCKETS=(hugegraph-store0 hugegraph-store1 hugegraph-store2) + +log() { + printf '[advanced-clear] %s\n' "$*" +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || die "missing required command: $1" +} + +resolve_hg_net() { + local net + + net=$(docker inspect cloud-storage-minio \ + --format '{{range $k, $v := .NetworkSettings.Networks}}{{println $k}}{{end}}' \ + 2>/dev/null | head -n1 || true) + + if [[ -z "$net" ]]; then + if docker network inspect cloud-storage-test_hg-net >/dev/null 2>&1; then + net="cloud-storage-test_hg-net" + elif docker network inspect cloud-storage-net >/dev/null 2>&1; then + net="cloud-storage-net" + fi + fi + + [[ -n "$net" ]] || die "no cloud-storage Docker network found" + export HG_NET="$net" + log "Using Docker network: $HG_NET" +} + +mc_exec() { + local cmd="$1" + + docker run --rm --entrypoint /bin/sh --network "$HG_NET" "$MC_IMAGE" -c " +if mc alias set local http://minio:9000 $MINIO_USER $MINIO_PASS >/dev/null 2>&1; then + : +elif mc alias set local http://cloud-storage-minio:9000 $MINIO_USER $MINIO_PASS >/dev/null 2>&1; then + : +else + echo 'ERROR: cannot reach MinIO at minio:9000 or cloud-storage-minio:9000' >&2 + exit 1 +fi +$cmd +" +} + +# Exact count via Gremlin: the REST /graph/vertices listing is capped by a default limit (100). +# The traversal-source alias key is "-" on graph-space aware servers and plain +# "" otherwise, so try both. +vertex_count() { + local key payload out count + for key in "${GRAPH_SPACE:-DEFAULT}-${GRAPH_NAME}" "${GRAPH_NAME}"; do + payload="{\"gremlin\":\"g.V().count()\",\"bindings\":{},\"language\":\"gremlin-groovy\",\"aliases\":{\"g\":\"__g_${key}\"}}" + out=$(curl -s --compressed -X POST "http://localhost:8080/gremlin" \ + -H 'Content-Type: application/json' -d "$payload" 2>/dev/null || true) + count=$(printf '%s' "$out" | python3 -c "import sys,json; print(int(json.load(sys.stdin)['result']['data'][0]))" \ + 2>/dev/null || true) + if [[ "$count" =~ ^[0-9]+$ ]]; then + echo "$count" + return 0 + fi + done + echo "ERROR: Gremlin vertex count query failed for graph '${GRAPH_NAME}'" >&2 + return 1 +} + +main() { + local bucket db_prefix candidate_bucket match + local before_count after_count immediate_count after_restart_count + + need_cmd docker + need_cmd curl + need_cmd grep + need_cmd head + need_cmd python3 + + resolve_hg_net + + bucket="" + db_prefix="" + + # Detect one graph DB prefix from CURRENT files in any store bucket. + for candidate_bucket in "${BUCKETS[@]}"; do + match=$(mc_exec "mc find local/${candidate_bucket} --name CURRENT 2>/dev/null" \ + | grep "/${GRAPH_NAME}/" | head -n1 || true) + if [[ -n "$match" ]]; then + bucket="$candidate_bucket" + db_prefix="${match#local/${candidate_bucket}/}" + db_prefix="${db_prefix%/CURRENT}" + break + fi + done + + [[ -n "$bucket" && -n "$db_prefix" ]] \ + || die "failed to detect DB prefix for graph '${GRAPH_NAME}' in any bucket" + + log "Detected DB prefix: bucket=${bucket}, prefix=${db_prefix}" + + before_count=$(mc_exec "mc ls --recursive local/${bucket}/${db_prefix} 2>/dev/null | wc -l") + before_count="${before_count//[^0-9]/}" + [[ -n "$before_count" ]] || before_count="0" + log "Objects before clear: ${before_count}" + + mc_exec "mc ls --recursive local/${bucket}/${db_prefix} 2>/dev/null" | head -20 || true + + if [[ "$before_count" == "0" ]]; then + die "objects before clear is 0; load data first (Core Steps 3-5)" + fi + + log "Calling clear API for graph '${GRAPH_NAME}'" + curl -s -X DELETE "http://localhost:8080/graphs/${GRAPH_NAME}/clear" \ + --get --data-urlencode "confirm_message=I'm sure to delete all data" \ + | head -c 200 + echo + + sleep 5 + + after_count=$(mc_exec "mc ls --recursive local/${bucket}/${db_prefix} 2>/dev/null | wc -l") + after_count="${after_count//[^0-9]/}" + [[ -n "$after_count" ]] || after_count="0" + log "Objects after clear (expected usually non-zero): ${after_count}" + + mc_exec "mc ls --recursive local/${bucket}/${db_prefix} 2>/dev/null" | head -20 || true + + immediate_count=$(vertex_count) + log "Vertex count immediately after clear: ${immediate_count}" + [[ "$immediate_count" == "0" ]] \ + || die "vertex count after clear is ${immediate_count}, expected 0" + + log "Restarting stores for no-orphan re-hydration check" + docker restart cloud-storage-store0 cloud-storage-store1 cloud-storage-store2 >/dev/null + sleep "$RESTART_WAIT_SECONDS" + + after_restart_count=$(vertex_count) + log "Vertex count after restart: ${after_restart_count}" + [[ "$after_restart_count" == "0" ]] \ + || die "vertex count after restart is ${after_restart_count}, expected 0" + + log "PASS: no orphan re-hydration observed" +} + +main "$@" + + diff --git a/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh b/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh new file mode 100755 index 0000000000..a7207a1755 --- /dev/null +++ b/docker/cloud-storage/scripts/test-graph-queries-and-sst.sh @@ -0,0 +1,1464 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STACK_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +REPO_ROOT="$(cd "${STACK_DIR}/../.." && pwd)" +GENERATED_DIR="${STACK_DIR}/.generated" +ARTIFACTS_DIR="${STACK_DIR}/.artifacts" +COMPOSE_FILE="${GENERATED_DIR}/docker-compose.yml" +FULL_TEST_REPORT="${GENERATED_DIR}/FULL-TEST-REPORT.txt" +TEST_REPORT="${GENERATED_DIR}/test-report.txt" +MINIO_REPORT="${GENERATED_DIR}/minio-verification.txt" +STORE_CLUSTER_LOG="${GENERATED_DIR}/store-cluster.log" +CLI_LOAD_LOG="${GENERATED_DIR}/cli-load.log" +LOAD_DATA_FILE="${GENERATED_DIR}/load-data.tsv" +export SERVER_GRAPHS_DIR="${GENERATED_DIR}/graphs" +export SERVER_GRAPH_CONF="${SERVER_GRAPHS_DIR}/hugegraph.properties" +export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-cloud-storage-test}" +HG_PD_IMAGE="${HG_PD_IMAGE:-hugegraph/pd:cloud-storage-local}" +HG_STORE_IMAGE="${HG_STORE_IMAGE:-hugegraph/store:cloud-storage-local}" +HG_SERVER_IMAGE="${HG_SERVER_IMAGE:-hugegraph/server:1.7.0}" +MINIO_IMAGE="${MINIO_IMAGE:-pgsty/minio:RELEASE.2026-08-04T00-00-00Z}" +MINIO_MC_IMAGE="${MINIO_MC_IMAGE:-pgsty/mc:RELEASE.2026-10-18T00-00-00Z}" +MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}" +MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-minioadmin}" +S3_BUCKET_STORE0="${S3_BUCKET_STORE0:-hugegraph-store0}" +S3_BUCKET_STORE1="${S3_BUCKET_STORE1:-hugegraph-store1}" +S3_BUCKET_STORE2="${S3_BUCKET_STORE2:-hugegraph-store2}" +S3_REGION="${S3_REGION:-us-east-1}" +S3_ENDPOINT="${S3_ENDPOINT:-http://minio:9000}" +GRAPH_API_BASE="${GRAPH_API_BASE:-http://localhost:8080/graphs/hugegraph}" +KEEP_UP="${KEEP_UP:-true}" +SKIP_SMOKE_TESTS="${SKIP_SMOKE_TESTS:-false}" +SCRIPT_START_TS="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +RECOVERY_STATUS="NOT_RUN" +RECREATE_STATUS="NOT_RUN" +TOMBSTONE_STATUS="NOT_RUN" +DELETE_CALLBACK_STATUS="NOT_RUN" +SMOKE_STATUS="NOT_RUN" +INFRA_READY="false" +NETWORK="" +# Tombstone evidence captured by run_db_recreation_no_orphan_test (a real partition delete) +# and asserted by run_tombstone_sibling_key_test. +DELETE_TOMBSTONE_BUCKET="" +DELETE_TOMBSTONE_KEY="" +DELETE_TOMBSTONE_INNER_KEY="" +DELETE_TOMBSTONE_SEEN="no" + +while [[ $# -gt 0 ]]; do + case "$1" in + --keep-stack) KEEP_UP=true ;; + --skip-smoke-tests|--infra-only) + SKIP_SMOKE_TESTS=true ;; + -h|--help) + cat <<'USAGE' + Usage: test-graph-queries-and-sst.sh [--keep-stack] [--skip-smoke-tests|--infra-only] + + This script runs a full end-to-end cloud storage test suite: + + 1. Recovery Test: + load data -> flush+compact -> verify a consistent + {CURRENT, MANIFEST, OPTIONS, SST} set in MinIO -> + wipe each store's local RocksDB state (raft/ preserved) -> + restart -> confirm data is recovered from cloud (not empty DB). + + 2. DB Deletion Cleanup Test: + create test graph -> load data -> sync to cloud -> + delete graph -> verify cloud storage prefix is cleaned up + (tests onDBDeleted() -> purgeRemotePrefix() behavior). + + --keep-stack Leave the stack running on exit (same as KEEP_UP=true). + --skip-smoke-tests Start infrastructure only; skip data load + validation tests. + --infra-only Alias of --skip-smoke-tests (skip scripted data/tests). +USAGE + exit 0 ;; + *) echo "unknown arg: $1 (see --help)" >&2; exit 2 ;; + esac + shift +done +log() { printf "[cloud-storage] %s\n" "$*"; } +need_cmd() { command -v "$1" >/dev/null 2>&1 || { echo "ERROR: $1 not found" >&2; exit 2; }; } + +report_line() { + local file="$1" + shift + printf "%s\n" "$*" >> "$file" +} + +report_event() { + local category="$1" + shift + report_line "$FULL_TEST_REPORT" "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] [$category] $*" +} + +record_phase() { + local phase="$1" + local status="$2" + local details="$3" + report_line "$TEST_REPORT" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")\t${phase}\t${status}\t${details}" + report_event "$phase" "${status} - ${details}" +} + +init_reports() { + mkdir -p "$GENERATED_DIR" + : > "$FULL_TEST_REPORT" + : > "$TEST_REPORT" + : > "$MINIO_REPORT" + : > "$STORE_CLUSTER_LOG" + : > "$CLI_LOAD_LOG" + : > "$LOAD_DATA_FILE" + + report_line "$FULL_TEST_REPORT" "Cloud Storage E2E Test Report" + report_line "$FULL_TEST_REPORT" "started_at=${SCRIPT_START_TS}" + report_line "$FULL_TEST_REPORT" "script=${BASH_SOURCE[0]}" + report_line "$FULL_TEST_REPORT" "compose_project=${COMPOSE_PROJECT_NAME}" + report_line "$FULL_TEST_REPORT" "" + + report_line "$TEST_REPORT" "timestamp_utc phase status details" + report_line "$MINIO_REPORT" "MinIO Verification Report" + report_line "$MINIO_REPORT" "started_at=${SCRIPT_START_TS}" + report_line "$MINIO_REPORT" "" + report_line "$CLI_LOAD_LOG" "CLI/Data Load Report" + report_line "$CLI_LOAD_LOG" "started_at=${SCRIPT_START_TS}" + report_line "$CLI_LOAD_LOG" "" + report_line "$LOAD_DATA_FILE" "name age city" +} + +collect_store_logs() { + if [[ -f "$COMPOSE_FILE" ]]; then + docker compose -f "$COMPOSE_FILE" logs store0 store1 store2 > "$STORE_CLUSTER_LOG" 2>&1 || true + fi +} + +finalize_reports() { + local exit_code="$1" + local end_ts + end_ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + + report_line "$FULL_TEST_REPORT" "" + report_line "$FULL_TEST_REPORT" "Summary" + report_line "$FULL_TEST_REPORT" "ended_at=${end_ts}" + report_line "$FULL_TEST_REPORT" "exit_code=${exit_code}" + report_line "$FULL_TEST_REPORT" "infra_ready=${INFRA_READY}" + report_line "$FULL_TEST_REPORT" "smoke_tests=${SMOKE_STATUS}" + report_line "$FULL_TEST_REPORT" "recovery_test=${RECOVERY_STATUS}" + report_line "$FULL_TEST_REPORT" "db_recreation_no_orphan_test=${RECREATE_STATUS}" + report_line "$FULL_TEST_REPORT" "tombstone_sibling_key_test=${TOMBSTONE_STATUS}" + report_line "$FULL_TEST_REPORT" "db_delete_callbacks_test=${DELETE_CALLBACK_STATUS}" + report_line "$FULL_TEST_REPORT" "" + report_line "$FULL_TEST_REPORT" "Generated artifacts:" + report_line "$FULL_TEST_REPORT" "- ${FULL_TEST_REPORT}" + report_line "$FULL_TEST_REPORT" "- ${TEST_REPORT}" + report_line "$FULL_TEST_REPORT" "- ${MINIO_REPORT}" + report_line "$FULL_TEST_REPORT" "- ${STORE_CLUSTER_LOG}" + report_line "$FULL_TEST_REPORT" "- ${CLI_LOAD_LOG}" + report_line "$FULL_TEST_REPORT" "- ${LOAD_DATA_FILE}" +} + +assert_all_reports_present() { + local missing=0 + for report_file in "$FULL_TEST_REPORT" "$TEST_REPORT" "$MINIO_REPORT" "$STORE_CLUSTER_LOG" "$CLI_LOAD_LOG" "$LOAD_DATA_FILE"; do + if [[ ! -f "$report_file" ]]; then + echo "ERROR: expected report file missing: $report_file" >&2 + missing=$((missing + 1)) + fi + done + if (( missing > 0 )); then + echo "ERROR: $missing expected report file(s) are missing; CI contract violated" >&2 + return 1 + fi + return 0 +} + +find_dist_dir() { + local glob="$1" + local d + for d in $glob; do + [[ -d "$d" ]] && { echo "$d"; return 0; } + done + return 1 +} + +find_plugin_jar() { + local f + for f in "${REPO_ROOT}"/hugegraph-store/hg-store-cloud-s3/target/hg-store-cloud-s3-*.jar; do + [[ -f "$f" ]] || continue + case "$f" in + *-sources.jar|*-javadoc.jar|*original*) continue ;; + esac + echo "$f" + return 0 + done + return 1 +} + +extract_embedded_rocksdbjni() { + local store_jar="$1" + local output_dir="$2" + python3 - "$store_jar" "$output_dir" <<'PY' +import os +import sys +import zipfile + +store_jar, output_dir = sys.argv[1], sys.argv[2] + +with zipfile.ZipFile(store_jar) as zf: + names = [name for name in zf.namelist() + if name.startswith("BOOT-INF/lib/rocksdbjni-") and name.endswith(".jar")] + if not names: + print(f"ERROR: no embedded rocksdbjni jar found in {store_jar}", file=sys.stderr) + sys.exit(1) + names.sort() + selected = names[-1] + target = os.path.join(output_dir, os.path.basename(selected)) + os.makedirs(output_dir, exist_ok=True) + with open(target, "wb") as fh: + fh.write(zf.read(selected)) +print(target) +PY +} + +prepare_artifacts() { + local pd_src store_src plugin_jar plugin_dep_dir dep_base + local store_fat_jar staged_rocksdb + + pd_src="$(find_dist_dir "${REPO_ROOT}/hugegraph-pd/apache-hugegraph-pd-*")" || { + echo "ERROR: PD dist not found under ${REPO_ROOT}/hugegraph-pd/apache-hugegraph-pd-*" >&2 + echo "Run: mvn clean package -DskipTests" >&2 + exit 2 + } + store_src="$(find_dist_dir "${REPO_ROOT}/hugegraph-store/apache-hugegraph-store-*")" || { + echo "ERROR: Store dist not found under ${REPO_ROOT}/hugegraph-store/apache-hugegraph-store-*" >&2 + echo "Run: mvn clean package -DskipTests -Ps3-cloud-storage" >&2 + exit 2 + } + plugin_jar="$(find_plugin_jar)" || { + echo "ERROR: S3 plugin jar not found under ${REPO_ROOT}/hugegraph-store/hg-store-cloud-s3/target/" >&2 + echo "Run: mvn clean package -DskipTests -Ps3-cloud-storage" >&2 + exit 2 + } + + log "preparing Docker artifacts in ${ARTIFACTS_DIR}" + rm -rf "${ARTIFACTS_DIR}" + mkdir -p "${ARTIFACTS_DIR}/pd-dist" "${ARTIFACTS_DIR}/store-dist" "${ARTIFACTS_DIR}/plugins" + + cp -R "${pd_src}/." "${ARTIFACTS_DIR}/pd-dist/" + cp -R "${store_src}/." "${ARTIFACTS_DIR}/store-dist/" + cp "${plugin_jar}" "${ARTIFACTS_DIR}/plugins/" + + store_fat_jar=$(echo "${ARTIFACTS_DIR}"/store-dist/lib/hg-store-node-*.jar) + if [[ ! -f "${store_fat_jar}" ]]; then + echo "ERROR: store fat-jar not found under ${ARTIFACTS_DIR}/store-dist/lib/" >&2 + exit 2 + fi + staged_rocksdb="$(extract_embedded_rocksdbjni "${store_fat_jar}" "${ARTIFACTS_DIR}/store-dist/lib")" || exit 2 + log "staged RocksDB JNI for system classpath: $(basename "${staged_rocksdb}")" + + plugin_dep_dir="${REPO_ROOT}/hugegraph-store/hg-store-cloud-s3/target/dependency" + if [[ -d "${plugin_dep_dir}" ]]; then + # Keep only external plugin deps; internal HugeGraph jars must come from /hugegraph-store/lib. + for dep in "${plugin_dep_dir}"/*.jar; do + [[ -f "${dep}" ]] || continue + dep_base="$(basename "${dep}")" + case "${dep_base}" in + hg-*.jar|hugegraph-*.jar|rocksdbjni-*.jar) continue ;; + esac + cp "${dep}" "${ARTIFACTS_DIR}/plugins/" + done + else + log "warning: plugin dependency dir not found at ${plugin_dep_dir}; continuing with plugin jar only" + fi + + # Prevent duplicate SLF4J binding clashes from plugin dependency staging. + rm -f "${ARTIFACTS_DIR}"/plugins/log4j-slf4j-impl-*.jar "${ARTIFACTS_DIR}"/plugins/slf4j-log4j12-*.jar || true +} + +ensure_image() { + local img="$1" + docker image inspect "$img" >/dev/null 2>&1 && return 0 + # Check if it's a local build image (contains "cloud-storage-local") + if [[ "$img" == *"cloud-storage-local"* ]]; then + log "image $img is local-build only (will be built via docker compose build)" + return 0 + fi + log "pulling $img..." + docker pull "$img" >/dev/null || exit 3 +} +ensure_minio_buckets() { + for bucket in "$S3_BUCKET_STORE0" "$S3_BUCKET_STORE1" "$S3_BUCKET_STORE2"; do + docker run --rm --network "$1" --entrypoint /bin/sh "$MINIO_MC_IMAGE" -c "mc alias set local http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD >/dev/null && mc mb --ignore-existing local/$bucket >/dev/null" + done +} +wait_svc() { + local svc max=120 i=0 + svc="$1" + while [[ $i -lt $max ]]; do + local cid status + cid=$(docker compose -f "$COMPOSE_FILE" ps -q "$svc" 2>/dev/null || true) + [[ -z "$cid" ]] && { sleep 2; i=$((i+1)); continue; } + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid" 2>/dev/null || true) + [[ "$status" == "healthy" || "$status" == "running" ]] && { log "✓ $svc"; return 0; } + sleep 2 + i=$((i+1)) + done + echo "ERROR: $svc timeout" >&2 + return 1 +} +wait_http() { + local url max=120 i=0 + url="$1" + while [[ $i -lt $max ]]; do + [[ $(curl -so /dev/null -w "%{http_code}" "$url" 2>/dev/null) == "200" ]] && { log "✓ $url ready"; return 0; } + sleep 2 + i=$((i+1)) + done + echo "ERROR: $url timeout" >&2 + return 1 +} + +delete_graph_with_confirm() { + local graph_api="$1" + local resp_file status + resp_file="/tmp/hg-delete-$$.json" + status=$(curl -s -o "$resp_file" -w "%{http_code}" -X DELETE "$graph_api" \ + --get --data-urlencode "confirm_message=I'm sure to drop the graph") + if [[ "$status" != 2* ]]; then + echo "ERROR: graph delete failed with HTTP ${status}: ${graph_api}" >&2 + head -80 "$resp_file" >&2 || true + rm -f "$resp_file" || true + return 1 + fi + rm -f "$resp_file" || true +} + +clear_graph_with_confirm() { + local graph_api="$1" + local resp_file status clear_api + resp_file="/tmp/hg-clear-$$.json" + clear_api="${graph_api}/clear" + status=$(curl -s -o "$resp_file" -w "%{http_code}" -X DELETE "$clear_api" \ + --get --data-urlencode "confirm_message=I'm sure to delete all data") + if [[ "$status" != 2* ]]; then + echo "ERROR: graph clear failed with HTTP ${status}: ${clear_api}" >&2 + head -80 "$resp_file" >&2 || true + rm -f "$resp_file" || true + return 1 + fi + rm -f "$resp_file" || true +} + +on_exit() { + local status=$? + local assertion_rc + collect_store_logs + finalize_reports "$status" + assert_all_reports_present + assertion_rc=$? + # Preserve original exit code unless test passed but assertion failed + [[ $status -eq 0 && $assertion_rc -ne 0 ]] && status=$assertion_rc + [[ "$KEEP_UP" == "true" ]] || (docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true) +} +trap on_exit EXIT + +# ----------------------------------------------------------------------------- +# Total-loss recovery E2E helpers +# ----------------------------------------------------------------------------- + +# Exact vertex count via Gremlin g.V().count() (0 if the query fails). The REST +# /graph/vertices listing is capped by a default limit (100), so it cannot count 150 vertices. +# The traversal-source alias is __g_; the key is "-" on +# graph-space aware servers and plain "" otherwise, so try both. +graph_vertex_count() { + local base graph key payload out count + base="${GRAPH_API_BASE%/graphs/*}" + graph="${GRAPH_API_BASE##*/}" + for key in "${GRAPH_SPACE:-DEFAULT}-${graph}" "${graph}"; do + payload="{\"gremlin\":\"g.V().count()\",\"bindings\":{},\"language\":\"gremlin-groovy\",\"aliases\":{\"g\":\"__g_${key}\"}}" + out=$(curl -s --compressed -X POST "${base}/gremlin" \ + -H 'Content-Type: application/json' -d "$payload" 2>/dev/null || true) + count=$(printf '%s' "$out" | python3 -c "import sys,json; print(int(json.load(sys.stdin)['result']['data'][0]))" \ + 2>/dev/null || true) + if [[ "$count" =~ ^[0-9]+$ ]]; then + echo "$count" + return 0 + fi + done + echo 0 +} + +# Number of vertices the last load_test_data call inserted successfully (HTTP 2xx). +LOADED_VERTEX_COUNT=0 + +# Create a minimal schema and insert enough vertices to generate SST files. +load_test_data() { + log "loading test data (schema + 150 vertices)..." + report_event "graph-load" "starting schema + vertex load" + local insert_ok=0 insert_fail=0 + + # Create deterministic input file promised by README. + for i in $(seq 1 150); do + printf "person_%s\t%s\tcity_%s\n" "$i" "$((20 + i % 50))" "$((i % 5))" >> "$LOAD_DATA_FILE" + done + + for pk in '{"name":"name","data_type":"TEXT","cardinality":"SINGLE"}' \ + '{"name":"age","data_type":"INT","cardinality":"SINGLE"}' \ + '{"name":"city","data_type":"TEXT","cardinality":"SINGLE"}'; do + curl -s -o /dev/null -X POST "${GRAPH_API_BASE}/schema/propertykeys" \ + -H 'Content-Type: application/json' -d "$pk" || true + done + curl -s -o /dev/null -X POST "${GRAPH_API_BASE}/schema/vertexlabels" \ + -H 'Content-Type: application/json' \ + -d '{"name":"person","id_strategy":"AUTOMATIC","properties":["name","age","city"]}' || true + + while IFS=$'\t' read -r name age city; do + [[ "$name" == "name" ]] && continue + local payload code + payload="{\"label\":\"person\",\"properties\":{\"name\":\"${name}\",\"age\":${age},\"city\":\"${city}\"}}" + code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${GRAPH_API_BASE}/graph/vertices" \ + -H 'Content-Type: application/json' -d "$payload" || true) + if [[ "$code" =~ ^2 ]]; then + insert_ok=$((insert_ok + 1)) + if (( insert_ok % 25 == 0 )); then + report_line "$CLI_LOAD_LOG" "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] inserted=${insert_ok} failed=${insert_fail}" + fi + else + insert_fail=$((insert_fail + 1)) + report_line "$CLI_LOAD_LOG" "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] vertex_insert_failed name=${name} http=${code}" + fi + done < "$LOAD_DATA_FILE" + + report_line "$CLI_LOAD_LOG" "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] load_complete inserted=${insert_ok} failed=${insert_fail}" + + if (( insert_ok == 0 )); then + record_phase "graph-load" "FAIL" "no vertices inserted" + echo "ERROR: failed to insert any vertices" >&2 + return 1 + fi + + LOADED_VERTEX_COUNT="$insert_ok" + local loaded_count + loaded_count="$(graph_vertex_count)" + log " ✓ inserted ${insert_ok} vertices (count now = ${loaded_count})" + if [[ "$loaded_count" != "$insert_ok" ]]; then + record_phase "graph-load" "FAIL" "inserted=${insert_ok};failed=${insert_fail};vertex_count=${loaded_count}" + echo "ERROR: Gremlin vertex count (${loaded_count}) != successfully inserted (${insert_ok})" >&2 + return 1 + fi + record_phase "graph-load" "PASS" "inserted=${insert_ok};failed=${insert_fail};vertex_count=${loaded_count}" +} + +# Assert every bucket holds a consistent {CURRENT, MANIFEST, OPTIONS, SST} set. +# This deterministic check verifies metadata objects from ordered +# CURRENT/MANIFEST/OPTIONS mirroring ran, so the SSTs are no longer orphans. +verify_metadata_in_minio() { + local out rc + set +e + out=$(docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + rc=0 + for b in '"$S3_BUCKET_STORE0 $S3_BUCKET_STORE1 $S3_BUCKET_STORE2"'; do + sst=$(mc find local/$b --name "*.sst" 2>/dev/null | wc -l | tr -d " ") + cur=$(mc find local/$b --name "CURRENT" 2>/dev/null | wc -l | tr -d " ") + man=$(mc find local/$b --name "MANIFEST-*" 2>/dev/null | wc -l | tr -d " ") + opt=$(mc find local/$b --name "OPTIONS-*" 2>/dev/null | wc -l | tr -d " ") + printf "### %s: sst=%s CURRENT=%s MANIFEST=%s OPTIONS=%s\n" "$b" "$sst" "$cur" "$man" "$opt" + if [ "$cur" -lt 1 ] || [ "$man" -lt 1 ] || [ "$sst" -lt 1 ]; then + echo " ✗ recovery metadata/SST set incomplete for $b"; rc=1 + else + echo " ✓ consistent {CURRENT, MANIFEST, OPTIONS, SST} set present" + fi + done + exit $rc + ' 2>&1) + rc=$? + set -e + + printf "%s\n" "$out" | tee -a "$MINIO_REPORT" + return $rc +} + +# Stop a store, wipe its RocksDB state-machine data (db/ + metadata graph) while +# preserving raft/ and snapshot/, then start it again. This models local +# state-machine loss where the node still rejoins its raft group but must +# re-hydrate its RocksDB data from cloud on open (pre-hydration path). +# +# All three phases (stop / wipe / start) are fail-closed: any error aborts +# the recovery test rather than silently leaving local data in place. +wipe_store_rocksdb_state() { + local idx="$1" + local vol="${COMPOSE_PROJECT_NAME}_hg-store${idx}-data" + log " wiping store${idx} local RocksDB state (volume: ${vol})..." + + docker compose -f "$COMPOSE_FILE" stop "store${idx}" || { + echo "ERROR: failed to stop store${idx} before wipe" >&2; return 1 + } + + docker run --rm --entrypoint /bin/sh -v "${vol}:/s" "$MINIO_MC_IMAGE" -c ' + set -e + # Guard: raft/ must exist to confirm we have the correct volume mounted. + if ! ls /s/raft >/dev/null 2>&1; then + echo "ERROR: /s/raft absent – wrong volume or mount failed for store'"${idx}"'" >&2 + exit 1 + fi + cd /s + # Assert that at least one non-raft directory exists (there is data to wipe). + found_data=0 + for d in *; do + case "$d" in + raft|snapshot) ;; + *) found_data=1 ;; + esac + done + if [ "$found_data" -eq 0 ]; then + echo "WARNING: no non-raft directories found in store'"${idx}"' volume – nothing to wipe" >&2 + fi + for d in *; do + case "$d" in + raft|snapshot) ;; # keep raft log + snapshot so the node rejoins cleanly + *) rm -rf "$d" ;; # drop db/ and the metadata graph -> must come back from cloud + esac + done + echo " store'"${idx}"' storage after wipe: $(ls -1 | tr "\n" " ")" + # Verify the wipe: db/ must be gone. + if ls /s/db >/dev/null 2>&1; then + echo "ERROR: db/ still present after wipe of store'"${idx}"'" >&2 + exit 1 + fi + ' || { + echo "ERROR: volume wipe failed for store${idx}" >&2; return 1 + } + + docker compose -f "$COMPOSE_FILE" start "store${idx}" || { + echo "ERROR: failed to restart store${idx} after wipe" >&2; return 1 + } +} + +count_objects_in_prefix() { + local bucket="$1" + local prefix="$2" + # Normalize to exactly one trailing slash so the listing counts only objects strictly + # UNDER this directory. Without it, S3/mc prefix-matching also matches sibling keys that + # share the string stem — critically the tombstone written by onDBDeleted (e.g. dir + # db/00000 emptied by purge, sibling db/00000_DELETED left behind). That sibling would keep + # the count at 1 forever, so run_db_delete_callbacks_test could never observe count==0. + local norm="${prefix%/}/" + docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + target="local/'"$bucket"'/'"$norm"'" + if mc ls --recursive "$target" >/tmp/prefix_ls.txt 2>/dev/null; then + wc -l < /tmp/prefix_ls.txt | tr -d " " + else + echo 0 + fi + ' + } + +prefix_has_only_recovery_metadata() { + local bucket="$1" + local prefix="$2" + local norm="${prefix%/}/" + docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + target="local/'"$bucket"'/'"$norm"'" + if ! mc ls --recursive "$target" >/tmp/prefix_ls.txt 2>/dev/null; then + echo no + exit 0 + fi + + total=0 + current=0 + manifest=0 + options=0 + bad=0 + while IFS= read -r line; do + [ -n "$line" ] || continue + key=${line##* } + base=${key##*/} + total=$((total + 1)) + case "$base" in + CURRENT) current=$((current + 1)) ;; + MANIFEST-*) manifest=$((manifest + 1)) ;; + OPTIONS-*) options=$((options + 1)) ;; + *.sst|*probe*) bad=1 ;; + *) bad=1 ;; + esac + done < /tmp/prefix_ls.txt + + if [ "$total" -eq 3 ] && [ "$current" -eq 1 ] && [ "$manifest" -eq 1 ] \ + && [ "$options" -eq 1 ] && [ "$bad" -eq 0 ]; then + echo yes + else + echo no + fi + ' +} + +list_object_keys_in_prefix() { + local bucket="$1" + local prefix="$2" + local norm="${prefix%/}/" + docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + target="local/'"$bucket"'/'"$norm"'" + mc ls --recursive "$target" 2>/dev/null + ' | awk '{print $NF}' +} + +object_exists_exact() { + local bucket="$1" + local key="$2" + docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + mc stat "local/'"$bucket"'/'"$key"'" >/dev/null 2>&1 && echo yes || echo no + ' 2>/dev/null || echo "no" +} + +find_graph_db_prefix() { + local bucket="$1" + local graph_name="$2" + local candidate rel + + candidate=$(docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + mc find local/'"$bucket"' --name "CURRENT" 2>/dev/null + ' | grep "/${graph_name}/" | head -n1 || true) + + if [[ -z "$candidate" ]]; then + candidate=$(docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + mc find local/'"$bucket"' --name "*.sst" 2>/dev/null + ' | grep "/${graph_name}/" | head -n1 || true) + fi + + [[ -z "$candidate" ]] && return 1 + rel="${candidate#local/${bucket}/}" + + if [[ "$rel" == "${graph_name}/"* ]]; then + : + elif [[ "$rel" == *"/${graph_name}/"* ]]; then + rel="${rel#*/${graph_name}/}" + rel="${graph_name}/${rel}" + else + return 1 + fi + + if [[ "$rel" == */CURRENT ]]; then + echo "${rel%/CURRENT}/" + else + echo "${rel%/*}/" + fi +} + +extract_partition_ids() { + python3 -c 'import sys, json +try: + data = json.load(sys.stdin) +except Exception: + raise SystemExit(0) + +def from_partitions(obj, out): + if isinstance(obj, dict): + parts = obj.get("partitions") + if isinstance(parts, list): + for item in parts: + if isinstance(item, dict) and "id" in item: + value = str(item.get("id", "")).strip() + if value.isdigit(): + out.append(value) + for value in obj.values(): + from_partitions(value, out) + elif isinstance(obj, list): + for item in obj: + from_partitions(item, out) + +items = [] +from_partitions(data, items) +seen = set() +for value in items: + if value not in seen: + seen.add(value) + print(value)' +} + +list_partition_targets() { + local port payload ids part + for port in 8520 8521 8522; do + payload=$(curl -s "http://127.0.0.1:${port}/v1/partitions" 2>/dev/null || true) + [[ -z "$payload" ]] && continue + ids=$(printf "%s" "$payload" | extract_partition_ids 2>/dev/null || true) + for part in $ids; do + part="${part//[^0-9]/}" + [[ -n "$part" ]] && printf "%s|%s\n" "$port" "$part" + done + done +} + +find_partition_db_prefix() { + local db_name="$1" + local bucket candidate rel + for bucket in "$S3_BUCKET_STORE0" "$S3_BUCKET_STORE1" "$S3_BUCKET_STORE2"; do + candidate=$(docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + mc find local/'"$bucket"' --name CURRENT 2>/dev/null + ' | grep "/${db_name}/CURRENT$" | head -n1 || true) + [[ -z "$candidate" ]] && continue + rel="${candidate#local/${bucket}/}" + rel="${rel%/CURRENT}" + printf "%s|%s" "$bucket" "$rel" + return 0 + + # not reached + done + for bucket in "$S3_BUCKET_STORE0" "$S3_BUCKET_STORE1" "$S3_BUCKET_STORE2"; do + candidate=$(docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + mc find local/'"$bucket"' --name "*.sst" 2>/dev/null + ' | grep "/db/${db_name}/" | head -n1 || true) + [[ -z "$candidate" ]] && continue + rel="${candidate#local/${bucket}/}" + rel="${rel%/*}" + printf "%s|%s" "$bucket" "$rel" + return 0 + done + for bucket in "$S3_BUCKET_STORE0" "$S3_BUCKET_STORE1" "$S3_BUCKET_STORE2"; do + candidate=$(docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + mc find local/'"$bucket"' 2>/dev/null + ' | grep "/db/${db_name}/" | head -n1 || true) + [[ -z "$candidate" ]] && continue + rel="${candidate#local/${bucket}/}" + rel="${rel%/*}" + printf "%s|%s" "$bucket" "$rel" + return 0 + done + return 1 +} + +detect_scope_prefix() { + local bucket="$1" + local prefix + prefix=$(docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + mc ls local/'"$bucket"' 2>/dev/null + ' 2>/dev/null | awk 'NR==1 {print $NF}' | sed 's#/$##' || true) + prefix="${prefix//$'\r'/}" + prefix="${prefix//$'\n'/}" + if [[ -z "$prefix" ]]; then + prefix="hugegraph" + fi + printf "%s" "$prefix" +} + +derive_partition_db_prefix() { + local port="$1" + local db_name="$2" + local bucket store_name scope + case "$port" in + 8520) bucket="$S3_BUCKET_STORE0"; store_name="store0" ;; + 8521) bucket="$S3_BUCKET_STORE1"; store_name="store1" ;; + 8522) bucket="$S3_BUCKET_STORE2"; store_name="store2" ;; + *) return 1 ;; + esac + scope=$(detect_scope_prefix "$bucket") + printf "%s|%s/store-%s_8510/db/%s" "$bucket" "$scope" "$store_name" "$db_name" +} + +run_recovery_test() { + log "=== Total-loss recovery E2E ===" + local before after + + before=$(graph_vertex_count) + log "baseline vertex count = ${before}" + if [[ "$before" == "0" ]]; then + echo "ERROR: no data to recover (baseline is 0); load data first" >&2 + return 1 + fi + if [[ "$before" != "$LOADED_VERTEX_COUNT" ]]; then + echo "ERROR: baseline vertex count (${before}) != successfully inserted (${LOADED_VERTEX_COUNT})" >&2 + return 1 + fi + + log "forcing flush + compaction (restart stores) so data lands in SST files..." + docker restart cloud-storage-store0 cloud-storage-store1 cloud-storage-store2 >/dev/null + wait_svc "store0" 240; wait_svc "store1" 240; wait_svc "store2" 240 + wait_http "${GRAPH_API_BASE}/graph/vertices" 180 + + + log "verifying a consistent metadata set is durable in MinIO..." + verify_metadata_in_minio || { echo "ERROR: recovery metadata not durable in cloud" >&2; return 1; } + + log "simulating local RocksDB state loss on all stores (raft/ preserved)..." + for i in 0 1 2; do wipe_store_rocksdb_state "$i"; done + wait_svc "store0" 240; wait_svc "store1" 240; wait_svc "store2" 240 + wait_http "${GRAPH_API_BASE}/graph/vertices" 240 + + log "checking store logs for pre-hydration / restore-consistency..." + if docker compose -f "$COMPOSE_FILE" logs store0 store1 store2 2>/dev/null \ + | grep -qi "Cloud restore inconsistent"; then + echo "ERROR: consistent-restore guard tripped (CURRENT referenced a missing manifest)" >&2 + return 1 + fi + # Verify atomic pre-hydration actually fired: look for the real log message emitted + # by preHydrateDbFiles() after a successful download loop. + local hydration_lines + hydration_lines=$(docker compose -f "$COMPOSE_FILE" logs store0 store1 store2 2>/dev/null \ + | grep -i "Pre-hydration completed" | wc -l | tr -d " ") + if [[ "$hydration_lines" -gt 0 ]]; then + log " ✓ pre-hydration log confirmed on ${hydration_lines} store partition(s)" + else + log " WARNING: no 'Pre-hydration completed' log lines found — check store logs manually" + fi + + # Check that the atomic download left no stale .hyd-tmp files in any store volume. + # A leftover temp file indicates the atomic-move step failed or a previous run crashed. + log "checking for stale .hyd-tmp files in store volumes..." + local stale_tmp_found=0 + for i in 0 1 2; do + local vol="${COMPOSE_PROJECT_NAME}_hg-store${i}-data" + local count + count=$(docker run --rm --entrypoint /bin/sh -v "${vol}:/s" "$MINIO_MC_IMAGE" -c ' + mc find /s --name "*.hyd-tmp-*" 2>/dev/null | wc -l | tr -d " " + ' 2>/dev/null || echo "0") + count="${count//[^0-9]/}" + if [[ -n "$count" && "$count" -gt 0 ]]; then + echo "ERROR: ${count} stale .hyd-tmp file(s) found in store${i} volume after recovery" >&2 + stale_tmp_found=$((stale_tmp_found + count)) + fi + done + if [[ "$stale_tmp_found" -gt 0 ]]; then + echo "ERROR: stale .hyd-tmp files remain — atomic pre-hydration did not complete cleanly" >&2 + return 1 + fi + log " ✓ no stale .hyd-tmp files in any store volume" + + after=$(graph_vertex_count) + log "recovered vertex count = ${after} (baseline was ${before})" + if [[ "$after" == "$before" && "$after" == "$LOADED_VERTEX_COUNT" ]]; then + log "✓ RECOVERY SUCCESS: data fully recovered from cloud after local state loss" + else + echo "ERROR: recovery mismatch (before=${before}, after=${after}, inserted=${LOADED_VERTEX_COUNT})" >&2 + return 1 + fi + } + + # Verifies that when a DB is deleted the tombstone key is written as a sibling of the + # data prefix (e.g. store-X/hugegraph/db_DELETED) and NOT inside it + # (store-X/hugegraph/db/_DELETED). The sibling placement is critical: it ensures the + # tombstone survives a partial deletePrefix run and prevents stale-data re-hydration. + # + # This phase consumes the evidence of a REAL delete: run_db_recreation_no_orphan_test issues + # raftDelete and records the tombstone key it expects. The tombstone is removed again after a + # successful purge, so the positive assertion is: the sibling key was observed in MinIO during + # the delete, or the store logs show it was written under exactly that key. Fails if no delete + # was exercised, if no evidence of the sibling key exists, or if an inner-path key appears. + # Return codes: 0 = pass, 1 = fail. + run_tombstone_sibling_key_test() { + log "=== Tombstone sibling-key E2E ===" + local tombstone_key inside_key written_logs sibling_logged="no" inside_found + + if [[ -z "$DELETE_TOMBSTONE_KEY" || -z "$DELETE_TOMBSTONE_BUCKET" ]]; then + echo "ERROR: no delete was exercised before the tombstone check; run_db_recreation_no_orphan_test must run first" >&2 + return 1 + fi + tombstone_key="$DELETE_TOMBSTONE_KEY" + inside_key="$DELETE_TOMBSTONE_INNER_KEY" + log " expected tombstone key (sibling): ${DELETE_TOMBSTONE_BUCKET}/${tombstone_key}" + log " forbidden tombstone key (child): ${DELETE_TOMBSTONE_BUCKET}/${inside_key}" + + written_logs=$(docker compose -f "$COMPOSE_FILE" logs store0 store1 store2 2>/dev/null \ + | grep -i "Cloud DB tombstone written" || true) + if [[ -n "$written_logs" ]]; then + echo "$written_logs" | while IFS= read -r line; do log " ${line}"; done + # The old inner-path format must never appear. + if echo "$written_logs" | grep -q "/_DELETED"; then + echo "ERROR: tombstone log shows old inner-path format '/_DELETED'; expected sibling '_DELETED'" >&2 + return 1 + fi + # Positive: this exact sibling key was written. + if echo "$written_logs" | grep -qF "key=${tombstone_key}"; then + sibling_logged="yes" + fi + fi + + if [[ "$DELETE_TOMBSTONE_SEEN" != "yes" && "$sibling_logged" != "yes" ]]; then + echo "ERROR: sibling tombstone '${tombstone_key}' was neither observed in MinIO during the delete nor logged as written" >&2 + return 1 + fi + + # Negative: the inner-path key must not exist now. + inside_found=$(object_exists_exact "$DELETE_TOMBSTONE_BUCKET" "$inside_key") + if [[ "$inside_found" == "yes" ]]; then + echo "ERROR: tombstone written inside data prefix at '${inside_key}' — sibling-key fix not effective" >&2 + return 1 + fi + log " ✓ no tombstone found at inner path '${inside_key}'" + log "✓ TOMBSTONE SIBLING-KEY PASS: tombstone written at sibling '${tombstone_key}' (observed=${DELETE_TOMBSTONE_SEEN};logged=${sibling_logged}) and not inside data prefix" + } + + # Verifies the DURABILITY of a genuine DB deletion against total local-state loss. A partition + # DB is genuinely deleted (raftDelete -> onDBDeleteBegin writes the tombstone, onDBDeleted purges + # the cloud prefix once the session refCount reaches 0). We then wipe every store's local RocksDB + # state and restart, forcing cloud pre-hydration to run for all partitions. The deleted + # partition's data must NOT resurrect from cloud: its purged prefix must not regain the pre-delete + # data objects (SSTs / probe). A fresh empty re-created partition (CURRENT/MANIFEST/OPTIONS only) + # is acceptable. Partitions are single-replica here (peerList has one peer), so a genuine delete + # is not healed from a peer — the only way old data could return is stale cloud re-hydration, + # which the tombstone/empty-prefix guard must prevent. This is the recovery-time counterpart of + # run_db_delete_callbacks_test (which verifies the purge itself). + run_db_recreation_no_orphan_test() { + log "=== Post-delete no-orphan rehydration E2E ===" + local part_id part_port db_name db_bucket db_prefix prefix_match target first_target + local delete_resp purged="no" i count_now data_survivors + + # 1) Find a partition that currently has a cloud DB prefix with data. + part_id=""; part_port=""; db_bucket=""; db_prefix=""; first_target="" + while IFS= read -r target; do + [[ -z "$target" ]] && continue + [[ -z "$first_target" ]] && first_target="$target" + part_port="${target%%|*}"; part_id="${target##*|}" + db_name=$(printf "%05d" "$part_id") + prefix_match=$(find_partition_db_prefix "$db_name" || true) + if [[ -n "$prefix_match" ]]; then + db_bucket="${prefix_match%%|*}"; db_prefix="${prefix_match##*|}" + break + fi + part_id="" + done < <(list_partition_targets) + + if [[ -z "$part_id" || -z "$db_bucket" || -z "$db_prefix" ]]; then + echo "ERROR: no partition with a cloud DB prefix found for no-orphan test" >&2 + return 1 + fi + db_name=$(printf "%05d" "$part_id") + log " target partition id=${part_id} (db=${db_name}), prefix=${db_bucket}/${db_prefix}" + + # 2) Drop a probe object so cloud resurrection would be unmistakable. + docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + printf "no-orphan-probe\n" | mc pipe local/'"$db_bucket"'/'"$db_prefix"'/no-orphan-probe.txt >/dev/null + ' || { echo "ERROR: failed to write probe object" >&2; return 1; } + log " objects under prefix before delete: $(count_objects_in_prefix "$db_bucket" "$db_prefix")" + + # 3) Genuinely delete the partition DB. + delete_resp=$(curl -s "http://127.0.0.1:${part_port}/test/raftDelete/${part_id}" 2>/dev/null || true) + if [[ "$delete_resp" != *"OK"* ]]; then + echo "ERROR: raftDelete did not return OK for partition ${part_id} on :${part_port}" >&2 + return 1 + fi + log " raftDelete issued; waiting for cloud prefix purge (destroy watcher ticks ~60s)..." + + # 3b) The delete tombstone is written by onDBDeleteBegin as a SIBLING of the data prefix + # (_DELETED) and removed again once the prefix purge succeeds, so it is only + # observable between raftDelete and the purge. Record what is seen for the dedicated + # tombstone-sibling-key phase. + DELETE_TOMBSTONE_BUCKET="$db_bucket" + DELETE_TOMBSTONE_KEY="${db_prefix%/}_DELETED" + DELETE_TOMBSTONE_INNER_KEY="${db_prefix%/}/_DELETED" + DELETE_TOMBSTONE_SEEN="no" + for i in $(seq 1 15); do + if [[ "$(object_exists_exact "$db_bucket" "$DELETE_TOMBSTONE_KEY")" == "yes" ]]; then + DELETE_TOMBSTONE_SEEN="yes" + break + fi + done + log " sibling tombstone '${DELETE_TOMBSTONE_KEY}' observed in MinIO during delete: ${DELETE_TOMBSTONE_SEEN}" + + # 4) Wait for the purge to empty the prefix. + for i in $(seq 1 60); do + count_now=$(count_objects_in_prefix "$db_bucket" "$db_prefix" || echo "0") + count_now="${count_now//[^0-9]/}" + [[ "${count_now:-0}" -eq 0 ]] && { purged="yes"; break; } + sleep 2 + done + if [[ "$purged" != "yes" ]]; then + echo "ERROR: cloud prefix not purged after delete (still ${count_now} objects)" >&2 + return 1 + fi + log " ✓ cloud prefix purged after delete" + + # 5) Simulate total local RocksDB loss on every store, then restart. + log " simulating total local RocksDB loss on all stores + restart..." + for i in 0 1 2; do wipe_store_rocksdb_state "$i"; done + wait_svc "store0" 240; wait_svc "store1" 240; wait_svc "store2" 240 + wait_http "${GRAPH_API_BASE}/graph/vertices" 240 + sleep 5 + + # 6) The deleted partition's data must NOT resurrect: no pre-delete data object (SST/probe) + # may reappear under the purged prefix. Fresh empty-DB metadata is acceptable. + data_survivors=$(list_object_keys_in_prefix "$db_bucket" "$db_prefix" \ + | grep -Ec '\.sst$|probe' || true) + data_survivors="${data_survivors//[^0-9]/}" + + if [[ "${data_survivors:-0}" -eq 0 ]]; then + log "✓ POST-DELETE NO-ORPHAN SUCCESS: deleted partition data did not resurrect after total local loss" + else + echo "ERROR: ${data_survivors} deleted data object(s) resurrected under '${db_prefix}' after restart" >&2 + log " listing surviving objects:" + list_object_keys_in_prefix "$db_bucket" "$db_prefix" | head -20 >&2 + return 1 + fi + } + +run_db_delete_callbacks_test() { + log "=== DB delete callbacks (onDBDeleteBegin + onDBDeleted) E2E ===" + local part_id part_port db_name db_prefix db_bucket count_before count_after delete_resp + local target prefix_match candidate_port candidate_id first_target + local callback_seen="no" + local metadata_only_remaining="no" + local before_keys_file="" after_keys_file="" + local probe_remaining="0" surviving_old_count="0" + + part_id="" + part_port="" + db_bucket="" + db_prefix="" + first_target="" + while IFS= read -r target; do + [[ -z "$target" ]] && continue + if [[ -z "$first_target" ]]; then + first_target="$target" + fi + candidate_port="${target%%|*}" + candidate_id="${target##*|}" + db_name=$(printf "%05d" "$candidate_id") + prefix_match=$(find_partition_db_prefix "$db_name" || true) + if [[ -n "$prefix_match" ]]; then + part_port="$candidate_port" + part_id="$candidate_id" + db_bucket="${prefix_match%%|*}" + db_prefix="${prefix_match##*|}" + break + fi + done < <(list_partition_targets) + + if [[ -z "$part_id" && -n "$first_target" ]]; then + part_port="${first_target%%|*}" + part_id="${first_target##*|}" + db_name=$(printf "%05d" "$part_id") + prefix_match=$(derive_partition_db_prefix "$part_port" "$db_name" || true) + db_bucket="${prefix_match%%|*}" + db_prefix="${prefix_match##*|}" + log " WARNING: no existing cloud prefix found for db=${db_name}; using derived prefix bucket=${db_bucket}, prefix=${db_prefix}" + fi + + if [[ -z "$part_id" ]]; then + echo "ERROR: no partition with cloud prefix found from /v1/partitions on store0/store1/store2" >&2 + for port in 8520 8521 8522; do + local sample + sample=$(curl -s "http://127.0.0.1:${port}/v1/partitions" 2>/dev/null | head -c 300 || true) + [[ -n "$sample" ]] && log " store endpoint :${port} sample: ${sample}" + done + return 1 + fi + db_name=$(printf "%05d" "$part_id") + log " selected partition id=${part_id} from store endpoint :${part_port} (db=${db_name})" + + if [[ -z "$db_bucket" || -z "$db_prefix" ]]; then + echo "ERROR: failed to resolve target cloud bucket/prefix for db '${db_name}'" >&2 + return 1 + fi + + log " detected partition DB cloud prefix: bucket=${db_bucket}, prefix=${db_prefix}" + + log " writing deterministic probe object under '${db_prefix}'..." + docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + printf "db-delete-callback-probe\n" | mc pipe local/'"$db_bucket"'/'"$db_prefix"'/manual-db-delete-probe-$(date +%s).txt >/dev/null + ' || { + echo "ERROR: failed to write callback probe object" >&2 + return 1 + } + + count_before=$(count_objects_in_prefix "$db_bucket" "$db_prefix" || echo "0") + count_before="${count_before//[^0-9]/}" + if [[ -z "$count_before" || "$count_before" -eq 0 ]]; then + echo "ERROR: callback probe not visible before raftDelete for prefix '${db_prefix}'" >&2 + return 1 + fi + log " ✓ objects in partition prefix before delete: ${count_before}" + + before_keys_file=$(mktemp) + list_object_keys_in_prefix "$db_bucket" "$db_prefix" | sort -u > "$before_keys_file" + + delete_resp=$(curl -s "http://127.0.0.1:${part_port}/test/raftDelete/${part_id}" 2>/dev/null || true) + log " raftDelete response: ${delete_resp}" + if [[ "$delete_resp" != *"OK"* ]]; then + echo "ERROR: raftDelete did not return OK for partition '${part_id}' on endpoint :${part_port}" >&2 + return 1 + fi + + # onDBDeleted is async (RocksDB destroy watcher runs every 60s). + # Poll object count as authoritative pass/fail; check log markers as supporting evidence. + local poll_start_ts + poll_start_ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + for i in $(seq 1 36); do + local callback_log + # Use --since + --tail to avoid dumping full log history on every iteration. + callback_log=$( + docker logs --since "$poll_start_ts" --tail 500 cloud-storage-store0 2>&1 + docker logs --since "$poll_start_ts" --tail 500 cloud-storage-store1 2>&1 + docker logs --since "$poll_start_ts" --tail 500 cloud-storage-store2 2>&1 + ) + if echo "$callback_log" | grep -q "Cloud DB tombstone written: db=${db_name}," \ + && echo "$callback_log" | grep -q "Cloud DB purge completed: db=${db_name},"; then + callback_seen="yes" + fi + count_after=$(count_objects_in_prefix "$db_bucket" "$db_prefix" || echo "0") + count_after="${count_after//[^0-9]/}" + if [[ "$count_after" == "0" || -z "$count_after" ]]; then + break + fi + sleep 2 + done + + if [[ "$callback_seen" == "yes" ]]; then + log " ✓ observed callback log markers for db=${db_name}" + else + log " WARNING: callback log markers not seen for db=${db_name} — may have fired before polling started or partition had no session" + log " checking recent logs for any tombstone/purge evidence..." + docker logs --tail 200 cloud-storage-store0 2>&1 | grep -E "db=${db_name}" | tail -10 || true + docker logs --tail 200 cloud-storage-store1 2>&1 | grep -E "db=${db_name}" | tail -10 || true + docker logs --tail 200 cloud-storage-store2 2>&1 | grep -E "db=${db_name}" | tail -10 || true + fi + + log " objects in partition prefix after delete: ${count_after}" + if [[ "$count_after" != "0" && -n "$count_after" && -n "$before_keys_file" ]]; then + after_keys_file=$(mktemp) + list_object_keys_in_prefix "$db_bucket" "$db_prefix" | sort -u > "$after_keys_file" + surviving_old_count=$(comm -12 "$before_keys_file" "$after_keys_file" | sed '/^$/d' | wc -l | tr -d " ") + fi + + if [[ "$count_after" == "3" ]]; then + metadata_only_remaining=$(prefix_has_only_recovery_metadata "$db_bucket" "$db_prefix" || echo "no") + if [[ "$metadata_only_remaining" == "yes" && "$surviving_old_count" == "0" ]]; then + log " ✓ remaining 3 objects are recovery metadata only " + log " (CURRENT + MANIFEST-* + OPTIONS-*); no probe/SST leftovers" + count_after="0" + elif [[ "$metadata_only_remaining" == "yes" ]]; then + log " metadata-only remainder detected, but old object keys survived" + log " (treating as failure to avoid false pass on incomplete purge)" + fi + fi + + if [[ "$count_after" != "0" && -n "$count_after" ]]; then + probe_remaining=$(docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + mc find local/'"$db_bucket"'/'"$db_prefix"' --name "manual-db-delete-probe-*.txt" 2>/dev/null | wc -l | tr -d " " + ' 2>/dev/null || echo "0") + probe_remaining="${probe_remaining//[^0-9]/}" + + if [[ "$probe_remaining" == "0" && "$surviving_old_count" == "0" ]]; then + log " ✓ probe removed and all pre-delete objects purged" + log " remaining objects appear to be newly created after delete " + log " (likely immediate partition re-create under same prefix)" + count_after="0" + fi + fi + + if [[ "$count_after" != "0" && -n "$count_after" ]]; then + echo "ERROR: partition cloud prefix not purged after callbacks (remaining=${count_after})" >&2 + docker run --rm --entrypoint /bin/sh --network "$NETWORK" "$MINIO_MC_IMAGE" -c ' + mc alias set local http://minio:9000 '"$MINIO_ROOT_USER $MINIO_ROOT_PASSWORD"' >/dev/null 2>&1 + mc ls --recursive local/'"$db_bucket"'/'"$db_prefix"' + ' | head -30 || true + rm -f "$before_keys_file" "$after_keys_file" >/dev/null 2>&1 || true + return 1 + fi + + rm -f "$before_keys_file" "$after_keys_file" >/dev/null 2>&1 || true + log "✓ DB DELETE CALLBACKS SUCCESS: onDBDeleteBegin + onDBDeleted observed and prefix purged" +} + +need_cmd docker curl python3 +init_reports +report_event "bootstrap" "initializing stack and reports" +log "pulling images..." +ensure_image "$MINIO_IMAGE" +ensure_image "$MINIO_MC_IMAGE" +ensure_image "$HG_PD_IMAGE" +ensure_image "$HG_STORE_IMAGE" +ensure_image "$HG_SERVER_IMAGE" +mkdir -p "$GENERATED_DIR" +mkdir -p "$SERVER_GRAPHS_DIR" +# Same HStore graph config the static docker-compose.yml mounts (backend=hstore, pd.peers=pd:8686). +cp "${STACK_DIR}/conf/graphs/hugegraph.properties" "$SERVER_GRAPH_CONF" +mkdir -p "$(dirname "$COMPOSE_FILE")" +docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true +log "generating docker-compose.yml..." +cat > "$COMPOSE_FILE" << 'YAML' +services: + minio: + image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z + container_name: cloud-storage-minio + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: ["9000:9000", "9001:9001"] + volumes: [hg-minio-data:/data] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9000/minio/health/live >/dev/null || exit 1"] + interval: 5s + timeout: 5s + retries: 40 + pd: + build: + context: ../../../ + dockerfile: docker/cloud-storage/images/pd.Dockerfile + image: hugegraph/pd:cloud-storage-local + container_name: cloud-storage-pd + hostname: pd + depends_on: + minio: + condition: service_healthy + environment: + HG_PD_GRPC_HOST: pd + HG_PD_GRPC_PORT: "8686" + HG_PD_REST_PORT: "8620" + HG_PD_RAFT_ADDRESS: pd:8610 + HG_PD_RAFT_PEERS_LIST: pd:8610 + HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 + HG_PD_INITIAL_STORE_COUNT: "3" + HG_PD_DATA_PATH: /hugegraph-pd/pd_data + ports: ["8620:8620", "8686:8686"] + volumes: [hg-pd-data:/hugegraph-pd/pd_data] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8620/v1/health >/dev/null || exit 1"] + interval: 10s + timeout: 5s + retries: 30 + store0: + build: + context: ../../../ + dockerfile: docker/cloud-storage/images/store.Dockerfile + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store0 + hostname: store0 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store0 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store0:8510 + HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_CLOUD_STORAGE_ENABLED: "true" + HG_CLOUD_STORAGE_BUCKET: "hugegraph-store0" + HG_CLOUD_STORAGE_ENDPOINT: "http://minio:9000" + HG_CLOUD_STORAGE_REGION: "us-east-1" + HG_CLOUD_STORAGE_ACCESS_KEY: "minioadmin" + HG_CLOUD_STORAGE_SECRET_KEY: "minioadmin" + ports: ["8520:8520"] + volumes: [hg-store0-data:/hugegraph-store/storage] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 10s + timeout: 10s + retries: 40 + start_period: 60s + store1: + build: + context: ../../../ + dockerfile: docker/cloud-storage/images/store.Dockerfile + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store1 + hostname: store1 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store1 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store1:8510 + HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_CLOUD_STORAGE_ENABLED: "true" + HG_CLOUD_STORAGE_BUCKET: "hugegraph-store1" + HG_CLOUD_STORAGE_ENDPOINT: "http://minio:9000" + HG_CLOUD_STORAGE_REGION: "us-east-1" + HG_CLOUD_STORAGE_ACCESS_KEY: "minioadmin" + HG_CLOUD_STORAGE_SECRET_KEY: "minioadmin" + ports: ["8521:8520"] + volumes: [hg-store1-data:/hugegraph-store/storage] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 10s + timeout: 10s + retries: 40 + start_period: 60s + store2: + build: + context: ../../../ + dockerfile: docker/cloud-storage/images/store.Dockerfile + image: hugegraph/store:cloud-storage-local + container_name: cloud-storage-store2 + hostname: store2 + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store2 + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store2:8510 + HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_CLOUD_STORAGE_ENABLED: "true" + HG_CLOUD_STORAGE_BUCKET: "hugegraph-store2" + HG_CLOUD_STORAGE_ENDPOINT: "http://minio:9000" + HG_CLOUD_STORAGE_REGION: "us-east-1" + HG_CLOUD_STORAGE_ACCESS_KEY: "minioadmin" + HG_CLOUD_STORAGE_SECRET_KEY: "minioadmin" + ports: ["8522:8520"] + volumes: [hg-store2-data:/hugegraph-store/storage] + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] + interval: 10s + timeout: 10s + retries: 40 + start_period: 60s + server: + image: hugegraph/server:1.7.0 + container_name: cloud-storage-server + hostname: server + depends_on: + store0: + condition: service_healthy + store1: + condition: service_healthy + store2: + condition: service_healthy + environment: + STORE_REST: store0:8520 + ports: ["8080:8080"] + volumes: + - ${SERVER_GRAPHS_DIR}:/hugegraph-server/conf/graphs + networks: [hg-net] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/versions >/dev/null || exit 1"] + interval: 10s + timeout: 5s + retries: 40 + start_period: 60s +networks: + hg-net: + driver: bridge +volumes: + hg-minio-data: + hg-pd-data: + hg-store0-data: + hg-store1-data: + hg-store2-data: +YAML +prepare_artifacts +log "building local cloud-storage images..." +BUILD_LOG="${GENERATED_DIR}/build.log" +# Build pd + a single store target only. store0/store1/store2 share the same +# Dockerfile and image tag, and concurrent multi-service build can race on tag export. +if ! docker compose -f "$COMPOSE_FILE" build --no-cache pd store0 > "$BUILD_LOG" 2>&1; then + echo "ERROR: docker compose build failed; see ${BUILD_LOG} for details" >&2 + grep -E "(Error|error|FAILED|failed)" "$BUILD_LOG" | head -30 >&2 || true + exit 1 +fi +grep -E "^(Building|FINISHED|\[|Successfully|Warning)" "$BUILD_LOG" || true +log " ✓ images built successfully" +log "starting minio and pd first..." +docker compose -f "$COMPOSE_FILE" up -d minio pd +wait_svc "minio" 120 +wait_svc "pd" 180 +NETWORK="${COMPOSE_PROJECT_NAME}_hg-net" +log "creating MinIO buckets before store startup..." +ensure_minio_buckets "$NETWORK" +log "starting stores and server..." +docker compose -f "$COMPOSE_FILE" up -d store0 store1 store2 server +log "waiting for stores..." +wait_svc "store0" 180 +wait_svc "store1" 180 +wait_svc "store2" 180 +wait_svc "server" 180 +log "waiting for graph backend..." +wait_http "$GRAPH_API_BASE/graph/vertices" 60 +INFRA_READY="true" +record_phase "infra-health" "PASS" "minio,pd,store0,store1,store2,server healthy" +log "✓ SUCCESS: Cloud storage infrastructure ready" + +if [[ "$SKIP_SMOKE_TESTS" == "true" ]]; then + SMOKE_STATUS="SKIPPED" + record_phase "smoke-tests" "SKIPPED" "SKIP_SMOKE_TESTS=true" + log "SKIP_SMOKE_TESTS=true -> skipping data creation and validation phases" + log "✓ SUCCESS: infrastructure-only mode complete" + exit 0 +fi + +SMOKE_STATUS="RUNNING" +if ! load_test_data; then + SMOKE_STATUS="FAILED" + exit 1 +fi + +if run_recovery_test; then + RECOVERY_STATUS="PASS" + record_phase "recovery-test" "PASS" "vertex count recovered after local state loss" +else + RECOVERY_STATUS="FAIL" + record_phase "recovery-test" "FAIL" "recovery workflow failed" + SMOKE_STATUS="FAILED" + exit 1 +fi + +if run_db_recreation_no_orphan_test; then + RECREATE_STATUS="PASS" + record_phase "db-recreation-no-orphan" "PASS" "deleted partition data did not resurrect after total local loss" +else + RECREATE_STATUS="FAIL" + record_phase "db-recreation-no-orphan" "FAIL" "deleted partition data resurrected from cloud" + SMOKE_STATUS="FAILED" + exit 1 +fi + +if run_tombstone_sibling_key_test; then + TOMBSTONE_STATUS="PASS" + record_phase "tombstone-sibling-key" "PASS" "tombstone written at sibling key ${DELETE_TOMBSTONE_KEY}, not inner path" +else + TOMBSTONE_STATUS="FAIL" + record_phase "tombstone-sibling-key" "FAIL" "tombstone sibling-key check failed" + SMOKE_STATUS="FAILED" + exit 1 +fi + +if run_db_delete_callbacks_test; then + DELETE_CALLBACK_STATUS="PASS" + record_phase "db-delete-callbacks" "PASS" "onDBDeleteBegin and onDBDeleted observed with cloud prefix purge" +else + DELETE_CALLBACK_STATUS="FAIL" + record_phase "db-delete-callbacks" "FAIL" "callback verification failed" + SMOKE_STATUS="FAILED" + exit 1 +fi + +SMOKE_STATUS="PASS" +record_phase "smoke-tests" "PASS" "all cloud storage E2E tests passed" +log "✓ SUCCESS: all cloud storage E2E tests passed" diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/ClusterConstant.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/ClusterConstant.java index e1bad5bf36..cc461bd7ec 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/ClusterConstant.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/base/ClusterConstant.java @@ -37,6 +37,7 @@ public class ClusterConstant { public static final String PD_JAR_PREFIX = "hg-pd-service"; public static final String STORE_PACKAGE_PREFIX = "apache-hugegraph-store"; public static final String STORE_JAR_PREFIX = "hg-store-node"; + public static final String ROCKSDB_JAR_PREFIX = "rocksdbjni-"; public static final String SERVER_PACKAGE_PREFIX = "apache-hugegraph-server"; public static final String CT_PACKAGE_PREFIX = "apache-hugegraph-ct"; public static final String APPLICATION_FILE = "application.yml"; diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java index ed7225f5a6..05ddfd2ba0 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java @@ -22,6 +22,7 @@ import static org.apache.hugegraph.ct.base.ClusterConstant.JVM_MODULE_OPTIONS_FILE; import static org.apache.hugegraph.ct.base.ClusterConstant.JAVA_CMD; import static org.apache.hugegraph.ct.base.ClusterConstant.LOG4J_FILE; +import static org.apache.hugegraph.ct.base.ClusterConstant.ROCKSDB_JAR_PREFIX; import static org.apache.hugegraph.ct.base.ClusterConstant.STORE_JAR_PREFIX; import static org.apache.hugegraph.ct.base.ClusterConstant.STORE_LIB_PATH; import static org.apache.hugegraph.ct.base.ClusterConstant.STORE_TEMPLATE_PATH; @@ -70,6 +71,13 @@ public void start() { .resolve(BIN_DIR).resolve(JVM_MODULE_OPTIONS_FILE))); String storeNodeJarPath = getFileInDir(workPath, STORE_JAR_PREFIX); + // Mirror start-hugegraph-store.sh: RocksDB JNI event callbacks run on native + // threads and resolve org.rocksdb.* via the system classloader, so rocksdbjni + // must be on the launcher classpath, not only inside BOOT-INF/lib. + String rocksdbJarPath = getFileInDir(workPath, ROCKSDB_JAR_PREFIX); + if (rocksdbJarPath.isEmpty()) { + throw new AssertionError("No rocksdbjni jar found in " + workPath); + } startCmd.addAll(Arrays.asList( "-Dname=HugeGraphStore" + this.index, "-Dlog4j.configurationFile=" + configPath + CONF_DIR @@ -84,7 +92,8 @@ public void start() { "-XX:HeapDumpPath=" + configPath + "logs", "-Dspring.config.location=" + configPath + CONF_DIR + File.separator + "application.yml", - "-jar", storeNodeJarPath)); + "-cp", rocksdbJarPath + File.pathSeparator + storeNodeJarPath, + "org.springframework.boot.loader.JarLauncher")); ProcessBuilder processBuilder = runCmd(startCmd, stdoutFile); this.instance = processBuilder.start(); } catch (IOException ex) { diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh index dffc44324e..5b0db15ce3 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh @@ -488,7 +488,7 @@ assert required_modules(pd_job) == { assert_order(store_job, [ "mvn clean package", "mvn editorconfig:check -pl hugegraph-store/hg-store-test -am -ntp", - "-Dtest=StoredRowIngressTest,ArthasSecurityTest", + "-Dtest='org/apache/hugegraph/store/node/**/*Test,StoredRowIngressTest'", "-P store-common-test -Djacoco.sessionId=store-common-test", "-P store-client-test -Djacoco.sessionId=store-client-test", "-P store-rocksdb-test -Djacoco.sessionId=store-rocksdb-test", diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-launcher-gc.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-launcher-gc.sh index 4d277596fc..66b54539dd 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-launcher-gc.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-launcher-gc.sh @@ -62,6 +62,8 @@ for component in server pd store; do launcher=start-hugegraph-store.sh rm -f "$FIXTURE/lib/"*.jar touch "$FIXTURE/lib/hg-store-node-test.jar" + # The store launcher requires one rocksdbjni jar for native callback classloading. + touch "$FIXTURE/lib/rocksdbjni-test.jar" ;; esac cp "$source_bin/$launcher" "$source_bin/util.sh" "$source_bin/jvm-module.options" "$FIXTURE/bin/" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh index 6c30369c61..031c0a2180 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh @@ -155,6 +155,8 @@ check_gc_options() { # Make default memory selection independent of host memory and OS. printf '%s\n' 'calc_xmx() { echo 1024; }' >> "$fixture/bin/util.sh" touch "$fixture/lib/hg-store-node-test.jar" + # The store launcher requires one rocksdbjni jar for native callback classloading. + touch "$fixture/lib/rocksdbjni-test.jar" cat > "$fixture/jdk/bin/java" <<'JAVA' #!/bin/bash if [[ "$1" == "-version" ]]; then diff --git a/hugegraph-store/README.md b/hugegraph-store/README.md index 60c5f0792b..12a11e9be4 100644 --- a/hugegraph-store/README.md +++ b/hugegraph-store/README.md @@ -126,6 +126,21 @@ unchanged. To run the Store server tests through the package phase: mvn clean package -pl hugegraph-store/hg-store-test -am -Pstore-server-test ``` +#### Enabling S3 on the stock distribution + +The S3 cloud storage provider is opt-in and is not bundled by default. To include it, add +`-Ps3-cloud-storage` to the build; it bundles `hg-store-cloud-s3` and the AWS SDK into the +`hg-store-node` executable JAR: + +```bash +mvn clean package -pl hugegraph-store/hg-store-dist -am -DskipTests -Ps3-cloud-storage +``` + +Then set `cloud.storage.enabled: true` and the `cloud.storage.s3.*` options in `conf/application.yml`. +The stock distribution does not scan a `plugins/` directory, so a provider jar cannot be dropped in +at runtime. See [Pluggable Cloud Storage](docs/pluggable-cloud-storage-architecture.md) for the +options, packaging details and operations guidance. + ### Configuration Extract the distribution package and edit `conf/application.yml`: @@ -444,6 +459,7 @@ Comprehensive documentation for HugeGraph Store: | [Integration Guide](docs/integration-guide.md) | Integrating Store with HugeGraph Server, client API usage, migrating from other backends | | [Query Engine](docs/query-engine.md) | Query pushdown mechanisms, multi-partition queries, gRPC API reference | | [Operations Guide](docs/operations-guide.md) | Monitoring and metrics, troubleshooting common issues, backup and recovery, rolling upgrades | +| [Pluggable Cloud Storage](docs/pluggable-cloud-storage-architecture.md) | Optional S3/object-storage mirroring of SSTs and metadata: configuration, packaging, observability and recovery | | [Best Practices](docs/best-practices.md) | Hardware sizing, performance tuning, security configuration, high availability design | | [Development Guide](docs/development-guide.md) | Development environment setup, module architecture, testing strategies, contribution workflow | diff --git a/hugegraph-store/docs/operations-guide.md b/hugegraph-store/docs/operations-guide.md index aafd8fa424..73bbfb505d 100644 --- a/hugegraph-store/docs/operations-guide.md +++ b/hugegraph-store/docs/operations-guide.md @@ -301,6 +301,14 @@ curl http://localhost:8620/v1/partitionsAndStats 4. **Wait**: Rebalancing is gradual (may take hours for large datasets) +**Note: disk space after a partition moves off a node**: When a partition leaves a Store node, its local +RocksDB is destroyed only once no reads or writes still hold it. The Store checks every 60 seconds. If +references are still held after 30 minutes, it force-releases them and deletes the directory on the next +check. Until then the DB keeps its RocksDB `LOCK` and disk space, and the Store logs +`DB has not been deleted refCount is `. Earlier releases checked these conditions in the wrong +order: they force-released live references on the first check and deleted the directory one to two +minutes after the destroy. The timing is the same with or without cloud storage. + --- ### Issue 3: Data Migration Slow diff --git a/hugegraph-store/docs/pluggable-cloud-storage-architecture.md b/hugegraph-store/docs/pluggable-cloud-storage-architecture.md new file mode 100644 index 0000000000..cb00110c79 --- /dev/null +++ b/hugegraph-store/docs/pluggable-cloud-storage-architecture.md @@ -0,0 +1,1120 @@ +# Pluggable Cloud Storage Architecture (HStore) + +This document explains: + +- The existing HStore workflow (without cloud storage) +- The new pluggable cloud storage workflow (cloud-backed SST lifecycle) +- Write path and read path behavior +- Failure handling and operational recovery scenarios + +## Overview + +HStore keeps partition data in local RocksDB on each Store node. The pluggable cloud-storage +feature extends this by mirroring SST file lifecycle events (create/delete) to an external +object store and hydrating missing files back when needed (startup and read-miss). + +- Existing path: local RocksDB is the primary online read/write path. +- New extension: cloud object storage acts as a pluggable SST mirror and recovery source. +- Provider model: SPI provider selected once at startup via `ServiceLoader` and injected into its consumers. + +## Architecture Diagram + +```text ++---------------------------+ +| Client Layer | +| Gremlin/REST/Cypher | ++-------------|-------------+ + v ++---------------------------+ +----------------+ +| HugeGraph Server |-->| PD Cluster | ++-------------|-------------+ +----------------+ + v ++----------------------------------------------+ +| Store Cluster (Raft replication) | +| | +| +-----------------------------------------+ | +| | Raft log + MemTable -> Local RocksDB SST| | +| +--------------------+--------------------+ | +| | | +| v | +| +-----------------------------------------+ | +| | Pluggable Cloud Storage Workflow (NEW) | | +| | CloudStorageEventListener (NEW) | | +| | -> CloudStorageProvider (NEW) | | +| +-----------------------------------------+ | ++----------------------|-----------------------+ + v + +-------------------------------+ + | Cloud Storage (NEW) | + | +-------------+ +----+ +----+ | + | |S3 compatible| |ADLS| |GCP | | + | +-------------+ +----+ +----+ | + +-------------------------------+ + +``` + +- `Client Layer`: External clients issuing Gremlin/REST/Cypher read-write requests. +- `HugeGraph Server`: API/query layer that routes graph requests to PD and Store. +- `PD Cluster`: Placement/metadata control plane (partition mapping, leader scheduling). +- `Store Cluster`: Raft-based data plane where writes are replicated and persisted. +- `Raft log + MemTable -> Local RocksDB SST`: Local write pipeline in each Store node. The Raft log is the durability source for recent writes (the RocksDB WAL is disabled under Raft); the MemTable is flushed/compacted into SSTs. +- `Pluggable Cloud Storage Workflow (NEW)`: SST lifecycle hook path for upload/delete/download/list. +- `CloudStorageEventListener (NEW)`: Captures RocksDB file events and triggers cloud operations. +- `CloudStorageProvider (NEW)`: Provider implementation (`s3`, etc.) used for object operations. +- `Cloud Storage (NEW)`: Remote object backend options (S3 compatible, ADSL, GCP). + + +## New Configuration Options + +The pluggable cloud-storage behavior is controlled from `application.yml` under +`cloud.storage`. + +| Configuration | Default | Description | +|------------------------------------------------------|-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `cloud.storage.enabled` | `false` | Enables/disables cloud storage integration. | +| `cloud.storage.provider` | `s3` | Active provider name. Must match `CloudStorageProvider#providerName()`. | +| `cloud.storage.path-prefix` | `hugegraph` | Prefix prepended to all remote object keys. | +| `cloud.storage.startup-hydration-enabled` | `true` | Downloads missing remote files on DB opening before normal serving. | +| `cloud.storage.read-miss-guard-window-ms` | `3000` | Guard window to throttle repeated read-miss hydration attempts per db/table. Values `<= 0` disable throttling. | +| `cloud.storage.upload-retry-max-attempts` | `3` | Whole-file retry attempts after a first upload failure. Default `3` retries are enabled to protect against transient network errors.
Set to `0` to disable whole-file retries (failures are given up on immediately) when the provider already has sufficient internal retry logic. `CloudStorageNonRetryableException` always bypasses retries and is given up on immediately regardless of this value. | +| `cloud.storage.upload-retry-initial-delay-ms` | `1000` | Delay before first whole-file retry; subsequent retries use exponential backoff. Only used when `upload-retry-max-attempts > 0`. | +| `cloud.storage.upload-retry-max-delay-ms` | `60000` | Upper bound for exponential backoff delay between whole-file retry attempts. Only used when `upload-retry-max-attempts > 0`. | +| `cloud.storage.metadata-sync-debounce-ms` | `1000` | Interval at which DBs with SST uploads since their last metadata publish are drained: each such DB gets at most one coalesced publish per interval, and a failed publish is retried on the next interval. Shutdown flushes pending DBs once. `<= 0` publishes on every SST. Event-driven publishes (delete guard, compaction, DB open) are **never** coalesced. | +| `cloud.storage.node-id` | _(blank)_ | Stable per-node identity used to derive the cloud key scope (`store-`). When set, it is authoritative and stable across restarts, IP/hostname changes, **and** local disk loss, so recovery always finds prior remote objects — recommended for production (e.g. a StatefulSet pod ordinal or provisioned node UUID). Blank falls back to a scope persisted in the data root (`.cloud-store-scope`), which is stable across network-identity drift but lost with the disk, and is in turn seeded from the runtime network address on first start. See "Cloud key scope resolution" below. | + +Notes: + +- Keep `path-prefix` stable across restarts to preserve object-key continuity. +- If `provider` is not found on classpath, initialization fails fast in `AppConfig`. +- Upload retry uses a two-layer model: S3 part-level retries (inside one `uploadFile()` call) are handled by the provider; whole-file retries are handled by `CloudUploadRetryQueue` when `upload-retry-max-attempts > 0` (default `3`). +- `CloudStorageNonRetryableException` thrown by a provider bypasses immediate retries. The file remains unconfirmed and is automatically retried on the next compaction via the delete guard. +- Failed uploads are tracked via metrics (`cloud_storage_upload_failures_total`) and structured logs. Automatic retry occurs through the whole-file retry queue, the deferred SST-delete retry, and startup SST backfill; no manual recovery needed. +- Provider-specific properties (`cloud.storage..*`) are bound only when `cloud.storage.enabled=true`; with cloud storage disabled they are never read, so a disabled node needs no bucket or credential settings. Credential placeholders in `application.yml` use empty defaults (for example `${AWS_ACCESS_KEY_ID:}`), so an unset variable resolves to blank and falls back to the AWS default credentials chain instead of failing placeholder resolution. +- Metadata sync publishes a consistent `CURRENT`/`MANIFEST`/`OPTIONS` snapshot so a full-disk-loss node can reopen from cloud. Capture forces a MemTable flush so the un-flushed tail is persisted into an SST and mirrored (the RocksDB WAL is disabled under Raft — the Raft log is the tail's durability source — so a flush is the only path that pushes recent writes to cloud). +- Metadata sync is always enabled when cloud storage is enabled. +- Metadata sync is triggered by storage events. Each SST upload marks its DB dirty, and a fixed-delay drain (`metadata-sync-debounce-ms`) publishes each dirty DB once per interval; event-driven publishes (delete guard, DB open) are never coalesced. +- Compaction completion (`onCompacted`) only marks the DB metadata-dirty; the publish runs on the single `cloud-metadata-sync` thread. `onTableFileDeleted` only queues the superseded SST; the live-set check, the metadata publish and the remote delete run on the same thread, in publish-before-delete order, with one metadata publish per DB for all deletes queued at that point. A failed publish backs off exponentially **per DB** (base `max(1s, metadata-sync-debounce-ms)`, doubling per consecutive failure, capped at 5 minutes) instead of redoing a checkpoint + upload every interval; a success clears the backoff. The backoff covers both the drain and held superseded-SST deletes, so the 500 ms delete retry does not re-publish a DB whose publishes are failing. +- **Publish before close.** When a DB is released (a partition moving away, or `releaseAllGraphDB()` at shutdown), RocksDB listeners get `onDBClosing` while the session is still open, and a DB with an unpublished metadata change is published then. At shutdown the store engine releases every DB before the cloud listener is closed, so a publish deferred to `close()` could no longer capture the DB. A DB being deleted is not published. +- The whole-file retry queue dedupes by `(db, remoteKey)`: a publish that re-checks an already-queued upload does not enqueue a second retry for the same object. +- Uploads that exhaust whole-file retries are given up on: logged at ERROR, counted in `cloud_storage_uploads_given_up_total`, and re-uploaded by startup SST backfill. +- Set a stable `cloud.storage.node-id` for production so the cloud key scope survives IP/hostname changes and local disk loss (see "Cloud key scope resolution"). +- Provider-specific properties (e.g., bucket, region, credentials) are configured under `cloud.storage..*` namespace. +- **Operational observability**: Monitor metrics and logs to track sync progress (see "Observability: Metrics & Logs" section above). + +### Metadata capture, the sync window, and RPO + +> **The RocksDB WAL is disabled in HStore.** The state-machine write path calls +> `dbSession.setDisableWAL(true)` (`BusinessHandlerImpl`, comment: *"raft mode, disable rocksdb +> log"*) because the **Raft log** — replicated across the partition's peers — is the durability +> source for recent writes, not the RocksDB WAL. RocksDB `*.log` segments therefore stay empty, and +> cloud storage never mirrors a WAL; the discussion below is about how recent writes reach cloud and +> how current the cloud copy is. + +#### How recent writes reach cloud (force-flush-and-mirror) + +Each metadata capture uses RocksDB `Checkpoint.createCheckpoint`, which flushes the MemTable, +persisting the un-flushed tail into a new L0 SST that is then mirrored. Because the RocksDB WAL is +disabled, this forced flush is the **only** mechanism that proactively pushes recent +(still-in-MemTable) writes into cloud. When the MemTable is empty the flush is a no-op. + +- *Cloud RPO for the recent tail:* bounded by the metadata-sync cadence + (`metadata-sync-debounce-ms`). +- *Cost:* a small extra L0 SST per non-empty capture (write amplification + cloud object churn + + compaction pressure), whose frequency is bounded by the drain interval. The Raft log is the + tail's durability source for any single/partial-node loss regardless of cloud cadence. + +#### Is metadata-sync coalescing a data-loss window? + +No uploaded SST bytes are ever lost to coalescing. In `SstUploadTask.run()` the SST object is +uploaded to cloud **first** (the code notes *"the SST is already durable in cloud"*); only the +subsequent `CURRENT`/`MANIFEST` **pointer publish** is coalesced. So the window is a **bounded +metadata / recovery-point lag, not a byte-loss hole**: + +- The newest uploaded SST(s) may exist in cloud but not yet be *referenced* by the mirrored MANIFEST + for about one `metadata-sync-debounce-ms` interval (default 1 s), longer only while publishes + fail. The dirty DB stays marked until a publish succeeds, so the final state is always published + even if writes then stop; `onDBCreated`, the delete guard and DB release (`onDBClosing`) + publish without coalescing. +- This lag becomes observable data loss **only** under a total, simultaneous loss of *all* Raft + peers' local state followed by a cloud-only restore that lands inside that ~1 s window — + then the newest unreferenced SSTs are orphans not in the restored set. +- For any lesser failure (single / partial node loss) the **Raft log** holds the committed tail and + is the primary recovery source, so the cloud metadata lag is irrelevant. Restore is also + fail-closed consistent (the "Cloud restore inconsistent" guard never restores a `CURRENT` that + references a missing MANIFEST — worst case an older but fully consistent point). + +To tighten the cloud recovery point — at the cost of more frequent metadata publishes and forced +flushes — lower `metadata-sync-debounce-ms` (`0` = publish on every SST). + +### S3 provider-specific options (`cloud.storage.s3.*`) + +For provider `s3`, configure these properties under `cloud.storage.s3`: + +| Configuration | Default | Description | +|---------------------------------------------------------|-----------|------------------------------------------------------------------------------------------------------| +| `cloud.storage.s3.bucket` | _(none)_ | Target S3 bucket name. Required when S3 provider is enabled. | +| `cloud.storage.s3.region` | _(none)_ | AWS region (for example `us-east-1`). Blank with `endpoint` set defaults to `us-east-1` (signing region only). Blank with no `endpoint` falls back to the AWS region chain (`AWS_REGION`, `aws.region`, profile, instance metadata); if none resolves, startup fails with an error naming `cloud.storage.s3.region`. | +| `cloud.storage.s3.endpoint` | _(none)_ | Optional custom endpoint for S3-compatible stores (MinIO/Ceph). | +| `cloud.storage.s3.access-key` | _(none)_ | Access key / AWS Access ID credential. Omit or leave blank to use AWS default credentials chain. | +| `cloud.storage.s3.secret-key` | _(none)_ | Secret key credential. Omit or leave blank to use AWS default credentials chain. | + +Request and response checksums use the AWS SDK `WHEN_REQUIRED` mode, so checksums are computed or +validated only for operations that mandate them, which keeps S3-compatible stores that reject the +SDK's default trailer checksums working. + +### Sample `application.yml` (`cloud.storage`) + +If you want to configure Store directly through `application.yml` (instead of env injection), +use a snippet like this: + +```yaml +cloud: + storage: + enabled: true + provider: s3 + path-prefix: hugegraph + + # Hydration controls + startup-hydration-enabled: true + read-miss-guard-window-ms: 3000 + + # Upload retry (whole-file level; common for all providers) + upload-retry-max-attempts: 3 # 0 to disable whole-file retries (give up immediately) + upload-retry-initial-delay-ms: 1000 + upload-retry-max-delay-ms: 60000 + + # Metadata durability (CURRENT/MANIFEST/OPTIONS) + # Metadata sync is always enabled when cloud storage is enabled + metadata-sync-debounce-ms: 1000 # coalesced per-SST metadata sync interval; <= 0 = publish on every SST + + # Stable per-node identity for the cloud key scope (store-). Blank => persist one in the + # data root (seeded from the network address). Set a deployment-stable value (e.g. StatefulSet + # ordinal / provisioned UUID) for guaranteed recovery after address change or local disk loss. + node-id: + + # S3 provider-specific configuration (cloud.storage.s3.*) + s3: + bucket: hugegraph-store0 + region: us-east-1 + endpoint: http://minio:9000 + access-key: minioadmin + secret-key: minioadmin +``` + +Notes: + +- `upload-retry-max-attempts` defaults to `3` (immediate retries enabled after first failure). Set to `0` only if the provider has sufficient built-in retry logic. Automatic retry also occurs on subsequent compactions via the delete guard, so failed uploads are never silently lost. +- In this docker stack, each Store node uses its own bucket (`hugegraph-store0/1/2`). +- For local MinIO, endpoint is typically `http://minio:9000` inside the docker network. +- Provider-specific properties are configured under `cloud.storage..*` namespace. +- **Monitor these metrics**: `cloud_storage_confirmed_files_total` (per-DB gauge; should track the live SST count), `cloud_storage_upload_failures_total` (should stay flat), `cloud_storage_sync_latency_ms_seconds` (the Prometheus name of the sync-latency timer; should be sub-second for typical files). + + +## 3) Write Path + +Two sub-flows exist: **SST upload** (on flush/compaction) and **SST delete** (on compaction cleanup). + +### 3a) SST Upload Flow + +```text +SST UPLOAD (onTableFileCreated) + +Client + | + | 1) Write request + v +HugeGraph Server + | + | 2) Route to partition leader + v +Store Node (RocksDB) + | + | 3) Raft log append + MemTable update (RocksDB WAL disabled) + | 4) Flush/compaction creates *.sst + v +CloudStorageEventListener + | + | 5) onTableFileCreated(db, cf, file) + | 6) toRelativeKey(file) -> / + | 7) uploadFile(localPath, remoteKey) + v +CloudStorageProvider + | + | 8) PUT object (full key: //) + v +Object Storage + | + | 9) ACK + v +CloudStorageEventListener + | + | 10) syncTracker.markConfirmed(dbName, fileNumber) <- bitmap updated +``` + +### 3b) SST Delete Flow + +```text +SST DELETE (onTableFileDeleted) — queued on the RocksDB callback thread, processed on cloud-metadata-sync + +RocksDB compaction completes: SST1 + SST2 -> MERGED_SST + | + | 1) onTableFileDeleted(db, cf, filePath=SST1 or SST2) + | -> queue the delete, schedule processDeferredSstDeletes; no cloud I/O on this thread + v +CloudStorageEventListener (cloud-metadata-sync thread), once per DB per pass: + | + | 2) ensureLiveSetUploaded(provider, dbName) + | -> syncTracker.allConfirmed(dbName, liveFiles) <- single lock, short-circuit on first miss + | If NOT all confirmed: + | -> for each unconfirmed live file: + | upload now (idempotent PUT, no existence probe) + | syncTracker.markConfirmed on success + | If any live file still not durable: return false -> skip delete (hold old SST) + | + | 3) [live set confirmed durable] + | syncMetadataSnapshotInline(dbName) <- publish MANIFEST/CURRENT BEFORE delete + | -> RocksDBFactory.captureMetadataSnapshot() + | -> uploadMetadataSnapshot(...) + | * verifies all manifest-referenced SSTs are in cloud + | * uploads OPTIONS / MANIFEST / CURRENT atomically + | If snapshot fails: return false -> skip delete (hold old SST) + | + | 4) [MANIFEST published] provider.deleteFile(remoteKey) <- for every queued SST of the DB + v +Object Storage: old SST(s) deleted + +Any "hold" above keeps the delete queued; it is retried on the +cloud-metadata-sync thread every 500 ms. Deletes still queued when the +listener closes are not persisted: the superseded objects stay in cloud +as unreferenced orphans (no data loss). +``` + +### Write-path notes + +- On DB creation (`onDBCreated`), existing local SST files are scanned and uploaded if missing in cloud. + A MemTable flush is triggered via `onDBCreated` (event-driven, not a background async task) so + in-memory data (recovered by replaying the Raft log on open) materializes into SST files and gets + mirrored to cloud. +- Remote object key is always scoped per store node: + **`//`** e.g. `hugegraph/store-127.0.0.1_8501/0/000001.sst`. + `store-scope-prefix` is `store-`, resolved at startup by + `AppConfig.resolveStableStoreScopePrefix()` (see "Cloud key scope resolution" below). + This guarantees key uniqueness across nodes even when multiple Store nodes share the same bucket. +- After every successful upload, `syncTracker.markConfirmed(dbName, fileNumber)` sets the corresponding + bit in a per-`dbName` in-memory bitmap (`CloudSyncTracker`). This bitmap is the fast path used by the + delete guard (`allConfirmed`) to avoid per-file cloud API calls on every delete pass. +- The delete guard (`ensureLiveSetUploaded`) checks the **entire current live SST set**, not just the file + being deleted. Reason: the old SST being confirmed says nothing about whether its compaction outputs + (replacement files) are durable. Only when every live file is confirmed does the delete proceed. +- MANIFEST/CURRENT is published to cloud **before** the old SST is deleted, so that a recovery attempt + always has a consistent metadata + data state to restore from. + +- The native RocksDB event listener is attached to `DBOptions` only when at least one registered `RocksdbChangedListener` reports `managesRemoteState()` (the cloud listener does). Nodes without cloud storage keep the pre-cloud behaviour. Only four native callbacks are enabled: `ON_COMPACTION_BEGIN`, `ON_COMPACTION_COMPLETED`, `ON_TABLE_FILE_CREATED` and `ON_TABLE_FILE_DELETED`, so other event kinds never cross the JNI boundary. +- Destroy timing changed for every node, with or without cloud storage. A destroyed DB that still has live references now waits up to 30 minutes before its references are force-released and the directory is deleted. Earlier releases did this one to two minutes after the destroy. See [Operations Guide](operations-guide.md#issue-2-partition-imbalance) for details. + +### Cloud key scope resolution + +The `store-scope-prefix` that isolates each node's objects is resolved by +`AppConfig.resolveStableStoreScopePrefix()` in precedence order, most-durable first: + +1. **Configured `cloud.storage.node-id`** — authoritative. Lives in deployment config, so it is stable + across restarts, IP/hostname changes, **and** local disk loss. Recovery always finds prior objects. + Recommended for production (e.g. StatefulSet pod ordinal, provisioned node UUID). When set, it is + also written to the persisted marker so a later removal of the config value still resolves the same scope. +2. **Persisted marker** `/.cloud-store-scope` — written on first start. Keeps the + scope stable across network-identity (IP/hostname) drift, but is **lost with the disk**. +3. **Runtime network address** — legacy fallback used to *seed* the marker on first start (keeps upgrades + migration-safe: an existing deployment already keyed by `store-` resolves to the same scope). + A `WARN` is logged advising an explicit `node-id` for durable recovery, since a later address change + would otherwise read a different scope. + +## 4) Read Path + +Two hydration modes exist: + +1. **Startup pre-hydration (`onDBOpening`)**: download missing remote files before serving, + then seed the sync-tracker bitmap from the remote listing so the first post-restart compaction + can use the fast-path bitmap check instead of issuing per-file cloud API calls. +2. **Read-miss hydration (`onReadMiss`)**: if a local read fails because a live SST is missing on disk, + download the missing SST back to its original path and retry the read once. A read that returns no + value is a genuine key-not-found (RocksDB raises an error when a file it needs is missing), so it + does not trigger hydration. + +```text +READ PATH (ANSI) + +Read request + | + | 1) get(key) + v +RocksDB + | + | 2) read error (missing live SST) -> onReadMiss(session, table, key) + v +CloudStorageEventListener + | + | 3) listFiles(//) + v +CloudStorageProvider + | + | 4) LIST objects + v +Object Storage + | + | 5) return key list (scoped to this store node's prefix) + v +CloudStorageProvider + | + | 6) for each missing *.sst: + | downloadFile(remoteKey, localPath) + | GET object -> receive object bytes + v +CloudStorageEventListener + | + | 7) ingestSstFile(downloaded) + v +RocksDB + | + | 8) retry get(key) + v +Read request result +``` + +### Startup pre-hydration detail (`onDBOpening`) + +```text +onDBOpening(dbName) + | + | 0) pending-delete marker present for this prefix? + | -> up to 3 bounded synchronous cleanup attempts (tryCompletePendingDelete) + | -> still failing: throw IllegalStateException("Cloud open blocked ..."), + | async purge retry re-armed. Runs even with startup-hydration-enabled=false. + | + | 1) listFiles(///) + | -> remoteFiles (scoped to this store node) + | + | 2) for each remoteFile not present locally: + | downloadFile(remoteKey, localPath) + | + | 3) [after download loop] + | for each *.sst in remoteFiles: + | syncTracker.markConfirmed(dbName, fileNumber) <- bitmap seeded from remote listing + | zero extra cloud API calls +``` + +### Read-path notes + +- **Open blocked by a pending delete.** If a pending-delete marker exists for the DB's prefix, `onDBOpening` makes three bounded synchronous cleanup attempts. If the purge still cannot complete, the open fails with `IllegalStateException("Cloud open blocked ...")` and the asynchronous retry is re-armed; opening is never allowed to proceed over a prefix that still holds the deleted generation. Cleanup is complete only once both the prefix purge and the deletion of the sibling tombstone succeed; until then the marker stays and the open stays blocked, because a tombstone left next to a re-created DB would make a later open purge that DB's live objects. This check runs even when `startup-hydration-enabled=false`. +- **Leftover tombstone at open.** If a tombstone exists without a local marker (for example the marker was lost with the disk), `onDBOpening` purges the prefix and deletes the tombstone before the DB can hydrate from or upload into it, and blocks the open if either step fails. This also runs when `startup-hydration-enabled=false`; in that mode a failed tombstone check is logged and the open proceeds, since nothing is read from cloud. Every successful purge also drops the purged DB's sync-tracker confirmations, so the next metadata publish re-uploads the SSTs it references. +- **Version-suffixed directories.** A partition can move to a version-suffixed directory (`_` or `__`), and its objects then live under that prefix. When the directory chosen for a DB holds no local DB (no `CURRENT`), the listener lists the remote siblings that share the base name (one `listFiles(/)` call, a plain key prefix) and the DB is opened at the newest restorable one, in the same version order `RocksDBSession` uses locally. A sibling is restorable when it has a remote `CURRENT`, no `_DELETED` tombstone and no local pending-delete marker. The redirect only applies with `startup-hydration-enabled=true`, only to a sibling of the chosen directory that is not awaiting deletion, and never when a local DB exists. If the listing fails, the chosen directory is kept and its hydration applies the usual failure rules. +- **Sweeps on open.** Leftover `_cloudmeta_*` metadata checkpoint directories (from a crash during a snapshot capture) and `.hyd-tmp-*` partial hydration download files are deleted when the DB is opened. +- All `listFiles` calls use the **`//`** namespace so each store node + only sees and downloads its own SST files, even when multiple nodes share the same bucket. +- A guard window (`read-miss-guard-window-ms`) throttles repeated hydration attempts for the same db/table pair. +- Only missing local SST files are downloaded. +- Non-SST objects are ignored for read-miss ingestion. +- After startup pre-hydration, `syncTracker` is fully seeded from the remote listing. Subsequent + compaction delete guards use the in-memory bitmap (`allConfirmed`) and do **not** issue + `provider.fileExists()` round-trips per delete. + +### Scope: managed delete vs accidental local loss + +- Current behavior intentionally treats a missing local DB directory as a **recovery** case and hydrates from cloud when remote objects exist. +- Intentional delete/reset is recognized through the managed deletion flow (`RocksDBFactory.destroyGraphDB()` -> `onDBDeleted`). +- **Anti-resurrection guard (implemented).** On managed delete, `onDBDeleted`: + 1. **First** persists a local *pending-delete marker* — this is the durable guard. If the marker cannot be durably written, the delete is **held** (`deleteMarkerHealthy` flips to `0`) rather than risking a re-hydration after a crash. + 2. Writes a sibling *tombstone* object (`//_DELETED`) alongside the data prefix, then purges the remote DB prefix (SSTs, metadata, tombstone). + - If the provider is unavailable, the purge/tombstone is deferred but the local pending-delete marker still blocks re-hydration (`onDBOpening`/`preHydrateDbFiles` detect the deleted generation and skip hydration), and an async retry completes the tombstone + purge once a provider returns. +- **Epoch-guarded purge retries.** The asynchronous pending-delete retry is tied to a per-prefix epoch captured when the retry is scheduled and re-checked under the per-prefix lock. If the prefix is opened or created in the meantime, the retry stands down rather than purging objects that now belong to the new DB. A retry scheduled at startup uses the RocksDB session name, which is the DB directory name without its `_` suffix, so an open of that DB invalidates it. +- Store tracks the last successfully published RocksDB **generation** per DB, so a re-create during a provider outage still cannot re-hydrate the deleted generation. +- If a local DB directory is removed **outside** the managed flow, no delete callback is emitted; hydration may still restore data from cloud for the same DB path — this remains an accepted scope tradeoff prioritizing accidental local-loss recovery. + +## 5) Failure Handling + +### Upload failures (write-critical) + +#### Two-layer retry model + +Upload failures use a two-layer retry strategy: + +| Layer | Where | What it retries | Config keys | +|----------------------------|----------------------------------------------|--------------------------------------------|---------------------------------------------------------------------------------------------------------| +| **Request-level** (S3 only) | AWS SDK client inside `S3CloudStorageProvider.uploadFile()` | Individual PUTs and multipart parts | None (SDK `STANDARD` retry mode, see `buildRetryOverride()`) | +| **File-level** (common) | `CloudUploadRetryQueue` (async) | Whole SST file after `uploadFile()` throws | `cloud.storage.upload-retry-max-attempts`, `upload-retry-initial-delay-ms`, `upload-retry-max-delay-ms` | + +The default `upload-retry-max-attempts=3` enables whole-file retries for transient failures. The S3 provider also relies on the AWS SDK's own retries for individual requests and multipart parts; only the file-level layer is configurable. + +#### Failure flow (default `upload-retry-max-attempts=3`) + +- `onTableFileCreated` calls `provider.uploadFile()`. +- The AWS SDK retries individual requests and multipart parts internally. +- If `uploadFile()` still throws a regular `IOException`, `CloudUploadRetryQueue.submit()` enqueues the task for up to `upload-retry-max-attempts` whole-file retries with exponential backoff. +- If `uploadFile()` throws `CloudStorageNonRetryableException`, the task bypasses retries and skips the upload queue. +- After all retry attempts are exhausted (or if non-retryable), the file remains unconfirmed in the bitmap. The file stays on disk and is retried by the delete guard (`ensureLiveSetUploaded`), which runs in the deferred SST-delete pass and on every metadata publish. + +#### Observability: Metrics & Logs + +The system provides real-time observability through metrics and structured logs: + +**Metrics (exposed to Prometheus/monitoring):** + +Names below are the Prometheus exposition names (Micrometer Prometheus naming: counters end in `_total`, timers export in seconds). + +- `cloud_storage_confirmed_files_total` (gauge, label `db_name`) — Number of SST files confirmed present in cloud for that DB (the bitmap size). Despite the `_total` suffix it is a gauge and can go down after compaction. A value well below the DB's live SST count means uploads are failing or slow. +- `cloud_storage_upload_failures_total` (counter, no labels) — Total upload failures (transient + permanent) across all DBs. Tracks upload problem frequency; use logs to identify the DB or file. +- `cloud_storage_retry_queue_size` (gauge) — Files waiting in the upload retry queue. Bound to the live queue (`retryQueue.getInFlightCount()`), so it reflects the real backlog rather than a constant `0`. +- `cloud_storage_uploads_given_up_total` (counter) — Uploads the retry queue gave up on (retries exhausted, `CloudStorageNonRetryableException`, or shutdown). Each file stays unconfirmed in the sync tracker and is re-uploaded by the next metadata sync, the delete guard, or startup SST backfill. Alert on a sustained increase. +- `cloud_storage_sync_latency_ms_seconds` (summary, no labels; series `{quantile="0.5|0.95|0.99"}`, `_count`, `_sum`, `_max`) — Time from SST file creation (`onTableFileCreated`) to bitmap confirmation (`markConfirmed`). The timer is registered as `cloud_storage_sync_latency_ms`; the Prometheus registry appends the `_seconds` base unit, so values are in seconds. +- `cloud_storage_delete_guard_reupload_count` (gauge, label `db_name`) — Cumulative number of files re-uploaded by the delete guard for that DB when the live set was not fully durable. Use `increase(...)` over a window; a rising value indicates frequent upload failures. +- `cloud_storage_delete_marker_healthy` (gauge, `1`=healthy / `0`=degraded) — Health of pending-delete marker persistence. `0` means a marker could not be durably written, so a DB delete whose remote cleanup fails may not be guarded against re-hydration after a crash. This is a hard error state: **delete progression is held while degraded.** Alert on `0`. + +**Structured Logs:** + +Log entries provide visibility into sync progress and failures: + +- **Startup (bitmap seeding)**: `"Seeded sync-tracker bitmap with N confirmed files from remote listing: dbName=..."` + - Indicates: bitmap is warm after restart, delete guard can use fast path. + +- **Delete guard checking**: `"Checking live set durability: dbName=..., liveFileCount=..., unconfirmedCount=..."` + - Indicates: delete guard evaluated M files, found K unconfirmed. + +- **Delete guard re-upload**: `"Re-uploading M unconfirmed live files (not yet confirmed in cloud): dbName=..., files=[...], reason=live_set_not_durable"` + - Indicates: files failed to sync on first attempt, being re-attempted now. + +- **Delete guard re-upload success**: `"Successfully confirmed L previously unconfirmed files; delete proceeding: dbName=..., oldSstFile=..."` + - Indicates: re-upload succeeded, old SST can now be safely deleted. + +- **Delete guard skipped**: `"Delete skipped (live set not fully durable in cloud): dbName=..., oldSstFile=..., unconfirmedFiles=[...], nextRetryAt=..."` + - Indicates: at least one live file is still not durable. Delete is held. Retry will occur on next compaction. + +- **Upload failure (first attempt)**: `"Upload failed (will retry): dbName=..., cfName=..., filePath=..., attempt=1/3, error=..."` + - Indicates: transient failure, will be retried. + +- **Upload failure (exhausted retries)**: `"Upload failed permanently after 3 retries: dbName=..., cfName=..., filePath=..., error=..., nextRetryOnCompaction=..."` + - Indicates: all retries exhausted. File stays on disk; will retry automatically on next compaction. + +**Example: Finding unsynced files** + +Query metrics in your monitoring system: + +```promql +# Delete-guard re-uploads per database over the last hour (live set was not durable) +increase(cloud_storage_delete_guard_reupload_count[1h]) > 0 + +# Uploads abandoned by the retry queue (re-uploaded later by backfill / delete guard) +increase(cloud_storage_uploads_given_up_total[1h]) > 0 +``` + +Or search logs: + +```bash +# Find all delete-guard re-uploads in the past hour +kubectl logs -l app=hugegraph-store --since=1h | grep "Re-uploading.*unconfirmed" + +# Find all permanently-failed uploads +kubectl logs -l app=hugegraph-store | grep "Upload failed permanently" + +# Track a specific file +kubectl logs -l app=hugegraph-store | grep "filePath=/path/to/000123.sst" +``` + +**Giving up on an upload:** + +- Uploads that exhaust all whole-file retries (or are thrown as `CloudStorageNonRetryableException`) are given up on: logged at ERROR, counted in `cloud_storage_uploads_given_up_total`, and their staging hard-link is released. There is no dead-letter queue; nothing retries a given-up upload. +- Given-up files are **not** lost: they stay recoverable via the delete guard and startup SST backfill. After a restart, `onDBCreated` re-runs the startup SST backfill, which uploads every live `*.sst` missing from the bucket, and orphaned staging hard-links under `/.cloud-upload-staging/` are deleted when the listener starts. + +**Automatic retry (delete guard, no manual replay):** + +- Files that fail to upload remain unconfirmed in the bitmap. +- On the next deferred delete pass for the same DB (triggered by a compaction deleting an SST, retried every 500 ms while held), the delete guard calls `ensureLiveSetUploaded`, which: + 1. Checks bitmap for all current live files + 2. For any unconfirmed file, attempts upload again (idempotent PUT) + 3. On success, updates bitmap; on failure, logs and holds delete +- This cycle repeats automatically; no manual replay needed. +- After a retry upload becomes durable, `onRetryUploadDurable` publishes `CURRENT`/`MANIFEST` so the mirrored recovery point advances even on an idle DB. +- Upload success rate and retry frequency are tracked via metrics and logs. + +**Tuning:** + +To control retry behavior: + +```yaml +cloud.storage.upload-retry-max-attempts: 3 # Immediate retries on first failure +cloud.storage.upload-retry-initial-delay-ms: 1000 # Start backoff at 1s +cloud.storage.upload-retry-max-delay-ms: 60000 # Cap backoff at 60s +``` + +If the provider handles all retry logic internally, disable whole-file retries: + +```yaml +cloud.storage.upload-retry-max-attempts: 0 # No immediate retries; rely on delete guard auto-retry +``` + +### Delete failures (non-fatal) + +- A failed superseded-SST delete is logged and stays queued for retry; processing continues. +- Impact is stale/orphaned cloud objects, not immediate read/write unavailability. + +### Hydration failures + +- Pre-hydration/list/download failures throw `IllegalStateException` and stop the flow for that DB open attempt. +- Read-miss hydration failures are logged and return `false`; the original read error propagates. + +### Provider lifecycle / config failures + +- Unknown provider name or missing plugin JAR fails initialization in `AppConfig`. +- **Resolved once, never reconfigured.** `AppConfig` selects the provider via `ServiceLoader`, calls `init()` once, and passes it to `CloudUploadRetryQueue` and `CloudStorageEventListener` as a final, non-null field. When `cloud.storage.enabled=false`, neither is constructed and no provider is loaded. +- **Failed `@PostConstruct` cleanup.** If cloud-storage wiring throws during startup, `AppConfig` closes the retry queue and the provider so a failed init does not leak the provider's client/threads (Spring does not invoke `@PreDestroy` when `@PostConstruct` throws), then fails startup. +- **`listFiles` and `deletePrefix` are required.** `CloudStorageProvider` has no default implementations for them: every provider must list keys and must throw an `IOException` from `deletePrefix()` if any object under the prefix could not be deleted, so callers such as the DB-purge path can detect an incomplete purge and keep the tombstone + local marker. + +### Non-retryable upload failures (`CloudStorageNonRetryableException`) + +- Providers can signal a permanently failed upload by throwing `CloudStorageNonRetryableException` from `uploadFile()`. +- This exception bypasses the whole-file retry loop and marks the file as unconfirmed (no bitmap entry). +- The file remains on disk and is retried by the delete guard's re-upload logic. +- The S3 provider throws it for permanent service-side errors (e.g. auth, permission, not-found, validation), so a request that cannot succeed is not retried as a whole file. +- Metrics (`cloud_storage_upload_failures_total`) and logs track the failure for operational visibility. + +### Delete guard failure (live set not fully durable) + +- Before deleting a superseded SST from cloud, the queued-delete pass (`processDeferredSstDeletes`) verifies the entire current live SST set is confirmed present in cloud (`ensureLiveSetUploaded`). +- If any live SST cannot be confirmed durable (e.g. a prior upload failed), the delete is held with a warning log and stays queued. +- The unconfirmed file remains on disk. The deferred delete is retried every 500 ms on the `cloud-metadata-sync` thread, and each attempt re-attempts upload for all unconfirmed files. +- **Observability**: Metrics `cloud_storage_delete_guard_reupload_count` (per DB) and `cloud_storage_confirmed_files_total` track the frequency and sync state. Logs show which files are unconfirmed and why the delete was held. +- **Impact**: Orphaned (but safe) old object may remain in cloud temporarily. Data integrity is preserved because the replacement files are not yet confirmed durable. + +### Metadata snapshot capture failure + +- `syncMetadataSnapshotInline` calls `RocksDBFactory.captureMetadataSnapshot()`. If this returns `null` (e.g. the DB is not open or the checkpoint could not be acquired), the metadata sync is skipped and returns `false`. +- When called from the deferred SST-delete pass, a `null` snapshot causes the delete to be held with a warning log, preserving the old SST in cloud until a valid snapshot can be published. + +### Metadata publish failure (SST durability check fails) + +- `uploadMetadataSnapshot` first ensures all manifest-referenced SSTs are present in cloud. If any SST cannot be made durable, the method aborts before uploading `MANIFEST` or `CURRENT`, returns `false`, and logs a warning. +- The previous durable generation (prior `CURRENT`/`MANIFEST`) is left intact in cloud, so a recovery attempt still has a valid consistent state to restore from. +- An `IOException` during upload of `OPTIONS`/`MANIFEST`/`CURRENT` also aborts the publish with a warning. + +### S3 multipart upload failures + +- Files larger than 512 MB are uploaded through `S3TransferManager`, which splits them into parts, uploads them concurrently and completes the multipart upload, or aborts it on failure. Part retries come from the AWS SDK's retry strategy; there is no provider-specific part-retry setting. +- If the upload still fails, `uploadFile()` throws an `IOException` (or `CloudStorageNonRetryableException` for permanent service errors), and the whole-file retry queue handles it as for any other upload. +- If the abort itself fails, the incomplete multipart upload can remain in the bucket (incurring storage cost). The provider does not sweep for these; configure an S3 lifecycle rule (`AbortIncompleteMultipartUpload`) on the bucket to remove them. + + +### Local file compacted away during retry + +- When a local SST file is compacted into a new merged SST (e.g., SST1 → MERGED_SST), the old SST1 may still be unconfirmed in the bitmap if its upload failed. +- On the next deferred delete pass involving MERGED_SST, the delete guard checks if MERGED_SST is confirmed. If not, it re-attempts upload. +- If SST1 was compacted away, its local file no longer exists, so no retry is attempted for SST1 itself. +- However, the data from SST1 is now present in MERGED_SST in cloud (or will be on next attempt). Once MERGED_SST is confirmed, SST1 can be safely deleted from cloud. +- Metrics track `cloud_storage_delete_guard_reupload_count` to monitor how often the delete guard needs to re-upload files. + +## 6) Recovery Scenarios + +Failure-mode and RPO-oriented recovery summary: + +| Failure scenario | Data loss? | RPO | Recovery mechanism | Mitigation | +|-----------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Single Store crash | No | 0s | 2/3 Raft quorum survives. Leader re-election continues service. In-flight (not SST-flushed) data is recovered from Raft logs; flushed data remains available via local/cloud SSTs. | Keep 3+ replicas across zones, alert on replica loss, and use durable PV-backed store nodes. | +| 2 of 3 Stores crash | No (service stalls until quorum restored) | 0s | Surviving replica Raft log bootstraps recovering nodes after restart. In-flight data is recovered from Raft log replay. | Enforce anti-affinity and failure-domain isolation to prevent correlated failures. | +| Catastrophic loss: all Store disks destroyed | Yes — writes committed after the last cloud metadata publish (`CURRENT`/`MANIFEST`) are unrecoverable because both the Raft log (the tail's durability source) and the local SSTs are gone. | Seconds to minutes (bounded by flush/compaction cadence + async upload latency + the metadata-sync window) | Nodes recover from the last cloud-mirrored `CURRENT`/`MANIFEST` + SST set. **Scope re-seed risk:** `.cloud-store-scope` is also lost; the node falls back to the runtime address to derive its cloud key scope. If the address changed (new VM, rescheduled pod), the node silently scopes to a wrong prefix and starts empty. Set `cloud.storage.node-id` to a stable value so the scope is config-driven and survives address changes (see "Cloud key scope resolution"). | Set `cloud.storage.node-id` (StatefulSet ordinal or node UUID). Use durable disks, scheduled backups, and a lower `metadata-sync-debounce-ms` for a smaller recovery point.| + +Notes: + +- In-flight data (not yet flushed to SST) is recovered from Raft logs when quorum-replicated. +- Cloud storage recovery primarily protects flushed SST state and disaster cases involving local disk loss. +- SST upload is always asynchronous and event-triggered: each `onTableFileCreated` (flush/compaction) + eagerly uploads the new SST on the shared upload executor, with failures routed to the retry queue. + There is no periodic/interval uploader and no synchronous-upload mode. The cloud recovery point + is bounded by how soon the SSTs upload and the subsequent coalesced `CURRENT`/`MANIFEST` publish + lands (`metadata-sync-debounce-ms`). +- Recovery correctness is protected by the metadata-before-delete invariant: before deleting superseded SSTs, Store confirms manifest-referenced SST durability and publishes updated `MANIFEST`/`CURRENT`, preventing stale or missing-manifest references after compaction retries. +- Policy note: for how managed delete vs accidental local-loss is distinguished during hydration, see `## 4) Read Path` -> `Scope: managed delete vs accidental local loss`. + +## 7) Operational Notes + +### Monitoring & Alerting + +Cloud storage health is exposed through metrics and logs. Set up monitoring for: + +**Alerts (suggest these thresholds):** + +```yaml +# Alert if the delete guard keeps re-uploading files for a database (uploads are not sticking) +- alert: CloudStorageDeleteGuardReuploads + expr: increase(cloud_storage_delete_guard_reupload_count[15m]) > 0 + for: 15m + annotations: + summary: "Store node {{ $labels.instance }} delete guard is re-uploading files in {{ $labels.db_name }}" + action: "Check upload logs for permission errors or network issues" + +# Alert if the retry queue is giving up on uploads +- alert: CloudStorageUploadsGivenUp + expr: increase(cloud_storage_uploads_given_up_total[15m]) > 0 + annotations: + summary: "Store node {{ $labels.instance }} gave up on {{ $value }} uploads" + action: "Given-up files are re-uploaded by backfill / delete guard; check provider status and credentials" + +# Alert if upload failures are increasing (transient or persistent failures) +- alert: CloudStorageUploadFailuresIncreasing + expr: rate(cloud_storage_upload_failures_total[5m]) > 0 + annotations: + summary: "Store node {{ $labels.instance }} is experiencing {{ $value }} upload failures/sec" + action: "Review logs for details; check S3/cloud provider status" + +# Alert if retry queue is growing (backlog of pending uploads) +- alert: CloudStorageRetryQueueBacklog + expr: cloud_storage_retry_queue_size > 10 + annotations: + summary: "Store node {{ $labels.instance }} has {{ $value }} files in retry queue" + action: "Monitor until queue drains" + +# Alert if sync latency is high (uploads taking too long) +- alert: CloudStorageSyncLatencyHigh + expr: cloud_storage_sync_latency_ms_seconds{quantile="0.95"} > 30 + annotations: + summary: "Store node {{ $labels.instance }} 95th percentile sync latency is {{ $value }}s" + action: "Check network link to cloud provider; consider tuning upload concurrency" + +# Alert if a pending-delete marker could not be persisted (DB delete progression is held) +- alert: CloudStorageDeleteMarkerDegraded + expr: cloud_storage_delete_marker_healthy == 0 + annotations: + summary: "Store node {{ $labels.instance }} pending-delete marker persistence is degraded" + action: "DB delete is held to preserve the anti-resurrection guard; check data-root disk/permissions" +``` + +**Logs to monitor:** + +```bash +# Watch for re-uploads (delete guard re-attempting failed uploads) +kubectl logs -f -l app=hugegraph-store | grep "Re-uploading.*unconfirmed" + +# Watch for permanently-failed uploads (still retried, but worth investigation) +kubectl logs -f -l app=hugegraph-store | grep "Upload failed permanently" + +# Watch for delete-skipped events (delete guard holding old SSTs) +kubectl logs -f -l app=hugegraph-store | grep "Delete skipped.*not fully durable" + +# Trace a specific unconfirmed file +kubectl logs -f -l app=hugegraph-store | grep "filePath=/path/to/000123.sst" +``` + +### Troubleshooting + +**Symptom: `cloud_storage_delete_guard_reupload_count` keeps increasing, or `cloud_storage_confirmed_files_total` stays below the DB's live SST count** + +- **Cause**: Uploads are failing and not recovering. +- **Action**: + 1. Check logs for "Upload failed permanently" messages — what's the error? + 2. Verify cloud provider credentials and connectivity: `curl https://s3-endpoint/bucket-name` + 3. Check if bucket exists and Store node has put/get/delete permissions. + 4. Verify `cloud.storage.path-prefix` matches actual S3 prefix where Store expects to write. + 5. If a specific file is stuck, wait for the next deferred delete pass / metadata publish (both re-attempt unconfirmed live files) or restart the node to run startup SST backfill. + +**Symptom: Delete operations are slow or stalling** + +- **Cause**: Delete guard is re-uploading many files. +- **Action**: + 1. Check logs for "Re-uploading M unconfirmed files" — how many? + 2. Monitor `cloud_storage_retry_queue_size`. Once it drains, delete resumes. + +**Symptom: High `cloud_storage_sync_latency_ms_seconds`** + +- **Cause**: Uploads are slow. Could be network, object size, or cloud provider latency. +- **Action**: + 1. Check average file size: `ls -lh /data/raft/*/db-*/` (larger files take longer) + 2. Measure network throughput: `iperf3` or cloud provider bandwidth test. + 3. Check if S3 multipart upload is enabled and tuned (reduces latency for large files). + +### General Best Practices + +- Prefer stable `path-prefix` and bucket naming; changing them affects object lookup continuity. +- Keep plugin JAR versions aligned with Store version to avoid SPI/API mismatch. +- Track logs around upload, hydration, and provider init to detect divergence early. +- For DR drills, test both node-level restart and full-cluster restart with cloud hydration enabled. +- Set up alerts on the key metrics (`delete_guard_reupload_count`, `uploads_given_up_total`, `upload_failures_total`, `sync_latency_ms_seconds`) to catch issues early. + +## Plugin Development: Adding Custom Cloud Storage Providers + +This section guides developers on building and deploying new cloud storage provider plugins (e.g., Azure Blob Storage, ADLS, GCP). + +### Plugin Architecture Overview + +HugeGraph uses a **Service Provider Interface (SPI)** pattern to discover and load cloud storage providers at runtime: + +1. **CloudStorageProvider interface**: Located in `hugegraph-store/hg-store-common`, defines the contract all providers must implement. +2. **ServiceLoader discovery**: Store node uses `java.util.ServiceLoader` to find all `CloudStorageProvider` implementations on the classpath. +3. **SPI selection**: At Store startup, `AppConfig` loads the provider specified in `cloud.storage.provider` config via `ServiceLoader`. +4. **Plugin packaging/runtime classpath**: Provider implementations are packaged as separate JAR modules and must be discoverable by Store at startup — either bundled into the distribution (Option A) or placed on the JVM launch classpath (Option B). There is no `plugins/` directory scan. See [Step 5](#step-5-package-for-runtime-classpath) below. + +### CloudStorageProvider Interface + +All custom providers must implement: + +```java +public interface CloudStorageProvider { + + /** + * Unique name for this provider (e.g., "s3", "azure", "adls"). + * Must match the `cloud.storage.provider` config value to be activated. + */ + String providerName(); + + /** + * Initialize the provider with configuration. + * Called once at Store startup. Throw exception if init fails. + */ + void init(CloudStorageConfig config) throws IOException; + + /** + * Upload a local file to cloud storage with the given remote key. + */ + void uploadFile(String localPath, String remoteKey) throws IOException; + + /** + * Download a remote file from cloud storage to the local path. + */ + void downloadFile(String remoteKey, String localPath) throws IOException; + + /** + * Delete a remote file from cloud storage. + */ + void deleteFile(String remoteKey) throws IOException; + + /** + * Check if a remote file exists. + */ + boolean fileExists(String remoteKey) throws IOException; + + /** + * List all remote keys that start with the given key prefix (a plain string match, as in + * S3 ListObjectsV2: pass a trailing "/" to list one directory). Returns keys relative to + * the provider root (provider path prefix stripped). + */ + List listFiles(String remoteDirPrefix) throws IOException; + + /** + * Close and release all provider resources. + */ + void close() throws IOException; +} +``` + +**Location:** `hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProvider.java` + +### Module Structure for a New Provider + +``` +hugegraph-store/ +├── hg-store-cloud-newprovider/ # New provider module +│ ├── pom.xml # Maven module definition +│ ├── src/main/java/org/apache/hugegraph/store/cloud/newprovider/ +│ │ └── NewProviderCloudStorageProvider.java # Implementation +│ ├── src/main/resources/META-INF/services/ +│ │ └── org.apache.hugegraph.store.cloud.CloudStorageProvider +│ └── src/test/java/... # Unit tests +``` + +Recommendation: + +- For providers maintained in this repository, prefer adding them as submodules under `hugegraph-store/` (same model as `hg-store-cloud-s3`). +- External providers are also supported if their jars are placed on runtime classpath. + +### Step 1: Create Module POM + +File: `hugegraph-store/hg-store-cloud-newprovider/pom.xml` + +```xml + + + 4.0.0 + + + org.apache.hugegraph + hugegraph-store + ${revision} + ../pom.xml + + + hg-store-cloud-newprovider + HugeGraph Store Cloud NewProvider + NewProvider Storage plugin for HugeGraph Store cloud storage integration + + + + + org.apache.hugegraph + hg-store-common + ${revision} + + + + + NewProvider + NewProvider + 1.0 + + + + +``` + +### Step 2: Implement the Provider + +File: `hugegraph-store/hg-store-cloud-newprovider/src/main/java/org/apache/hugegraph/store/cloud/newprovider/NewProviderCloudStorageProvider.java` + +```java +package org.apache.hugegraph.store.cloud.newprovider; + +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings("RedundantThrows") +@Slf4j +public class NewProviderCloudStorageProvider implements CloudStorageProvider { + + public static final String PROVIDER_NAME = "newprovider"; + + private Object providerClient; + private String pathPrefix; + + @Override + public String providerName() { + return PROVIDER_NAME; + } + + @Override + public void init(CloudStorageConfig config) throws IOException { + //Steps to initialize the NewProvider client using config parameters + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + //Steps to upload file to NewProvider cloud storage + } + + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + //Steps to download file from NewProvider cloud storage + } + + @Override + public void deleteFile(String remoteKey) throws IOException { + //Steps to delete file from NewProvider cloud storage + } + + @Override + public boolean fileExists(String remoteKey) throws IOException { + //Steps to check if file exists in NewProvider cloud storage + return false; + } + + @Override + public List listFiles(String remoteDirPrefix) throws IOException { + //Steps to list files in NewProvider cloud storage + return new ArrayList<>(); + } + + @Override + public void close() throws IOException { + //Steps to close and cleanup NewProvider client resources + } +} +``` + +### Step 3: Register via SPI + +File: `hugegraph-store/hg-store-cloud-newprovider/src/main/resources/META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider` + +``` +org.apache.hugegraph.store.cloud.newprovider.NewProviderCloudStorageProvider +``` + +This file tells Java's `ServiceLoader` to discover this provider at runtime. + +### Step 4: Build and Test + +```bash +# Build the provider module +cd hugegraph-store +mvn clean package -pl hg-store-cloud-newprovider -am -DskipTests + +# Find the generated JAR +find . -name "hg-store-cloud-newprovider-*.jar" +# Output: hugegraph-store/hg-store-cloud-newprovider/target/hg-store-cloud-newprovider-1.0.jar +``` + +### Step 5: Package for Runtime Classpath + +Use one of these runtime models: + +**Option A (recommended for in-repo providers): add as Store submodule dependency** + +1. Add module `hg-store-cloud-newprovider` under `hugegraph-store/`. +2. Add dependency from `hg-store-node` to `hg-store-cloud-newprovider`. +3. Rebuild distribution so provider + transitive dependencies are packaged with Store runtime. + +**Option B (external provider): provide jars on runtime classpath** + +- Supply `hg-store-cloud-newprovider-*.jar` **and** all required dependency jars, or +- supply a single shaded/uber provider jar that already contains transitive dependencies. + +Notes: + +- Simply copying a provider jar without its runtime dependencies can cause `ClassNotFoundException` during SPI loading. +- The stock distribution does not scan a `plugins/` directory (the test harness image under `docker/cloud-storage` is the exception: it launches Spring `PropertiesLauncher` with `-Dloader.path=/hugegraph-store/plugins`, which is a harness-only arrangement). `AppConfig` calls + `ServiceLoader.load(CloudStorageProvider.class)` once at startup, which only sees jars inside the + `hg-store-node` boot jar (`BOOT-INF/lib`, i.e. Option A) or on the JVM launch classpath. +- The shipped `bin/start-hugegraph-store.sh` builds the launch classpath from only the rocksdbjni + jar and the `hg-store-node` boot jar (`-cp ":" JarLauncher`), so there + is no supported drop-in location for an external provider; Option A is the supported path. +- Appending a provider jar to that `-cp` is not enough by itself: classes on `-cp` are loaded by + the system classloader, which cannot see `CloudStorageProvider` (it lives in `hg-store-common` + inside `BOOT-INF/lib`), so loading the provider fails with `NoClassDefFoundError`. The + `hg-store-common` jar would have to be on `-cp` as well. +- The provider is resolved once, so adding or replacing a provider jar requires a restart. +- To ship S3 in the stock distribution, build `hg-store-node` with the opt-in `-Ps3-cloud-storage` profile, which bundles `hg-store-cloud-s3` and the AWS SDK into the node fat jar (Option A). See "Enabling S3 on the stock distribution" in the [Store README](../README.md). +- For the harness plugin directory, the build stages only the SDK runtime closure in `hg-store-cloud-s3/target/dependency`: `hg-store-common` is `provided` scope there, because the node boot jar already supplies it and a second copy on `loader.path` would split `CloudStorageProvider` across classloaders. + +### Step 6: Configure and Activate + +In Store `application.yml`: + +```yaml +cloud: + storage: + enabled: true + provider: newprovider # Must match NewProviderCloudStorageProvider.providerName() + path-prefix: hugegraph + + # Provider-specific configuration (cloud.storage.newprovider.*) + newprovider: + bucket: my-container + region: myaccount # e.g., Azure account name + endpoint: https://myaccount.blob.core.windows.net + access-key: ${NEWPROVIDER_KEY} + secret-key: ${NEWPROVIDER_SECRET} +``` + +Or via Docker env (for S3 example): + +```bash +HG_CLOUD_STORAGE_ENABLED=true \ +HG_CLOUD_STORAGE_PROVIDER=s3 \ +HG_CLOUD_STORAGE_PATH_PREFIX=hugegraph \ +HG_CLOUD_STORAGE_S3_BUCKET=hugegraph-store0 \ +HG_CLOUD_STORAGE_S3_REGION=us-east-1 \ +HG_CLOUD_STORAGE_S3_ENDPOINT=http://minio:9000 \ +HG_CLOUD_STORAGE_S3_ACCESS_KEY=minioadmin \ +HG_CLOUD_STORAGE_S3_SECRET_KEY=minioadmin \ +docker run hugegraph-store:latest +``` + +Or via Docker env (for custom newprovider): + +```bash +HG_CLOUD_STORAGE_ENABLED=true \ +HG_CLOUD_STORAGE_PROVIDER=newprovider \ +HG_CLOUD_STORAGE_PATH_PREFIX=hugegraph \ +HG_CLOUD_STORAGE_NEWPROVIDER_BUCKET=my-container \ +HG_CLOUD_STORAGE_NEWPROVIDER_REGION=myaccount \ +HG_CLOUD_STORAGE_NEWPROVIDER_ENDPOINT=https://myaccount.blob.core.windows.net \ +HG_CLOUD_STORAGE_NEWPROVIDER_ACCESS_KEY=${NEWPROVIDER_KEY} \ +HG_CLOUD_STORAGE_NEWPROVIDER_SECRET_KEY=${NEWPROVIDER_SECRET} \ +docker run hugegraph-store:latest +``` + +Notes on property naming conventions: + +- **YAML properties**: Use kebab-case with provider-specific namespacing (e.g., `cloud.storage.s3.bucket`, `cloud.storage.newprovider.access-key`) +- **Environment variables**: Use uppercase with underscores and `HG_CLOUD_STORAGE_*` prefix. Provider-specific options use `HG_CLOUD_STORAGE__*` pattern +- Kebab-case YAML properties are converted to uppercase with underscores (e.g., `access-key` → `ACCESS_KEY`) + +### Testing Your Provider + +Add unit tests in `hg-store-cloud-newprovider/src/test/`: + +```java +@Test +public void testInit() { + // Test provider initialization with mock config +} + +@Test +public void testUploadDownload() { + // Test upload and download of a sample file +} + +@Test +public void testDeleteAndList() { + // Test delete and list operations +} + +@Test +public void testFileExists() { + // Test file existence check +} + +@Test +public void testClose() { + // Test provider close and resource cleanup +} +``` + +### Troubleshooting Provider Discovery + +If your provider isn't loaded: + +1. **Check SPI registration file exists:** + ```bash + jar tf hg-store-cloud-newprovider-1.0.jar | grep "META-INF/services" + ``` + Should show: `META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider` + +2. **Check provider name matches config:** + ```bash + jar xf hg-store-cloud-newprovider-1.0.jar META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider + cat META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider + ``` + +3. **Enable debug logging:** + ```yaml + logging: + level: + org.apache.hugegraph.store.cloud: DEBUG + org.apache.hugegraph.store.node.cloud: DEBUG + ``` + +4. **Verify provider jars are on classpath:** + ```bash + # Check Store startup logs for provider discovery / class-loading errors + docker logs hugegraph-store | grep -i "ServiceLoader\|provider" + ``` + +### Key Implementation Tips + +1. **Thread safety**: Ensure provider is thread-safe for concurrent upload/download/delete operations. +2. **Connection pooling**: Reuse client connections; initialize once in `init()`, close in `close()`. +3. **Path normalization**: Always use `pathPrefix` correctly (see S3 example for reference). +4. **Error handling**: Throw `IOException` for operational (retryable) issues, or `CloudStorageNonRetryableException` for permanent failures that should skip whole-file retries. Implement your own internal retry logic where useful (see `S3CloudStorageProvider` for reference). The common `CloudUploadRetryQueue` performs whole-file retries by default (`upload-retry-max-attempts=3`) and gives up on (logs and counts) uploads that exhaust them. +5. **Logging**: Use SLF4J (via `@Slf4j`) for consistent log levels with Store. +6. **Configuration validation**: Validate all required fields in `init()` and fail fast. + +### Contributing Your Provider + +To upstream your new provider (e.g., `hg-store-cloud-newprovider`): + +1. Create PR to `hugegraph-store/` with new provider module +2. Add tests in `hg-store-test/` that validate provider behavior +3. Update `docker/cloud-storage/` examples with new provider setup steps +4. Update `install-dist/` license/notice if adding third-party dependencies diff --git a/hugegraph-store/hg-store-cloud-s3/pom.xml b/hugegraph-store/hg-store-cloud-s3/pom.xml new file mode 100644 index 0000000000..56821d7ded --- /dev/null +++ b/hugegraph-store/hg-store-cloud-s3/pom.xml @@ -0,0 +1,172 @@ + + + + + 4.0.0 + + + org.apache.hugegraph + hugegraph-store + ${revision} + ../pom.xml + + + hg-store-cloud-s3 + + + S3 cloud storage provider plugin for HugeGraph Store. + Bundled into the hg-store-node jar by the opt-in -Ps3-cloud-storage build profile; set + cloud.storage.provider=s3 to enable S3 offload. A copy on the launch classpath cannot load. + + + + 2.33.8 + + + + + + org.apache.hugegraph + hg-store-common + provided + + + + + org.rocksdb + rocksdbjni + runtime + + + + + software.amazon.awssdk + s3 + ${aws.sdk.version} + + + + + software.amazon.awssdk + s3-transfer-manager + ${aws.sdk.version} + + + + org.projectlombok + lombok + provided + + + + + junit + junit + 4.13.2 + test + + + org.slf4j + slf4j-simple + 1.7.36 + test + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-plugin-runtime-deps + package + + copy-dependencies + + + + org.apache.hugegraph + + runtime + + + rocksdbjni,log4j-slf4j-impl,slf4j-log4j12 + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + **/*Benchmark.java + **/*E2ETest.java + + + @{argLine} -Xmx512m + + 21600 + + + + + diff --git a/hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageConfig.java b/hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageConfig.java new file mode 100644 index 0000000000..7ddd6f4b6d --- /dev/null +++ b/hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageConfig.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud.s3; + +import lombok.Data; +import lombok.ToString; + +/** + * Amazon S3 / S3-compatible provider configuration, bound from + * {@code cloud.storage.s3.*} in application.yml. + * + *

This is a plain Java POJO (no Spring annotations) so it can be used + * both as a Spring {@code @ConfigurationProperties} target (via standard setters) + * and directly in tests without a Spring context. + * + *

+ * cloud:
+ *   storage:
+ *     s3:
+ *       bucket: hugegraph-store
+ *       region: us-east-1
+ *       endpoint:                             # optional custom endpoint (MinIO, Ceph…)
+ *       access-key: AKIAIOSFODNN7EXAMPLE      # omit to use AWS default credentials chain
+ *       secret-key: wJalrXUtnFEMI/…           # omit to use AWS default credentials chain
+ * 
+ */ +@Data +public class S3CloudStorageConfig { + + public static final String KEY_BUCKET = "bucket"; + public static final String KEY_REGION = "region"; + public static final String KEY_ENDPOINT = "endpoint"; + public static final String KEY_ACCESS_KEY = "access-key"; + public static final String KEY_SECRET_KEY = "secret-key"; + + /** + * S3 bucket name. + */ + private String bucket; + + /** + * AWS region (e.g. "us-east-1"). + */ + private String region; + + /** + * Optional custom endpoint URL for S3-compatible stores (e.g. MinIO, Ceph). + * Leave empty to use the default AWS endpoint. + */ + private String endpoint; + + /** + * AWS access key ID. Omit to use the default AWS credentials chain + * (env vars, instance profile, ~/.aws/credentials, etc.). + * Excluded from {@code toString()} so credentials are never written to logs. + */ + @ToString.Exclude + private String accessKey; + + /** + * AWS secret access key. Omit to use the default AWS credentials chain. + * Excluded from {@code toString()} so credentials are never written to logs. + */ + @ToString.Exclude + private String secretKey; + +} + diff --git a/hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProvider.java b/hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProvider.java new file mode 100644 index 0000000000..a5e09d72bf --- /dev/null +++ b/hugegraph-store/hg-store-cloud-s3/src/main/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProvider.java @@ -0,0 +1,722 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud.s3; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Consumer; + +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageNonRetryableException; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; + +import lombok.extern.slf4j.Slf4j; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; +import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.retry.RetryMode; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedFileUpload; +import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.UploadFileRequest; + +/** + * Amazon S3 (and S3-compatible) implementation of {@link CloudStorageProvider}. + * + *

Activation

+ * Place {@code hg-store-cloud-s3-*.jar} on the classpath and configure: + *
+ * cloud:
+ *   storage:
+ *     enabled: true
+ *     provider: s3
+ *     s3:
+ *       bucket: my-bucket
+ *       region: us-east-1
+ * 
+ * + *

Credentials

+ *
    + *
  • If {@code cloud.storage.s3.access-key} / {@code secret-key} are set, + * they are used directly.
  • + *
  • Otherwise the standard AWS Default Credentials chain is followed + * (env vars, instance profile, ~/.aws/credentials, etc.).
  • + *
+ * + *

S3-compatible endpoints (MinIO, Ceph, etc.)

+ * Set {@code cloud.storage.s3.endpoint} to the custom HTTP/HTTPS endpoint URL. + * + *

Large-file (multipart) uploads

+ * Files larger than {@link #MULTIPART_THRESHOLD_BYTES} ({@value #MULTIPART_THRESHOLD_BYTES} MB) + * are uploaded via {@link S3TransferManager}, which splits them into parts and uploads them + * concurrently using the S3 Multipart Upload API — required for files larger than 5 GB, S3's + * hard per-PUT limit. Smaller files use a direct single PUT. Retry (including per-part retry + * for multipart uploads) is handled entirely by the AWS SDK's own client-level retry strategy + * (see {@link #buildRetryOverride()}) rather than by application code. + * + *

Timing metrics

+ * Every upload logs the file size, elapsed time, and throughput at INFO level: + *
+ *   S3 upload complete: db/000042.sst | size=64.0 MB | elapsed=830 ms | throughput=77.11 MB/s
+ * 
+ */ +@Slf4j +public class S3CloudStorageProvider implements CloudStorageProvider { + + /** Provider name as referenced in {@link CloudStorageConfig#getProvider()}. */ + public static final String PROVIDER_NAME = "s3"; + + /** + * Files larger than this are uploaded via {@link S3TransferManager} (multipart). + * S3's hard per-PUT limit is 5 GB; we start multipart well below that. + */ + static final long MULTIPART_THRESHOLD_BYTES = 512L * 1024 * 1024; // 512 MB + + /** Signing region used when only a custom endpoint is configured. */ + static final String DEFAULT_ENDPOINT_REGION = "us-east-1"; + + private S3Client s3Client; + private S3AsyncClient s3AsyncClient; + private S3TransferManager transferManager; + private String bucket; + private String pathPrefix; + + // ----------------------------------------------------------------------- + // CloudStorageProvider + // ----------------------------------------------------------------------- + + @Override + public String providerName() { + return PROVIDER_NAME; + } + + @Override + public void init(CloudStorageConfig config) { + Map props = config.getProviderProperties(); + if (props == null || props.isEmpty()) { + throw new IllegalArgumentException( + "S3 provider selected but providerProperties are empty"); + } + + this.bucket = props.get(S3CloudStorageConfig.KEY_BUCKET); + if (this.bucket == null || this.bucket.isBlank()) { + throw new IllegalArgumentException("S3 bucket is required: cloud.storage.s3.bucket"); + } + this.pathPrefix = config.getPathPrefix(); + + String ak = props.get(S3CloudStorageConfig.KEY_ACCESS_KEY); + String sk = props.get(S3CloudStorageConfig.KEY_SECRET_KEY); + String region = props.get(S3CloudStorageConfig.KEY_REGION); + String endpoint = props.get(S3CloudStorageConfig.KEY_ENDPOINT); + + region = resolveRegion(region, endpoint); + + S3ClientBuilder builder = S3Client.builder(); + builder.overrideConfiguration(buildRetryOverride()); + // Only compute/validate checksums when the operation requires it. SDK 2.30+ defaults to + // WHEN_SUPPORTED, which adds CRC32 trailer/aws-chunked payloads that many S3-compatible + // endpoints (MinIO, Ceph, older gateways) reject on PutObject / multipart uploads. + builder.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED); + builder.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED); + applyCredentials(builder::credentialsProvider, ak, sk); + if (region != null && !region.isEmpty()) { + builder.region(Region.of(region)); + } + if (endpoint != null && !endpoint.isEmpty()) { + builder.endpointOverride(URI.create(endpoint)); + // Path-style required for most non-AWS S3 services + builder.forcePathStyle(true); + } + + // Java-based multipart async client (no native CRT dependency); backs S3TransferManager. + S3AsyncClientBuilder asyncBuilder = S3AsyncClient.builder().multipartEnabled(true); + asyncBuilder.overrideConfiguration(buildRetryOverride()); + asyncBuilder.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED); + asyncBuilder.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED); + applyCredentials(asyncBuilder::credentialsProvider, ak, sk); + if (region != null && !region.isEmpty()) { + asyncBuilder.region(Region.of(region)); + } + if (endpoint != null && !endpoint.isEmpty()) { + asyncBuilder.endpointOverride(URI.create(endpoint)); + asyncBuilder.forcePathStyle(true); + } + + // Close any clients from a previous init() so a re-initialization (e.g. Spring context + // restart, which re-runs the same singleton provider instance) does not leak the old + // clients' connection pools and SDK threads. + closeQuietly(); + + try { + this.s3Client = builder.build(); + this.s3AsyncClient = asyncBuilder.build(); + } catch (SdkClientException e) { + closeQuietly(); + if (e.getMessage() != null && e.getMessage().contains("region")) { + throw new IllegalArgumentException( + "Cannot determine the S3 region: set cloud.storage.s3.region (or " + + "AWS_REGION / aws.region), or set cloud.storage.s3.endpoint for an " + + "S3-compatible store. Cause: " + e.getMessage(), e); + } + throw e; + } + this.transferManager = S3TransferManager.builder().s3Client(this.s3AsyncClient).build(); + log.info("S3CloudStorageProvider initialized: bucket='{}', region='{}', endpoint='{}'", + bucket, region, endpoint); + } + + /** + * Resolves the region to configure on the SDK clients. A configured region is used as-is. With + * a custom endpoint and no region, defaults to {@link #DEFAULT_ENDPOINT_REGION} because + * S3-compatible stores ignore it but the SDK still requires one for request signing. With + * neither, returns the (blank) input so the SDK's own region provider chain applies. + */ + static String resolveRegion(String region, String endpoint) { + if (region != null && !region.isBlank()) { + return region.trim(); + } + if (endpoint != null && !endpoint.isBlank()) { + log.info("cloud.storage.s3.region is not set; defaulting to '{}' for custom endpoint " + + "'{}' (S3-compatible stores ignore the region)", + DEFAULT_ENDPOINT_REGION, endpoint); + return DEFAULT_ENDPOINT_REGION; + } + return region; + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + Path path = Paths.get(localPath); + long fileSize; + try { + fileSize = Files.size(path); + } catch (IOException e) { + throw new IOException("Cannot stat local file: " + localPath, e); + } + + String fullKey = buildKey(remoteKey); + long startNs = System.nanoTime(); + + if (fileSize > MULTIPART_THRESHOLD_BYTES) { + uploadViaTransferManager(path, fullKey); + } else { + uploadSinglePart(path, fullKey); + } + + long elapsedMs = (System.nanoTime() - startNs) / 1_000_000; + double throughputMBps = elapsedMs > 0 + ? (fileSize / 1_048_576.0) / (elapsedMs / 1000.0) + : 0.0; + log.info("S3 upload complete: {} | size={} | elapsed={} ms | throughput={} MB/s", + remoteKey, + humanSize(fileSize), + elapsedMs, + String.format(Locale.US, "%.2f", throughputMBps)); + } + + @Override + public void deleteFile(String remoteKey) throws IOException { + String fullKey = buildKey(remoteKey); + try { + s3Client.deleteObject( + DeleteObjectRequest.builder().bucket(bucket).key(fullKey).build()); + log.debug("S3 delete: s3://{}/{}", bucket, fullKey); + } catch (SdkException e) { + throw classifySdkException("deleteObject", fullKey, e); + } + } + + /** + * Deletes all objects under a prefix using S3's DeleteObjects (batch delete) API. + * + *

Much more efficient than individual deletes, especially for prefixes with many objects. + * Handles pagination internally if the prefix contains more than 1000 objects. + * + * @param remoteDirPrefix directory/prefix inside bucket (without provider pathPrefix) + * @return number of objects deleted + * @throws IOException on I/O or network failure + */ + @Override + public int deletePrefix(String remoteDirPrefix) throws IOException { + String fullPrefix = buildKey(remoteDirPrefix == null ? "" : remoteDirPrefix); + int totalDeleted = 0; + + try { + String token = null; + do { + ListObjectsV2Request.Builder listReq = + ListObjectsV2Request.builder().bucket(bucket).prefix(fullPrefix); + if (token != null) { + listReq.continuationToken(token); + } + ListObjectsV2Response listResp = s3Client.listObjectsV2(listReq.build()); + + List toDelete = new ArrayList<>(); + for (S3Object obj : listResp.contents()) { + String key = obj.key(); + if (key != null && !key.endsWith("/")) { + toDelete.add(ObjectIdentifier.builder().key(key).build()); + } + } + + if (!toDelete.isEmpty()) { + try { + DeleteObjectsResponse deleteResp = s3Client.deleteObjects( + DeleteObjectsRequest.builder() + .bucket(bucket) + .delete(Delete.builder() + .objects(toDelete) + .build()) + .build()); + totalDeleted += deleteResp.deleted().size(); + log.debug("S3 batch delete: deleted {} objects from prefix {}", + deleteResp.deleted().size(), remoteDirPrefix); + + // S3 returns HTTP 200 even when individual keys fail; always inspect. + if (!deleteResp.errors().isEmpty()) { + log.warn("S3 batch delete partial failure: {}/{} key(s) failed in " + + "prefix '{}' — retrying individually", + deleteResp.errors().size(), toDelete.size(), remoteDirPrefix); + List stillFailed = new ArrayList<>(); + for (software.amazon.awssdk.services.s3.model.S3Error err + : deleteResp.errors()) { + log.warn(" S3 DeleteObjects error: key={} code={} message={}", + err.key(), err.code(), err.message()); + try { + s3Client.deleteObject(DeleteObjectRequest.builder() + .bucket(bucket) + .key(err.key()) + .build()); + totalDeleted++; + log.debug("S3 individual retry delete succeeded: key={}", + err.key()); + } catch (SdkException ex) { + stillFailed.add(err.key()); + log.warn("S3 individual retry delete failed: key={}: {}", + err.key(), ex.getMessage()); + } + } + if (!stillFailed.isEmpty()) { + throw new IOException( + "S3 DeleteObjects: " + stillFailed.size() + + " key(s) could not be deleted from prefix '" + + remoteDirPrefix + "': " + stillFailed); + } + } + } catch (SdkException e) { + log.warn("S3 batch delete failed for prefix='{}': {}", + fullPrefix, e.getMessage()); + // Fall back to individual deletes for any remaining objects. + // Failures are collected and re-thrown so callers (e.g. purgeRemotePrefix) + // can correctly preserve the tombstone guard when the purge is incomplete. + List stillFailed = new ArrayList<>(); + for (ObjectIdentifier obj : toDelete) { + try { + s3Client.deleteObject(DeleteObjectRequest.builder() + .bucket(bucket) + .key(obj.key()) + .build()); + totalDeleted++; + } catch (SdkException ex) { + stillFailed.add(obj.key()); + log.debug("S3 fallback delete failed for key='{}': {}", + obj.key(), ex.getMessage()); + } + } + if (!stillFailed.isEmpty()) { + throw new IOException( + "S3 fallback delete: " + stillFailed.size() + + " key(s) could not be deleted from prefix '" + + remoteDirPrefix + "': " + stillFailed); + } + } + } + + token = listResp.nextContinuationToken(); + // Some S3-compatible gateways return isTruncated=true without a usable + // continuation token. A prefix purge that silently stops here would report + // success while objects remain — and the caller (onDBDeleted / truncate purge) + // would then remove its tombstone guard, leaving stale objects that can be + // re-hydrated as live data. + // Fail loudly so the purge is marked incomplete and the guard is preserved. + if (Boolean.TRUE.equals(listResp.isTruncated()) + && (token == null || token.isEmpty())) { + throw new IOException( + "S3 deletePrefix: listing for prefix '" + remoteDirPrefix + + "' is truncated but returned no continuation token; the purge is " + + "incomplete and cannot be confirmed. Deleted " + totalDeleted + + " object(s) so far."); + } + } while (token != null && !token.isEmpty()); + + if (totalDeleted > 0) { + log.info("S3 prefix delete completed: prefix={}, deleted={}", remoteDirPrefix, + totalDeleted); + } + return totalDeleted; + + } catch (SdkException e) { + throw classifySdkException("deletePrefix", fullPrefix, e); + } + } + + @Override + public boolean fileExists(String remoteKey) throws IOException { + String fullKey = buildKey(remoteKey); + try { + s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(fullKey).build()); + return true; + } catch (NoSuchKeyException e) { + return false; + } catch (AwsServiceException e) { + // A missing bucket is a misconfiguration, not a missing key. Treat it as a hard error + // so callers (e.g. tombstone check in preHydrateDbFiles) surface the problem rather + // than silently skipping hydration and starting with an empty database. + String errorCode = e.awsErrorDetails() != null ? e.awsErrorDetails().errorCode() : ""; + if ("NoSuchBucket".equals(errorCode) || "InvalidBucketName".equals(errorCode)) { + throw new IOException( + "S3 bucket not found or misconfigured (key=" + fullKey + "): " + errorCode, + e); + } + // Some S3-compatible providers return generic service exceptions for 404. + if (e.statusCode() == 404) { + return false; + } + throw classifySdkException("headObject", fullKey, e); + } catch (SdkException e) { + throw classifySdkException("headObject", fullKey, e); + } + } + + @Override + public List listFiles(String remoteDirPrefix) throws IOException { + String fullPrefix = buildKey(remoteDirPrefix == null ? "" : remoteDirPrefix); + List keys = new ArrayList<>(); + try { + String token = null; + do { + ListObjectsV2Request.Builder req = + ListObjectsV2Request.builder().bucket(bucket).prefix(fullPrefix); + if (token != null) { + req.continuationToken(token); + } + ListObjectsV2Response resp = s3Client.listObjectsV2(req.build()); + for (S3Object obj : resp.contents()) { + String key = obj.key(); + if (key == null || key.endsWith("/")) { + continue; + } + keys.add(stripPathPrefix(key)); + } + token = resp.nextContinuationToken(); + // A truncated listing with no continuation token would silently return a PARTIAL + // key set. Startup hydration relies on a complete listing (e.g. to find CURRENT); + // a partial set could let a DB open on incomplete local state. Fail loudly so the + // caller blocks rather than proceeding on a partial listing. + if (Boolean.TRUE.equals(resp.isTruncated()) + && (token == null || token.isEmpty())) { + throw new IOException( + "S3 listFiles: listing for prefix '" + fullPrefix + "' is truncated " + + "but returned no continuation token; refusing to return a partial " + + "listing (" + keys.size() + " key(s) seen so far)."); + } + } while (token != null && !token.isEmpty()); + return keys; + } catch (SdkException e) { + throw classifySdkException("listObjectsV2", fullPrefix, e); + } + } + + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + String fullKey = buildKey(remoteKey); + long startNs = System.nanoTime(); + Path destinationPath = Paths.get(localPath); + try { + + s3Client.getObject( + GetObjectRequest.builder().bucket(bucket).key(fullKey).build(), + destinationPath); + } catch (SdkException e) { + throw classifySdkException("getObject", fullKey, e); + } + long elapsedMs = (System.nanoTime() - startNs) / 1_000_000; + long fileSize = 0; + try { + fileSize = Files.size(destinationPath); + } catch (IOException ignored) { + // best-effort; don't fail download reporting + } + double throughputMBps = elapsedMs > 0 + ? (fileSize / 1_048_576.0) / (elapsedMs / 1000.0) + : 0.0; + log.info("S3 download complete: {} | size={} | elapsed={} ms | throughput={} MB/s", + remoteKey, + humanSize(fileSize), + elapsedMs, + String.format(Locale.US, "%.2f", throughputMBps)); + } + + @Override + public void close() { + closeQuietly(); + log.info("S3CloudStorageProvider closed"); + } + + // ----------------------------------------------------------------------- + // Internal – client construction + // ----------------------------------------------------------------------- + + /** + * Retry policy applied to both the sync and async S3 clients. {@code STANDARD} mode retries + * transient failures (including individual multipart part uploads) with exponential backoff + * and jitter — the AWS SDK owns retry entirely; no application-level retry loop is layered + * on top. + */ + private static ClientOverrideConfiguration buildRetryOverride() { + return ClientOverrideConfiguration.builder() + .retryStrategy(RetryMode.STANDARD) + .build(); + } + + private static void applyCredentials( + Consumer sink, + String accessKey, String secretKey) { + if (accessKey != null && !accessKey.isEmpty() + && secretKey != null && !secretKey.isEmpty()) { + sink.accept(StaticCredentialsProvider.create( + AwsBasicCredentials.create(accessKey, secretKey))); + } else { + sink.accept(DefaultCredentialsProvider.builder().build()); + } + } + + private void closeQuietly() { + if (this.transferManager != null) { + try { + this.transferManager.close(); + } catch (Exception e) { + log.warn("Failed to close previous S3 transfer manager: {}", e.getMessage()); + } + this.transferManager = null; + } + if (this.s3AsyncClient != null) { + try { + this.s3AsyncClient.close(); + } catch (Exception e) { + log.warn("Failed to close previous S3 async client: {}", e.getMessage()); + } + this.s3AsyncClient = null; + } + if (this.s3Client != null) { + try { + this.s3Client.close(); + } catch (Exception e) { + log.warn("Failed to close previous S3 client: {}", e.getMessage()); + } + this.s3Client = null; + } + } + + // ----------------------------------------------------------------------- + // Internal – upload strategies + // ----------------------------------------------------------------------- + + /** + * Single-PUT upload for files ≤ {@link #MULTIPART_THRESHOLD_BYTES}. Transient failures are + * retried by the SDK's own retry strategy (see {@link #buildRetryOverride()}); this method + * makes one call and lets exceptions propagate. + */ + private void uploadSinglePart(Path path, String fullKey) throws IOException { + try { + s3Client.putObject( + PutObjectRequest.builder().bucket(bucket).key(fullKey).build(), + path); + } catch (SdkException e) { + throw classifySdkException("putObject", fullKey, e); + } + } + + /** + * Multipart upload for files > {@link #MULTIPART_THRESHOLD_BYTES}, delegated entirely to + * {@link S3TransferManager}. The Transfer Manager handles splitting into parts, uploading + * them concurrently, completing (or aborting, on failure) the multipart upload, and retrying + * transient per-part failures via the underlying async client's SDK-level retry strategy. + */ + private void uploadViaTransferManager(Path path, String fullKey) throws IOException { + UploadFileRequest request = UploadFileRequest.builder() + .putObjectRequest(b -> b.bucket(bucket).key(fullKey)) + .source(path) + .build(); + FileUpload fileUpload = transferManager.uploadFile(request); + try { + CompletedFileUpload completed = fileUpload.completionFuture().join(); + log.info("S3 multipart upload completed: key={} eTag={}", + fullKey, completed.response().eTag()); + } catch (java.util.concurrent.CompletionException e) { + Throwable cause = e.getCause() != null ? e.getCause() : e; + if (cause instanceof SdkException) { + throw classifySdkException("uploadFile(multipart)", fullKey, (SdkException) cause); + } + throw new IOException("S3 multipart upload failed for key='" + fullKey + "'", cause); + } + } + + /** + * Classifies SDK exceptions into retryable/non-retryable provider exceptions. + * + *

Retryable: + *

    + *
  • client-side transport failures ({@link SdkException});
  • + *
  • service throttling / transient statuses (408/425/429/500/502/503/504);
  • + *
  • AWS error codes that the SDK classifies as retryable regardless of HTTP status + * (e.g. {@code RequestTimeout} at HTTP 400, {@code PriorRequestNotComplete}).
  • + *
+ * Non-retryable: + *
    + *
  • permanent service-side statuses (e.g. auth/permission/not-found/validation).
  • + *
+ */ + private IOException classifySdkException(String operation, String key, SdkException e) { + if (e instanceof AwsServiceException ase) { + int status = ase.statusCode(); + AwsErrorDetails errorDetails = ase.awsErrorDetails(); + String code = errorDetails != null ? errorDetails.errorCode() : ""; + String requestId = ase.requestId(); + String message = String.format(Locale.US, + "S3 %s failed: key=%s status=%d code=%s requestId=%s", + operation, key, status, code, requestId); + if (isRetryableServiceFailure(ase)) { + return new IOException(message, ase); + } + return new CloudStorageNonRetryableException(message, ase); + } + + // Client-side network/IO/timeout failures are retryable by default. + return new IOException("S3 " + operation + " failed for key='" + key + "'", e); + } + + /** + * AWS error codes that are retryable regardless of HTTP status code. + * For example, {@code RequestTimeout} arrives as HTTP 400 but is transient and should + * be retried. This list is sourced from the AWS SDK retry-condition documentation. + */ + private static final java.util.Set RETRYABLE_ERROR_CODES = + new java.util.HashSet<>(java.util.Arrays.asList( + "RequestTimeout", + "RequestTimeoutException", + "PriorRequestNotComplete", + "InternalError", + "ServiceUnavailable", + "SlowDown", + "ProvisionedThroughputExceededException" + )); + + private static boolean isRetryableServiceFailure(AwsServiceException e) { + if (e.isThrottlingException()) { + return true; + } + AwsErrorDetails errorDetails = e.awsErrorDetails(); + String code = errorDetails != null ? errorDetails.errorCode() : ""; + if (code != null && RETRYABLE_ERROR_CODES.contains(code)) { + return true; + } + int status = e.statusCode(); + return status == 408 || status == 425 || status == 429 || + status == 500 || status == 502 || status == 503 || status == 504; + } + + // ----------------------------------------------------------------------- + // Internal – key helpers + // ----------------------------------------------------------------------- + + /** + * Prepends {@link #pathPrefix} to the supplied key, using "/" as separator. + * If the prefix is null or empty, the key is returned unchanged. + */ + private String buildKey(String key) { + if (pathPrefix == null || pathPrefix.isEmpty()) { + return key; + } + // Normalise leading slashes + String normalKey = key.startsWith("/") ? key.substring(1) : key; + return pathPrefix.endsWith("/") + ? pathPrefix + normalKey + : pathPrefix + "/" + normalKey; + } + + private String stripPathPrefix(String fullKey) { + if (fullKey == null) { + return ""; + } + if (pathPrefix == null || pathPrefix.isEmpty()) { + return fullKey.startsWith("/") ? fullKey.substring(1) : fullKey; + } + String normalizedPrefix = pathPrefix.endsWith("/") ? pathPrefix : pathPrefix + "/"; + if (fullKey.startsWith(normalizedPrefix)) { + return fullKey.substring(normalizedPrefix.length()); + } + return fullKey; + } + + // ----------------------------------------------------------------------- + // Internal – formatting + // ----------------------------------------------------------------------- + + static String humanSize(long bytes) { + if (bytes < 1024L) return bytes + " B"; + if (bytes < 1024L * 1024) return String.format(Locale.US, "%.1f KB", + bytes / 1024.0); + if (bytes < 1024L * 1024 * 1024) return String.format(Locale.US, "%.1f MB", + bytes / (1024.0 * 1024)); + return String.format(Locale.US, "%.2f GB", bytes / (1024.0 * 1024 * 1024)); + } +} diff --git a/hugegraph-store/hg-store-cloud-s3/src/main/resources/META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider b/hugegraph-store/hg-store-cloud-s3/src/main/resources/META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider new file mode 100644 index 0000000000..241d82992b --- /dev/null +++ b/hugegraph-store/hg-store-cloud-s3/src/main/resources/META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider @@ -0,0 +1,2 @@ +org.apache.hugegraph.store.cloud.s3.S3CloudStorageProvider + diff --git a/hugegraph-store/hg-store-cloud-s3/src/test/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProviderTest.java b/hugegraph-store/hg-store-cloud-s3/src/test/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProviderTest.java new file mode 100644 index 0000000000..bfa2add179 --- /dev/null +++ b/hugegraph-store/hg-store-cloud-s3/src/test/java/org/apache/hugegraph/store/cloud/s3/S3CloudStorageProviderTest.java @@ -0,0 +1,750 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud.s3; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageNonRetryableException; +import org.junit.Assume; +import org.junit.Test; + +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.DeletedObject; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Error; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.transfer.s3.S3TransferManager; + +/** + * Unit-level tests for {@link S3CloudStorageProvider} and {@link S3CloudStorageConfig}, all driven + * without a live S3/MinIO via a dynamic-{@link java.lang.reflect.Proxy} {@code S3Client}: + *
    + *
  • SDK-exception classification (retryable vs non-retryable);
  • + *
  • client lifecycle across re-init (sync client, async client, and transfer manager are all + * closed on re-init);
  • + *
  • operations: single-PUT upload, delete, prefix purge (multi-page, partial-batch-failure, + * batch-exception fallback, truncated-without-token safety), listing, {@code fileExists} + * status handling, download, and human-size formatting;
  • + *
  • the {@link S3CloudStorageConfig} bean (key constants, accessors/equality).
  • + *
+ * Retry (including per-part retry for multipart uploads) is delegated entirely to the AWS SDK's + * own client-level retry strategy, so there is no app-level retry loop left to test here. The full + * single-large-file (multipart, via {@link S3TransferManager}) upload/download round trip lives in + * {@link S3SingleLargeFileE2ETest}. + */ +@SuppressWarnings({"resource", "SuspiciousInvocationHandlerImplementation"}) +public class S3CloudStorageProviderTest { + + // ========================================================================= + // SDK exception classification + // ========================================================================= + + @Test + public void testClassifySdkExceptionRetryableServiceStatusReturnsIOException() throws Exception { + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + AwsServiceException retryable503 = s3ServiceException(503, "SlowDown", "req-503"); + + IOException classified = invokeClassify(provider, retryable503); + + assertFalse("503 should be treated as retryable IOException", + classified instanceof CloudStorageNonRetryableException); + } + + @Test + public void testClassifySdkExceptionNonRetryableServiceStatusReturnsNonRetryable() throws Exception { + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + AwsServiceException forbidden403 = s3ServiceException(403, "AccessDenied", "req-403"); + + IOException classified = invokeClassify(provider, forbidden403); + + assertTrue("403 should be treated as non-retryable", + classified instanceof CloudStorageNonRetryableException); + } + + @Test + public void testClassifySdkExceptionRetryable429StatusIsRetryable() throws Exception { + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + AwsServiceException throttled = s3ServiceException(429, "TooManyRequests", "req-throttle"); + + IOException classified = invokeClassify(provider, throttled); + + assertFalse("429 should be treated as retryable", + classified instanceof CloudStorageNonRetryableException); + } + + @Test + public void testClassifySdkExceptionClientSideFailureIsRetryableIOException() throws Exception { + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + SdkClientException clientFailure = + SdkClientException.builder().message("connection reset").build(); + + IOException classified = invokeClassify(provider, clientFailure); + + assertFalse("Client-side failures should stay retryable", + classified instanceof CloudStorageNonRetryableException); + } + + // ========================================================================= + // Client lifecycle: init() closes all previous clients on re-init (no leak) + // ========================================================================= + + /** + * {@code S3CloudStorageProvider} is a singleton discovered once via SPI; a Spring context + * restart re-runs {@code init()} on the same instance. If {@code init()} overwrote the clients + * without closing the previous ones, the old SDK connection pools and threads would leak. + */ + @Test + public void testInitClosesPreviousClientsOnReinit() throws Exception { + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + + AtomicBoolean staleSyncClosed = new AtomicBoolean(false); + S3Client staleSync = (S3Client) java.lang.reflect.Proxy.newProxyInstance( + S3Client.class.getClassLoader(), + new Class[]{S3Client.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + staleSyncClosed.set(true); + return null; + case "serviceName": + return "s3"; + default: + return null; + } + }); + AtomicBoolean staleAsyncClosed = new AtomicBoolean(false); + S3AsyncClient staleAsync = (S3AsyncClient) java.lang.reflect.Proxy.newProxyInstance( + S3AsyncClient.class.getClassLoader(), + new Class[]{S3AsyncClient.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + staleAsyncClosed.set(true); + return null; + case "serviceName": + return "s3"; + default: + return null; + } + }); + AtomicBoolean staleTransferManagerClosed = new AtomicBoolean(false); + S3TransferManager staleTransferManager = (S3TransferManager) java.lang.reflect.Proxy + .newProxyInstance( + S3TransferManager.class.getClassLoader(), + new Class[]{S3TransferManager.class}, + (proxy, method, args) -> { + if (method.getName().equals("close")) { + staleTransferManagerClosed.set(true); + return null; + } + return null; + }); + + setField(provider, "s3Client", staleSync); + setField(provider, "s3AsyncClient", staleAsync); + setField(provider, "transferManager", staleTransferManager); + + // Re-initialize. A region is set so the real clients build without needing a resolvable + // default region. + CloudStorageConfig cfg = new CloudStorageConfig(); + cfg.setEnabled(true); + cfg.setProvider("s3"); + cfg.setPathPrefix(""); + cfg.getProviderProperties().put("bucket", "test-bucket"); + cfg.getProviderProperties().put("region", "us-east-1"); + + try { + provider.init(cfg); + assertTrue("init() must close the previous sync S3 client to avoid leaking its " + + "connection pool/threads on re-init", staleSyncClosed.get()); + assertTrue("init() must close the previous async S3 client on re-init", + staleAsyncClosed.get()); + assertTrue("init() must close the previous transfer manager on re-init", + staleTransferManagerClosed.get()); + } finally { + provider.close(); + } + } + + // ========================================================================= + // Region defaulting for S3-compatible endpoints + // ========================================================================= + + @Test + public void testResolveRegionKeepsConfiguredRegion() { + assertEquals("eu-west-1", + S3CloudStorageProvider.resolveRegion("eu-west-1", "http://minio:9000")); + assertEquals("eu-west-1", S3CloudStorageProvider.resolveRegion("eu-west-1", null)); + } + + @Test + public void testResolveRegionDefaultsWhenEndpointSetWithoutRegion() { + assertEquals("us-east-1", S3CloudStorageProvider.resolveRegion(null, "http://minio:9000")); + assertEquals("us-east-1", S3CloudStorageProvider.resolveRegion("", "http://minio:9000")); + assertEquals("us-east-1", S3CloudStorageProvider.resolveRegion(" ", "http://minio:9000")); + } + + @Test + public void testResolveRegionLeavesBlankWithoutEndpoint() { + assertEquals(null, S3CloudStorageProvider.resolveRegion(null, null)); + assertEquals("", S3CloudStorageProvider.resolveRegion("", "")); + } + + @Test + public void testInitWithEndpointAndNoRegionSucceeds() { + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + CloudStorageConfig cfg = newInitConfig(); + cfg.getProviderProperties().put("endpoint", "http://localhost:9000"); + cfg.getProviderProperties().put("access-key", "ak"); + cfg.getProviderProperties().put("secret-key", "sk"); + try { + provider.init(cfg); + } finally { + provider.close(); + } + } + + @Test + public void testInitWithoutRegionOrEndpointNamesRegionKey() { + // Only meaningful when the environment cannot supply a region (env, system property, + // ~/.aws/config, EC2 metadata); skip otherwise. + boolean sdkResolvesRegion; + try { + DefaultAwsRegionProviderChain.builder().build().getRegion(); + sdkResolvesRegion = true; + } catch (SdkClientException e) { + sdkResolvesRegion = false; + } + Assume.assumeFalse("environment provides an AWS region", sdkResolvesRegion); + + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + CloudStorageConfig cfg = newInitConfig(); + cfg.getProviderProperties().put("access-key", "ak"); + cfg.getProviderProperties().put("secret-key", "sk"); + try { + IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> provider.init(cfg)); + assertTrue(e.getMessage(), e.getMessage().contains("cloud.storage.s3.region")); + } finally { + provider.close(); + } + } + + // ========================================================================= + // Pagination safety: truncated listing without a continuation token must fail loudly + // ========================================================================= + + /** + * Some S3-compatible gateways return {@code isTruncated=true} but omit the continuation token. + * A prefix purge that stops there would report success while objects remain, causing the caller + * to drop its tombstone guard and later re-hydrate stale data. deletePrefix() must instead throw. + */ + @Test + public void testDeletePrefixTruncatedWithoutTokenFailsLoudly() throws Exception { + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + S3Client s3 = (S3Client) java.lang.reflect.Proxy.newProxyInstance( + S3Client.class.getClassLoader(), + new Class[]{S3Client.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "listObjectsV2": + // Truncated, but NO continuation token: the malformed-gateway case. + return ListObjectsV2Response.builder().isTruncated(true).build(); + case "close": + return null; + default: + throw new UnsupportedOperationException( + "Unexpected S3Client method: " + method.getName()); + } + }); + setField(provider, "s3Client", s3); + setField(provider, "bucket", "test-bucket"); + + IOException ex = assertThrows("Incomplete purge must fail loudly, not report success", + IOException.class, () -> provider.deletePrefix("db/prefix/")); + assertTrue("Failure must indicate a truncated/incomplete purge: " + ex.getMessage(), + ex.getMessage().toLowerCase().contains("truncated")); + } + + /** + * Startup hydration relies on a complete listing (e.g. to find CURRENT). A truncated listing + * with no continuation token would return a partial key set; listFiles() must throw instead. + */ + @Test + public void testListFilesTruncatedWithoutTokenFailsLoudly() throws Exception { + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + S3Client s3 = (S3Client) java.lang.reflect.Proxy.newProxyInstance( + S3Client.class.getClassLoader(), + new Class[]{S3Client.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "listObjectsV2": + return ListObjectsV2Response.builder() + .isTruncated(true) + .contents(S3Object.builder() + .key("db/prefix/000001.sst").build()) + .build(); + case "close": + return null; + default: + throw new UnsupportedOperationException( + "Unexpected S3Client method: " + method.getName()); + } + }); + setField(provider, "s3Client", s3); + setField(provider, "bucket", "test-bucket"); + + IOException ex = assertThrows("Partial listing must fail loudly", IOException.class, + () -> provider.listFiles("db/prefix/")); + assertTrue("Failure must indicate a truncated/partial listing: " + ex.getMessage(), + ex.getMessage().toLowerCase().contains("truncated")); + } + + /** + * Regression guard: the new truncation check must NOT break the normal (complete) listing path — + * an {@code isTruncated=false} response with no token is a valid terminal page. + */ + @Test + public void testDeletePrefixNotTruncatedCompletesWithoutError() throws Exception { + S3CloudStorageProvider provider = new S3CloudStorageProvider(); + S3Client s3 = (S3Client) java.lang.reflect.Proxy.newProxyInstance( + S3Client.class.getClassLoader(), + new Class[]{S3Client.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "listObjectsV2": + return ListObjectsV2Response.builder().isTruncated(false).build(); + case "close": + return null; + default: + throw new UnsupportedOperationException( + "Unexpected S3Client method: " + method.getName()); + } + }); + setField(provider, "s3Client", s3); + setField(provider, "bucket", "test-bucket"); + + assertEquals("A complete (non-truncated) empty listing must succeed with 0 deletions", + 0, provider.deletePrefix("db/prefix/")); + } + + // ========================================================================= + // S3CloudStorageConfig bean: key constants, accessors/equality + // ========================================================================= + + @Test + public void testConfigKeyConstants() { + assertEquals("bucket", S3CloudStorageConfig.KEY_BUCKET); + assertEquals("region", S3CloudStorageConfig.KEY_REGION); + assertEquals("endpoint", S3CloudStorageConfig.KEY_ENDPOINT); + assertEquals("access-key", S3CloudStorageConfig.KEY_ACCESS_KEY); + assertEquals("secret-key", S3CloudStorageConfig.KEY_SECRET_KEY); + } + + @Test + public void testConfigSettersGettersAndEquality() { + S3CloudStorageConfig a = newPopulatedConfig(); + assertEquals("b", a.getBucket()); + assertEquals("us-east-1", a.getRegion()); + assertEquals("http://localhost:9000", a.getEndpoint()); + assertEquals("ak", a.getAccessKey()); + assertEquals("sk", a.getSecretKey()); + + assertNotEquals(new S3CloudStorageConfig(), a); + S3CloudStorageConfig b = newPopulatedConfig(); + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + assertTrue(a.toString().contains("bucket")); + } + + @Test + public void testConfigToStringOmitsCredentials() { + S3CloudStorageConfig c = newPopulatedConfig(); + c.setAccessKey("AKIAIOSFODNN7EXAMPLE"); + c.setSecretKey("super-secret-value"); + + String str = c.toString(); + assertTrue(str.contains("bucket=b")); + assertFalse(str.contains("accessKey")); + assertFalse(str.contains("secretKey")); + assertFalse(str.contains("AKIAIOSFODNN7EXAMPLE")); + assertFalse(str.contains("super-secret-value")); + } + + // ========================================================================= + // Operations via a dynamic-proxy S3Client (no live S3): upload / delete / + // deletePrefix / listFiles / fileExists / download / humanSize + // ========================================================================= + + @Test + public void testUploadFileSinglePartSucceeds() throws Exception { + AtomicInteger puts = new AtomicInteger(); + S3CloudStorageProvider p = proxyProvider(handler(m -> { + if (m.equals("putObject")) { + puts.incrementAndGet(); + return PutObjectResponse.builder().build(); + } + return null; + })); + Path f = tmpFile(); + try { + p.uploadFile(f.toString(), "db/000001.sst"); + assertEquals("one PUT expected", 1, puts.get()); + } finally { + Files.deleteIfExists(f); + } + } + + @Test + public void testUploadFileSinglePartPropagatesNonRetryableFailure() throws Exception { + S3CloudStorageProvider p = proxyProvider(handler(m -> { + if (m.equals("putObject")) { + throw s3ServiceException(403, "AccessDenied", "r"); + } + return null; + })); + Path f = tmpFile(); + try { + assertThrows("A non-retryable PUT failure must propagate without any app-level retry", + CloudStorageNonRetryableException.class, + () -> p.uploadFile(f.toString(), "db/3.sst")); + } finally { + Files.deleteIfExists(f); + } + } + + @Test + public void testDeleteFileSucceeds() throws Exception { + AtomicInteger deletes = new AtomicInteger(); + S3CloudStorageProvider p = proxyProvider(handler(m -> { + if (m.equals("deleteObject")) { + deletes.incrementAndGet(); + return DeleteObjectResponse.builder().build(); + } + return null; + })); + p.deleteFile("db/old.sst"); + assertEquals(1, deletes.get()); + } + + @Test + public void testDeleteFileNonRetryableIsClassified() throws Exception { + S3CloudStorageProvider p = proxyProvider(handler(m -> { + if (m.equals("deleteObject")) { + throw s3ServiceException(403, "AccessDenied", "r"); + } + return null; + })); + assertThrows(IOException.class, () -> p.deleteFile("db/old.sst")); + } + + @Test + public void testDeletePrefixDeletesAcrossPages() throws Exception { + AtomicInteger lists = new AtomicInteger(); + S3CloudStorageProvider p = proxyProvider(handler((m, args) -> { + switch (m) { + case "listObjectsV2": + if (lists.incrementAndGet() == 1) { + return ListObjectsV2Response.builder() + .contents(obj("db/p/1.sst"), obj("db/p/2.sst"), obj("db/p/")) + .isTruncated(true).nextContinuationToken("t1").build(); + } + return ListObjectsV2Response.builder() + .contents(obj("db/p/3.sst")) + .isTruncated(false).build(); + case "deleteObjects": + software.amazon.awssdk.services.s3.model.DeleteObjectsRequest req = + (software.amazon.awssdk.services.s3.model.DeleteObjectsRequest) args[0]; + List deleted = req.delete().objects().stream() + .map(o -> DeletedObject.builder().key(o.key()).build()) + .collect(java.util.stream.Collectors.toList()); + return DeleteObjectsResponse.builder().deleted(deleted).build(); + default: + return null; + } + })); + // Directory marker "db/p/" must be skipped; 3 real objects deleted across 2 pages. + assertEquals(3, p.deletePrefix("db/p/")); + } + + @Test + public void testDeletePrefixPartialBatchFailureRetriedIndividually() throws Exception { + S3CloudStorageProvider p = proxyProvider(handler((m, args) -> { + switch (m) { + case "listObjectsV2": + return ListObjectsV2Response.builder() + .contents(obj("db/p/1.sst"), obj("db/p/2.sst")) + .isTruncated(false).build(); + case "deleteObjects": + // One key deleted, one reported as an error (retried individually below). + return DeleteObjectsResponse.builder() + .deleted(DeletedObject.builder().key("db/p/1.sst").build()) + .errors(S3Error.builder().key("db/p/2.sst") + .code("InternalError").message("x").build()) + .build(); + case "deleteObject": + return DeleteObjectResponse.builder().build(); + default: + return null; + } + })); + assertEquals("batch-deleted + individually-retried", 2, p.deletePrefix("db/p/")); + } + + @Test + public void testDeletePrefixBatchExceptionFallsBackToIndividualDeletes() throws Exception { + S3CloudStorageProvider p = proxyProvider(handler((m, args) -> { + switch (m) { + case "listObjectsV2": + return ListObjectsV2Response.builder() + .contents(obj("db/p/1.sst"), obj("db/p/2.sst")) + .isTruncated(false).build(); + case "deleteObjects": + throw s3ServiceException(500, "InternalError", "r"); // batch API fails entirely + case "deleteObject": + return DeleteObjectResponse.builder().build(); // fallback individual deletes ok + default: + return null; + } + })); + assertEquals("fallback individual deletes", 2, p.deletePrefix("db/p/")); + } + + @Test + public void testListFilesReturnsKeysAcrossPagesSkippingDirMarkers() throws Exception { + AtomicInteger lists = new AtomicInteger(); + S3CloudStorageProvider p = proxyProvider(handler(m -> { + if (m.equals("listObjectsV2")) { + if (lists.incrementAndGet() == 1) { + return ListObjectsV2Response.builder() + .contents(obj("db/p/CURRENT"), obj("db/p/")) + .isTruncated(true).nextContinuationToken("t1").build(); + } + return ListObjectsV2Response.builder() + .contents(obj("db/p/000001.sst")) + .isTruncated(false).build(); + } + return null; + })); + List keys = p.listFiles("db/p/"); + assertEquals(List.of("db/p/CURRENT", "db/p/000001.sst"), keys); + } + + @Test + public void testFileExistsTrueWhenHeadSucceeds() throws Exception { + S3CloudStorageProvider p = proxyProvider(handler(m -> + m.equals("headObject") ? HeadObjectResponse.builder().build() : null)); + assertTrue(p.fileExists("db/x.sst")); + } + + @Test + public void testFileExistsFalseOnNoSuchKey() throws Exception { + S3CloudStorageProvider p = proxyProvider(handler(m -> { + if (m.equals("headObject")) { + throw NoSuchKeyException.builder().message("missing").build(); + } + return null; + })); + assertFalse(p.fileExists("db/x.sst")); + } + + @Test + public void testFileExistsFalseOnGeneric404() throws Exception { + S3CloudStorageProvider p = proxyProvider(handler(m -> { + if (m.equals("headObject")) { + throw s3ServiceException(404, "NotFound", "r"); // S3-compatible generic 404 + } + return null; + })); + assertFalse(p.fileExists("db/x.sst")); + } + + @Test + public void testFileExistsThrowsOnMissingBucket() throws Exception { + S3CloudStorageProvider p = proxyProvider(handler(m -> { + if (m.equals("headObject")) { + throw s3ServiceException(404, "NoSuchBucket", "r"); // misconfig, not a missing key + } + return null; + })); + IOException ex = assertThrows(IOException.class, () -> p.fileExists("db/x.sst")); + assertTrue(ex.getMessage().contains("bucket")); + } + + @Test + public void testDownloadFileInvokesGetObject() throws Exception { + AtomicInteger gets = new AtomicInteger(); + S3CloudStorageProvider p = proxyProvider(handler(m -> { + if (m.equals("getObject")) { + gets.incrementAndGet(); + return GetObjectResponse.builder().build(); + } + return null; + })); + Path dest = Files.createTempDirectory("hg-s3-dl").resolve("out.sst"); + p.downloadFile("db/x.sst", dest.toString()); + assertEquals(1, gets.get()); + } + + @Test + public void testHumanSizeUnits() { + assertEquals("512 B", S3CloudStorageProvider.humanSize(512)); + assertTrue(S3CloudStorageProvider.humanSize(2048).endsWith(" KB")); + assertTrue(S3CloudStorageProvider.humanSize(5L * 1024 * 1024).endsWith(" MB")); + assertTrue(S3CloudStorageProvider.humanSize(3L * 1024 * 1024 * 1024).endsWith(" GB")); + } + + // ========================================================================= + // Helpers + // ========================================================================= + + private static CloudStorageConfig newInitConfig() { + CloudStorageConfig cfg = new CloudStorageConfig(); + cfg.setEnabled(true); + cfg.setProvider("s3"); + cfg.setPathPrefix(""); + cfg.getProviderProperties().put("bucket", "test-bucket"); + return cfg; + } + + private static S3CloudStorageConfig newPopulatedConfig() { + S3CloudStorageConfig c = new S3CloudStorageConfig(); + c.setBucket("b"); + c.setRegion("us-east-1"); + c.setEndpoint("http://localhost:9000"); + c.setAccessKey("ak"); + c.setSecretKey("sk"); + return c; + } + + /** Builds a provider with the given proxy S3Client and a fixed bucket, no path-prefix. */ + private static S3CloudStorageProvider proxyProvider(InvocationHandler handler) throws Exception { + S3CloudStorageProvider p = new S3CloudStorageProvider(); + S3Client s3 = (S3Client) java.lang.reflect.Proxy.newProxyInstance( + S3Client.class.getClassLoader(), new Class[]{S3Client.class}, handler); + setField(p, "s3Client", s3); + setField(p, "bucket", "test-bucket"); + return p; + } + + /** Handler dispatching on method name only. */ + private interface ByName { + Object apply(String method) throws Throwable; + } + + /** Handler dispatching on method name and raw args. */ + private interface ByNameArgs { + Object apply(String method, Object[] args) throws Throwable; + } + + private static InvocationHandler handler(ByName fn) { + return (proxy, method, args) -> { + if (method.getName().equals("close")) { + return null; + } + Object r = fn.apply(method.getName()); + if (r != null) { + return r; + } + throw new UnsupportedOperationException("Unexpected S3Client call: " + method.getName()); + }; + } + + private static InvocationHandler handler(ByNameArgs fn) { + return (proxy, method, args) -> { + if (method.getName().equals("close")) { + return null; + } + Object r = fn.apply(method.getName(), args); + if (r != null) { + return r; + } + throw new UnsupportedOperationException("Unexpected S3Client call: " + method.getName()); + }; + } + + private static S3Object obj(String key) { + return S3Object.builder().key(key).build(); + } + + private static Path tmpFile() throws IOException { + Path f = Files.createTempFile("hg-s3-op", ".sst"); + Files.write(f, new byte[16]); + return f; + } + + private static IOException invokeClassify(S3CloudStorageProvider provider, SdkException e) + throws Exception { + Method classify = S3CloudStorageProvider.class.getDeclaredMethod( + "classifySdkException", String.class, String.class, SdkException.class); + classify.setAccessible(true); + return (IOException) classify.invoke(provider, "uploadFile(multipart)", "k.sst", e); + } + + private static void setField(S3CloudStorageProvider provider, + String fieldName, + Object value) throws Exception { + Field field = S3CloudStorageProvider.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(provider, value); + } + + private static AwsServiceException s3ServiceException(int statusCode, + String errorCode, + String requestId) { + AwsErrorDetails details = AwsErrorDetails.builder() + .serviceName("S3") + .errorCode(errorCode) + .errorMessage("simulated") + .build(); + return S3Exception.builder() + .statusCode(statusCode) + .requestId(requestId) + .awsErrorDetails(details) + .message("simulated") + .build(); + } +} diff --git a/hugegraph-store/hg-store-cloud-s3/src/test/java/org/apache/hugegraph/store/cloud/s3/S3SingleLargeFileE2ETest.java b/hugegraph-store/hg-store-cloud-s3/src/test/java/org/apache/hugegraph/store/cloud/s3/S3SingleLargeFileE2ETest.java new file mode 100644 index 0000000000..f89895b65c --- /dev/null +++ b/hugegraph-store/hg-store-cloud-s3/src/test/java/org/apache/hugegraph/store/cloud/s3/S3SingleLargeFileE2ETest.java @@ -0,0 +1,478 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud.s3; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.Socket; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Locale; +import java.util.UUID; + +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; + +/** + * End-to-end large-file S3 test focused on one realistic scenario: + * generate one 20 GB SST-like file, upload (multipart), check remote existence, + * download it back, and validate size. + * + *

Setup: The test requires a running S3-compatible service (e.g., MinIO). + * It automatically skips if the endpoint is unreachable or required properties are missing. + * + *

Zero-config local run: If no VM options are set, the test defaults to a local + * MinIO instance at {@code http://localhost:9000} with credentials {@code minioadmin/minioadmin}. + * Simply start MinIO and run the test directly from your IDE: + *

+ * docker run -p 9000:9000 \
+ *   -e MINIO_ROOT_USER=minioadmin \
+ *   -e MINIO_ROOT_PASSWORD=minioadmin \
+ *   minio/minio server /data
+ * 
+ * + *

Custom run with explicit properties: + *

+ * # Start MinIO in Docker:
+ * docker run -p 9000:9000 -e MINIO_ROOT_USER=minioadmin \
+ *   -e MINIO_ROOT_PASSWORD=minioadmin minio/minio server /data
+ *
+ * # Run test:
+ * mvn test -pl hugegraph-store/hg-store-cloud-s3 \
+ *   -Dtest=S3SingleLargeFileE2ETest \
+ *   -Ds3.perf.endpoint=... \
+ *   -Ds3.perf.bucket=hugegraph-perf \
+ *   -Ds3.perf.accessKey=minioadmin \
+ *   -Ds3.perf.secretKey=minioadmin \
+ *   -Ds3.perf.region=us-east-1 \
+ *   -Ds3.perf.singleFileSizeGb=20
+ * 
+ * + *

Properties: + *

    + *
  • {@code s3.perf.bucket} (required) — S3 bucket name + *
  • {@code s3.perf.endpoint} (required if MinIO) — Service endpoint URL + *
  • {@code s3.perf.accessKey} — Access key (default: empty, uses default credentials) + *
  • {@code s3.perf.secretKey} — Secret key (default: empty, uses default credentials) + *
  • {@code s3.perf.region} — AWS region (default: us-east-1) + *
  • {@code s3.perf.pathPrefix} — Path prefix in bucket (default: hugegraph-perf) + *
  • {@code s3.perf.singleFileSizeGb} — File size in GB for test (default: 20) + *
  • {@code s3.perf.skipCleanup} — Skip remote file cleanup after test (default: false) + *
+ * + *

The test auto-skips if: + *

    + *
  • The endpoint (default: {@code http://localhost:9000}) is not reachable + *
+ */ +@SuppressWarnings("SameParameterValue") +public class S3SingleLargeFileE2ETest { + + private static final String PROP_BUCKET = "s3.perf.bucket"; + private static final String PROP_REGION = "s3.perf.region"; + private static final String PROP_ENDPOINT = "s3.perf.endpoint"; + private static final String PROP_ACCESS_KEY = "s3.perf.accessKey"; + private static final String PROP_SECRET_KEY = "s3.perf.secretKey"; + private static final String PROP_PATH_PREFIX = "s3.perf.pathPrefix"; + private static final String PROP_SINGLE_FILE_GB = "s3.perf.singleFileSizeGb"; + private static final String PROP_SKIP_CLEANUP = "s3.perf.skipCleanup"; + /** + * When set truthy (system property {@code s3.perf.integration} or env + * {@code S3_PERF_INTEGRATION}), an unreachable endpoint FAILS the test instead of skipping it. + * CI's S3-integration profile must set this so a broken endpoint/credentials cannot masquerade + * as a green build. Unset (local/IDE runs) preserves the graceful skip when MinIO is absent. + */ + private static final String PROP_INTEGRATION_REQUIRED = "s3.perf.integration"; + private static final String ENV_INTEGRATION_REQUIRED = "S3_PERF_INTEGRATION"; + + // Defaults for zero-config local MinIO runs (e.g. running the test directly from an IDE). + // These values match the standard MinIO Docker quickstart: + // docker run -p 9000:9000 -e MINIO_ROOT_USER=minioadmin \ + // -e MINIO_ROOT_PASSWORD=minioadmin minio/minio server /data + private static final String DEFAULT_ENDPOINT = "http://localhost:9000"; + private static final String DEFAULT_BUCKET = "hugegraph-perf"; + private static final String DEFAULT_REGION = "us-east-1"; + private static final String DEFAULT_ACCESS_KEY = "minioadmin"; + private static final String DEFAULT_SECRET_KEY = "minioadmin"; + + private S3CloudStorageProvider provider; + private Path tmpDir; + private String remoteKey; + private boolean skipCleanup; + + /** S3 client used exclusively for bucket-lifecycle management in tests. */ + private S3Client adminS3Client; + + @Before + public void setUp() throws IOException { + String endpoint = System.getProperty(PROP_ENDPOINT, DEFAULT_ENDPOINT); + String bucket = System.getProperty(PROP_BUCKET, DEFAULT_BUCKET); + String region = System.getProperty(PROP_REGION, DEFAULT_REGION); + String ak = System.getProperty(PROP_ACCESS_KEY, DEFAULT_ACCESS_KEY); + String sk = System.getProperty(PROP_SECRET_KEY, DEFAULT_SECRET_KEY); + + // Endpoint reachability gate. In an integration profile (flag set) an unreachable endpoint + // is a HARD FAILURE — CI must not stay green while real S3 integration is broken. Otherwise + // (local/IDE runs, default CI) skip gracefully when MinIO / S3 is not running. + boolean reachable = isEndpointReachable(endpoint); + if (integrationRequired()) { + Assert.assertTrue( + "S3 integration profile is enabled (" + PROP_INTEGRATION_REQUIRED + "/" + + ENV_INTEGRATION_REQUIRED + ") but the S3 endpoint is not reachable at " + endpoint + + " — failing hard so a broken S3 integration cannot pass CI.", + reachable); + } else { + Assume.assumeTrue( + "Skipping S3SingleLargeFileE2ETest: S3 endpoint not reachable at " + endpoint + + ". Start MinIO with: docker run -p 9000:9000 " + + "-e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin " + + "minio/minio server /data. To require this test (fail instead of skip), set -D" + + PROP_INTEGRATION_REQUIRED + "=true.", + reachable); + } + + // Build an admin S3 client for bucket management + this.adminS3Client = buildAdminS3Client(endpoint, region, ak, sk); + + // Auto-create the bucket if it does not exist yet + ensureBucketExists(this.adminS3Client, bucket, region, endpoint); + + CloudStorageConfig s3Cfg = new CloudStorageConfig(); + s3Cfg.setEnabled(true); + s3Cfg.setProvider("s3"); + s3Cfg.setPathPrefix(System.getProperty(PROP_PATH_PREFIX, "hugegraph-perf")); + s3Cfg.getProviderProperties().put("bucket", bucket); + s3Cfg.getProviderProperties().put("region", region); + s3Cfg.getProviderProperties().put("endpoint", endpoint); + s3Cfg.getProviderProperties().put("access-key", ak); + s3Cfg.getProviderProperties().put("secret-key", sk); + + this.skipCleanup = boolProp(PROP_SKIP_CLEANUP, false); + this.provider = new S3CloudStorageProvider(); + this.provider.init(s3Cfg); + this.tmpDir = Files.createTempDirectory("hg-s3-e2e-"); + } + + @After + public void tearDown() throws Exception { + if (this.provider != null) { + if (!this.skipCleanup && this.remoteKey != null) { + try { + this.provider.deleteFile(this.remoteKey); + } catch (Exception ignored) { + // Best-effort cleanup only. + } + } + this.provider.close(); + } + if (this.adminS3Client != null) { + this.adminS3Client.close(); + } + if (this.tmpDir != null) { + deleteDirectory(this.tmpDir.toFile()); + } + } + + @Test + public void uploadDownloadLargeFileEndToEnd() throws Exception { + long fileSizeGb = longProp(PROP_SINGLE_FILE_GB, 20L); + long fileSizeBytes = fileSizeGb * 1024L * 1024L * 1024L; + + Assert.assertTrue("file size must trigger multipart upload", + fileSizeBytes > S3CloudStorageProvider.MULTIPART_THRESHOLD_BYTES); + + Path localSrc = this.tmpDir.resolve("src-" + UUID.randomUUID() + ".sst"); + Path localDst = this.tmpDir.resolve("dst-" + UUID.randomUUID() + ".sst"); + this.remoteKey = "perf-single-file/hugegraph-large-" + UUID.randomUUID() + ".sst"; + + System.out.printf(Locale.US, + "%n[E2E] generating file: size=%d GB (%s)%n", + fileSizeGb, humanSize(fileSizeBytes)); + generateLargeFile(localSrc, fileSizeBytes); + + long uploadStart = System.nanoTime(); + this.provider.uploadFile(localSrc.toString(), this.remoteKey); + long uploadMs = (System.nanoTime() - uploadStart) / 1_000_000; + + Assert.assertTrue("uploaded object not found in remote storage", + this.provider.fileExists(this.remoteKey)); + + long downloadStart = System.nanoTime(); + this.provider.downloadFile(this.remoteKey, localDst.toString()); + long downloadMs = (System.nanoTime() - downloadStart) / 1_000_000; + + long srcSize = Files.size(localSrc); + long dstSize = Files.size(localDst); + Assert.assertEquals("downloaded file size mismatch", srcSize, dstSize); + + byte[] srcDigest = sha256(localSrc); + byte[] dstDigest = sha256(localDst); + Assert.assertArrayEquals("downloaded file content mismatch (SHA-256 differs)", srcDigest, + dstDigest); + + double uploadMBps = srcSize > 0 && uploadMs > 0 + ? (srcSize / 1_048_576.0) / (uploadMs / 1000.0) + : 0.0; + double downloadMBps = dstSize > 0 && downloadMs > 0 + ? (dstSize / 1_048_576.0) / (downloadMs / 1000.0) + : 0.0; + + System.out.printf(Locale.US, + "[E2E] upload=%s (%.2f MB/s), download=%s (%.2f MB/s)%n", + humanDuration(uploadMs), uploadMBps, + humanDuration(downloadMs), downloadMBps); + } + + private static void generateLargeFile(Path dest, long sizeBytes) throws IOException { + final int blockSize = 64 * 1024 * 1024; + byte[] block = buildSstBlock(blockSize); + long written = 0L; + + try (BufferedOutputStream out = new BufferedOutputStream( + new FileOutputStream(dest.toFile()), blockSize)) { + while (written < sizeBytes) { + long toWrite = Math.min(blockSize, sizeBytes - written); + block[0] = (byte) (written >>> 20); + out.write(block, 0, (int) toWrite); + written += toWrite; + } + } + } + + private static byte[] buildSstBlock(int size) { + byte[] block = new byte[size]; + block[0] = (byte) 0x57; + block[1] = (byte) 0xfb; + block[2] = (byte) 0x80; + block[3] = (byte) 0x8b; + for (int i = 4; i < size; i++) { + block[i] = (byte) ((i * 1103515245 + 12345) & 0xFF); + } + return block; + } + + private static byte[] sha256(Path file) throws IOException { + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + final int bufSize = 64 * 1024 * 1024; + byte[] buf = new byte[bufSize]; + try (InputStream in = Files.newInputStream(file)) { + int n; + while ((n = in.read(buf)) != -1) { + md.update(buf, 0, n); + } + } + return md.digest(); + } catch (NoSuchAlgorithmException e) { + throw new IOException("SHA-256 not available", e); + } + } + + private static String humanSize(long bytes) { + if (bytes < 1024L) { + return bytes + " B"; + } + if (bytes < 1024L * 1024L) { + return String.format(Locale.US, "%.1f KB", bytes / 1024.0); + } + if (bytes < 1024L * 1024L * 1024L) { + return String.format(Locale.US, "%.1f MB", bytes / (1024.0 * 1024.0)); + } + return String.format(Locale.US, "%.2f GB", bytes / (1024.0 * 1024.0 * 1024.0)); + } + + private static String humanDuration(long millis) { + if (millis < 1000L) { + return millis + " ms"; + } + if (millis < 60_000L) { + return String.format(Locale.US, "%.1f s", millis / 1000.0); + } + long m = millis / 60_000L; + long s = (millis % 60_000L) / 1000L; + return String.format(Locale.US, "%dm %02ds", m, s); + } + + private static long longProp(String key, long def) { + String v = System.getProperty(key); + if (v == null || v.isBlank()) { + return def; + } + try { + return Long.parseLong(v.trim()); + } catch (NumberFormatException e) { + return def; + } + } + + private static boolean boolProp(String key, boolean def) { + String v = System.getProperty(key); + if (v == null || v.isBlank()) { + return def; + } + return Boolean.parseBoolean(v.trim()); + } + + /** + * True when an S3 integration profile is active (system property {@code s3.perf.integration} or + * env {@code S3_PERF_INTEGRATION} set truthy), in which case an unreachable endpoint must FAIL + * rather than skip. + */ + private static boolean integrationRequired() { + if (boolProp(PROP_INTEGRATION_REQUIRED, false)) { + return true; + } + String env = System.getenv(ENV_INTEGRATION_REQUIRED); + return env != null && !env.isBlank() && Boolean.parseBoolean(env.trim()); + } + + /** + * Builds an S3 admin client for bucket management during tests. + * Mirrors the same credential / endpoint / region logic used by {@link S3CloudStorageProvider}. + */ + private static S3Client buildAdminS3Client(String endpoint, String region, + String ak, String sk) { + S3ClientBuilder builder = S3Client.builder(); + + if (ak != null && !ak.isEmpty() && sk != null && !sk.isEmpty()) { + builder.credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(ak, sk))); + } else { + builder.credentialsProvider(DefaultCredentialsProvider.builder().build()); + } + + if (region != null && !region.isEmpty()) { + builder.region(Region.of(region)); + } + + if (endpoint != null && !endpoint.isEmpty()) { + builder.endpointOverride(URI.create(endpoint)); + builder.serviceConfiguration( + software.amazon.awssdk.services.s3.S3Configuration.builder() + .pathStyleAccessEnabled(true) + .build()); + } + + return builder.build(); + } + + /** + * Creates the bucket if it does not already exist. + * For AWS S3, {@code CreateBucketRequest} needs a {@code LocationConstraint} for + * regions other than {@code us-east-1}; for MinIO (path-style) no constraint is needed. + * + * @param s3 admin S3 client + * @param bucket bucket name to ensure + * @param region AWS region (used for location constraint on real AWS) + * @param endpoint custom endpoint (empty for real AWS) + */ + private static void ensureBucketExists(S3Client s3, String bucket, + String region, String endpoint) { + try { + s3.headBucket(HeadBucketRequest.builder().bucket(bucket).build()); + System.out.printf("[E2E] bucket already exists: %s%n", bucket); + } catch (software.amazon.awssdk.services.s3.model.S3Exception e) { + // Bucket does not exist — create it + System.out.printf("[E2E] bucket '%s' not found, creating...%n", bucket); + + CreateBucketRequest.Builder createReq = CreateBucketRequest.builder().bucket(bucket); + + // Real AWS: us-east-1 must NOT have a LocationConstraint; every other region must. + boolean isRealAws = endpoint == null || endpoint.isEmpty(); + if (isRealAws && region != null && !region.isEmpty() + && !region.equals("us-east-1")) { + createReq.createBucketConfiguration( + software.amazon.awssdk.services.s3.model.CreateBucketConfiguration.builder() + .locationConstraint(region) + .build()); + } + + s3.createBucket(createReq.build()); + System.out.printf("[E2E] bucket created: %s%n", bucket); + } + } + + /** + * Checks if the S3 endpoint (e.g., ...) is reachable. + * Parses the URL to extract host and port, then attempts a socket connection. + * + * @param endpoint the endpoint URL (e.g., ...) + * @return true if reachable, false otherwise + */ + private static boolean isEndpointReachable(String endpoint) { + try { + // Parse endpoint URL to extract host and port + // Expected format: http://host:port or https://host:port + java.net.URL url = new java.net.URL(endpoint); + String host = url.getHost(); + int port = url.getPort(); + if (port == -1) { + port = url.getDefaultPort(); // 80 for http, 443 for https + } + + if (host == null || host.isBlank() || port <= 0) { + return false; + } + + // Attempt socket connection with 2-second timeout + try (Socket socket = new Socket()) { + socket.connect(new java.net.InetSocketAddress(host, port), 2000); + return true; + } + } catch (Exception e) { + return false; + } + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + private static void deleteDirectory(java.io.File dir) { + if (!dir.exists()) { + return; + } + java.io.File[] children = dir.listFiles(); + if (children != null) { + for (java.io.File child : children) { + if (child.isDirectory()) { + deleteDirectory(child); + } else { + child.delete(); + } + } + } + dir.delete(); + } +} + diff --git a/hugegraph-store/hg-store-common/pom.xml b/hugegraph-store/hg-store-common/pom.xml index f49fcd4628..495ddf881a 100644 --- a/hugegraph-store/hg-store-common/pom.xml +++ b/hugegraph-store/hg-store-common/pom.xml @@ -46,5 +46,11 @@ guava 32.0.1-android + + org.mockito + mockito-core + 4.1.0 + test + diff --git a/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageConfig.java b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageConfig.java new file mode 100644 index 0000000000..fa0b6b4bc9 --- /dev/null +++ b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageConfig.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Data; +import lombok.ToString; + +/** + * Configuration for pluggable cloud storage providers. + * + *

Mapped from application.yml under the {@code cloud.storage} prefix. +

+ * cloud:
+ *   storage:
+ *     enabled: true
+ *     provider: s3
+ *     path-prefix: hugegraph
+ *     s3:
+ *       bucket: hugegraph-store
+ *       region: us-east-1
+ *       access-key: AKIAIOSFODNN7EXAMPLE
+ *       secret-key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+ * 
+ */ +@Data +public class CloudStorageConfig { + + /** + * Whether cloud storage is enabled. Defaults to false. + */ + private boolean enabled = false; + + /** + * Name of the provider to activate. Must match {@link CloudStorageProvider#providerName()}. + * The provider JAR must be on the classpath. Defaults to "s3". + */ + private String provider = "s3"; + + /** + * Key prefix prepended to every object stored in the bucket. + * Defaults to "hugegraph". + */ + private String pathPrefix = "hugegraph"; + + /** + * Whether startup pre-hydration (cloud -> local before DB open) is enabled. + */ + private boolean startupHydrationEnabled = true; + + /** + * Guard window in milliseconds for repeated read-miss hydration attempts on the + * same db/table pair. Values <= 0 disable the guard. + */ + private long readMissGuardWindowMs = 3000L; + + + /** + * Maximum number of whole-file upload retries after a first failure. Default is {@code 3} + * (whole-file retries enabled). Under the primary-durability model an SST that never reaches + * cloud is at risk on the ephemeral local disk, so whole-file retries are on by default. After + * all attempts are exhausted the upload is given up on (logged and counted); the file stays + * unconfirmed and is re-uploaded by the next metadata sync, delete guard, or startup SST + * backfill. + * + *

Set to {@code 0} to disable whole-file retries (failures are given up on immediately) + * when the provider already implements a sufficient internal retry strategy. + */ + private int uploadRetryMaxAttempts = 3; + + /** + * Delay before the first whole-file retry attempt, in milliseconds. + * Subsequent retries use exponential back-off up to {@link #uploadRetryMaxDelayMs}. + * Only used when {@link #uploadRetryMaxAttempts} > 0. Default: 1 000 ms. + */ + private long uploadRetryInitialDelayMs = 1_000L; + + /** + * Upper bound for the exponential back-off delay for whole-file retries, in milliseconds. + * Only used when {@link #uploadRetryMaxAttempts} > 0. Default: 60 000 ms. + */ + private long uploadRetryMaxDelayMs = 60_000L; + + /** + * Interval in milliseconds at which DBs with SST uploads since their last metadata publish are + * drained: each such DB gets at most one metadata publish per interval, coalescing the + * checkpoint + S3 list/prune cost under write-heavy load. Failed publishes are retried on the + * next interval. Values {@code <= 0} publish inline on every SST. Default: {@code 1000} ms. + * Event-driven publishes (delete guard, compaction, DB open) are never coalesced. + */ + private long metadataSyncDebounceMs = 1_000L; + + /** + * Stable per-node identity used to derive the cloud key scope ({@code store-}). + * + *

When set, this is the authoritative scope key and is stable across restarts, + * IP/hostname changes, and local disk loss — so a node can always find its prior remote + * objects during recovery. Leave blank to fall back to a scope persisted in the data directory + * (stable across network-identity drift but lost with the disk), which is in turn seeded from + * the runtime network address on first start. Operators who need guaranteed post-disk-loss + * recovery should set an explicit, deployment-stable value here (e.g. a Kubernetes + * StatefulSet pod ordinal or a provisioned node UUID). + */ + private String nodeId = ""; + + + /** + * Provider-specific properties, mapped from {@code cloud.storage..*} in the + * configuration. These are passed verbatim to the provider implementation and may include + * credentials, endpoint overrides, regions, or SDK tuning keys expected by that provider. + * Excluded from {@code toString()} so credentials are never written to logs. + */ + @ToString.Exclude + private Map providerProperties = new HashMap<>(); +} diff --git a/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageNonRetryableException.java b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageNonRetryableException.java new file mode 100644 index 0000000000..40b535e8a5 --- /dev/null +++ b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageNonRetryableException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud; + +import java.io.IOException; +import java.io.Serial; + +/** + * Marker exception indicating a cloud upload failure should not be retried at + * the whole-SST level and can be given up on immediately. + */ +public class CloudStorageNonRetryableException extends IOException { + + @Serial + private static final long serialVersionUID = 1L; + + public CloudStorageNonRetryableException(String message, Throwable cause) { + super(message, cause); + } +} + diff --git a/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProvider.java b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProvider.java new file mode 100644 index 0000000000..b592d24332 --- /dev/null +++ b/hugegraph-store/hg-store-common/src/main/java/org/apache/hugegraph/store/cloud/CloudStorageProvider.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; + +/** + * SPI interface for pluggable cloud storage backends. + * + *

Implementations are discovered via {@link java.util.ServiceLoader}: + * each provider JAR must contain + * {@code META-INF/services/org.apache.hugegraph.store.cloud.CloudStorageProvider} + * listing the fully-qualified implementation class. + * + *

The provider whose {@link #providerName()} matches {@link CloudStorageConfig#getProvider()} + * is selected once at store startup, initialized via {@link #init(CloudStorageConfig)}, and passed + * to its consumers; it is never reconfigured at runtime. + */ +public interface CloudStorageProvider extends Closeable { + + /** + * Returns the unique, lower-case name of this provider (e.g. {@code "s3"}, {@code "gcs"}). + * Must match the value set in {@link CloudStorageConfig#getProvider()}. + */ + String providerName(); + + /** + * Initializes the provider with the supplied configuration. + * Called once before any upload/download/delete operations. + * + * @param config cloud storage configuration + */ + void init(CloudStorageConfig config); + + /** + * Uploads a local file to cloud storage. + * + * @param localPath absolute path of the local source file + * @param remoteKey destination key / path inside the bucket (without prefix) + * @throws IOException on I/O or network failure + */ + void uploadFile(String localPath, String remoteKey) throws IOException; + + /** + * Deletes an object from cloud storage. + * + * @param remoteKey key / path of the object to delete (without prefix) + * @throws IOException on I/O or network failure + */ + void deleteFile(String remoteKey) throws IOException; + + /** + * Checks whether an object exists in cloud storage. + * + * @param remoteKey key / path to check (without prefix) + * @return {@code true} if the object exists + * @throws IOException on I/O or network failure + */ + boolean fileExists(String remoteKey) throws IOException; + + /** + * Lists object keys that start with a remote key prefix. The prefix is matched as a plain + * string, as S3 {@code ListObjectsV2} does: callers pass a trailing {@code /} to list one + * directory, and omit it to list sibling directories that share a name prefix (used to find a + * DB's version-suffixed directories, e.g. {@code 00001} and {@code 00001_5}). + * + * @param remoteDirPrefix key prefix inside bucket (without provider pathPrefix) + * @return object keys relative to provider root (without provider pathPrefix) + * @throws IOException on I/O or network failure + */ + List listFiles(String remoteDirPrefix) throws IOException; + + /** + * Downloads an object from cloud storage to a local file. + * + * @param remoteKey destination key / path inside the bucket (without prefix) + * @param localPath absolute path of the local destination file + * @throws IOException on I/O or network failure + */ + void downloadFile(String remoteKey, String localPath) throws IOException; + + /** + * Deletes all objects under a prefix. Implementations should use bulk-delete semantics where + * the underlying storage backend supports them (e.g. S3 DeleteObjects API). + * + * @param remoteDirPrefix directory/prefix inside bucket (without provider pathPrefix) + * @return number of objects deleted + * @throws IOException on I/O or network failure, including a partial delete + */ + int deletePrefix(String remoteDirPrefix) throws IOException; + + /** + * Releases resources held by the provider (e.g. HTTP clients, connections). + */ + @Override + void close() throws IOException; +} + diff --git a/hugegraph-store/hg-store-common/src/test/java/org/apache/hugegraph/store/cloud/CloudStorageConfigTest.java b/hugegraph-store/hg-store-common/src/test/java/org/apache/hugegraph/store/cloud/CloudStorageConfigTest.java new file mode 100644 index 0000000000..161644bb45 --- /dev/null +++ b/hugegraph-store/hg-store-common/src/test/java/org/apache/hugegraph/store/cloud/CloudStorageConfigTest.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; + +@SuppressWarnings("ObviousNullCheck") +public class CloudStorageConfigTest { + + private CloudStorageConfig config; + + @Before + public void setUp() { + config = new CloudStorageConfig(); + } + + // ---- Common fields ---- + + @Test + public void testDefaults() { + assertFalse(config.isEnabled()); + assertEquals("s3", config.getProvider()); + assertEquals("hugegraph", config.getPathPrefix()); + assertTrue(config.isStartupHydrationEnabled()); + assertEquals(3000L, config.getReadMissGuardWindowMs()); + assertNotNull(config.getProviderProperties()); + } + + @Test + public void testEnabledFlag() { + assertFalse(config.isEnabled()); + + config.setEnabled(true); + assertTrue(config.isEnabled()); + + config.setEnabled(false); + assertFalse(config.isEnabled()); + } + + @Test + public void testProvider() { + assertEquals("s3", config.getProvider()); + config.setProvider("gcs"); + assertEquals("gcs", config.getProvider()); + config.setProvider("adls"); + assertEquals("adls", config.getProvider()); + } + + @Test + public void testPathPrefix() { + assertEquals("hugegraph", config.getPathPrefix()); + + config.setPathPrefix("myapp/data"); + assertEquals("myapp/data", config.getPathPrefix()); + } + + @Test + public void testStartupHydrationEnabled() { + assertTrue(config.isStartupHydrationEnabled()); + + config.setStartupHydrationEnabled(false); + assertFalse(config.isStartupHydrationEnabled()); + + config.setStartupHydrationEnabled(true); + assertTrue(config.isStartupHydrationEnabled()); + } + + @Test + public void testReadMissGuardWindowMs() { + assertEquals(3000L, config.getReadMissGuardWindowMs()); + + config.setReadMissGuardWindowMs(5000L); + assertEquals(5000L, config.getReadMissGuardWindowMs()); + + config.setReadMissGuardWindowMs(0L); + assertEquals(0L, config.getReadMissGuardWindowMs()); + + config.setReadMissGuardWindowMs(-1L); + assertEquals(-1L, config.getReadMissGuardWindowMs()); + } + + @Test + public void testUploadRetryDefaults() { + // Default 3: whole-file retries enabled under the primary-durability model. + assertEquals(3, config.getUploadRetryMaxAttempts()); + assertEquals(1_000L, config.getUploadRetryInitialDelayMs()); + assertEquals(60_000L, config.getUploadRetryMaxDelayMs()); + } + + @Test + public void testUploadRetryMaxAttempts() { + config.setUploadRetryMaxAttempts(3); + assertEquals(3, config.getUploadRetryMaxAttempts()); + + config.setUploadRetryMaxAttempts(0); + assertEquals(0, config.getUploadRetryMaxAttempts()); + + config.setUploadRetryMaxAttempts(10); + assertEquals(10, config.getUploadRetryMaxAttempts()); + } + + @Test + public void testUploadRetryInitialDelayMs() { + config.setUploadRetryInitialDelayMs(500L); + assertEquals(500L, config.getUploadRetryInitialDelayMs()); + + config.setUploadRetryInitialDelayMs(2_000L); + assertEquals(2_000L, config.getUploadRetryInitialDelayMs()); + } + + @Test + public void testUploadRetryMaxDelayMs() { + config.setUploadRetryMaxDelayMs(30_000L); + assertEquals(30_000L, config.getUploadRetryMaxDelayMs()); + + config.setUploadRetryMaxDelayMs(120_000L); + assertEquals(120_000L, config.getUploadRetryMaxDelayMs()); + } + + @Test + public void testMetadataSyncDebounceMs() { + assertEquals(1_000L, config.getMetadataSyncDebounceMs()); + + config.setMetadataSyncDebounceMs(250L); + assertEquals(250L, config.getMetadataSyncDebounceMs()); + + config.setMetadataSyncDebounceMs(0L); + assertEquals(0L, config.getMetadataSyncDebounceMs()); + } + + @Test + public void testNodeId() { + assertEquals("", config.getNodeId()); + + config.setNodeId("store-node-7"); + assertEquals("store-node-7", config.getNodeId()); + } + + /** + * Sanity check of the value semantics provided by {@code @Data}. + */ + @Test + public void testEqualsHashCodeAndToString() { + CloudStorageConfig a = new CloudStorageConfig(); + CloudStorageConfig b = new CloudStorageConfig(); + + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + assertEquals(a, a); + assertNotEquals(null, a); + assertNotEquals("not-a-config", a); + assertNotNull(a.toString()); + + b.setProvider("gcs"); + assertNotEquals(a, b); + assertNotEquals(a.hashCode(), b.hashCode()); + } + + @Test + public void testToStringOmitsProviderProperties() { + Map props = new HashMap<>(); + props.put("access-key", "AKIAIOSFODNN7EXAMPLE"); + props.put("secret-key", "super-secret-value"); + config.setProviderProperties(props); + + String str = config.toString(); + assertTrue(str.contains("provider=s3")); + assertFalse(str.contains("providerProperties")); + assertFalse(str.contains("AKIAIOSFODNN7EXAMPLE")); + assertFalse(str.contains("super-secret-value")); + } + + // ---- Provider properties (cloud.storage..* flattened) ---- + + @Test + public void testProviderPropertiesDefaults() { + assertNotNull(config.getProviderProperties()); + assertTrue(config.getProviderProperties().isEmpty()); + } + + @Test + public void testProviderPropertiesSetAndGet() { + Map props = new HashMap<>(); + props.put("bucket", "my-bucket"); + props.put("region", "us-east-1"); + props.put("endpoint", "https://s3.amazonaws.com"); + props.put("access-key", "AKIAIOSFODNN7EXAMPLE"); + props.put("secret-key", "secret"); + props.put("multipart-part-retry-max-attempts", "7"); + + config.setProviderProperties(props); + + assertEquals("my-bucket", config.getProviderProperties().get("bucket")); + assertEquals("us-east-1", config.getProviderProperties().get("region")); + assertEquals("7", config.getProviderProperties() + .get("multipart-part-retry-max-attempts")); + } + + @Test + public void testCompleteConfiguration() { + config.setEnabled(true); + config.setProvider("s3"); + config.setPathPrefix("production/data"); + config.setStartupHydrationEnabled(true); + config.setReadMissGuardWindowMs(5000L); + config.setUploadRetryMaxAttempts(3); + config.setUploadRetryInitialDelayMs(500L); + config.setUploadRetryMaxDelayMs(30_000L); + + Map providerProps = new HashMap<>(); + providerProps.put("bucket", "hugegraph-backup"); + providerProps.put("region", "us-west-2"); + providerProps.put("endpoint", "https://s3-us-west-2.amazonaws.com"); + providerProps.put("access-key", "test-access-key"); + providerProps.put("secret-key", "test-secret-key"); + providerProps.put("multipart-part-retry-max-attempts", "5"); + providerProps.put("multipart-part-retry-base-backoff-ms", "1500"); + config.setProviderProperties(providerProps); + + // Common fields + assertTrue(config.isEnabled()); + assertEquals("s3", config.getProvider()); + assertEquals("production/data", config.getPathPrefix()); + assertTrue(config.isStartupHydrationEnabled()); + assertEquals(5000L, config.getReadMissGuardWindowMs()); + assertEquals(3, config.getUploadRetryMaxAttempts()); + assertEquals(500L, config.getUploadRetryInitialDelayMs()); + assertEquals(30_000L, config.getUploadRetryMaxDelayMs()); + + // Provider-specific fields (cloud.storage..*) + assertEquals("hugegraph-backup", config.getProviderProperties().get("bucket")); + assertEquals("us-west-2", config.getProviderProperties().get("region")); + assertEquals("https://s3-us-west-2.amazonaws.com", + config.getProviderProperties().get("endpoint")); + assertEquals("test-access-key", config.getProviderProperties().get("access-key")); + assertEquals("test-secret-key", config.getProviderProperties().get("secret-key")); + assertEquals("5", config.getProviderProperties().get("multipart-part-retry-max-attempts")); + assertEquals("1500", + config.getProviderProperties().get("multipart-part-retry-base-backoff-ms")); + } + + @Test + public void testMultiplePropertyUpdates() { + Map props1 = new HashMap<>(); + props1.put("key1", "value1"); + config.setProviderProperties(props1); + assertEquals("value1", config.getProviderProperties().get("key1")); + + Map props2 = new HashMap<>(); + props2.put("key2", "value2"); + config.setProviderProperties(props2); + assertEquals("value2", config.getProviderProperties().get("key2")); + } + + @Test + public void testNegativeReadMissGuardWindow() { + config.setReadMissGuardWindowMs(-5000L); + assertEquals(-5000L, config.getReadMissGuardWindowMs()); + } + + @Test + public void testProviderPropertiesImmutabilityBehavior() { + Map props = new HashMap<>(); + props.put("bucket", "original"); + config.setProviderProperties(props); + + // Modify the original map + props.put("bucket", "modified"); + + // The config should have the modified value since it references the same map + assertEquals("modified", config.getProviderProperties().get("bucket")); + } +} diff --git a/hugegraph-store/hg-store-common/src/test/java/org/apache/hugegraph/store/cloud/CloudStorageNonRetryableExceptionTest.java b/hugegraph-store/hg-store-common/src/test/java/org/apache/hugegraph/store/cloud/CloudStorageNonRetryableExceptionTest.java new file mode 100644 index 0000000000..d1e3b24830 --- /dev/null +++ b/hugegraph-store/hg-store-common/src/test/java/org/apache/hugegraph/store/cloud/CloudStorageNonRetryableExceptionTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.cloud; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.junit.Test; + +public class CloudStorageNonRetryableExceptionTest { + + @Test + public void testConstructorPreservesMessageAndCause() { + IOException cause = new IOException("root cause"); + + CloudStorageNonRetryableException exception = + new CloudStorageNonRetryableException("non-retryable", cause); + + assertEquals("non-retryable", exception.getMessage()); + assertSame(cause, exception.getCause()); + } + + @Test + public void testDirectSuperclassIsIOException() { + assertSame(IOException.class, CloudStorageNonRetryableException.class.getSuperclass()); + } + + @Test + public void testSerialVersionUID() { + try { + java.lang.reflect.Field field = + CloudStorageNonRetryableException.class.getDeclaredField("serialVersionUID"); + field.setAccessible(true); + Long serialVersionUID = (Long) field.get(null); + assertEquals(Long.valueOf(1L), serialVersionUID); + } catch (NoSuchFieldException | IllegalAccessException e) { + // serialVersionUID field exists as expected + } + } + + @Test + public void testConstructorWithNullMessage() { + IOException cause = new IOException("cause"); + + CloudStorageNonRetryableException exception = + new CloudStorageNonRetryableException(null, cause); + + assertNull(exception.getMessage()); + assertSame(cause, exception.getCause()); + } + + @Test + public void testConstructorWithNullCause() { + CloudStorageNonRetryableException exception = + new CloudStorageNonRetryableException("message", null); + + assertEquals("message", exception.getMessage()); + assertNull(exception.getCause()); + } + + @Test + public void testConstructorWithBothNull() { + CloudStorageNonRetryableException exception = + new CloudStorageNonRetryableException(null, null); + + assertNull(exception.getMessage()); + assertNull(exception.getCause()); + } + + @Test + public void testExceptionThrowableAndCatch() { + IOException cause = new IOException("original cause"); + CloudStorageNonRetryableException exception = + new CloudStorageNonRetryableException("cloud error", cause); + + try { + throw exception; + } catch (IOException e) { + assertEquals("cloud error", e.getMessage()); + assertSame(cause, e.getCause()); + } + } + + @Test + public void testExceptionStackTrace() { + IOException cause = new IOException("root cause"); + CloudStorageNonRetryableException exception = + new CloudStorageNonRetryableException("non-retryable", cause); + + assertTrue(exception.toString().contains("CloudStorageNonRetryableException")); + assertTrue(exception.toString().contains("non-retryable")); + } +} diff --git a/hugegraph-store/hg-store-core/pom.xml b/hugegraph-store/hg-store-core/pom.xml index 6d456af251..061f672d43 100644 --- a/hugegraph-store/hg-store-core/pom.xml +++ b/hugegraph-store/hg-store-core/pom.xml @@ -39,6 +39,12 @@ hugegraph-struct ${revision} + + org.mockito + mockito-core + 4.1.0 + test + org.projectlombok diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java index 65e97b5ab4..f340770863 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java @@ -1113,7 +1113,12 @@ public void batchGet(String graph, String table, SupplierA partition's RocksDB instance is shared by every graph mapped to that partition, so this + * clears only {@code graphName}'s key range via {@code deleteRange}. The range-tombstoned SSTs + * produced here converge in cloud through the normal + * {@code onTableFileCreated}/{@code onTableFileDeleted} mirroring. */ @Override public void truncate(String graphName, int partId) throws HgStoreException { diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionManager.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionManager.java index 85a281e940..d9350281a6 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionManager.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/meta/PartitionManager.java @@ -134,6 +134,14 @@ public void addPartitionChangedListener(PartitionChangedListener listener) { partitionChangedListeners.add(listener); } + /** + * Whether a DB-directory name is a transient artifact of snapshot loading ("_temp_" suffix) or + * of cloud metadata capture ("_cloudmeta_" infix), not a partition's data directory. + */ + private static boolean isTransientDbDir(String name) { + return name.endsWith("_temp_") || name.contains("_cloudmeta_"); + } + /** * Judge the storage path as either the partition id or starting with partition id_ * @@ -149,6 +157,9 @@ private Boolean checkPathContains(File[] detections, int partitionId, boolean ch // Must be in a folder named after the partition id if (detections[x].isDirectory()) { String tmp = detections[x].getName(); + if (isTransientDbDir(tmp)) { + continue; + } if (tmp.equals(partitionDirectory) || tmp.startsWith(partitionDirectory + "_")) { if (checkLogDir) { String logDir = detections[x].getAbsolutePath() + "/log"; @@ -252,7 +263,7 @@ private void loadPartitions() { } for (File f : dirs) { - if (f.isDirectory()) { + if (f.isDirectory() && !isTransientDbDir(f.getName())) { try { partIds.add(Integer.parseInt(f.getName().split("_")[0])); } catch (Exception e) { diff --git a/hugegraph-store/hg-store-core/src/test/java/org/apache/hugegraph/store/business/BusinessHandlerImplTest.java b/hugegraph-store/hg-store-core/src/test/java/org/apache/hugegraph/store/business/BusinessHandlerImplTest.java new file mode 100644 index 0000000000..faaa576aa4 --- /dev/null +++ b/hugegraph-store/hg-store-core/src/test/java/org/apache/hugegraph/store/business/BusinessHandlerImplTest.java @@ -0,0 +1,469 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.business; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.SessionOperator; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.pd.PdProvider; +import org.apache.hugegraph.store.util.HgStoreException; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.config.Configurator; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.rocksdb.Cache; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.MemoryUsageType; + +/** + * Consolidated unit tests for {@link BusinessHandlerImpl}, including the former + * {@code BusinessHandlerImplExtendedTest} scenarios. + * + *

Covers static utility methods, partition operations, and core business logic. + */ +public class BusinessHandlerImplTest { + + private static class SessionOverridingBusinessHandler extends BusinessHandlerImpl { + + private final RocksDBSession session; + + SessionOverridingBusinessHandler(PartitionManager partitionManager, RocksDBSession session) { + super(partitionManager); + this.session = session; + } + + @Override + public RocksDBSession getSession(int partId) throws HgStoreException { + return this.session; + } + } + + /** Reads the private static {@code indexDataSize} field so setter behavior is observable. */ + private static long readIndexDataSize() throws Exception { + Field field = BusinessHandlerImpl.class.getDeclaredField("indexDataSize"); + field.setAccessible(true); + return (Long) field.get(null); + } + + private BusinessHandlerImpl handler; + private PartitionManager mockPartitionManager; + + @BeforeClass + public static void setUpClass() { + // Initialize static resources if needed + } + + @Before + public void setUp() { + mockPartitionManager = mock(PartitionManager.class); + RocksDBSession mockSession = mock(RocksDBSession.class); + + handler = new BusinessHandlerImpl(mockPartitionManager); + + // Setup common mock behaviors + when(mockSession.sessionOp()).thenReturn(mock(SessionOperator.class)); + } + + @Test + public void testFnvHashIsDeterministicAndSensitiveToInput() { + byte[] keyA = "abc".getBytes(StandardCharsets.UTF_8); + byte[] keyB = "abd".getBytes(StandardCharsets.UTF_8); + + Long hashA1 = BusinessHandlerImpl.fnvHash(keyA); + Long hashA2 = BusinessHandlerImpl.fnvHash(keyA); + Long hashB = BusinessHandlerImpl.fnvHash(keyB); + + assertEquals(hashA1, hashA2); + assertNotEquals(hashA1, hashB); + } + + @Test + public void testFnvHashEmptyInputUsesOffsetBasis() { + Long hash = BusinessHandlerImpl.fnvHash(new byte[0]); + + assertEquals(Long.valueOf(0xcbf29ce484222325L), hash); + } + + @Test + public void testFnvHashIsSensitiveToByteOrder() { + byte[] key1 = new byte[]{1, 2, 3}; + byte[] key2 = new byte[]{3, 2, 1}; + + Long hash1 = BusinessHandlerImpl.fnvHash(key1); + Long hash2 = BusinessHandlerImpl.fnvHash(key2); + + assertNotEquals(hash1, hash2); + } + + @Test + public void testFnvHashLargeInputReturnsHash() { + byte[] largeInput = new byte[10000]; + for (int i = 0; i < largeInput.length; i++) { + largeInput[i] = (byte) (i % 256); + } + + Long hash = BusinessHandlerImpl.fnvHash(largeInput); + + assertNotNull(hash); + } + + @Test + public void testGetDbNameFormatsAndCachesPartitionId() { + String dbName = BusinessHandlerImpl.getDbName(12); + + assertEquals("00012", dbName); + assertEquals(dbName, BusinessHandlerImpl.getDbName(12)); + } + + @Test + public void testGetDbNameFormatsPadding() { + assertEquals("00001", BusinessHandlerImpl.getDbName(1)); + assertEquals("00010", BusinessHandlerImpl.getDbName(10)); + assertEquals("00100", BusinessHandlerImpl.getDbName(100)); + assertEquals("01000", BusinessHandlerImpl.getDbName(1000)); + assertEquals("10000", BusinessHandlerImpl.getDbName(10000)); + } + + @Test + public void testGetDbNameReturnsCachedValueForSameId() { + String first = BusinessHandlerImpl.getDbName(999); + String second = BusinessHandlerImpl.getDbName(999); + String third = BusinessHandlerImpl.getDbName(999); + + assertEquals(first, second); + assertEquals(second, third); + assertEquals("00999", first); + } + + @Test + public void testSetIndexDataSizeStoresPositiveValue() throws Exception { + long original = readIndexDataSize(); + try { + BusinessHandlerImpl.setIndexDataSize(100 * 1024L); + assertEquals(100 * 1024L, readIndexDataSize()); + } finally { + BusinessHandlerImpl.setIndexDataSize(original); + } + } + + @Test + public void testSetIndexDataSizeIgnoresNonPositiveValue() throws Exception { + long original = readIndexDataSize(); + try { + BusinessHandlerImpl.setIndexDataSize(64 * 1024L); + // Zero and negative values are documented no-ops: the previous value must survive. + BusinessHandlerImpl.setIndexDataSize(0L); + assertEquals(64 * 1024L, readIndexDataSize()); + BusinessHandlerImpl.setIndexDataSize(-10L); + assertEquals(64 * 1024L, readIndexDataSize()); + } finally { + BusinessHandlerImpl.setIndexDataSize(original); + } + } + + @Test + public void testGetCompactionPoolReturnsInitializedPool() { + var pool = BusinessHandlerImpl.getCompactionPool(); + + assertNotNull(pool); + assertTrue(pool.getCorePoolSize() > 0); + assertTrue(pool.getMaximumPoolSize() > 0); + } + + @Test + public void testGetCompactionPoolReturnsSameInstance() { + var pool1 = BusinessHandlerImpl.getCompactionPool(); + var pool2 = BusinessHandlerImpl.getCompactionPool(); + + assertEquals(pool1, pool2); + } + + // ========== Tests for doPut/doGet ========== + + @Test + public void testDoPutInternalErrorWrapsAsHgStoreException() { + PartitionManager partitionManager = mock(PartitionManager.class); + PdProvider pdProvider = mock(PdProvider.class); + when(partitionManager.getPdProvider()).thenReturn(pdProvider); + + Metapb.Partition partition = Metapb.Partition.newBuilder().setId(10).build(); + when(pdProvider.getPartitionByCode("g", 1)).thenReturn(partition); + + RocksDBSession session = mock(RocksDBSession.class); + SessionOperator op = mock(SessionOperator.class); + when(session.sessionOp()).thenReturn(op); + doThrow(new RuntimeException("prepare boom")).when(op).prepare(); + + BusinessHandlerImpl localHandler = + new SessionOverridingBusinessHandler(partitionManager, session); + + // doPut logs this exception at ERROR before wrapping it; silence it so the expected + // failure doesn't print a misleading stack trace into the test run's output. + String loggerName = BusinessHandlerImpl.class.getName(); + Level originalLevel = + ((org.apache.logging.log4j.core.Logger) LogManager.getLogger(loggerName)) + .getLevel(); + Configurator.setLevel(loggerName, Level.OFF); + try { + localHandler.doPut("g", 1, "g+v", new byte[]{1}, new byte[]{2}); + fail("Expected HgStoreException"); + } catch (HgStoreException e) { + assertEquals(HgStoreException.EC_RKDB_DOPUT_FAIL, e.getCode()); + assertTrue(e.getMessage().contains("prepare boom")); + verify(op, times(1)).rollback(); + } finally { + Configurator.setLevel(loggerName, originalLevel); + } + } + + @Test + public void testDoGetPartitionNotManagedReturnsNull() throws HgStoreException { + PartitionManager partitionManager = mock(PartitionManager.class); + PdProvider pdProvider = mock(PdProvider.class); + when(partitionManager.getPdProvider()).thenReturn(pdProvider); + + Metapb.Partition partition = Metapb.Partition.newBuilder().setId(7).build(); + when(pdProvider.getPartitionByCode("g", 1)).thenReturn(partition); + // Partition 7 is not managed by this store, so doGet must short-circuit to null without + // ever opening a RocksDB session. + when(partitionManager.hasPartition("g", 7)).thenReturn(false); + + RocksDBSession session = mock(RocksDBSession.class); + BusinessHandlerImpl localHandler = + new SessionOverridingBusinessHandler(partitionManager, session); + + assertNull(localHandler.doGet("g", 1, "g+v", new byte[]{1})); + verify(partitionManager, times(1)).hasPartition("g", 7); + verify(session, never()).sessionOp(); + } + + // ========== Tests for getLeaderPartitionIds ========== + + @Test + public void testGetLeaderPartitionIdsDelegatesToPartitionManager() { + String graph = "test-graph"; + List expectedIds = Arrays.asList(1, 2, 3); + when(mockPartitionManager.getLeaderPartitionIds(graph)).thenReturn(expectedIds); + + List result = handler.getLeaderPartitionIds(graph); + + assertEquals(expectedIds, result); + verify(mockPartitionManager, times(1)).getLeaderPartitionIds(graph); + } + + @Test + public void testGetLeaderPartitionIdsReturnsEmptyListWhenNoLeaders() { + String graph = "empty-graph"; + when(mockPartitionManager.getLeaderPartitionIds(graph)).thenReturn(Collections.emptyList()); + + List result = handler.getLeaderPartitionIds(graph); + + assertTrue(result.isEmpty()); + verify(mockPartitionManager, times(1)).getLeaderPartitionIds(graph); + } + + // ========== Tests for getLeaderPartitionIdSet ========== + + @Test + public void testGetLeaderPartitionIdSetDelegatesToPartitionManager() { + when(mockPartitionManager.getLeaderPartitionIdSet()).thenReturn( + Collections.singleton(1)); + + var result = handler.getLeaderPartitionIdSet(); + + assertNotNull(result); + assertTrue(result.contains(1)); + verify(mockPartitionManager, times(1)).getLeaderPartitionIdSet(); + } + + // ========== Tests for Table operations ========== + + @Test + public void testExistsTableReturnsSessionResult() { + RocksDBSession session = mock(RocksDBSession.class); + when(session.tableIsExist("g+v")).thenReturn(true); + when(session.tableIsExist("missing")).thenReturn(false); + + BusinessHandlerImpl localHandler = + new SessionOverridingBusinessHandler(mockPartitionManager, session); + + assertTrue(localHandler.existsTable("g", 1, "g+v")); + assertFalse(localHandler.existsTable("g", 1, "missing")); + } + + @Test + public void testGetTableNamesReturnsSessionTableKeys() { + Map tableMap = new HashMap<>(); + for (String t : Arrays.asList("g+v", "g+e", "g+index")) { + tableMap.put(t, mock(ColumnFamilyHandle.class)); + } + + RocksDBSession session = mock(RocksDBSession.class); + when(session.getTables()).thenReturn(tableMap); + BusinessHandlerImpl localHandler = + new SessionOverridingBusinessHandler(mockPartitionManager, session); + + List names = localHandler.getTableNames("g", 1); + + assertNotNull(names); + assertEquals(3, names.size()); + assertTrue(names.containsAll(Arrays.asList("g+v", "g+e", "g+index"))); + } + + // ========== Tests for Partition operations ========== + + // ========== Tests for Metric operations ========== + + @Test + public void testGetApproximateMemoryUsageByTypeReturnsNonNullMap() { + List caches = new ArrayList<>(); + + Map result = handler.getApproximateMemoryUsageByType(caches); + + assertNotNull(result); + // Empty map on exception is expected behavior + } + + // ========== Tests for transaction operations ========== + + @Test + public void testTxBuilderReturnsBuilderBackedBySession() throws HgStoreException { + RocksDBSession session = mock(RocksDBSession.class); + // TxBuilderImpl's constructor opens and prepares an operator on the session. + SessionOperator op = mock(SessionOperator.class); + when(session.sessionOp()).thenReturn(op); + BusinessHandlerImpl localHandler = + new SessionOverridingBusinessHandler(mockPartitionManager, session); + + BusinessHandler.TxBuilder txBuilder = localHandler.txBuilder("g", 1); + assertNotNull(txBuilder); + verify(op, times(1)).prepare(); + + // The constructor takes graphLock("g", 1).readLock() and only releases it when the + // built Tx is committed or rolled back; leaving it held here would deadlock any later + // test that takes the write lock for the same (graph, partId), e.g. truncate("g", 1). + txBuilder.build().rollback(); + verify(op, times(1)).rollback(); + } + + // ========== Tests for database operations ========== + + @Test + public void testCloseDbBasicPathIsInvocable() { + int partId = 1; + + // Verifies the method can be called + // Full testing requires RocksDBFactory setup + handler.closeDB(partId); + } + + @Test + public void testFlushAllBasicPathIsInvocable() { + // Verifies the method can be called without throwing + handler.flushAll(); + } + + @Test + public void testCloseAllBasicPathIsInvocable() { + // Verifies the method can be called without throwing + handler.closeAll(); + } + + @Test + public void testTruncateClearsOnlyGraphKeyRange() + throws Exception { + RocksDBSession session = mock(RocksDBSession.class); + SessionOperator op = mock(SessionOperator.class); + when(session.sessionOp()).thenReturn(op); + + BusinessHandlerImpl localHandler = + new SessionOverridingBusinessHandler(mockPartitionManager, session); + + // Replace the real InnerKeyCreator with a mock so getStartKey/getEndKey/delGraphId do not + // touch RocksDB or the graph-id metadata store. + InnerKeyCreator mockKeyCreator = mock(InnerKeyCreator.class); + Field keyCreatorField = BusinessHandlerImpl.class.getDeclaredField("keyCreator"); + keyCreatorField.setAccessible(true); + keyCreatorField.set(localHandler, mockKeyCreator); + + localHandler.truncate("g", 1); + + // Only this graph's key range is cleared... + verify(op, times(1)).deleteRange(any(), any()); + // ...and its graph id is released. + verify(mockKeyCreator, times(1)).delGraphId(1, "g"); + // A partition's RocksDB instance is shared across graphs, so the clear must not drop or + // recreate the whole instance. + verify(session, never()).truncate(); + // The session opened by truncate() is closed by try-with-resources. + verify(session, times(1)).close(); + } + + @Test + public void testGetPartitionIdsDelegatesToPartitionManager() { + String graph = "test-graph"; + List expectedIds = Arrays.asList(1, 2, 3); + when(mockPartitionManager.getPartitionIds(graph)).thenReturn(expectedIds); + + List result = handler.getPartitionIds(graph); + + assertEquals(expectedIds, result); + } + + // ========== Tests for lock operations ========== + + @Test + public void testGetLockPathDelegatesToPartitionManager() { + int partitionId = 1; + when(mockPartitionManager.getDbDataPath(partitionId)) + .thenReturn("/data/partition/00001"); + + String lockPath = handler.getLockPath(partitionId); + + assertNotNull(lockPath); + verify(mockPartitionManager, times(1)).getDbDataPath(partitionId); + } +} + diff --git a/hugegraph-store/hg-store-dist/pom.xml b/hugegraph-store/hg-store-dist/pom.xml index ab53403f3f..45d7dec751 100644 --- a/hugegraph-store/hg-store-dist/pom.xml +++ b/hugegraph-store/hg-store-dist/pom.xml @@ -51,6 +51,11 @@ exec runtime + + org.rocksdb + rocksdbjni + runtime + diff --git a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml index ee2b6af9bf..2da4c744ae 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml +++ b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml @@ -89,6 +89,18 @@ org.apache.hugegraph:${executable.jar.name}:jar:exec:* + + + /lib + ${artifact.artifactId}-${artifact.version}.jar + false + runtime + false + + org.rocksdb:rocksdbjni + + diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 41844e7e04..2852fb13a5 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -278,10 +278,19 @@ if [ "${#BOOT_JARS[@]}" -ne 1 ] || [ ! -f "${BOOT_JARS[0]}" ]; then echo "Error: expected one component executable JAR in $LIB" >&2 exit 1 fi -JAVA_MAIN=(-jar "${BOOT_JARS[0]}") +# RocksDB fires JNI event callbacks on native threads, which resolve org.rocksdb.* through the +# system classloader; keep a rocksdbjni jar on the launcher classpath, not only in BOOT-INF/lib. if [ -n "${TOPLING_RUNTIME_CLASSPATH:-}" ]; then - JAVA_MAIN=(-cp "${TOPLING_RUNTIME_CLASSPATH}:${BOOT_JARS[0]}" org.springframework.boot.loader.JarLauncher) + ROCKSDB_CLASSPATH="${TOPLING_RUNTIME_CLASSPATH}" +else + ROCKSDB_JARS=("${LIB}"/rocksdbjni-*.jar) + if [ "${#ROCKSDB_JARS[@]}" -ne 1 ] || [ ! -f "${ROCKSDB_JARS[0]}" ]; then + echo "Error: expected one rocksdbjni JAR in $LIB; rebuild the distribution" >&2 + exit 1 + fi + ROCKSDB_CLASSPATH="${ROCKSDB_JARS[0]}" fi +JAVA_MAIN=(-cp "${ROCKSDB_CLASSPATH}:${BOOT_JARS[0]}" org.springframework.boot.loader.JarLauncher) # Turn on security check if [[ $DAEMON == "true" ]]; then diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application.yml index d36579d7c5..085371d126 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application.yml @@ -66,5 +66,70 @@ logging: level: root: info +# --------------------------------------------------------------------------- +# Cloud Storage (pluggable provider – disabled by default) +# --------------------------------------------------------------------------- +# To enable, build the distribution with the opt-in -Ps3-cloud-storage profile (it bundles +# hg-store-cloud-s3 and the AWS SDK into the node jar) and set enabled=true. The default build +# ships no provider, and a provider jar added to the launch classpath cannot load (see "Enabling +# S3 on the stock distribution" in hugegraph-store/README.md). +# +# cloud: +# storage: +# enabled: true +# provider: s3 # must match CloudStorageProvider.providerName() +# path-prefix: hugegraph # key prefix inside the bucket +# # --- Upload retry (common for all providers) --- +# upload-retry-max-attempts: 3 # whole-file retries after first failure; default 3 (0 disables) +# upload-retry-initial-delay-ms: 1000 # delay before first retry (ms); default 1000 +# upload-retry-max-delay-ms: 60000 # exponential-backoff ceiling (ms); default 60000 +# metadata-sync-debounce-ms: 1000 # coalesced per-SST metadata sync interval; <= 0 publishes on every SST +# node-id: # stable per-node identity for the cloud key scope; blank = persist +# # one in the data dir (seeded from address). Set a stable value +# # (e.g. StatefulSet ordinal / provisioned UUID) for recovery after +# # IP/hostname change or local disk loss. +# # Metadata sync is triggered by storage events; SST-upload syncs are coalesced per interval. +# # --- S3 provider settings (cloud.storage.s3.*) --- +# s3: +# bucket: hugegraph-store # S3 bucket name +# region: us-east-1 # AWS region +# endpoint: # custom S3-compatible endpoint (MinIO, Ceph...) +# access-key: AKIAIOSFODNN7EXAMPLE # omit to use the AWS default credentials chain +# secret-key: wJalrXUtnFEMI/... # omit to use the AWS default credentials chain +cloud: + storage: + enabled: false + provider: s3 + path-prefix: hugegraph + # Hydration controls + startup-hydration-enabled: true # download missing remote files on DB opening before serving + read-miss-guard-window-ms: 3000 # guard window (ms) to throttle repeated read-miss hydration attempts + # Whole-file upload retries after a first failure (default 3). Set 0 to disable + # (failures are given up on immediately) when the provider has sufficient internal retry. + upload-retry-max-attempts: 3 + upload-retry-initial-delay-ms: 1000 + upload-retry-max-delay-ms: 60000 + # Interval (ms) at which DBs with new SST uploads get one coalesced metadata publish; failed + # publishes retry on the next interval. <= 0 publishes on every SST. Event-driven publishes + # are never coalesced. + metadata-sync-debounce-ms: 1000 + # Mirror a consistent CURRENT/MANIFEST/OPTIONS snapshot so cloud objects stay + # recoverable (a node that lost its local disk reopens from cloud, not an empty DB). + # Metadata sync is always enabled when cloud storage is enabled. + # Sync is triggered by storage events; SST-upload syncs are coalesced per the interval above. + # Capture forces a MemTable flush so the un-flushed tail is persisted into an SST and mirrored; + # the RocksDB WAL is disabled under Raft (the Raft log is the tail's durability source), so a + # flush is the only way to push recent writes into cloud. + # Stable per-node identity for the cloud key scope (store-). Blank => a scope is + # persisted in the data dir on first start (seeded from the network address), which is stable + # across IP/hostname drift but lost with the disk. Set a deployment-stable value here for + # guaranteed recovery of prior remote data after an address change or local disk loss. + node-id: + s3: + bucket: + region: + endpoint: + access-key: + secret-key: # HTTP Arthas activation is disabled. Use bin/arthas.sh locally without restarting Store. diff --git a/hugegraph-store/hg-store-dist/src/assembly/travis/test-store-preloads.sh b/hugegraph-store/hg-store-dist/src/assembly/travis/test-store-preloads.sh index e5ad4e2495..296458c02c 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/travis/test-store-preloads.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/travis/test-store-preloads.sh @@ -60,7 +60,8 @@ download_and_verify() { touch "$2" } MOCK - touch "$top/lib/hg-store-node-fixture.jar" "$top/topling/rocksdbjni.jar" \ + touch "$top/lib/hg-store-node-fixture.jar" "$top/lib/rocksdbjni-fixture.jar" \ + "$top/topling/rocksdbjni.jar" \ "$top/topling/librocksdbjni-linux64.so" (cd "$top/topling" && sha256sum rocksdbjni.jar librocksdbjni-linux64.so > easy-migrate.sha256) printf '%s\n' 'http: {auto_start_http: false}' > "$top/conf/toplingdb.yaml" @@ -90,7 +91,7 @@ for arch in x86_64 aarch64 arm64; do } test "$(cat "$top/version-preloads")" = "$expected" test "$(cat "$top/main-preloads")" = "$expected" - grep -qx -- '-jar' "$top/main-preloads.args" + grep -qx 'org.springframework.boot.loader.JarLauncher' "$top/main-preloads.args" default_top="$TEST_ROOT/$name-default" default_expected="${expected//$top/$default_top}" diff --git a/hugegraph-store/hg-store-node/pom.xml b/hugegraph-store/hg-store-node/pom.xml index 2a20008500..cb91038a35 100644 --- a/hugegraph-store/hg-store-node/pom.xml +++ b/hugegraph-store/hg-store-node/pom.xml @@ -120,7 +120,6 @@ org.apache.hugegraph hg-store-core - org.apache.logging.log4j log4j-core @@ -170,4 +169,22 @@ + + + + s3-cloud-storage + + + org.apache.hugegraph + hg-store-cloud-s3 + + + + + diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java index ad60d00780..0a9ab16617 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java @@ -17,22 +17,44 @@ package org.apache.hugegraph.store.node; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.ServiceLoader; +import java.util.stream.Collectors; import javax.annotation.PostConstruct; - +import javax.annotation.PreDestroy; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.store.node.cloud.CloudStorageEventListener; +import org.apache.hugegraph.store.node.cloud.CloudStorageMetrics; +import org.apache.hugegraph.store.node.cloud.CloudSyncTracker; +import org.apache.hugegraph.store.node.cloud.CloudUploadRetryQueue; +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; import org.apache.hugegraph.store.options.JobOptions; import org.apache.hugegraph.store.options.RocksdbMemoryBudget; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.bind.Bindable; +import org.springframework.boot.context.properties.bind.Binder; import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; +import io.micrometer.core.instrument.MeterRegistry; + import lombok.Data; +import lombok.extern.slf4j.Slf4j; @Data +@Slf4j @Component public class AppConfig { @@ -90,6 +112,24 @@ public class AppConfig { @Autowired private QueryPushDownConfig queryPushDownConfig; + @Autowired + private Environment environment; + + @Autowired(required = false) + private MeterRegistry meterRegistry; + + /** Provider resolved and initialized during {@link #initCloudStorage()}; closed last on + * {@link #onDestroy()}. */ + private volatile CloudStorageProvider cloudStorageProvider; + + /** Retry queue created during {@link #initCloudStorage()}; closed on {@link #onDestroy()}. */ + private volatile CloudUploadRetryQueue cloudUploadRetryQueue; + + /** Listener registered with RocksDBFactory during {@link #initCloudStorage()}; deregistered on + * {@link #onDestroy()} so a context restart does not leave a stale listener in the static + * RocksDBFactory listener list. */ + private volatile CloudStorageEventListener cloudStorageListener; + public String getRaftPath() { if (raftPath == null || raftPath.length() == 0) { return dataPath; @@ -115,6 +155,299 @@ public void init() { "0".equals(rocksdb.get("write_buffer_size"))) { rocksdb.put("write_buffer_size", Long.toString(writeBufferSize(totalMemory))); } + + // ---- Cloud storage initialization ---- + initCloudStorage(); + } + + /** + * Initialises the cloud storage provider (if enabled) and registers + * {@link CloudStorageEventListener} with {@link RocksDBFactory} so that + * SST file creation/deletion events are forwarded to cloud storage. + * + *

The resolved absolute data-path is passed to the listener so that + * S3 object keys are relative to the store's storage root rather than + * being full container-specific absolute paths. + */ + private void initCloudStorage() { + CloudStorageConfig cfg = bindCloudStorageConfig(environment); + if (!cfg.isEnabled()) { + log.info("Cloud storage disabled (cloud.storage.enabled=false)"); + return; + } + try { + CloudStorageProvider provider = ServiceLoader.load(CloudStorageProvider.class).stream() + .map(ServiceLoader.Provider::get) + .filter(p -> p.providerName().equals(cfg.getProvider())) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "No cloud storage provider '" + cfg.getProvider() + + "' on the classpath")); + this.cloudStorageProvider = provider; + provider.init(cfg); + // Parse comma-separated dataPath into individual roots. Filter blank tokens so a + // trailing/duplicate comma (e.g. "store," or "a,,b") does not resolve "" to the JVM + // working directory and inject it as a bogus data root. + List resolvedDataRoots = Arrays.stream(dataPath.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(path -> Paths.get(path).toAbsolutePath().normalize().toString()) + .collect(Collectors.toList()); + if (resolvedDataRoots.isEmpty()) { + throw new IllegalStateException( + "Cloud storage enabled but app.data-path resolved to no valid roots: '" + + dataPath + "'"); + } + + // Shared sync tracker: the listener's delete guard and the retry queue's success + // callback both update it, so a superseded cloud object is deleted only once every + // live SST file of that DB is confirmed present in cloud. + CloudSyncTracker syncTracker = new CloudSyncTracker(); + + // The first root holds the persisted cloud key-scope marker. + String primaryDataRoot = resolvedDataRoots.get(0); + + CloudUploadRetryQueue retryQueue = new CloudUploadRetryQueue( + provider, + cfg.getUploadRetryMaxAttempts(), + cfg.getUploadRetryInitialDelayMs(), + cfg.getUploadRetryMaxDelayMs(), + syncTracker::markConfirmedIfEpoch); + this.cloudUploadRetryQueue = retryQueue; + + String storeScopePrefix = resolveStableStoreScopePrefix(cfg.getNodeId(), + primaryDataRoot); + + CloudStorageEventListener listener = new CloudStorageEventListener( + provider, + resolvedDataRoots, + cfg.isStartupHydrationEnabled(), + cfg.getReadMissGuardWindowMs(), + retryQueue, + syncTracker, + storeScopePrefix, + cfg.getMetadataSyncDebounceMs()); + + // After a retry upload becomes durable, publish CURRENT/MANIFEST so the + // mirrored recovery point advances even on an idle DB (the tracker-confirm callback + // alone does not trigger a metadata sync). Wired here since it needs the listener. + retryQueue.setMetadataSyncTrigger(listener::onRetryUploadDurable); + + // Initialize metrics if MeterRegistry is available + if (meterRegistry != null) { + CloudStorageMetrics.init(meterRegistry, syncTracker); + // Wire the retry-queue-size gauge to the live queue so it reflects the real upload + // backlog (the gauge otherwise reports a constant 0). + CloudStorageMetrics.bindRetryQueueSizeSupplier(retryQueue::getInFlightCount); + CloudStorageMetrics.bindUploadsGivenUpSupplier(retryQueue::getGivenUpCount); + // Surface pending-delete marker persistence health (1=healthy, 0=degraded) so a + // DB delete held for lack of a durable anti-resurrection guard is alertable. + CloudStorageMetrics.bindDeleteMarkerHealthySupplier( + () -> listener.isDeleteMarkerHealthy() ? 1 : 0); + } + + RocksDBFactory.getInstance().addRocksdbChangedListener(listener); + this.cloudStorageListener = listener; + log.info("Cloud storage provider '{}' registered with RocksDBFactory " + + "(dataRoots={}, storeScopePrefix='{}', startupHydration={}, " + + "readMissHydration=true, " + + "readMissGuardWindowMs={}, uploadRetryMaxAttempts={}, " + + "uploadRetryInitialDelayMs={}, uploadRetryMaxDelayMs={}, " + + "metadataSyncDebounceMs={})", + cfg.getProvider(), resolvedDataRoots, storeScopePrefix, + cfg.isStartupHydrationEnabled(), + cfg.getReadMissGuardWindowMs(), + cfg.getUploadRetryMaxAttempts(), + cfg.getUploadRetryInitialDelayMs(), + cfg.getUploadRetryMaxDelayMs(), + cfg.getMetadataSyncDebounceMs()); + } catch (Exception e) { + log.error("Failed to initialize cloud storage provider '{}': {}", + cfg.getProvider(), e.getMessage(), e); + // Release whatever was already started so a failed @PostConstruct does not leak the + // provider's client/threads — Spring does not invoke @PreDestroy when @PostConstruct + // throws. + if (this.cloudUploadRetryQueue != null) { + try { + this.cloudUploadRetryQueue.close(); + } catch (Exception ignore) { + // best-effort cleanup on the failure path + } + this.cloudUploadRetryQueue = null; + } + closeCloudStorageProvider(); + // Fail startup: an explicitly-enabled provider that cannot initialize leaves + // the node silently uploading nothing, making durability failures invisible. + throw new IllegalStateException( + "Cloud storage initialization failed for provider '" + + cfg.getProvider() + "': " + e.getMessage(), e); + } + } + + /** File in the primary data root that persists the resolved cloud key scope across restarts. */ + static final String CLOUD_SCOPE_MARKER_FILE = ".cloud-store-scope"; + + /** + * Resolves the cloud key scope prefix with stability across restarts, giving precedence to the + * most durable source available: + * + *

    + *
  1. Configured {@code cloud.storage.node-id} — authoritative and survives IP drift + * AND local disk loss (it lives in deployment config), so recovery always finds prior + * objects. This is the recommended setting for production.
  2. + *
  3. Persisted marker in the primary data root — written on first start, it keeps the + * scope stable across network-identity (IP/hostname) drift, though it is lost with the + * disk.
  4. + *
  5. Runtime network address — legacy behavior, used to seed the marker on first + * start. If a node's address later changes it would read a different scope, so we warn + * that an explicit node-id is advisable for durable recovery.
  6. + *
+ * + *

Seeding the marker from the network address on first start keeps upgrades migration-safe: + * an existing deployment whose objects are already keyed by {@code store-} resolves + * to the same scope and keeps finding its data. + */ + String resolveStableStoreScopePrefix(String configuredNodeId, String primaryDataRoot) { + if (configuredNodeId != null && !configuredNodeId.trim().isEmpty()) { + String prefix = "store-" + sanitizeCloudKeySegment(configuredNodeId.trim()); + // Persist so a later removal of the config value still resolves to the same scope. + persistCloudScopeMarker(primaryDataRoot, prefix); + log.info("Cloud key scope from configured cloud.storage.node-id: '{}'", prefix); + return prefix; + } + + String persisted = readCloudScopeMarker(primaryDataRoot); + if (persisted != null && !persisted.isEmpty()) { + log.info("Cloud key scope loaded from persisted marker in data root: '{}'", persisted); + return persisted; + } + + // First start (or the marker was lost with the disk): seed from the network identity. + String prefix = buildIdentityScopePrefix(); + persistCloudScopeMarker(primaryDataRoot, prefix); + log.warn("Cloud key scope seeded from runtime network address: '{}'. This scope is only " + + "stable while the node's address does not change. For guaranteed recovery after " + + "an IP/hostname change or local disk loss, set a stable cloud.storage.node-id.", + prefix); + return prefix; + } + + /** Reads the persisted cloud scope prefix from the data root, or {@code null} if unavailable. */ + private static String readCloudScopeMarker(String primaryDataRoot) { + if (primaryDataRoot == null || primaryDataRoot.isEmpty()) { + return null; + } + java.nio.file.Path marker = Paths.get(primaryDataRoot, CLOUD_SCOPE_MARKER_FILE); + try { + if (!java.nio.file.Files.exists(marker)) { + return null; + } + String content = java.nio.file.Files.readString(marker).trim(); + return content.isEmpty() ? null : content; + } catch (IOException e) { + log.warn("Failed to read cloud scope marker {}: {}", marker, e.getMessage()); + return null; + } + } + + /** Best-effort persist of the resolved cloud scope prefix into the data root. */ + private static void persistCloudScopeMarker(String primaryDataRoot, String prefix) { + if (primaryDataRoot == null || primaryDataRoot.isEmpty()) { + return; + } + java.nio.file.Path marker = Paths.get(primaryDataRoot, CLOUD_SCOPE_MARKER_FILE); + try { + java.nio.file.Files.createDirectories(marker.getParent()); + String existing = readCloudScopeMarker(primaryDataRoot); + if (prefix.equals(existing)) { + return; // already persisted + } + java.nio.file.Files.writeString(marker, prefix, + java.nio.file.StandardOpenOption.CREATE, + java.nio.file.StandardOpenOption.TRUNCATE_EXISTING); + } catch (IOException e) { + log.warn("Failed to persist cloud scope marker {}: {} — scope stability across restarts " + + "is not guaranteed until this succeeds", marker, e.getMessage()); + } + } + + /** + * Builds a deterministic per-store cloud key prefix from the runtime network identity so + * distributed store nodes can share a bucket/path-prefix without key collisions. Used only to + * seed {@link #resolveStableStoreScopePrefix} on first start. + */ + private String buildIdentityScopePrefix() { + String identity = raft != null ? raft.getAddress() : null; + if (identity == null || identity.trim().isEmpty()) { + identity = getStoreServerAddress(); + } + return "store-" + sanitizeCloudKeySegment(identity); + } + + /** Converts an address-like identifier into a cloud-key-safe path segment. */ + private static String sanitizeCloudKeySegment(String raw) { + if (raw == null || raw.trim().isEmpty()) { + return "unknown"; + } + StringBuilder sb = new StringBuilder(raw.length()); + for (int i = 0; i < raw.length(); i++) { + char ch = raw.charAt(i); + if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') || ch == '-' || ch == '_' || ch == '.') { + sb.append(ch); + } else { + sb.append('_'); + } + } + return sb.toString(); + } + + /** + * Gracefully tears down cloud storage on application stop. Order: deregister the listener, then + * close it (draining its upload executor) so no new SST dispatches occur, then close the retry + * queue (which drains its own independent scheduler — retries do NOT run in the upload + * executor), and finally close the provider so in-flight uploads/retries could still use it. + */ + @PreDestroy + public void onDestroy() { + // Deregister the listener first so RocksDB stops dispatching cloud events to it before we + // tear down the executor/provider it depends on — and so a context restart in the same JVM + // does not leave a stale listener registered in the static RocksDBFactory listener list + // (which would double-dispatch events to a defunct instance). + CloudStorageEventListener listener = this.cloudStorageListener; + if (listener != null) { + RocksDBFactory.getInstance().removeRocksdbChangedListener(listener); + this.cloudStorageListener = null; + // Shut down the listener's upload executor so no new SST upload tasks are dispatched + // after this point. close() gives in-flight uploads a chance to complete before the + // JVM exits; any that don't finish are re-uploaded by startup SST backfill. + listener.close(); + } + + if (cloudUploadRetryQueue != null) { + log.info("Shutting down CloudUploadRetryQueue (givenUp={}) …", + cloudUploadRetryQueue.getGivenUpCount()); + cloudUploadRetryQueue.close(); + } + + // Close the active provider LAST — after uploads and retries have drained — so its client + // (e.g. the S3 SDK connection pool and threads) is released rather than leaked on context + // shutdown/restart. Doing it last ensures in-flight uploads/retries above could still use it. + closeCloudStorageProvider(); + } + + private void closeCloudStorageProvider() { + CloudStorageProvider provider = this.cloudStorageProvider; + if (provider == null) { + return; + } + this.cloudStorageProvider = null; + try { + provider.close(); + log.info("Cloud storage provider '{}' closed", provider.providerName()); + } catch (IOException e) { + log.warn("Error closing cloud storage provider '{}'", provider.providerName(), e); + } } /** @@ -330,6 +663,72 @@ public class RocksdbConfig { private Map rocksdb = new HashMap<>(); } + /** + * Binds {@code cloud.storage.*} from the Spring {@link Environment} directly onto + * {@link CloudStorageConfig}, so its field defaults are the only defaults. Relaxed binding + * applies, so env vars such as {@code CLOUD_STORAGE_ENABLED} resolve as usual. + * + *

Provider-specific keys are forwarded from the sub-namespace named by + * {@code cloud.storage.provider}, keeping this class unaware of any specific cloud provider: + * + *

+     * cloud:
+     *   storage:
+     *     enabled: true
+     *     provider: s3          # selects which sub-namespace to forward to the provider
+     *     path-prefix: hugegraph
+     *     s3:                   # all keys here are forwarded as-is to the S3 provider
+     *       bucket: my-bucket
+     *       region: us-east-1
+     *       access-key: ${AWS_ACCESS_KEY_ID:}
+     *       secret-key: ${AWS_SECRET_ACCESS_KEY:}
+     * 
+ * + *

Provider-specific properties are bound only when {@code cloud.storage.enabled=true}: a + * disabled node must start even if a provider placeholder (e.g. the example above without the + * trailing {@code :} default) cannot be resolved. Give placeholders an empty default + * ({@code ${VAR:}}) so an unset variable also does not fail an enabled node at bind time. + */ + static CloudStorageConfig bindCloudStorageConfig(Environment environment) { + Binder binder = Binder.get(environment); + CloudStorageConfig cfg = binder.bind("cloud.storage", CloudStorageConfig.class) + .orElseGet(CloudStorageConfig::new); + if (cfg.isEnabled()) { + cfg.setProviderProperties(readProviderProperties(binder, cfg.getProvider())); + } + return cfg; + } + + /** + * Reads all {@code cloud.storage..*} keys and returns them as a flat map with the + * provider sub-prefix stripped. + * + *

For example, with {@code provider=s3}, the YAML key {@code cloud.storage.s3.bucket} + * becomes {@code bucket} in the returned map, and relaxed binding lets the env var + * {@code CLOUD_STORAGE_S3_ACCESS_KEY} resolve the same way. Multi-word keys bound from a + * {@code SCREAMING_SNAKE_CASE} source come back dot-separated (e.g. {@code secret.key}), so + * they are normalized to the kebab-case form ({@code secret-key}) that provider configs such + * as {@code S3CloudStorageConfig.KEY_*} expect. + * + *

The same key may arrive in both forms (application.yml {@code secret-key} plus the env + * var bound as {@code secret.key}); the env-var form wins, matching Spring's precedence of + * environment variables over application.yml. + */ + private static Map readProviderProperties(Binder binder, String provider) { + Map props = binder + .bind("cloud.storage." + provider, Bindable.mapOf(String.class, String.class)) + .orElseGet(Collections::emptyMap); + Map normalised = new HashMap<>(); + for (Map.Entry e : props.entrySet()) { + String key = e.getKey().replace('.', '-'); + boolean fromEnvVar = !key.equals(e.getKey()); + if (fromEnvVar || !normalised.containsKey(key)) { + normalised.put(key, e.getValue()); + } + } + return normalised; + } + public JobOptions getJobOptions() { JobOptions jobOptions = new JobOptions(); jobOptions.setCore(jobConfig.getCore() == 0 ? cpus : jobConfig.getCore()); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListener.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListener.java new file mode 100644 index 0000000000..0fce23e234 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListener.java @@ -0,0 +1,2671 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import lombok.Getter; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.LiveSstFile; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.MetadataSnapshot; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; + +import lombok.extern.slf4j.Slf4j; + +/** + * {@link RocksdbChangedListener} that bridges RocksDB table-file lifecycle events + * to the active {@link CloudStorageProvider}. + * + *

When cloud storage is enabled: + *

    + *
  • {@link #onDBCreated} uploads any SST files that already exist in the DB directory + * (e.g. surviving from a previous run), triggers a non-blocking MemTable flush so + * WAL-recovered or recently-written data is written to SST files (completion is signalled + * event-driven via {@link #onTableFileCreated}), then mirrors metadata inline.
  • + *
  • {@link #onTableFileCreated} pins newly created SST files via a hard link, dispatches + * upload work to a bounded background executor, and mirrors metadata after upload completes. + * If the hard link fails (e.g. cross-device mount, filesystem limits), the upload is routed + * directly to the retry queue using the original SST path — no copy is made, so no extra + * disk space is consumed.
  • + *
  • {@link #onTableFileDeleted} mirrors metadata first, then removes the superseded SST object.
  • + *
+ * + *

Remote key construction

+ * The remote key is derived by stripping the {@code dataRoot} prefix from the absolute + * local file path. This keeps the object layout clean and independent of the container + * filesystem layout: + *
+ *   dataRoot  = /hugegraph-store/storage
+ *   filePath  = /hugegraph-store/storage/hgstore-metadata/000008.sst
+ *   remoteKey = store-127.0.0.1_8501/hgstore-metadata/000008.sst
+ *   (with path-prefix "hugegraph") → hugegraph/store-127.0.0.1_8501/hgstore-metadata/000008.sst
+ * 
+ * + * This listener is registered with {@link RocksDBFactory} during application startup + * (see {@link org.apache.hugegraph.store.node.AppConfig}). + */ +@Slf4j +public class CloudStorageEventListener implements RocksdbChangedListener, AutoCloseable { + + /** Absolute, normalised paths of configured store data roots (one per configured partition root). */ + private final String primaryDataRoot; + private final List allDataRoots; + + /** Optional per-store namespace prefix prepended to every remote cloud key. */ + private final String storeScopePrefix; + + static final long DEFAULT_READ_MISS_GUARD_WINDOW_MS = 3000L; + + private final boolean startupHydrationEnabled; + private final long readMissGuardWindowMs; + private final Map readMissAttemptTs; + + /** + * Tracks which {@code (dbName, remoteKey)} pairs are currently being restored by + * {@link #restoreMissingLiveFiles} so a thundering herd of concurrent read misses on the + * same cold SST does not download the same object many times over. This is a best-effort + * de-duplication; correctness against concurrent restores of the same file is guaranteed by + * the per-attempt unique temp name plus an atomic replace-move, not by this set. + */ + private final Set inFlightRestores = ConcurrentHashMap.newKeySet(); + + /** + * Optional retry queue; when non-null, upload failures are submitted here instead + * of just being logged. When null, failures are only logged (no retry). + */ + private final CloudUploadRetryQueue retryQueue; + + /** Initialized cloud storage provider; resolved once at startup and never reconfigured. */ + private final CloudStorageProvider provider; + + /** Tracks which SST files are confirmed present in cloud (per-DB confirmed-file-number set). */ + private final CloudSyncTracker syncTracker; + + /** + * Hard health flag for pending-delete marker durability. Flips to {@code false} when a marker + * cannot be durably persisted (write + fsync) — a state in which a DB delete whose remote + * cleanup fails may be unguarded against re-hydration after a crash — and back to + * {@code true} once a marker is durably persisted again. Surfaced via + * {@link CloudStorageMetricsConst#DELETE_MARKER_HEALTHY}. {@code volatile}: written from delete + * callbacks, read from metric-scrape threads. + * -- GETTER -- + * Whether pending-delete marker persistence is currently healthy. + * indicates a + * marker could not be durably written, so a delete whose remote cleanup fails may not + * survive a crash as a hydration guard. Bound to + *

+ * . + + */ + @Getter + private volatile boolean deleteMarkerHealthy = true; + + /** Directory fsync is not supported on Windows directory handles. */ + private static final boolean WINDOWS_OS = + System.getProperty("os.name", "").toLowerCase().contains("win"); + + /** Bounded async upload dispatcher so RocksDB callbacks return quickly. */ + private static final int ASYNC_UPLOAD_THREADS = 2; + private static final int ASYNC_UPLOAD_QUEUE_CAPACITY = 256; + /** Upload pool owned by this listener; drained by {@link #close()}. */ + private final ThreadPoolExecutor uploadExecutor; + + // ----------------------------------------------------------------------- + // Metadata-sync coalescing + // ----------------------------------------------------------------------- + // onTableFileCreated fires once per flushed/compacted SST; syncing metadata inline on every + // one triggers a full RocksDB checkpoint + S3 list/prune per SST. Instead each upload marks + // its DB dirty and a fixed-delay drain publishes each dirty DB at most once per interval. + // Event-driven callers that need an immediate publish (delete guard, onCompacted, + // onDBCreated) still call syncMetadataSnapshotInline directly and are NOT coalesced. + + /** Default drain interval for the post-upload metadata sync. */ + static final long DEFAULT_METADATA_SYNC_DEBOUNCE_MS = 1_000L; + + /** + * Scheduler for the metadata-sync drain and purge/delete retries. Owned by this listener and + * shut down by {@link #close()}; a schedule after that throws + * {@link RejectedExecutionException}, which every call site handles. + */ + private final ScheduledExecutorService metadataSyncScheduler; + + /** Drain interval (ms); {@code <= 0} publishes inline on every SST upload instead. */ + private final long metadataSyncDebounceMs; + + /** DBs with uploads not yet reflected in a published manifest; drained by the scheduler. */ + private final Set dirtyMetadataDbs = ConcurrentHashMap.newKeySet(); + + // ----------------------------------------------------------------------- + // Metadata (CURRENT/MANIFEST/OPTIONS) mirroring & consistent restore + // ----------------------------------------------------------------------- + + /** Resolved DB directory -> logical DB name, so {@link #onCompacted} resolves path events. */ + private final Map dbNameByDir = new ConcurrentHashMap<>(); + + /** Per-DB mutexes to serialize metadata capture/publication/pruning. */ + private final Map metadataSyncLocks = new ConcurrentHashMap<>(); + + /** + * Per-remote-prefix mutexes serializing pending-delete cleanup (inline at open vs. the async + * retry). Ensures the purge runs at most once and only while the marker is present, so it can + * never delete data a re-created DB uploads to the same prefix after cleanup completes. + */ + private final Map pendingDeleteLocks = new ConcurrentHashMap<>(); + + /** + * Per-remote-prefix generation, bumped every time a DB is opened or created at that prefix. + * An asynchronous pending-delete purge captures the value when it is scheduled and aborts + * (without purging) if it changed, so a retry can never delete objects that a re-created DB + * has since taken over the prefix with. Entries are never removed: dropping one would reset + * the counter and let a stale retry match again. + */ + private final Map prefixEpochs = new ConcurrentHashMap<>(); + + /** Per-DB metadata-publish failure state used to back off a persistently failing publish. */ + private final Map publishBackoffs = new ConcurrentHashMap<>(); + + /** Cap for the exponential publish-retry backoff. */ + static final long MAX_PUBLISH_BACKOFF_MS = 5 * 60 * 1000L; + /** Minimum base for the publish-retry backoff, so a tiny debounce does not hot-loop failures. */ + static final long MIN_PUBLISH_BACKOFF_BASE_MS = 1_000L; + + /** Bounded synchronous attempts at completing a pending delete while a DB is being opened. */ + private static final int OPEN_PENDING_DELETE_ATTEMPTS = 3; + private static final long OPEN_PENDING_DELETE_BACKOFF_MS = 200L; + /** Bounded attempts at flushing dirty metadata on close(). */ + private static final int CLOSE_FLUSH_ATTEMPTS = 3; + private static final long CLOSE_FLUSH_BACKOFF_MS = 200L; + + /** Consecutive publish failures and the earliest time the next attempt is allowed. */ + private static final class PublishBackoff { + + final int failures; + final long nextAttemptAtMs; + + PublishBackoff(int failures, long nextAttemptAtMs) { + this.failures = failures; + this.nextAttemptAtMs = nextAttemptAtMs; + } + } + + /** Last successfully published RocksDB generation per DB. */ + private final Map lastPublishedMetadataGeneration = new ConcurrentHashMap<>(); + + /** + * Suffix appended to the DB prefix (not inside it) when a database is deleted. + * Placing the tombstone as a sibling of the data prefix means the + * {@link #purgeRemotePrefix} call in {@link #onDBDeleted} cannot accidentally remove it + * while stale SST or metadata objects remain, so {@link #preHydrateDbFiles} can still + * detect the deleted generation and skip hydration. + * + *

Example: data prefix = {@code store-host_8500/hugegraph/db}, + * tombstone key = {@code store-host_8500/hugegraph/db_DELETED}. + */ + static final String DB_TOMBSTONE_SUFFIX = "_DELETED"; + + /** Infix of hydration temp files ({@code .hyd-tmp--}). */ + private static final String HYDRATION_TEMP_MARKER = ".hyd-tmp-"; + + /** + * Fully-parameterised constructor: the listener is completely configured the moment it is + * constructed, so there is no window in which a registered listener can be observed + * partially configured. + * + * @param provider initialized cloud storage provider + * @param dataRoots configured store data roots (typically parsed from comma-separated + * {@code app.data-path}) + * @param readMissGuardWindowMs guard window in ms for repeated read-miss hydration attempts + * for the same db/table pair (cloud.storage.read-miss-guard-window-ms) + * @param retryQueue optional {@link CloudUploadRetryQueue}; when non-null, upload failures + * are retried asynchronously and eventually given up on. + * Pass {@code null} to disable retries (failures are only logged). + * @param syncTracker tracks SST files confirmed present in cloud; the delete guard uses it + * to avoid deleting a superseded object before replacements are durable. + * Must be shared with the retry queue. Pass {@code null} for a fresh + * {@link CloudSyncTracker}. + * @param storeScopePrefix optional per-store key prefix to isolate cloud objects + * @param metadataSyncDebounceMs drain interval (ms) for the per-SST metadata sync; + * {@code <= 0} publishes inline on every SST upload + */ + public CloudStorageEventListener(CloudStorageProvider provider, + List dataRoots, + boolean startupHydrationEnabled, + long readMissGuardWindowMs, + CloudUploadRetryQueue retryQueue, + CloudSyncTracker syncTracker, + String storeScopePrefix, + long metadataSyncDebounceMs) { + // Fail fast on a missing/empty data-root list: primaryDataRoot is derived from index 0 + // below, and every key<->path conversion depends on at least one root. Without this guard + // the constructor would throw an opaque IndexOutOfBoundsException at + // allDataRoots.get(0), which is far harder to diagnose than a config error. + if (dataRoots == null || dataRoots.isEmpty()) { + throw new IllegalArgumentException( + "CloudStorageEventListener requires at least one data root; none configured. " + + "Check the store data-path configuration (e.g. rocksdb.data_path / " + + "raft.path) used to derive the cloud storage data roots."); + } + // Normalize each root + this.allDataRoots = new ArrayList<>(); + for (String root : dataRoots) { + String normalised = Paths.get(root).toAbsolutePath().normalize().toString(); + // Strip trailing separator so substring arithmetic is consistent. + normalised = normalised.endsWith(File.separator) + ? normalised.substring(0, normalised.length() - 1) + : normalised; + this.allDataRoots.add(normalised); + } + this.primaryDataRoot = this.allDataRoots.get(0); + this.provider = Objects.requireNonNull(provider, "provider"); + this.startupHydrationEnabled = startupHydrationEnabled; + this.readMissGuardWindowMs = Math.max(0L, readMissGuardWindowMs); + this.readMissAttemptTs = new ConcurrentHashMap<>(); + this.retryQueue = retryQueue; + this.syncTracker = syncTracker != null ? syncTracker : new CloudSyncTracker(); + this.storeScopePrefix = normaliseKeyPrefix(storeScopePrefix); + this.uploadExecutor = new ThreadPoolExecutor( + ASYNC_UPLOAD_THREADS, + ASYNC_UPLOAD_THREADS, + 60L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(ASYNC_UPLOAD_QUEUE_CAPACITY), + newUploadThreadFactory(), + new ThreadPoolExecutor.AbortPolicy()); + ScheduledThreadPoolExecutor syncScheduler = new ScheduledThreadPoolExecutor(1, r -> { + Thread t = new Thread(r, "cloud-metadata-sync"); + t.setDaemon(true); + return t; + }); + // close() uses a graceful shutdown() so an in-flight publish completes; not-yet-due delayed + // tasks (purge/delete retries) are cancelled instead of holding close() open. Their intent + // is crash-durable (local markers) and is re-driven by the next startup scan. + syncScheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + this.metadataSyncScheduler = syncScheduler; + this.metadataSyncDebounceMs = metadataSyncDebounceMs; + if (metadataSyncDebounceMs > 0) { + this.metadataSyncScheduler.scheduleWithFixedDelay(this::drainDirtyMetadata, + metadataSyncDebounceMs, + metadataSyncDebounceMs, + TimeUnit.MILLISECONDS); + } + + // A crash may leave local pending-delete markers whose remote cleanup never completed. + // Kick off bounded async retries for each so stale remote data is eventually purged even if + // the affected DB is never re-opened (an open would otherwise be the only trigger). + scanMarkers(pendingDeleteDir(), (db, prefix) -> { + log.warn("Cloud startup: found pending remote-delete marker for prefix={} — " + + "scheduling deferred purge to prevent stale-data re-hydration.", prefix); + scheduleDeletePurgeRetry(db, prefix, 1); + }); + // Upload intent is not persisted across restarts, so any staging pin left by the previous + // process is orphaned; the live SSTs it pinned are re-uploaded by startup backfill. + sweepStagingDirsOnStartup(); + } + + private static ThreadFactory newUploadThreadFactory() { + return r -> { + Thread t = new Thread(r, "cloud-upload-dispatch"); + t.setDaemon(true); + return t; + }; + } + + /** Shuts down this listener's executors, waiting up to 10s for in-flight uploads. */ + @Override + public void close() { + close(10, TimeUnit.SECONDS); + } + + /** + * Shuts down the SST upload executor, waiting up to {@code timeout} for in-flight uploads to + * complete, then the metadata-sync scheduler. Called from + * {@link org.apache.hugegraph.store.node.AppConfig#onDestroy()} after the listener has been + * deregistered, so uploads started before shutdown have a chance to finish. + */ + void close(long timeout, TimeUnit unit) { + // Drain the upload executor FIRST so DBs marked dirty by in-flight uploads are flushed below. + uploadExecutor.shutdown(); + try { + if (!uploadExecutor.awaitTermination(timeout, unit)) { + log.warn("CloudStorageEventListener: upload executor did not terminate within " + + "{}{}; forcing shutdown", timeout, unit); + handoffDroppedUploadTasks(uploadExecutor.shutdownNow()); + } + } catch (InterruptedException e) { + handoffDroppedUploadTasks(uploadExecutor.shutdownNow()); + Thread.currentThread().interrupt(); + } + // Graceful stop: an in-flight drain/publish finishes instead of being interrupted mid-upload, + // and the final flush below runs only after the drain thread is gone so the two cannot + // publish the same DB concurrently. + metadataSyncScheduler.shutdown(); + try { + if (!metadataSyncScheduler.awaitTermination(timeout, unit)) { + log.warn("CloudStorageEventListener: metadata-sync scheduler did not terminate " + + "within {}{}; forcing shutdown", timeout, unit); + metadataSyncScheduler.shutdownNow(); + metadataSyncScheduler.awaitTermination(1, TimeUnit.SECONDS); + } + } catch (InterruptedException e) { + metadataSyncScheduler.shutdownNow(); + Thread.currentThread().interrupt(); + } + // Flush dirty DBs so uploads that completed during the drain are referenced by the + // mirrored CURRENT/MANIFEST. Later requests (e.g. retry-queue completions) see the + // scheduler shut down in markMetadataDirty and publish inline. + flushDirtyMetadataOnClose(); + } + + /** + * Final bounded flush of every dirty DB. A failed publish is retried a few times; DBs that + * still cannot be published are logged so the gap is visible. + */ + private void flushDirtyMetadataOnClose() { + List remaining = new ArrayList<>(dirtyMetadataDbs); + for (int attempt = 1; attempt <= CLOSE_FLUSH_ATTEMPTS && !remaining.isEmpty(); attempt++) { + for (String dbName : new ArrayList<>(remaining)) { + dirtyMetadataDbs.remove(dbName); + if (publishDirtyMetadata(dbName)) { + remaining.remove(dbName); + } else { + dirtyMetadataDbs.add(dbName); + } + } + if (remaining.isEmpty() || attempt == CLOSE_FLUSH_ATTEMPTS) { + break; + } + try { + Thread.sleep(CLOSE_FLUSH_BACKOFF_MS * attempt); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + if (!remaining.isEmpty()) { + log.error("CloudStorageEventListener: closed with metadata NOT published for {} DB(s): " + + "{} — the mirrored CURRENT/MANIFEST may lag the uploaded SSTs until the next " + + "start publishes them.", remaining.size(), remaining); + } + } + + /** + * Routes upload tasks discarded by a forced {@code shutdownNow()} into their retry queue + * so an SST accepted locally but never mirrored is not silently lost from the durability + * pipeline. Only {@link SstUploadTask}s carry the metadata needed for handoff; any other + * runnable (there should be none) is ignored. + */ + private static void handoffDroppedUploadTasks(java.util.List dropped) { + if (dropped == null || dropped.isEmpty()) { + return; + } + int handed = 0; + for (Runnable r : dropped) { + if (r instanceof SstUploadTask) { + try { + ((SstUploadTask) r).handoffOnShutdown(); + handed++; + } catch (Exception e) { + log.warn("Failed to hand off a dropped upload task to the retry queue: {}", + e.getMessage()); + } + } + } + if (handed > 0) { + log.warn("CloudStorageEventListener: handed off {} queued upload task(s) to the retry queue " + + "during shutdown so their upload intent survives.", handed); + } + } + + /** + * Finds which configured data roots contain the given file path. + * Returns the matching root, or the primary root if no exact match is found. + * + *

Uses {@link Path#startsWith} so that {@code /data/store1} never incorrectly + * matches {@code /data/store10/...} (a raw string prefix match would). + * + * @param filePath absolute file path + * @return the matched configured data root for this file + */ + private String findMatchingDataRoot(String filePath) { + Path fileNormalized = Paths.get(filePath).toAbsolutePath().normalize(); + for (String root : allDataRoots) { + if (fileNormalized.startsWith(Paths.get(root))) { + return root; + } + } + // Fallback to primary root (should not happen in normal operation) + return primaryDataRoot; + } + + // ----------------------------------------------------------------------- + // RocksdbChangedListener + // ----------------------------------------------------------------------- + + @Override + public boolean managesRemoteState() { + return true; + } + + /** + * Picks the directory to open when the local one holds no DB. A partition can move to a + * version-suffixed directory ({@code _[_]}) and its objects then live under that + * prefix, so after a disk loss the default path would hydrate nothing. Lists the remote + * siblings that share the base name and returns the newest restorable one, matching the order + * {@code RocksDBSession} uses locally. A sibling is restorable when it has a remote + * {@code CURRENT}, no delete tombstone and no local pending-delete marker. If listing fails the + * path is kept, and the hydration of that path applies its own failure rules. + */ + @Override + public String resolveOpenPath(String dbName, String dbPath) { + if (!startupHydrationEnabled) { + return dbPath; + } + Path candidate = Paths.get(dbPath).toAbsolutePath().normalize(); + Path parent = candidate.getParent(); + if (parent == null) { + return dbPath; + } + String name = candidate.getFileName().toString(); + int sep = name.indexOf('_'); + String baseName = sep < 0 ? name : name.substring(0, sep); + String basePrefix = dbPrefix(parent.resolve(baseName).toString()); + if (!basePrefix.endsWith(baseName)) { + return dbPath; + } + String parentKeyPrefix = basePrefix.substring(0, basePrefix.length() - baseName.length()); + List keys; + try { + keys = provider.listFiles(basePrefix); + } catch (IOException e) { + log.warn("Cloud open: listing version siblings of db={} failed, keeping {}: {}", + dbName, dbPath, e.getMessage()); + return dbPath; + } + String newest = newestRestorableSibling(parentKeyPrefix, baseName, keys); + if (newest == null || newest.equals(name)) { + return dbPath; + } + log.info("Cloud open: db={} has no local DB at {}; restoring newest remote sibling '{}'", + dbName, dbPath, newest); + return parent.resolve(newest).toString(); + } + + /** + * Newest restorable directory name among {@code baseName} and its version-suffixed siblings + * found in {@code keys}, or {@code null} if none (package-private for tests). + */ + String newestRestorableSibling(String parentKeyPrefix, String baseName, List keys) { + Pattern dirName = Pattern.compile(Pattern.quote(baseName) + "(?:_(\\d+)(?:_(\\d+))?)?"); + Set withCurrent = new HashSet<>(); + Set tombstoned = new HashSet<>(); + for (String key : keys) { + if (!key.startsWith(parentKeyPrefix)) { + continue; + } + String rel = key.substring(parentKeyPrefix.length()); + int slash = rel.indexOf('/'); + if (slash < 0) { + if (rel.endsWith(DB_TOMBSTONE_SUFFIX)) { + tombstoned.add(rel.substring(0, rel.length() - DB_TOMBSTONE_SUFFIX.length())); + } + continue; + } + String dir = rel.substring(0, slash); + if ("CURRENT".equals(rel.substring(slash + 1)) && dirName.matcher(dir).matches()) { + withCurrent.add(dir); + } + } + String newest = null; + long[] newestVersion = null; + for (String dir : withCurrent) { + if (tombstoned.contains(dir) || hasPendingDeleteMarker(parentKeyPrefix + dir)) { + continue; + } + Matcher m = dirName.matcher(dir); + if (!m.matches()) { + continue; + } + long[] version; + try { + version = new long[]{m.group(1) == null ? -1L : Long.parseLong(m.group(1)), + m.group(2) == null ? -1L : Long.parseLong(m.group(2))}; + } catch (NumberFormatException e) { + continue; + } + if (newestVersion == null || version[0] > newestVersion[0] + || (version[0] == newestVersion[0] && version[1] > newestVersion[1])) { + newest = dir; + newestVersion = version; + } + } + return newest; + } + + @Override + public void onDBOpening(String dbName, String dbPath) { + // A name reopened at a NEW dir (snapshot install, destroy-then-recreate) must not inherit + // upload confirmations or the published generation of the previous directory. This runs + // before recordDb and preHydrateDbFiles, which seed the tracker for the new dir. + resetStateIfDbMovedDir(dbName, dbPath); + // Register the dir -> logical-name mapping first: native events that arrive before + // onDBCreated (or on a disabled-hydration node) must resolve to the logical name. + recordDb(dbName, dbPath); + // Stale hydration temp files can only be left by a crashed download; sweep them from the + // local dir on every open, not just when something is about to be hydrated. + sweepStaleHydrationTemps(Paths.get(dbPath)); + + String prefix = dbPrefix(dbPath); + // Invalidate every async purge scheduled for the previous generation at this prefix BEFORE + // anything below can write to it. Retries compare against this epoch under the prefix lock. + bumpEpoch(prefixEpochs, prefix); + + // Pending-delete guard, independent of hydration: a prior delete's remote cleanup may not + // be confirmed, so the reopened DB must not upload into (or hydrate from) a prefix that + // still holds the deleted generation and a delete marker. + if (hasPendingDeleteMarker(prefix)) { + completePendingDeleteOrBlockOpen(dbName, prefix); + } + if (!startupHydrationEnabled) { + // Nothing is hydrated, but the DB will upload into this prefix: a leftover tombstone + // must be resolved now, or a later open with hydration enabled would purge its data. + purgeTombstonedPrefix(provider, dbName, Paths.get(dbPath), prefix, false); + return; + } + preHydrateDbFiles(provider, dbName, dbPath); + } + + /** + * Finishes a pending remote delete synchronously (bounded attempts) before the open proceeds. + * If it cannot be completed the open is BLOCKED: the marker means the prefix still holds the + * deleted generation, and opening anyway would let the new DB upload into it while the async + * retry (or a restart) later purges the new data. The async retry is re-armed so the delete + * finishes when the cloud recovers, and a subsequent open attempt simply tries again. + */ + private void completePendingDeleteOrBlockOpen(String dbName, String prefix) { + for (int attempt = 1; attempt <= OPEN_PENDING_DELETE_ATTEMPTS; attempt++) { + if (tryCompletePendingDelete(provider, dbName, prefix, null)) { + return; + } + if (attempt < OPEN_PENDING_DELETE_ATTEMPTS) { + try { + Thread.sleep(OPEN_PENDING_DELETE_BACKOFF_MS * attempt); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } + scheduleDeletePurgeRetry(dbName, prefix, 1); + throw new IllegalStateException(String.format( + "Cloud open blocked for db=%s: remote-delete cleanup of prefix '%s' is not yet " + + "confirmed. Opening now would let the new DB upload into a prefix still holding " + + "the deleted generation; the cleanup is being retried in the background.", + dbName, prefix)); + } + + /** Deletes stale {@code .hyd-tmp-} download temp files left under a local DB dir by a crash. */ + private void sweepStaleHydrationTemps(Path root) { + if (!Files.isDirectory(root)) { + return; + } + try (Stream stale = Files.walk(root, 10)) { + stale.filter(p -> p.getFileName() != null + && p.getFileName().toString().contains(HYDRATION_TEMP_MARKER)) + .forEach(p -> deleteIfExistsQuietly(p, "stale hydration temp cleanup")); + } catch (IOException e) { + log.debug("Failed to scan stale hydration temp files under {}: {}", + root, e.getMessage()); + } + } + + private static long bumpEpoch(Map epochs, String key) { + return epochs.computeIfAbsent(key, k -> new AtomicLong()).incrementAndGet(); + } + + private static long epochOf(Map epochs, String key) { + AtomicLong epoch = epochs.get(key); + return epoch == null ? 0L : epoch.get(); + } + + /** Current open/create generation of {@code prefix} (package-private for tests). */ + long prefixEpoch(String prefix) { + return epochOf(prefixEpochs, prefix); + } + + /** + * Called when a RocksDB read fails, typically because a live SST is missing on local disk. + * + *

We restore only the SST files that RocksDB references as live in its manifest but + * that are physically missing on local disk, downloading each back to its exact original + * path so RocksDB finds it on the next access. This deliberately avoids + * {@code ingestExternalFile}, which would (a) risk placing a file into the wrong column family + * and (b) assign a fresh sequence number that can resurrect deleted keys. Restricting to the + * live set also guarantees superseded / compacted-away objects are never resurrected. + * + *

If no live file is missing (the error has another cause) this returns {@code false} + * without any cloud I/O, and the original error propagates. + */ + @Override + public boolean onReadMiss(RocksDBSession session, String table, byte[] key) { + String dbName = session.getGraphName(); + if (!shouldAttemptReadMissHydration(dbName, table)) { + return false; + } + int restored = restoreMissingLiveFiles(provider, dbName, + RocksDBFactory.getInstance().getLiveSstFiles(dbName)); + if (restored > 0) { + log.info("Cloud read-miss hydration: restored {} missing live SST file(s) for db={}", + restored, dbName); + return true; + } + return false; + } + + /** + * Downloads back any SST files that are live in RocksDB's manifest but missing on local disk, + * writing each to its original path. Returns the number of files restored. + * + *

Package-private testable seam: caller supplies the live-file set. + */ + int restoreMissingLiveFiles(CloudStorageProvider provider, String dbName, + List liveFiles) { + int restored = 0; + for (LiveSstFile live : liveFiles) { + Path localPath = Paths.get(live.getAbsolutePath()); + if (Files.exists(localPath)) { + continue; + } + String remoteKey = toRelativeKey(live.getAbsolutePath()); + // De-dup concurrent restores of the same object (thundering herd on a cold read + // miss). If another thread already holds the slot, skip: it will produce the file. + String restoreGuardKey = dbName + "::" + remoteKey; + if (!inFlightRestores.add(restoreGuardKey)) { + continue; + } + try { + // Re-check after acquiring the slot: a concurrent restore may have just finished. + if (Files.exists(localPath)) { + continue; + } + if (!provider.fileExists(remoteKey)) { + log.warn("Cloud read-miss: live file missing locally AND absent in cloud: " + + "db={}, key={}", dbName, remoteKey); + continue; + } + downloadAtomically(provider, remoteKey, localPath); + syncTracker.markConfirmed(dbName, live.getAbsolutePath()); + restored++; + } catch (IOException e) { + log.warn("Cloud read-miss restore failed: db={}, key={}, reason={}", + dbName, remoteKey, e.getMessage()); + } finally { + inFlightRestores.remove(restoreGuardKey); + } + } + return restored; + } + + /** + * Called when a new RocksDB instance is opened for the first time. + * + *

Uploads any SST files that already exist in {@code dbPath} (e.g. from a previous run) + * and then triggers a MemTable flush so that WAL-recovered data is also written to + * SST files and eventually forwarded here via {@link #onTableFileCreated}. + * + * @param dbName logical name of the graph / partition + * @param dbPath absolute path of the RocksDB directory + */ + @Override + public void onDBCreated(String dbName, String dbPath) { + recordDb(dbName, dbPath); + // The DB is live at this prefix now: cancel any purge scheduled for an earlier generation. + bumpEpoch(prefixEpochs, dbPrefix(dbPath)); + // Do not propagate upload failures: the session is already live in dbSessionMap at + // this point, so throwing here would cause the createGraphDB caller to receive an + // error while the DB is actually open and usable by other threads (split-brain). + try { + uploadExistingSstFiles(provider, dbName, dbPath); + } catch (Exception e) { + log.warn("Cloud initial-upload failed for db={}: {} — DB is open, " + + "existing SSTs may not be in cloud yet", dbName, e.getMessage()); + } + flushDb(dbName); + // Mirror metadata immediately after initial upload/flush to keep cloud state recoverable. + syncMetadataSnapshotInline(provider, dbName); + } + + /** + * Called just before the local RocksDB directory is removed. Writes a small tombstone object + * (key = {@code dbPrefix + }{@value #DB_TOMBSTONE_SUFFIX}) outside the data prefix so that + * any subsequent {@link #preHydrateDbFiles} call for the same path will detect the deleted + * generation and skip hydration rather than re-ingesting stale objects. + * + *

This callback fires while the session is still in a pending-destroy list (refcount may + * be non-zero). The tombstone write is best-effort: a failure is logged but does not block + * the deletion. The cloud purge in {@link #onDBDeleted} provides a second line of defence. + * + * @param dbName logical graph/partition name + * @param dbPath absolute path of the RocksDB directory being destroyed + */ + @Override + public void onDBDeleteBegin(String dbName, String dbPath) { + String prefix = dbPrefix(dbPath); + // ALWAYS persist a LOCAL pending-delete marker first. This is the durable anti-resurrection + // guard that survives a failed remote cleanup: even if we cannot write the remote + // tombstone or purge below, the marker blocks hydration of this DB (see onDBOpening) and + // drives async cleanup until confirmed. It is removed once the remote purge succeeds. + // + // Marker durability is a hard precondition for a safe delete: if we cannot fsync it, a crash + // could lose the guard and let stale remote SST/metadata be re-hydrated as live data. So on a + // persistence failure we flip the health signal to degraded and HOLD delete progression + // (throw) rather than proceeding unguarded — the caller can retry once local storage recovers. + try { + writeMarker(pendingDeleteDir(), prefix); + deleteMarkerHealthy = true; + log.debug("Cloud pending-delete marker persisted (fsync'd): db={}, prefix={}", + dbName, prefix); + } catch (IOException e) { + deleteMarkerHealthy = false; + log.error("Cloud pending-delete marker could not be durably persisted for db={} " + + "prefix={}: {} — HOLDING delete to avoid an unguarded delete that a crash " + + "could let re-hydrate. Delete-marker health is DEGRADED.", + dbName, prefix, e.getMessage()); + throw new IllegalStateException( + "Cannot durably persist pending-delete marker for db=" + dbName + + "; holding delete to preserve the anti-resurrection guard", e); + } + // The delete is going ahead and the prefix will be purged, so publishing the DB's + // metadata when its session closes (onDBClosing) would be wasted work. + dirtyMetadataDbs.remove(dbName); + + // Tombstone lives OUTSIDE the data prefix (sibling, not child) so it is not + // accidentally deleted by purgeRemotePrefix when SST objects are still present. + String tombstoneKey = prefix + DB_TOMBSTONE_SUFFIX; + try { + writeTombstone(prefix); + log.info("Cloud DB tombstone written: db={}, key={}", dbName, tombstoneKey); + } catch (Exception e) { + log.warn("Cloud DB tombstone write failed (onDBDeleted will still purge): " + + "db={}, key={}, reason={}", dbName, tombstoneKey, e.getMessage()); + } + } + + /** + * Publishes a still-dirty DB's metadata while its session is open. The release path (a + * partition moving away, and {@code releaseAllGraphDB()} at shutdown, which runs before this + * listener is closed) would otherwise close the DB first, and the deferred publish could then + * never capture it, leaving the cloud CURRENT/MANIFEST behind the uploaded SSTs. + */ + @Override + public void onDBClosing(String dbName) { + if (!dirtyMetadataDbs.remove(dbName)) { + return; + } + if (!publishDirtyMetadata(dbName)) { + // Not re-marked: the DB is closing, so a deferred publish would only fail again. + log.warn("Cloud metadata publish before close failed for db={}: the mirrored " + + "CURRENT/MANIFEST may lag the uploaded SSTs until the DB is next opened.", + dbName); + } + } + + /** Uploads the sibling delete-tombstone for {@code prefix} from a short-lived temp file. */ + private void writeTombstone(String prefix) throws IOException { + Path tmp = Files.createTempFile("hgstore-tombstone-", ".tmp"); + try { + Files.writeString(tmp, "deleted"); + provider.uploadFile(tmp.toString(), prefix + DB_TOMBSTONE_SUFFIX); + } finally { + deleteIfExistsQuietly(tmp, "tombstone temp cleanup"); + } + } + + /** + * Called after the local RocksDB directory has been physically removed. Purges all cloud + * objects under the DB prefix (SSTs, metadata, tombstone) so a future creation at the same + * path starts with a clean remote state. Also clears all in-memory state for this DB. + * + *

The purge is best-effort: individual delete failures are logged at DEBUG level and do + * not throw. Any objects that survive the purge are neutralised by the tombstone check in + * {@link #preHydrateDbFiles}: the next open will find the tombstone (or an empty prefix if + * the purge was complete), skip hydration, and clean up any leftovers. + * + * @param dbName logical graph/partition name + * @param dbPath absolute path of the now-deleted RocksDB directory + */ + @Override + public void onDBDeleted(String dbName, String dbPath) { + String deletedDir = normaliseDir(dbPath); + dbNameByDir.remove(deletedDir); + if (dbNameByDir.containsValue(dbName)) { + // The same logical DB is live at another directory (it was re-created there while this + // one awaited destruction). Its name-keyed state belongs to the live copy, so only + // the work queued for files of the deleted directory is dropped. + deferredSstDeletes.removeIf(t -> dbName.equals(t.dbName) + && deletedDir.equals(parentDir(t.filePath))); + log.info("Cloud DB deleted at {} while db={} is live elsewhere; keeping its state", + deletedDir, dbName); + } else { + dirtyMetadataDbs.remove(dbName); + publishBackoffs.remove(dbName); + // Clear in-memory tracking so no stale state bleeds into a recreated DB. + syncTracker.clearDb(dbName); + readMissAttemptTs.entrySet().removeIf(e -> e.getKey().startsWith(dbName + "::")); + dbNameByDir.values().removeIf(dbName::equals); + metadataSyncLocks.remove(dbName); + lastPublishedMetadataGeneration.remove(dbName); + deferredSstDeletes.removeIf(t -> dbName.equals(t.dbName)); + // Remove per-DB meters so meter cardinality does not grow without bound as DBs churn. + CloudStorageMetrics.unregisterDatabaseMetrics(dbName); + } + + String prefix = dbPrefix(dbPath); + if (purgeRemotePrefix(provider, dbName, prefix) + && deleteTombstone(provider, dbName, prefix)) { + // Remote data and the sibling tombstone are gone: drop the local marker (cleanup done). + removeMarker(pendingDeleteDir(), prefix); + } else { + // Purge or tombstone delete failed. Keep the local marker so a future onDBOpening is + // blocked until both succeed, and retry (bounded). The marker must outlive the + // tombstone: a tombstone left next to a re-created DB would make a later open purge + // that DB's live objects. + log.warn("Cloud DB delete cleanup failed for db={}: local marker preserved to block " + + "re-opening the prefix; scheduling bounded retries.", dbName); + scheduleDeletePurgeRetry(dbName, prefix, 1); + } + } + + /** + * Deletes the sibling delete-tombstone once the prefix purge has succeeded. Returns + * {@code false} on failure: the caller must then keep the pending-delete marker (and so keep + * the prefix blocked), because a tombstone that outlives the marker would make the next open + * purge the live objects of a DB re-created at the same prefix. + */ + private boolean deleteTombstone(CloudStorageProvider provider, String dbName, String prefix) { + String tombstoneKey = prefix + DB_TOMBSTONE_SUFFIX; + try { + provider.deleteFile(tombstoneKey); + log.debug("Cloud DB tombstone cleaned up: db={}, key={}", dbName, tombstoneKey); + return true; + } catch (IOException e) { + log.warn("Cloud DB tombstone cleanup failed: db={}, key={}: {}", + dbName, tombstoneKey, e.getMessage()); + return false; + } + } + + // ----------------------------------------------------------------------- + // DB-deletion remote cleanup: local pending-delete marker + bounded purge retry + // ----------------------------------------------------------------------- + + /** Local subdirectory of the primary data root holding pending-remote-delete markers. */ + static final String PENDING_DELETE_DIR = ".cloud-pending-delete"; + /** Local subdirectory of each data root holding hard-linked SSTs awaiting async upload. */ + static final String STAGING_DIR = ".cloud-upload-staging"; + /** Max async retries for a failed delete purge before giving up (logging loudly). */ + private static final int MAX_DELETE_PURGE_RETRIES = 8; + /** Base backoff (ms) for delete-purge retries; doubles each attempt up to a small cap. */ + private static final long DELETE_PURGE_RETRY_BASE_MS = 500L; + /** Retry delay for superseded-SST deletes deferred by transient metadata/cloud failures. */ + private static final long DEFERRED_SST_DELETE_RETRY_MS = 500L; + /** Emit one WARN when deferred delete backlog crosses this size; otherwise keep DEBUG-only. */ + private static final int DEFERRED_SST_DELETE_WARN_THRESHOLD = 64; + + /** Pending superseded-SST deletes held until metadata/cloud preconditions are satisfied. */ + private final Set deferredSstDeletes = ConcurrentHashMap.newKeySet(); + /** Coalescing guard so only one deferred-delete retry task is scheduled at a time. */ + private final AtomicBoolean deferredSstDeleteRetryScheduled = new AtomicBoolean(false); + /** Ensures backlog WARN is emitted only once per threshold crossing. */ + private final AtomicBoolean deferredSstDeleteBacklogWarned = new AtomicBoolean(false); + + /** Immutable payload for a deferred superseded-SST delete. */ + private static final class DeferredSstDelete { + + final String dbName; + final String filePath; + + DeferredSstDelete(String dbName, String filePath) { + this.dbName = dbName; + this.filePath = filePath; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DeferredSstDelete that)) { + return false; + } + return java.util.Objects.equals(this.dbName, that.dbName) + && java.util.Objects.equals(this.filePath, that.filePath); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(this.dbName, this.filePath); + } + } + + private Path pendingDeleteDir() { + return Paths.get(primaryDataRoot, PENDING_DELETE_DIR); + } + + private static void deleteIfExistsQuietly(Path path, String context) { + try { + Files.deleteIfExists(path); + } catch (IOException cleanupError) { + log.debug("Failed to cleanup temp file during {}: path={}, reason={}", + context, path, cleanupError.getMessage()); + } + } + + /** + * Marker file path for a remote prefix. The remote prefix contains '/', so the filename is a + * URL-safe Base64 of the prefix — reversible (for the startup scan) and collision-free. The + * filename is the only payload; the marker file itself is empty. + */ + private static Path markerPath(Path dir, String prefix) { + return dir.resolve(encodeMarkerName(prefix)); + } + + private static String encodeMarkerName(String prefix) { + return java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString(prefix.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + + /** Decodes the marker filename (Base64 URL, no padding) back to the original remote prefix. */ + private static String decodeMarkerName(String encoded) { + if (encoded == null || encoded.isEmpty()) { + return null; + } + try { + byte[] bytes = java.util.Base64.getUrlDecoder().decode(encoded); + return new String(bytes, java.nio.charset.StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + return null; + } + } + + /** + * Crash-safely persists an empty marker for {@code prefix} under {@code dir}. Pending-delete + * markers are the durable anti-resurrection guard, so this is NOT best-effort: the marker file + * is created and fsynced, and the containing directory is fsynced so the new directory entry + * itself survives a crash. Throws {@link IOException} if durability cannot be guaranteed. + */ + private static void writeMarker(Path dir, String prefix) throws IOException { + Files.createDirectories(dir); + // TRUNCATE_EXISTING drops the payload of markers written by older versions. + try (FileChannel ch = FileChannel.open(markerPath(dir, prefix), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING)) { + ch.force(true); + } + // fsync the directory so the new marker entry itself is durable (a file fsync does not + // guarantee the directory entry pointing at it survives a crash on many filesystems). + try (FileChannel dirCh = FileChannel.open(dir, StandardOpenOption.READ)) { + dirCh.force(true); + } catch (IOException e) { + // Windows rejects directory-handle fsync; elsewhere it is a real durability error. + if (WINDOWS_OS) { + log.debug("Directory fsync of {} not supported on this platform: {}", + dir, e.getMessage()); + } else { + throw e; + } + } + } + + private static void removeMarker(Path dir, String prefix) { + try { + Files.deleteIfExists(markerPath(dir, prefix)); + } catch (IOException e) { + log.debug("Failed to remove marker in {} for prefix={}: {}", + dir, prefix, e.getMessage()); + } + } + + /** Whether a local pending-remote-delete marker exists for {@code prefix}. */ + boolean hasPendingDeleteMarker(String prefix) { + return Files.exists(markerPath(pendingDeleteDir(), prefix)); + } + + /** + * Completes a pending remote delete under a per-prefix lock: if the marker is still present, + * ensure the tombstone exists, purge the prefix, and on success drop the tombstone + marker. + * Returns {@code true} when cleanup is confirmed complete (marker absent after this call) or + * the call was cancelled, {@code false} when the purge failed and should be retried. + * + *

The lock + marker re-check make the purge run at most once and ONLY while the marker is + * present. {@code expectedEpoch} (non-null for the async retry) is the {@link #prefixEpoch} + * captured when the retry was scheduled: a DB opened or created at the prefix since then bumped + * it, so the retry stands down without purging — the prefix may already hold the new DB's + * objects. An open never passes an epoch: it bumped the epoch itself and runs the cleanup + * inline, before its DB can upload anything. + */ + private boolean tryCompletePendingDelete(CloudStorageProvider provider, String dbName, + String prefix, Long expectedEpoch) { + Object lock = pendingDeleteLocks.computeIfAbsent(prefix, k -> new Object()); + synchronized (lock) { + if (expectedEpoch != null && prefixEpoch(prefix) != expectedEpoch) { + log.info("Cloud pending-delete purge cancelled for db={} (prefix={}): the prefix " + + "was opened/created since it was scheduled.", dbName, prefix); + return true; + } + if (!hasPendingDeleteMarker(prefix)) { + return true; // already cleaned up by another caller + } + ensureTombstonePresent(provider, dbName, prefix); + if (purgeRemotePrefix(provider, dbName, prefix) + && deleteTombstone(provider, dbName, prefix)) { + removeMarker(pendingDeleteDir(), prefix); + log.info("Cloud pending-delete cleanup completed for db={} (prefix={})", + dbName, prefix); + return true; + } + return false; + } + } + + /** + * Retries the remote purge for a deleted DB with capped exponential backoff. While retries are + * pending, the local pending-delete marker keeps {@link #onDBOpening} from re-hydrating stale + * data. After {@link #MAX_DELETE_PURGE_RETRIES} failures it logs loudly for operator action; + * the marker and tombstone remain, so re-hydration stays blocked until manual cleanup. + */ + private void scheduleDeletePurgeRetry(String dbName, String prefix, int attempt) { + scheduleDeletePurgeRetry(dbName, prefix, attempt, prefixEpoch(prefix)); + } + + private void scheduleDeletePurgeRetry(String dbName, String prefix, int attempt, + long prefixEpochAtSchedule) { + if (attempt > MAX_DELETE_PURGE_RETRIES) { + log.error("Cloud DB delete purge for db={} still failing after {} attempt(s) — remote " + + "prefix '{}' may retain stale objects. The local pending-delete marker and " + + "tombstone remain (re-hydration stays blocked); manual cleanup is required.", + dbName, MAX_DELETE_PURGE_RETRIES, prefix); + return; + } + long delay = Math.min(DELETE_PURGE_RETRY_BASE_MS * (1L << (attempt - 1)), 8_000L); + try { + metadataSyncScheduler.schedule(() -> { + // Cancelled (returns true) when the marker is already cleared OR a DB was opened / + // created at the prefix since this retry was scheduled, so the retry never purges a + // prefix a re-created DB has taken over. + if (!tryCompletePendingDelete(provider, dbName, prefix, prefixEpochAtSchedule)) { + scheduleDeletePurgeRetry(dbName, prefix, attempt + 1, prefixEpochAtSchedule); + } + }, delay, TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException e) { + log.error("Cloud DB delete purge retry could not be scheduled for db={} (scheduler " + + "shutting down) — local marker preserved; remote prefix '{}' may retain " + + "stale objects.", dbName, prefix); + } + } + + /** Writes the delete-tombstone if it is not already present (best-effort). */ + private void ensureTombstonePresent(CloudStorageProvider provider, String dbName, String prefix) { + String tombstoneKey = prefix + DB_TOMBSTONE_SUFFIX; + try { + if (provider.fileExists(tombstoneKey)) { + return; + } + writeTombstone(prefix); + log.info("Cloud DB tombstone written (deferred): db={}, key={}", dbName, tombstoneKey); + } catch (Exception e) { + log.warn("Deferred tombstone write failed for db={}, key={}: {}", + dbName, tombstoneKey, e.getMessage()); + } + } + + /** Deletes staging hard-links left in every data root by a previous process. */ + private void sweepStagingDirsOnStartup() { + for (String root : allDataRoots) { + Path dir = Paths.get(root, STAGING_DIR); + if (!Files.isDirectory(dir)) { + continue; + } + int removed = 0; + try (Stream pins = Files.list(dir)) { + for (Path pin : (Iterable) pins::iterator) { + try { + if (Files.deleteIfExists(pin)) { + removed++; + } + } catch (IOException e) { + log.warn("Failed to delete orphaned staging file {}: {}", pin, + e.getMessage()); + } + } + } catch (IOException e) { + log.warn("Failed to scan staging dir {}: {}", dir, e.getMessage()); + } + if (removed > 0) { + log.info("Cloud startup: removed {} orphaned staging file(s) from {}", removed, dir); + } + } + } + + /** + * Scans {@code dir} for markers left by a previous process and calls {@code onMarker} with + * (inferred dbName, prefix) for each valid one. Markers whose name does not decode or whose + * prefix is out of this listener's scope are logged and left in place for operator inspection. + */ + private void scanMarkers(Path dir, BiConsumer onMarker) { + if (!Files.isDirectory(dir)) { + return; + } + try (Stream markers = Files.list(dir)) { + markers.forEach(marker -> { + Path fileName = marker.getFileName(); + String prefix = fileName == null ? null : decodeMarkerName(fileName.toString()); + if (prefix == null) { + log.error("Cloud startup: ignoring marker {} with undecodable name", marker); + return; + } + if (isMarkerPrefixOutOfScope(prefix)) { + log.error("Cloud startup: ignoring out-of-scope marker {} for prefix='{}'", + marker, prefix); + return; + } + onMarker.accept(inferDbNameFromPrefix(prefix), prefix); + }); + } catch (IOException e) { + log.warn("Failed to scan markers in {}: {}", dir, e.getMessage()); + } + } + + /** + * Returns {@code true} if a marker prefix must be rejected: empty, absolute, not normalized, or + * outside this listener's store scope. + */ + private boolean isMarkerPrefixOutOfScope(String prefix) { + if (prefix == null || prefix.isEmpty()) { + return true; + } + String normalized = prefix.replace('\\', '/'); + if (normalized.startsWith("/") || normalized.contains("//")) { + return true; + } + String[] parts = normalized.split("/"); + for (String p : parts) { + if (p.isEmpty() || ".".equals(p) || "..".equals(p)) { + return true; + } + } + if (storeScopePrefix.isEmpty()) { + return false; + } + String scopedPrefix = storeScopePrefix + "/"; + return !normalized.equals(storeScopePrefix) && !normalized.startsWith(scopedPrefix); + } + + /** Best-effort DB-name inference for startup marker logs/scheduling. */ + private String inferDbNameFromPrefix(String prefix) { + if (prefix == null || prefix.isEmpty()) { + return prefix; + } + try { + String stripped = stripStoreScope(prefix); + return stripped.isEmpty() ? prefix : stripped; + } catch (IllegalArgumentException e) { + return prefix; + } + } + + /** + * Deletes every remote object under {@code prefix} using an optimized prefix-level delete + * if available, falling back to individual file deletion if necessary. + * This is called during DB destruction to prevent a recreated DB from hydrating stale data. + * + * @return {@code true} if the purge completed without error; {@code false} if it failed. + * The caller uses the return value to decide whether it is safe to remove the + * tombstone — the tombstone must survive any partial purge. + */ + private boolean purgeRemotePrefix(CloudStorageProvider provider, String dbName, String prefix) { + String normalizedPrefix = prefix.endsWith("/") ? prefix : prefix + "/"; + try { + int deleted = provider.deletePrefix(normalizedPrefix); + if (deleted > 0) { + log.info("Cloud DB purge completed: db={}, prefix={}, deleted={}", + dbName, prefix, deleted); + } + // Every confirmation for this DB now names a deleted object. Drop them (bumping the + // tracker epoch so in-flight confirmations are discarded too); otherwise the snapshot + // SST check would skip re-uploading them and publish a MANIFEST pointing at nothing. + // Skipped when the name is live under another prefix: those confirmations are valid. + if (!liveUnderOtherPrefix(dbName, prefix)) { + syncTracker.clearDb(dbName); + } + return true; + } catch (IOException e) { + log.warn("Cloud DB purge failed for db={}, prefix={}: {}", + dbName, prefix, e.getMessage()); + return false; + } + } + + /** + * Pins and asynchronously uploads the newly created SST file to the active cloud + * storage provider. + * + * @param dbNameOrPath RocksDB instance name (partition id) or DB directory path + * @param cfName column-family name + * @param filePath absolute local path of the new SST file + * @param fileSize file size in bytes (informational) + */ + @Override + public void onTableFileCreated(String dbNameOrPath, String cfName, + String filePath, long fileSize) { + String dbName = resolveDbName(dbNameOrPath, filePath); + recordDb(dbName, parentDir(filePath)); + CloudStorageMetrics.registerDatabaseMetrics(dbName); + String remoteKey = toRelativeKey(filePath); + + // Capture the epoch before we hand off to a background thread. The async callback + // uses markConfirmedIfEpoch so a late confirmation after clearDb() + DB recreation + // with reused file numbers is silently dropped instead of producing stale durability state. + long uploadEpoch = syncTracker.currentEpoch(dbName); + + + Path pinned; + try { + pinned = pinForAsyncUpload(filePath); + } catch (Exception e) { + String errorType = e.getClass().getSimpleName(); + CloudStorageMetrics.recordUploadFailure(dbName, cfName, errorType); + // Hard link failed — no copy fallback, no extra disk use. Route original SST path + // to retry queue; retry will upload directly from the original file if still present. + log.warn("Cloud upload staging (hard link failed): db={}, cf={}, path={} " + + "— routing original SST to retry queue: {}", + dbName, cfName, filePath, e.getMessage()); + if (retryQueue != null) { + // Pass the captured epoch so a successful retry is confirmed via the epoch-guarded + // callback; the plain submit() would use epoch 0 and be silently dropped. + retryQueue.submit(dbName, cfName, filePath, remoteKey, uploadEpoch, e); + } + return; + } + + try { + uploadExecutor.execute(new SstUploadTask(provider, dbName, cfName, pinned, filePath, + remoteKey, uploadEpoch, fileSize)); + } catch (RejectedExecutionException e) { + // Queue is full. Keep the hard-link pin alive and submit IT to the retry queue so + // the file survives even if RocksDB compacts the original SST before the retry fires. + // Pass both the pinned path (for the actual upload) and the original SST path (for + // confirmation and cleanup) so CloudSyncTracker can parse the file number correctly. + CloudStorageMetrics.recordUploadFailure(dbName, cfName, "UploadQueueFull"); + log.error("Cloud upload dispatch rejected (queue full): db={}, cf={}, path={}", + dbName, cfName, filePath); + if (retryQueue != null) { + retryQueue.submitPinned(dbName, cfName, pinned.toString(), filePath, remoteKey, + uploadEpoch, + new IOException("cloud upload dispatch queue full", e)); + } else { + // No retry queue — nothing more we can do; clean up the pin. + try { + Files.deleteIfExists(pinned); + } catch (IOException ioe) { + log.debug("Failed to cleanup staged upload file {}: {}", pinned, ioe.getMessage()); + } + } + } + } + + /** + * A single asynchronous SST upload, as a typed {@link Runnable} rather than a lambda so that a + * forced {@code shutdownNow()} of the upload executor returns the actual task objects + * (a {@link ThreadPoolExecutor} returns the exact runnables it had queued). {@link + * #handoffOnShutdown()} then routes any never-run task into the retry queue so an SST that + * was accepted locally but not yet mirrored is not silently dropped from the durability + * pipeline. {@code run()} preserves the original inline behaviour exactly. + */ + private final class SstUploadTask implements Runnable { + + private final CloudStorageProvider provider; + private final String dbName; + private final String cfName; + private final Path pinned; + private final String filePath; + private final String remoteKey; + private final long uploadEpoch; + private final long fileSize; + + SstUploadTask(CloudStorageProvider provider, String dbName, String cfName, Path pinned, + String filePath, String remoteKey, long uploadEpoch, long fileSize) { + this.provider = provider; + this.dbName = dbName; + this.cfName = cfName; + this.pinned = pinned; + this.filePath = filePath; + this.remoteKey = remoteKey; + this.uploadEpoch = uploadEpoch; + this.fileSize = fileSize; + } + + @Override + public void run() { + long startTimeMs = System.currentTimeMillis(); + // When the upload fails and is handed to the retry queue, OWNERSHIP of the pinned + // hard-link transfers to the retry queue lifecycle — it must NOT be deleted here, or a + // retry firing after RocksDB has compacted away the original SST would find no source + // and be silently dropped (retry intent lost). We only delete the pin when we still + // own it (successful upload, or no retry queue to hand off to). + boolean pinHandedOff = false; + try { + // ---- Upload proper: ONLY a failure here is an upload failure that should retry. ---- + try { + provider.uploadFile(pinned.toString(), remoteKey); + } catch (Exception e) { + String errorType = e.getClass().getSimpleName(); + CloudStorageMetrics.recordUploadFailure(dbName, cfName, errorType); + log.error("Cloud upload failed (will retry on next compaction): " + + "db={}, cf={}, path={}, error={}", dbName, cfName, + filePath, e.getMessage()); + if (retryQueue != null) { + // Hand the SURVIVING pinned hard-link to the retry queue (submitPinned), not + // the original SST path (plain submit): compaction may delete the original + // before the retry fires, but the pin keeps the byteset alive. + // sourceSstPath=filePath is still used for epoch-guarded confirmation and + // staging cleanup on success. + retryQueue.submitPinned(dbName, cfName, pinned.toString(), filePath, + remoteKey, uploadEpoch, e); + pinHandedOff = true; + } + return; + } + // ---- Post-upload: the upload SUCCEEDED. Errors below (e.g. an inline metadata + // publish failing during shutdown) must NOT be reclassified as an upload failure + // and re-uploaded or given up on; the SST is already durable in cloud. ---- + syncTracker.markConfirmedIfEpoch(dbName, filePath, uploadEpoch); + long syncLatencyMs = System.currentTimeMillis() - startTimeMs; + CloudStorageMetrics.recordSyncLatency(dbName, syncLatencyMs); + try { + // Coalesce the per-SST metadata sync into the periodic drain. + markMetadataDirty(dbName); + } catch (Exception e) { + log.warn("Cloud upload succeeded but post-upload metadata-sync scheduling " + + "failed (SST is durable; CURRENT/MANIFEST will catch up on the next " + + "sync): db={}, path={}: {}", dbName, filePath, e.getMessage()); + } + log.debug("Cloud upload success: db={}, cf={}, path={}, size={}, latencyMs={}", + dbName, cfName, filePath, fileSize, syncLatencyMs); + } finally { + if (!pinHandedOff) { + try { + Files.deleteIfExists(pinned); + } catch (IOException e) { + log.debug("Failed to cleanup staged upload file {}: {}", + pinned, e.getMessage()); + } + } + } + } + + /** + * Called for a task that was queued but never ran because the executor was force-stopped at + * shutdown. Hands the still-pinned file to the retry queue so its upload intent survives: + * the pin is kept alive (not deleted) so the retry queue can upload it even if RocksDB has + * since compacted the original SST away. + */ + void handoffOnShutdown() { + if (retryQueue == null) { + // No retry queue: best-effort cleanup of the pin; nothing else we can do. + try { + Files.deleteIfExists(pinned); + } catch (IOException ignore) { + // best-effort + } + return; + } + log.warn("Cloud upload task not executed before shutdown; handing off to retry queue: " + + "db={}, cf={}, path={}", dbName, cfName, filePath); + retryQueue.submitPinned(dbName, cfName, pinned.toString(), filePath, remoteKey, + uploadEpoch, + new IOException("upload executor stopped before task ran")); + } + } + + /** + * Creates a stable hard-link snapshot of the SST file for async upload, so the upload worker + * can read a consistent source even after RocksDB deletes the original during compaction. + * + *

Hard links share the same inode — no extra data blocks are consumed. If the original is + * deleted by compaction before the worker runs, the hard-link still holds the inode alive so + * the upload can proceed normally. + * + *

If the hard link fails (e.g. cross-device mount, filesystem hard-link limits), an + * {@link IOException} is thrown. The caller ({@link #onTableFileCreated}) catches this and + * routes the upload to the retry queue using the original SST path — no copy is made and no + * extra disk space is consumed. The retry will succeed as long as the original file still + * exists when it fires; if it has been compacted away the retry queue silently drops it. + */ + private Path pinForAsyncUpload(String filePath) throws IOException { + Path source = Paths.get(filePath); + // Use the staging dir that lives on the same filesystem as the source SST to avoid + // cross-device hard-link failures in multi-disk deployments. + String matchingRoot = findMatchingDataRoot(source.toAbsolutePath().normalize().toString()); + Path stagingDir = Paths.get(matchingRoot, STAGING_DIR); + Files.createDirectories(stagingDir); + String fileName = source.getFileName().toString(); + Path staged = stagingDir.resolve(fileName + ".upload-" + System.nanoTime()); + try { + Files.createLink(staged, source); + return staged; + } catch (Exception linkEx) { + throw new IOException( + "Hard link failed; upload will be retried from original SST path: " + + linkEx.getMessage(), linkEx); + } + } + + /** + * Removes the deleted SST file from the active cloud storage provider. + * + *

This callback runs on a RocksDB background (compaction/flush) thread, so it performs no + * cloud I/O itself: the delete is queued and {@link #processDeferredSstDeletes} runs the + * live-set durability guard, the MANIFEST publish and the remote delete on the metadata-sync + * thread, publishing metadata once per DB for every delete a compaction produced. + * + * @param dbNameOrPath RocksDB instance name (partition id) or DB directory path + * @param cfName column-family name + * @param filePath absolute local path of the deleted SST file + */ + @Override + public void onTableFileDeleted(String dbNameOrPath, String cfName, String filePath) { + // RocksDB delivers the DB *directory path* here (from db.getName()), not the logical name — + // exactly as it does for onTableFileCreated. Resolve it the same way: without this, the + // delete guard's live-set lookup (RocksDBFactory.getLiveSstFiles), the metadata-sync + // capture (captureMetadataSnapshot), and the sync-tracker updates all key off a path that + // is absent from dbSessionMap, so every superseded-SST removal is silently skipped and the + // object leaks in cloud forever (with an endless deferred-delete retry loop). + String dbName = resolveDbName(dbNameOrPath, filePath); + if (deferredSstDeletes.add(new DeferredSstDelete(dbName, filePath))) { + log.debug("Cloud delete queued: db={}, cf={}, path={}", dbName, cfName, filePath); + updateDeferredSstDeleteWarnState(); + } + scheduleDeferredSstDeleteRetry(0L); + } + + /** Emits one WARN only when deferred backlog crosses the threshold, then rearms below it. */ + private void updateDeferredSstDeleteWarnState() { + int backlog = deferredSstDeletes.size(); + if (backlog >= DEFERRED_SST_DELETE_WARN_THRESHOLD) { + if (deferredSstDeleteBacklogWarned.compareAndSet(false, true)) { + log.warn("Deferred cloud-delete backlog crossed threshold: pending={}, threshold={}", + backlog, DEFERRED_SST_DELETE_WARN_THRESHOLD); + } + return; + } + deferredSstDeleteBacklogWarned.set(false); + } + + private void scheduleDeferredSstDeleteRetry(long delayMs) { + if (deferredSstDeletes.isEmpty()) { + return; + } + if (!deferredSstDeleteRetryScheduled.compareAndSet(false, true)) { + return; + } + long safeDelay = Math.max(0L, delayMs); + try { + metadataSyncScheduler.schedule(() -> { + deferredSstDeleteRetryScheduled.set(false); + processDeferredSstDeletes(); + }, safeDelay, TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException e) { + deferredSstDeleteRetryScheduled.set(false); + log.debug("Deferred cloud-delete retry scheduling rejected (scheduler shutting down): {}", + e.getMessage()); + } + } + + /** + * Deletes queued superseded SSTs from cloud, on the metadata-sync thread. For each DB the + * preconditions are checked once per pass, whatever the number of queued deletes: + * + *

    + *
  1. DATA-LOSS GUARD: every SST currently live in the DB is confirmed present in cloud, so + * a compaction input is never removed before its outputs are durable.
  2. + *
  3. MANIFEST-BEFORE-DELETE: an updated MANIFEST+CURRENT reflecting the post-compaction + * live set is published first; otherwise a crash between the SST delete and the next + * metadata sync leaves a cloud MANIFEST that references a deleted object.
  4. + *
+ * + *

Deletes whose DB fails either check, or whose remote delete fails, stay queued and are + * retried. Package-private (and serialised) so tests can drive a pass deterministically. + */ + synchronized void processDeferredSstDeletes() { + if (deferredSstDeletes.isEmpty()) { + return; + } + Map readyByDb = new HashMap<>(); + int succeeded = 0; + for (DeferredSstDelete task : new ArrayList<>(deferredSstDeletes)) { + boolean ready = readyByDb.computeIfAbsent(task.dbName, this::readyForSstDeletes); + if (!ready) { + continue; + } + String remoteKey = toRelativeKey(task.filePath); + try { + provider.deleteFile(remoteKey); + syncTracker.clearConfirmed(task.dbName, task.filePath); + if (deferredSstDeletes.remove(task)) { + succeeded++; + } + } catch (Exception e) { + log.debug("Deferred cloud-delete retry failed: db={}, path={}, reason={}", + task.dbName, task.filePath, e.getMessage()); + } + } + if (succeeded > 0) { + log.debug("Cloud delete removed {} superseded SST(s)", succeeded); + } + updateDeferredSstDeleteWarnState(); + if (!deferredSstDeletes.isEmpty()) { + scheduleDeferredSstDeleteRetry(DEFERRED_SST_DELETE_RETRY_MS); + } + } + + /** Whether superseded SSTs of {@code dbName} may be deleted from cloud right now. */ + private boolean readyForSstDeletes(String dbName) { + // Share the drain's per-DB publish backoff: while a DB's publishes keep failing, the 500 ms + // delete retry must not redo a checkpoint (which flushes the memtable) and an upload. + PublishBackoff backoff = publishBackoffs.get(dbName); + if (backoff != null && System.currentTimeMillis() < backoff.nextAttemptAtMs) { + return false; + } + if (ensureLiveSetUploaded(provider, dbName)) { + log.warn("Cloud delete held (live set not fully durable in cloud): db={}", dbName); + return false; + } + // The compaction is complete, so its outputs are already on disk and the MANIFEST already + // reflects the post-compaction state; a failure here is recorded for the backoff. + if (!publishDirtyMetadata(dbName)) { + log.warn("Cloud delete held (metadata sync failed, MANIFEST not yet updated): db={}", + dbName); + return false; + } + return true; + } + + /** + * Ensures every live SST file of {@code dbName} is confirmed present in cloud, uploading any + * that are not yet confirmed. Returns {@code true} only when the entire live set is durable. + * + *

Fast path (common case)

+ * {@link #preHydrateDbFiles} seeds {@link #syncTracker} from the cloud listing on startup, + * and {@link #onTableFileCreated} marks every successful upload confirmed. By the time + * {@link #onTableFileDeleted} fires the entire live set is normally already confirmed, so + * {@link CloudSyncTracker#allConfirmed} returns {@code true} after one lock acquisition with + * zero cloud I/O — regardless of live-set size. + * + *

Why we must check the whole live set, not just the deleted {@code filePath}

+ * {@code filePath} is the old compaction input being removed from cloud. Its bit + * being set in the bitmap only confirms it was uploaded previously — it says nothing about + * whether the compaction outputs (the replacement files, which form the new live + * set) are also in cloud. Deleting the input before the outputs are durable would leave + * data irretrievably lost on a node crash. + * + *

Slow path (rare)

+ * Entered only when {@code allConfirmed} returns {@code false} (e.g. a previous upload + * failed). Files present locally are uploaded directly without a {@code fileExists} probe + * (idempotent PUT). Files absent locally and not confirmed are treated as not-yet-durable + * and the delete is held. + */ + private boolean ensureLiveSetUploaded(CloudStorageProvider provider, String dbName) { + return !ensureLiveSetUploaded(provider, dbName, currentLiveSstFiles(dbName)); + } + + /** + * Returns the DB's current live SST set. Package-private and overridable purely as a test seam + * so the {@link #onTableFileDeleted} delete-guard path can be exercised with a controlled + * live-set (the production implementation reads the RocksDB singleton, which a unit test without + * a live session cannot populate). + */ + List currentLiveSstFiles(String dbName) { + return RocksDBFactory.getInstance().getLiveSstFiles(dbName); + } + + /** Testable seam: caller supplies the live-file set instead of reading the RocksDB singleton. */ + boolean ensureLiveSetUploaded(CloudStorageProvider provider, String dbName, + List liveFiles) { + // Fast path: single lock acquisition, zero cloud I/O — the common case. + if (syncTracker.allConfirmed(dbName, liveFiles)) { + return true; + } + + // Slow path: one or more live files not yet confirmed — find and upload them. + log.info("Checking live set durability: db={}, liveFileCount={}", dbName, liveFiles.size()); + + java.util.List unconfirmedFiles = new java.util.ArrayList<>(); + boolean allDurable = true; + + for (LiveSstFile live : liveFiles) { + String localPath = live.getAbsolutePath(); + if (syncTracker.isConfirmed(dbName, localPath)) { + continue; + } + unconfirmedFiles.add(localPath); + Path localFile = Paths.get(localPath); + String remoteKey = toRelativeKey(localPath); + long uploadEpoch = syncTracker.currentEpoch(dbName); + if (!Files.exists(localFile)) { + // File absent locally and absent from bitmap. Startup hydration (preHydrateDbFiles) + // seeds the bitmap from the cloud listing, so reaching here means this file is + // genuinely not durable in cloud — hold the delete. + allDurable = false; + log.warn("Delete guard: live file absent locally and not confirmed in cloud: " + + "db={}, filePath={}", dbName, localPath); + continue; + } + try { + // Upload directly — no fileExists probe (idempotent PUT is cheaper than a probe). + provider.uploadFile(localPath, remoteKey); + if (!syncTracker.markConfirmedIfEpoch(dbName, localPath, uploadEpoch)) { + allDurable = false; + log.warn("Delete guard: stale live-file confirmation dropped by epoch guard: " + + "db={}, filePath={}, epoch={}", dbName, localPath, uploadEpoch); + } + } catch (Exception e) { + allDurable = false; + log.warn("Delete guard: failed to upload live file: db={}, filePath={}, error={}", + dbName, localPath, e.getMessage()); + if (retryQueue != null) { + // Epoch captured now so a successful retry confirms via the epoch-guarded + // callback (plain submit() uses epoch 0 → dropped, so the file would never be + // marked durable and the delete guard would keep re-uploading it). + retryQueue.submit(dbName, live.getCfName(), localPath, remoteKey, + uploadEpoch, e); + } + } + } + + if (!unconfirmedFiles.isEmpty()) { + CloudStorageMetrics.incrementDeleteGuardReupload(dbName); + if (allDurable) { + log.info("Re-uploaded {} previously unconfirmed live files (delete proceeding): " + + "db={}, files={}", unconfirmedFiles.size(), dbName, unconfirmedFiles); + } else { + log.warn("Re-upload attempted for {} unconfirmed live files (some still not durable, " + + "delete held): db={}, files={}", unconfirmedFiles.size(), dbName, + unconfirmedFiles); + } + } + + return allDurable; + } + + /** + * A compaction changed this DB's live SST set; schedule a (coalesced) metadata publish. + * + *

Note: RocksDB delivers the DB directory path here (from {@code db.getName()}), + * not the logical name, so we resolve it against {@link #dbNameByDir}. + */ + @Override + public void onCompacted(String dbNameOrPath) { + String dbName = resolveDbName(dbNameOrPath, null); + // Runs on a RocksDB compaction thread: only mark the DB dirty, the coalesced drain does + // the checkpoint + upload. + markMetadataDirty(dbName); + } + + // ----------------------------------------------------------------------- + // Metadata mirroring + // ----------------------------------------------------------------------- + + /** + * Drops per-name upload state when {@code dbName} is already mapped to a different directory. + * The old dir's mapping is kept so its native events still resolve; onDBDeleted removes it. + * Reopening at the same dir, or a name with no mapping, leaves the state untouched. + */ + private void resetStateIfDbMovedDir(String dbName, String dbPath) { + if (dbName == null || dbPath == null) { + return; + } + String dir = normaliseDir(dbPath); + for (Map.Entry e : dbNameByDir.entrySet()) { + if (dbName.equals(e.getValue()) && !dir.equals(e.getKey())) { + log.info("Cloud DB {} reopened at {} (previously {}); resetting upload confirmations " + + "and published generation", dbName, dir, e.getKey()); + // clearDb bumps the epoch, so in-flight confirmations from the old dir are dropped. + syncTracker.clearDb(dbName); + lastPublishedMetadataGeneration.remove(dbName); + return; + } + } + } + + /** Records the resolved DB directory for a logical DB name (idempotent). */ + private void recordDb(String dbName, String dbDir) { + if (dbName == null || dbDir == null) { + return; + } + dbNameByDir.put(normaliseDir(dbDir), dbName); + } + + private static String normaliseDir(String dir) { + return Paths.get(dir).toAbsolutePath().normalize().toString(); + } + + /** Whether {@code dbName} is mapped to a DB directory whose remote prefix is not {@code prefix}. */ + private boolean liveUnderOtherPrefix(String dbName, String prefix) { + for (Map.Entry e : dbNameByDir.entrySet()) { + if (dbName.equals(e.getValue()) && !prefix.equals(dbPrefix(e.getKey()))) { + return true; + } + } + return false; + } + + /** Absolute, normalised parent directory of a file path (the DB dir of an SST). */ + private String parentDir(String filePath) { + Path parent = Paths.get(filePath).toAbsolutePath().normalize().getParent(); + return parent == null ? null : parent.toString(); + } + + /** Resolves callback DB identifiers that may be path-form into logical DB names. */ + private String resolveDbName(String dbNameOrPath, String filePath) { + if (dbNameOrPath == null || dbNameOrPath.isBlank()) { + String parent = parentDir(filePath); + if (parent == null) { + return dbNameOrPath; + } + Path name = Paths.get(parent).getFileName(); + return name == null ? dbNameOrPath : name.toString(); + } + + if (!isPathLike(dbNameOrPath)) { + return dbNameOrPath; + } + + String normalised = Paths.get(dbNameOrPath).toAbsolutePath().normalize().toString(); + String mapped = dbNameByDir.get(normalised); + if (mapped != null && !mapped.isBlank()) { + return mapped; + } + + Path name = Paths.get(normalised).getFileName(); + return name == null ? dbNameOrPath : name.toString(); + } + + private static boolean isPathLike(String value) { + return value.contains("/") || value.contains("\\"); + } + + /** + * Coalesces the high-frequency, per-SST metadata sync triggered by {@link #onTableFileCreated} + * by marking {@code dbName} dirty for {@link #drainDirtyMetadata}. This does not weaken the + * durability contract: SST objects are uploaded eagerly regardless — only the (idempotent, + * generation-guarded) metadata pointer publish is coalesced. Publishes inline instead when + * coalescing is disabled or the scheduler has shut down. + */ + void markMetadataDirty(String dbName) { + if (metadataSyncDebounceMs <= 0) { + publishDirtyMetadata(dbName); + return; + } + dirtyMetadataDbs.add(dbName); + // Re-check after adding so a mark racing close()'s final flush is not stranded. + if (metadataSyncScheduler.isShutdown() && dirtyMetadataDbs.remove(dbName)) { + publishDirtyMetadata(dbName); + } + } + + /** + * Trigger invoked by {@link CloudUploadRetryQueue} after an upload becomes durable via retry. + * Publishes CURRENT/MANIFEST (coalesced) so the mirrored recovery point advances + * even when the retry succeeded during an otherwise-quiet period on an idle DB — without this, + * cloud could hold the SST but keep exposing a stale CURRENT. Runs on the retry-queue thread. + */ + public void onRetryUploadDurable(String dbName) { + // During shutdown the retry queue is closed AFTER this listener and BEFORE the provider, so + // a retry that becomes durable in that window must still advance the pointer. Once the + // scheduler is shut down, markMetadataDirty falls back to an inline publish. + markMetadataDirty(dbName); + // A newly durable upload may unblock one or more deferred superseded-SST deletes. + scheduleDeferredSstDeleteRetry(0L); + } + + /** + * Periodic drain on {@link #metadataSyncScheduler}: publishes each dirty DB once. A failed + * publish re-marks the DB and backs off exponentially per DB (see {@link #publishBackoffMs}), + * so a persistently failing DB does not redo a checkpoint + upload every interval. + */ + private void drainDirtyMetadata() { + long now = System.currentTimeMillis(); + for (String dbName : new ArrayList<>(dirtyMetadataDbs)) { + PublishBackoff backoff = publishBackoffs.get(dbName); + if (backoff != null && now < backoff.nextAttemptAtMs) { + continue; + } + if (!dirtyMetadataDbs.remove(dbName)) { + continue; + } + if (publishDirtyMetadata(dbName)) { + publishBackoffs.remove(dbName); + } else { + dirtyMetadataDbs.add(dbName); + } + } + } + + /** Backoff before the next publish attempt after {@code failures} consecutive failures. */ + long publishBackoffMs(int failures) { + long base = Math.max(MIN_PUBLISH_BACKOFF_BASE_MS, metadataSyncDebounceMs); + int exp = Math.min(Math.max(failures - 1, 0), 30); + long delay = base * (1L << exp); + return delay <= 0L || delay > MAX_PUBLISH_BACKOFF_MS ? MAX_PUBLISH_BACKOFF_MS : delay; + } + + /** Consecutive publish failures currently recorded for {@code dbName} (test seam). */ + int publishFailureCount(String dbName) { + PublishBackoff backoff = publishBackoffs.get(dbName); + return backoff == null ? 0 : backoff.failures; + } + + /** + * Publishes metadata for a dirty DB. Returns {@code true} when published, {@code false} when + * the publish failed (the failure is recorded for backoff; success clears it). + */ + private boolean publishDirtyMetadata(String dbName) { + boolean published; + try { + published = syncMetadataSnapshotInline(provider, dbName); + } catch (Exception e) { + log.warn("Deferred cloud metadata-sync failed for db={}: {}", dbName, e.getMessage()); + published = false; + } + if (published) { + publishBackoffs.remove(dbName); + } else { + publishBackoffs.compute(dbName, (k, prev) -> { + int failures = prev == null ? 1 : prev.failures + 1; + return new PublishBackoff(failures, System.currentTimeMillis() + + publishBackoffMs(failures)); + }); + } + return published; + } + + /** + * Captures a consistent metadata snapshot (via a RocksDB {@link org.rocksdb.Checkpoint}) and + * mirrors it to cloud. Safe to call from any thread, including a RocksDB event/compaction + * callback (it does not call {@code flushSession(wait=true)}, which would deadlock there). + * + *

Flush behavior: the capture uses RocksJava {@code Checkpoint.createCheckpoint}, + * which flushes the memtable (verified by {@code RocksDBFactoryTest}). That flush is intentional + * and beneficial here: it persists un-flushed MemTable data into an SST that is then mirrored, so + * recent writes are recoverable from cloud (the RocksDB WAL is disabled under Raft, so a flush is + * the only way to push the tail into cloud; the Raft log is the tail's local durability source). + * The cost is a small extra L0 SST (write amplification) whose frequency is bounded by + * the metadata-sync drain interval; when the memtable is empty the flush is a no-op. + * + *

Package-private for testability. + */ + boolean syncMetadataSnapshotInline(CloudStorageProvider provider, String dbName) { + Object lock = metadataSyncLocks.computeIfAbsent(dbName, ignored -> new Object()); + synchronized (lock) { + MetadataSnapshot snapshot = captureMetadataSnapshot(dbName); + if (snapshot == null) { + return false; + } + try { + return publishSnapshotWithGenerationCheck(provider, dbName, snapshot); + } finally { + snapshot.cleanup(); + } + } + } + + MetadataSnapshot captureMetadataSnapshot(String dbName) { + return RocksDBFactory.getInstance().captureMetadataSnapshot(dbName); + } + + private boolean publishSnapshotWithGenerationCheck(CloudStorageProvider provider, + String dbName, + MetadataSnapshot snapshot) { + long snapshotGeneration = snapshot.getGeneration(); + long lastPublished = lastPublishedMetadataGeneration.getOrDefault(dbName, Long.MIN_VALUE); + if (snapshotGeneration >= 0L && snapshotGeneration < lastPublished) { + log.warn("Cloud metadata-sync rejected stale snapshot: db={}, generation={}, lastPublished={}", + dbName, snapshotGeneration, lastPublished); + return false; + } + boolean published = uploadMetadataSnapshot(provider, dbName, snapshot); + if (published && snapshotGeneration >= 0L) { + lastPublishedMetadataGeneration.merge(dbName, snapshotGeneration, Math::max); + } + return published; + } + + /** + * Uploads a captured metadata snapshot to cloud in strict consistency order: + *

    + *
  1. every SST the captured manifest references (from the checkpoint's hard-links);
  2. + *
  3. the {@code OPTIONS-*} and {@code MANIFEST-} blobs;
  4. + *
  5. last, {@code CURRENT} — the pointer — so a restore that fetches {@code CURRENT} + * never sees it referencing a manifest whose SSTs are not all in cloud;
  6. + *
  7. only then prune superseded remote {@code MANIFEST-*}/{@code OPTIONS-*}.
  8. + *
+ * If any referenced SST cannot be made durable, the method aborts before publishing + * {@code MANIFEST}/{@code CURRENT} and returns {@code false}, leaving the previous durable + * generation intact. + */ + boolean uploadMetadataSnapshot(CloudStorageProvider provider, String dbName, + MetadataSnapshot snapshot) { + String dbDir = snapshot.getDbDir(); + String tempDir = snapshot.getTempDir(); + + // (0) Defense in depth: a snapshot without both CURRENT and MANIFEST is degenerate. If we + // proceeded, steps (3)/(4) would upload nothing (both null-guarded) but step (5) would still + // run pruneRemoteMetadata, which deletes remote MANIFEST-* objects not in the (empty) keep + // set — destroying the previously-valid remote MANIFEST while the remote CURRENT still + // points at it, corrupting the cloud copy irrecoverably. Treat it as a failed publish. + if (snapshot.getManifestFileName() == null || snapshot.getCurrentFileName() == null) { + log.warn("Cloud metadata-sync: snapshot for db={} missing {} — skipping publish/prune " + + "to avoid deleting the valid remote MANIFEST", dbName, + snapshot.getManifestFileName() == null ? "MANIFEST" : "CURRENT"); + return false; + } + + // (1) Confirm every manifest-referenced SST is present in cloud. + if (!ensureSnapshotSstsUploaded(provider, dbName, snapshot)) { + log.warn("Cloud metadata-sync: SST set not fully durable, holding metadata publish " + + "for db={}", dbName); + return false; + } + + try { + // (2) OPTIONS-* then MANIFEST-. + for (String optionsName : snapshot.getOptionsFileNames()) { + uploadMetaFile(provider, dbDir, tempDir, optionsName); + } + uploadMetaFile(provider, dbDir, tempDir, snapshot.getManifestFileName()); + // (3) CURRENT last — the atomic pointer publish. + uploadMetaFile(provider, dbDir, tempDir, snapshot.getCurrentFileName()); + } catch (IOException e) { + log.warn("Cloud metadata-sync: failed to publish metadata for db={}: {}", + dbName, e.getMessage()); + return false; + } + + // (4) Prune superseded remote metadata now that the new CURRENT is durable. + pruneRemoteMetadata(provider, dbDir, snapshot); + log.debug("Cloud metadata-sync published: db={}, manifest={}, options={}", + dbName, snapshot.getManifestFileName(), snapshot.getOptionsFileNames().size()); + return true; + } + + /** + * Ensures every SST the captured manifest references (the checkpoint's hard-linked {@code *.sst} + * set) is present in cloud, uploading from the hard-link when not. The hard-link pins the + * content even if compaction removed the original, so this cannot race a concurrent delete. + */ + private boolean ensureSnapshotSstsUploaded(CloudStorageProvider provider, String dbName, + MetadataSnapshot snapshot) { + String dbDir = snapshot.getDbDir(); + String tempDir = snapshot.getTempDir(); + boolean allDurable = true; + for (String sstName : snapshot.getSstFileNames()) { + String realPath = joinPath(dbDir, sstName); + String remoteKey = toRelativeKey(realPath); + long uploadEpoch = syncTracker.currentEpoch(dbName); + if (syncTracker.isConfirmed(dbName, realPath)) { + continue; + } + try { + if (provider.fileExists(remoteKey)) { + if (!syncTracker.markConfirmedIfEpoch(dbName, realPath, uploadEpoch)) { + allDurable = false; + log.warn("Cloud metadata-sync: stale SST confirmation dropped by epoch guard: " + + "db={}, key={}, epoch={}", dbName, remoteKey, uploadEpoch); + } + continue; + } + provider.uploadFile(joinPath(tempDir, sstName), remoteKey); + if (!syncTracker.markConfirmedIfEpoch(dbName, realPath, uploadEpoch)) { + allDurable = false; + log.warn("Cloud metadata-sync: stale SST upload confirmation dropped by epoch " + + "guard: db={}, key={}, epoch={}", dbName, remoteKey, uploadEpoch); + } + } catch (Exception e) { + allDurable = false; + log.warn("Cloud metadata-sync: failed to confirm SST db={}, key={}, reason={}", + dbName, remoteKey, e.getMessage()); + if (retryQueue != null && Files.exists(Paths.get(realPath)) + && !retryQueue.isRetryPending(dbName, remoteKey)) { + // Epoch captured now so a successful retry confirms via the epoch-guarded + // callback (plain submit() uses epoch 0 → dropped). Skipped when a retry for + // this object is already queued, so every failed publish does not stack + // another retry chain. + retryQueue.submit(dbName, null, realPath, remoteKey, uploadEpoch, e); + } + } + } + return allDurable; + } + + /** Uploads one metadata file from the checkpoint temp dir to its real-path-derived remote key. */ + private void uploadMetaFile(CloudStorageProvider provider, String dbDir, String tempDir, + String fileName) throws IOException { + provider.uploadFile(joinPath(tempDir, fileName), toRelativeKey(joinPath(dbDir, fileName))); + } + + /** + * Deletes remote {@code MANIFEST-*}/{@code OPTIONS-*} objects for this DB that are not part of + * the just-published snapshot, bounding remote metadata growth. {@code CURRENT} and + * {@code *.sst} are never pruned here (SST lifecycle is the delete guard's). WAL {@code *.log} + * files are never mirrored to cloud (the snapshot only uploads CURRENT/MANIFEST/OPTIONS/SST), + * so there is nothing to prune for them. + */ + private void pruneRemoteMetadata(CloudStorageProvider provider, String dbDir, + MetadataSnapshot snapshot) { + Set keep = new HashSet<>(); + if (snapshot.getManifestFileName() != null) { + keep.add(snapshot.getManifestFileName()); + } + keep.addAll(snapshot.getOptionsFileNames()); + String prefix = dbPrefix(dbDir); + List remoteKeys; + try { + remoteKeys = provider.listFiles(prefix.endsWith("/") ? prefix : prefix + "/"); + } catch (IOException e) { + log.debug("Cloud metadata-sync prune: list failed for prefix={}: {}", + prefix, e.getMessage()); + return; + } + for (String remoteKey : remoteKeys) { + int slash = remoteKey.lastIndexOf('/'); + String base = slash >= 0 ? remoteKey.substring(slash + 1) : remoteKey; + boolean isSupersededMeta = (base.startsWith("MANIFEST-") || base.startsWith("OPTIONS-")) + && !keep.contains(base); + if (!isSupersededMeta) { + continue; + } + try { + provider.deleteFile(remoteKey); + log.debug("Cloud metadata-sync prune: deleted superseded {}", remoteKey); + } catch (IOException e) { + log.debug("Cloud metadata-sync prune: delete failed for {}: {}", remoteKey, e.getMessage()); + } + } + } + + /** Joins a directory and a file name with the platform separator. */ + private static String joinPath(String dir, String name) { + return dir.endsWith(File.separator) || dir.endsWith("/") + ? dir + name + : dir + File.separator + name; + } + + // ----------------------------------------------------------------------- + // Internal helpers + // ----------------------------------------------------------------------- + + /** + * Converts an absolute local file path to a remote key by stripping the matching configured + * data-root prefix. + * + *
+     *   dataRoot = /hugegraph-store/storage
+     *   filePath = /hugegraph-store/storage/hgstore-metadata/000008.sst
+     *   result   = hgstore-metadata/000008.sst
+     * 
+ * + * If {@code filePath} does not start with any configured data roots the leading slash is simply + * stripped so the key is still valid (though possibly not ideally formatted). + */ + String toRelativeKey(String filePath) { + String matchingRoot = findMatchingDataRoot(filePath); + // Use Path.startsWith for boundary-safe matching so /data/store1 does not match + // /data/store10/... the way a raw String.startsWith would. + Path fileNormalized = Paths.get(filePath).toAbsolutePath().normalize(); + Path rootPath = Paths.get(matchingRoot); + String relative; + if (fileNormalized.startsWith(rootPath)) { + Path relPath = rootPath.relativize(fileNormalized); + relative = relPath.toString(); + return withStoreScope(relative); + } + // Fallback: strip any leading slash so the key does not start with '/'. + String normalized = fileNormalized.toString(); + relative = normalized.startsWith("/") ? normalized.substring(1) : normalized; + return withStoreScope(relative); + } + + /** + * STALE-DATA GUARD: if a tombstone exists (written by {@link #onDBDeleteBegin} outside the data + * prefix), the previous generation at this prefix was destroyed but its cloud objects may not + * have been fully removed. Purge them and the tombstone before the opening DB can hydrate from, + * or upload into, the prefix; otherwise the stale objects would resurrect deleted data, and the + * leftover tombstone would make a later open purge the new DB's live objects. The open is + * blocked if either step fails. + * + *

If the tombstone check itself fails, the generation state is unknown. With + * {@code requireConsistentLocal} (startup hydration) the open is blocked unless local metadata + * is self-consistent; without it (no hydration, so nothing remote is read) the open proceeds. + * + * @return {@code true} if a tombstone was found and the prefix purged (do not hydrate) + */ + private boolean purgeTombstonedPrefix(CloudStorageProvider provider, String dbName, Path root, + String prefix, boolean requireConsistentLocal) { + String tombstoneKey = prefix + DB_TOMBSTONE_SUFFIX; + boolean tombstoned; + try { + tombstoned = provider.fileExists(tombstoneKey); + } catch (IOException e) { + // Tombstone check itself failed — cannot safely determine generation, and cloud cannot + // validate the DB. Proceed from local state ONLY if it is self-consistent (valid + // CURRENT→MANIFEST); orphan SSTs without a valid lineage must not silently boot. + if (requireConsistentLocal && hasInconsistentLocalMetadata(root)) { + throw new IllegalStateException( + String.format("Cloud pre-hydration: tombstone check failed for db=%s and " + + "local metadata is not self-consistent (no valid " + + "CURRENT→MANIFEST lineage) — cannot safely open DB without " + + "knowing generation state: %s", dbName, e.getMessage()), e); + } + log.warn("Cloud open: tombstone check failed for db={}, proceeding with local state: {}", + dbName, e.getMessage()); + return false; + } + if (!tombstoned) { + return false; + } + log.warn("Cloud open: tombstone found for db={} — previous generation was deleted. " + + "Purging stale remote objects; hydration is skipped.", dbName); + Object lock = pendingDeleteLocks.computeIfAbsent(prefix, k -> new Object()); + synchronized (lock) { + if (!purgeRemotePrefix(provider, dbName, prefix)) { + // Leave the tombstone in place so the next open attempts the purge again and does + // not accidentally hydrate stale objects. + throw new IllegalStateException( + String.format("Cloud open: prefix purge failed for db=%s — tombstone " + + "preserved to guard against stale-data re-hydration. " + + "DB open is blocked.", dbName)); + } + // The tombstone lives outside the slash-terminated purged prefix, so purgeRemotePrefix + // never touches it — it must be deleted explicitly. If that fails the open stays + // blocked: the next open re-purges the (already clean) prefix and retries the delete. + if (!deleteTombstone(provider, dbName, prefix)) { + throw new IllegalStateException( + String.format("Cloud open: tombstone deletion failed for db=%s after a " + + "successful prefix purge — opening now would let a later " + + "open purge this DB's new objects. DB open is blocked " + + "until tombstone '%s' is removed.", dbName, tombstoneKey)); + } + } + log.info("Cloud open: tombstone cleaned up: db={}", dbName); + return true; + } + + /** + * Walks {@code dbPath} and uploads every {@code *.sst} file that is not already + * present in cloud storage. This handles restarts where SST files from a previous + * run were never uploaded (e.g. cloud storage was enabled after the last shutdown). + */ + private void uploadExistingSstFiles(CloudStorageProvider provider, String dbName, + String dbPath) { + Path root = Paths.get(dbPath); + if (!root.toFile().isDirectory()) { + return; + } + try (Stream paths = Files.walk(root)) { + paths.filter(p -> p.toString().endsWith(".sst")) + .forEach(p -> { + String localPath = p.toString(); + String remoteKey = toRelativeKey(localPath); + try { + if (!provider.fileExists(remoteKey)) { + provider.uploadFile(localPath, remoteKey); + log.info("Cloud initial-upload: {} -> {}", localPath, remoteKey); + } + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud initial-upload failed for db=%s path=%s", + dbName, localPath), e); + } + }); + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud initial-upload scan failed for db=%s dbPath=%s", + dbName, dbPath), e); + } + } + + private void preHydrateDbFiles(CloudStorageProvider provider, String dbName, String dbPath) { + Path root = Paths.get(dbPath); + try { + Files.createDirectories(root); + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud pre-hydration mkdir failed for db=%s path=%s", + dbName, dbPath), e); + } + + CloudStorageMetrics.registerDatabaseMetrics(dbName); + + // Snapshot whether a local CURRENT existed BEFORE hydration. Only then can the download loop + // (which skips already-present files) leave a stale pointer in place; if CURRENT was absent, + // the loop fetches the authoritative remote copy and no reconciliation is needed. Capturing + // it here also avoids a redundant compare-download of a CURRENT we just hydrated. + boolean localCurrentPreexisted = Files.exists(root.resolve("CURRENT")); + + String prefix = dbPrefix(dbPath); + + if (purgeTombstonedPrefix(provider, dbName, root, prefix, true)) { + return; + } + + List remoteFiles = listRemoteKeys(provider, prefix, root); + if (remoteFiles.isEmpty()) { + log.debug("Cloud pre-hydration skipped: no remote files for db={} prefix={}", + dbName, prefix); + return; + } + + // Stale hydration temp files were already swept at the start of onDBOpening. + + int downloaded = 0; + for (String remoteKey : remoteFiles) { + Path localPath; + try { + localPath = resolveLocalPath(remoteKey, dbPath); + } catch (IllegalArgumentException e) { + // Key doesn't belong to the current store scope (e.g. left over from a previous + // node identity after an IP change). Skip it rather than aborting hydration. + log.warn("Cloud pre-hydration: skipping remote key outside current scope " + + "(db={}, key={}): {}", dbName, remoteKey, e.getMessage()); + continue; + } + if (Files.exists(localPath)) { + continue; + } + try { + downloadAtomically(provider, remoteKey, localPath); + downloaded++; + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud pre-hydration failed for db=%s key=%s", + dbName, remoteKey), e); + } + } + if (downloaded > 0) { + log.info("Pre-hydration completed: db={}, downloadedFiles={}", dbName, downloaded); + } + + int confirmed = 0; + for (String remoteKey : remoteFiles) { + if (remoteKey.endsWith(".sst")) { + Path localSst; + try { + localSst = resolveLocalPath(remoteKey, dbPath); + } catch (IllegalArgumentException e) { + continue; // scope mismatch already warned above + } + syncTracker.markConfirmed(dbName, localSst.toString()); + confirmed++; + } + } + if (confirmed > 0) { + log.info("Seeded sync-tracker bitmap with {} confirmed files from remote listing: " + + "db={}", confirmed, dbName); + } + + // Converge the CURRENT pointer to the newest generation. The download loop above skips any + // file already present locally, which is correct for immutable metadata/SSTs but would + // leave a STALE local CURRENT in place when cloud holds a newer generation — a silent + // rollback. Only relevant when CURRENT pre-existed (otherwise the loop already fetched the + // authoritative remote copy). Reconcile explicitly (generation-compared) before validating. + if (localCurrentPreexisted) { + reconcileCurrentPointer(provider, dbName, root); + } + + verifyMetadataConsistency(dbName, root); + } + + /** + * Reconciles the local {@code CURRENT} pointer with the remote one when both exist. + * + *

{@link #preHydrateDbFiles}'s download loop skips any file already present locally. That is + * correct for RocksDB's immutable, number-addressed metadata ({@code MANIFEST-}, + * {@code OPTIONS-}) and SSTs — a same-named local copy is byte-identical to the remote one. + * It is WRONG for {@code CURRENT}, whose name is fixed while its content advances every + * generation. A stale local {@code CURRENT} (e.g. left by a partial disk rollback) while cloud + * holds a newer generation would make the node silently open the older generation. + * + *

We compare generations (parsed from the {@code MANIFEST-} each CURRENT references) and + * adopt the remote pointer only when it is strictly newer. When the local generation is newer + * (recent writes not yet mirrored) the local pointer wins, so hydration never rolls the DB back + * to an older cloud generation. The newer generation's MANIFEST/OPTIONS/SSTs carry new numbers + * and were therefore already fetched by the main download loop. + */ + private void reconcileCurrentPointer(CloudStorageProvider provider, String dbName, + Path root) { + Path localCurrent = root.resolve("CURRENT"); + if (!Files.exists(localCurrent)) { + // No local pointer to reconcile — the loop already fetched the remote CURRENT if any. + return; + } + String currentRemoteKey = toRelativeKey(localCurrent.toString()); + Path remoteCurrent = hydrationTempFor(localCurrent); + try { + if (!provider.fileExists(currentRemoteKey)) { + return; // no remote CURRENT to compare against + } + provider.downloadFile(currentRemoteKey, remoteCurrent.toString()); + String remoteManifest = Files.readString(remoteCurrent).trim(); + String localManifest = Files.readString(localCurrent).trim(); + long remoteGen = parseManifestGeneration(remoteManifest); + long localGen = parseManifestGeneration(localManifest); + if (remoteGen < 0L || localGen < 0L) { + // An unparseable pointer — do not guess. Keep local; verifyMetadataConsistency and + // RocksDB's own open still guard against a genuinely broken CURRENT. + log.warn("Cloud pre-hydration: unparseable CURRENT during reconciliation for db={} " + + "(local='{}', remote='{}') — keeping local pointer", + dbName, localManifest, remoteManifest); + return; + } + if (remoteGen > localGen) { + moveReplacing(remoteCurrent, localCurrent); + log.warn("Cloud pre-hydration: local CURRENT for db={} pointed at {} (gen {}) but " + + "cloud holds {} (gen {}); adopted the newer remote generation to avoid a " + + "silent rollback.", dbName, localManifest, localGen, remoteManifest, + remoteGen); + } else if (remoteGen < localGen) { + log.debug("Cloud pre-hydration: keeping newer local CURRENT (gen {} > remote gen " + + "{}) for db={}", localGen, remoteGen, dbName); + } + } catch (IOException e) { + log.warn("Cloud pre-hydration: CURRENT reconciliation failed for db={}: {} — keeping " + + "local pointer", dbName, e.getMessage()); + } finally { + deleteIfExistsQuietly(remoteCurrent, "CURRENT reconciliation temp cleanup"); + } + } + + /** + * Downloads {@code remoteKey} to a unique sibling temp file, then atomically moves it to + * {@code dest}. The per-thread/per-attempt suffix prevents two concurrent restorers from + * writing the same temp file (which would interleave into a corrupt SST), and REPLACE_EXISTING + * makes a late second mover a harmless idempotent overwrite. A crash mid-download never leaves + * RocksDB reading a truncated file at the expected path; the temp file is removed on failure. + */ + private static void downloadAtomically(CloudStorageProvider provider, String remoteKey, + Path dest) throws IOException { + Files.createDirectories(dest.getParent()); + Path tmp = hydrationTempFor(dest); + try { + provider.downloadFile(remoteKey, tmp.toString()); + moveReplacing(tmp, dest); + } finally { + deleteIfExistsQuietly(tmp, "hydration temp cleanup"); + } + } + + /** Unique sibling temp path for {@code dest}, swept by {@link #preHydrateDbFiles} on startup. */ + private static Path hydrationTempFor(Path dest) { + return dest.resolveSibling(dest.getFileName() + HYDRATION_TEMP_MARKER + + Thread.currentThread().getId() + "-" + System.nanoTime()); + } + + /** Atomically replaces {@code dest} with {@code src}, falling back to a non-atomic replace. */ + private static void moveReplacing(Path src, Path dest) throws IOException { + try { + Files.move(src, dest, StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + // Cross-filesystem / FS without atomic-move support: still a replace. + Files.move(src, dest, StandardCopyOption.REPLACE_EXISTING); + } + } + + /** + * Parses the RocksDB generation encoded in a {@code MANIFEST-} file name, or {@code -1} if + * {@code manifestName} is null / not a well-formed manifest name. + */ + private static long parseManifestGeneration(String manifestName) { + if (manifestName == null || !manifestName.startsWith("MANIFEST-")) { + return -1L; + } + try { + return Long.parseLong(manifestName.substring("MANIFEST-".length())); + } catch (NumberFormatException e) { + return -1L; + } + } + + /** + * Consistent-restore guard: if a {@code CURRENT} pointer was mirrored, the {@code MANIFEST-} + * it references must be present locally after pre-hydration. Opening RocksDB with a + * {@code CURRENT} pointing at a missing manifest would silently yield an empty / partial DB, so + * we fail loudly instead. (A manifest-referenced SST that is absent is caught loudly by + * RocksDB's own open, which refuses to start on a missing referenced file.) + */ + private void verifyMetadataConsistency(String dbName, Path dbRoot) { + Path current = dbRoot.resolve("CURRENT"); + if (!Files.exists(current)) { + // No mirrored metadata (or a genuinely empty prefix) — nothing to verify. + return; + } + String manifestName; + try { + manifestName = Files.readString(current).trim(); + } catch (IOException e) { + throw new IllegalStateException( + String.format("Cloud restore: unreadable CURRENT for db=%s", dbName), e); + } + if (manifestName.isEmpty() || !Files.exists(dbRoot.resolve(manifestName))) { + throw new IllegalStateException(String.format( + "Cloud restore inconsistent for db=%s: CURRENT references manifest '%s' which is " + + "absent locally and in cloud. Refusing to open a partial DB.", + dbName, manifestName)); + } + } + + private List listRemoteKeys(CloudStorageProvider provider, String prefix, + Path localRoot) { + try { + return provider.listFiles(prefix.endsWith("/") ? prefix : prefix + "/"); + } catch (IOException e) { + // Cloud listing failed during startup hydration; cloud cannot validate the DB. Fall back + // to local state ONLY if the local metadata is self-consistent (valid CURRENT→MANIFEST). + // A weaker "any SST present" check would let a stale/partial directory (orphan SSTs, no + // valid CURRENT lineage) boot silently — risking a rollback/partial open. Require the + // stronger predicate; otherwise fail startup loudly. + if (hasInconsistentLocalMetadata(localRoot)) { + throw new IllegalStateException( + String.format("Cloud pre-hydration list failed for prefix=%s and local " + + "metadata is not self-consistent (no valid CURRENT→MANIFEST " + + "lineage) — refusing to open a partial/unvalidated DB while " + + "cloud is unreachable: %s", prefix, e.getMessage()), e); + } + log.warn("Cloud pre-hydration list failed for prefix={}, proceeding with self-consistent " + + "local state (valid CURRENT→MANIFEST present): {}", prefix, e.getMessage()); + return java.util.Collections.emptyList(); + } + } + + /** + * Returns {@code true} when the local DB metadata is NOT self-consistent — i.e. there is no + * usable {@code CURRENT} → {@code MANIFEST-*} lineage: {@code CURRENT} is absent, empty, or + * unreadable, or it references a {@code MANIFEST-*} that is missing locally. Returns + * {@code false} only when {@code CURRENT} points at a {@code MANIFEST-*} that exists locally + * (a valid lineage). + * + *

This is a stronger predicate than a "local SST exists" check — orphan SSTs without a valid + * CURRENT→MANIFEST lineage (a partial/rolled-back directory) count as inconsistent. Callers use + * it to refuse opening a DB from local state when cloud is unreachable and cannot validate it: + * an inconsistent directory ({@code true}) must not silently boot as a partial/rolled-back DB. + */ + private static boolean hasInconsistentLocalMetadata(Path root) { + Path current = root.resolve("CURRENT"); + if (!Files.exists(current)) { + return true; + } + try { + String manifest = Files.readString(current).trim(); + return manifest.isEmpty() || !Files.exists(root.resolve(manifest)); + } catch (IOException e) { + return true; + } + } + + private String dbPrefix(String dbPath) { + String relative = toRelativeKey(dbPath); + return relative.endsWith("/") ? relative.substring(0, relative.length() - 1) : relative; + } + + /** + * Resolves the download destination + * under the data root that actually contains {@code dbPath}, rather than always using + * the primary root. Used by {@link #preHydrateDbFiles} so that files from a secondary + * data root are written back to the correct root on restore. + */ + private Path resolveLocalPath(String remoteKey, String dbPath) { + String relative = stripStoreScope(remoteKey); + String matchingRoot = findMatchingDataRoot( + Paths.get(dbPath).toAbsolutePath().normalize().toString()); + Path root = Paths.get(matchingRoot); + Path local = root.resolve(relative).normalize(); + if (!local.startsWith(root)) { + throw new IllegalArgumentException("Invalid remote key outside configured data roots: " + + remoteKey); + } + return local; + } + + private String withStoreScope(String relativeKey) { + if (storeScopePrefix.isEmpty()) { + return relativeKey; + } + if (relativeKey == null || relativeKey.isEmpty()) { + return storeScopePrefix; + } + return storeScopePrefix + "/" + relativeKey; + } + + private String stripStoreScope(String remoteKey) { + if (storeScopePrefix.isEmpty()) { + return remoteKey; + } + String scopedPrefix = storeScopePrefix + "/"; + if (remoteKey.startsWith(scopedPrefix)) { + return remoteKey.substring(scopedPrefix.length()); + } + if (remoteKey.equals(storeScopePrefix)) { + return ""; + } + throw new IllegalArgumentException(String.format( + "Remote key '%s' does not belong to store scope '%s'", remoteKey, storeScopePrefix)); + } + + private static String normaliseKeyPrefix(String prefix) { + if (prefix == null) { + return ""; + } + String trimmed = prefix.trim(); + if (trimmed.isEmpty()) { + return ""; + } + String normalized = trimmed.replace('\\', '/'); + while (normalized.startsWith("/")) { + normalized = normalized.substring(1); + } + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + return normalized; + } + + /** + * Triggers a non-blocking MemTable flush for the named DB via {@link RocksDBFactory}. + * The call returns immediately (fire-and-forget); when RocksDB completes the flush it + * fires {@link #onTableFileCreated} via its event-listener mechanism, which then uploads + * the resulting SST file to cloud storage. The overall flow is event-driven, not async + * in the sense of a background thread managed by this class. + */ + private void flushDb(String dbName) { + try { + RocksDBFactory.getInstance().flushSession(dbName, false); + log.debug("Cloud storage: triggered flush (non-blocking, event-driven) for db={}", dbName); + } catch (Exception e) { + log.warn("Cloud storage: flush failed for db={}: {}", dbName, e.getMessage()); + } + } + + boolean shouldAttemptReadMissHydration(String dbName, String table) { + if (readMissGuardWindowMs <= 0) { + return true; + } + long now = System.currentTimeMillis(); + String guardKey = dbName + "::" + table; + // Atomically decide admission AND update the timestamp under the per-key lock of the + // ConcurrentHashMap. A non-atomic get-then-put lets two concurrent read-misses both observe + // no/expired entry and both get admitted, defeating the throttle and triggering redundant + // live-set scans + cloud checks under a read storm. compute() runs the remap exactly once + // per key with mutual exclusion, so only the first caller in a window flips it to admitted. + boolean[] admitted = {false}; + readMissAttemptTs.compute(guardKey, (k, prev) -> { + if (prev == null || now - prev >= readMissGuardWindowMs) { + admitted[0] = true; + return now; // open a fresh window for this caller + } + return prev; // keep the existing window; do not extend it + }); + if (!admitted[0]) { + log.debug("Skip read-miss hydration due to guard window: db={}, table={}, nowMs={}", + dbName, table, now); + } + return admitted[0]; + } +} diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageMetrics.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageMetrics.java new file mode 100644 index 0000000000..caf1253c2d --- /dev/null +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageMetrics.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.IntSupplier; +import java.util.function.LongSupplier; + +import org.apache.hugegraph.store.node.util.HgAssert; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.FunctionCounter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; + +/** + * Cloud Storage Metrics registration and management. + * Exposes metrics for cloud storage operations including upload success/failure, + * sync latency, confirmed files (SSTs synced to cloud), and delete guard re-upload activity. + */ +@Slf4j +public final class CloudStorageMetrics { + + private static MeterRegistry registry; + private static CloudSyncTracker syncTracker; + + // Per-database counters for delete guard re-uploads + private static final ConcurrentHashMap deleteGuardReuploadPerDb = + new ConcurrentHashMap<>(); + + // Per-database registered gauges, tracked so they can be removed when a DB is deleted. + // Without this, per-DB (db_name-tagged) meters accumulate forever as DBs are created and + // destroyed (partition rebalancing, graph drops), causing unbounded meter cardinality. + private static final ConcurrentHashMap> perDbMeters = + new ConcurrentHashMap<>(); + + // Overall upload failure counter (will be incremented with tags) + private static Counter uploadFailuresCounter; + + // Sync latency timer + private static Timer syncLatencyTimer; + + // Live source for the RETRY_QUEUE_SIZE gauge. Bound to the retry queue via + // bindRetryQueueSizeSupplier once it is constructed; until then the gauge reports 0. + // volatile so the gauge (read on scrape threads) sees the binding published by startup wiring. + private static volatile IntSupplier retryQueueSizeSupplier = () -> 0; + + // Live source for the UPLOADS_GIVEN_UP counter. Bound to the retry queue via + // bindUploadsGivenUpSupplier once it is constructed; until then the counter reports 0. + private static volatile LongSupplier uploadsGivenUpSupplier = () -> 0L; + + // Live source for the DELETE_MARKER_HEALTHY gauge (1 = healthy, 0 = degraded). Bound to the + // listener via bindDeleteMarkerHealthySupplier once it is constructed; until then reports 1. + private static volatile IntSupplier deleteMarkerHealthySupplier = () -> 1; + + private CloudStorageMetrics() { + } + + /** + * Initialize cloud storage metrics. + * Must be called once at application startup with MeterRegistry and CloudSyncTracker instances. + */ + public static void init(final MeterRegistry meterRegistry, final CloudSyncTracker tracker) { + HgAssert.isArgumentNotNull(meterRegistry, "meterRegistry"); + HgAssert.isArgumentNotNull(tracker, "CloudSyncTracker"); + + if (registry != null) { + return; + } + + registry = meterRegistry; + syncTracker = tracker; + + // Register global metrics + uploadFailuresCounter = Counter.builder(CloudStorageMetricsConst.UPLOAD_FAILURES) + .description("Total upload failures (transient and permanent)") + .register(registry); + + syncLatencyTimer = Timer.builder(CloudStorageMetricsConst.SYNC_LATENCY_MS) + .description("Time in milliseconds from SST file creation to cloud confirmation") + .publishPercentiles(0.5, 0.95, 0.99) + .register(registry); + + // Register gauge for retry queue size (monitored at runtime). The supplier is read on every + // scrape and delegates to retryQueueSizeSupplier, which is bound to the live retry queue via + // bindRetryQueueSizeSupplier once it is constructed (defaults to 0 until then). + Gauge.builder(CloudStorageMetricsConst.RETRY_QUEUE_SIZE, + () -> retryQueueSizeSupplier.getAsInt()) + .description("Number of files waiting in the upload retry queue") + .register(registry); + + // Uploads the retry queue gave up on (retries exhausted, non-retryable, or shutdown). + FunctionCounter.builder(CloudStorageMetricsConst.UPLOADS_GIVEN_UP, uploadsGivenUpSupplier, + s -> uploadsGivenUpSupplier.getAsLong()) + .description("Number of SST uploads the retry queue gave up on") + .register(registry); + + // Pending-delete marker persistence health: 1 = healthy, 0 = degraded (a marker could not be + // durably fsynced, so a DB delete may be unguarded against re-hydration after a crash). Bound + // to the live listener via bindDeleteMarkerHealthySupplier; defaults to 1 until then. + Gauge.builder(CloudStorageMetricsConst.DELETE_MARKER_HEALTHY, + () -> deleteMarkerHealthySupplier.getAsInt()) + .description("Pending-delete marker persistence health (1=healthy, 0=degraded)") + .register(registry); + + log.info("Cloud storage metrics initialized"); + } + + /** + * Register a per-database metrics gauges. + * Should be called when a database opens for the first time. + */ + public static void registerDatabaseMetrics(final String dbName) { + if (registry == null) { + return; + } + + try { + // Micrometer registration is idempotent per registry: registering the same name+tags + // returns the existing meter. We (re-)capture the returned meters into perDbMeters so + // unregisterDatabaseMetrics can remove precisely these when the DB is deleted. Using + // put() (overwrite) keeps exactly one tracking entry per DB regardless of how often + // this is called, and correctly re-tracks if the underlying registry was swapped. + List meters = new CopyOnWriteArrayList<>(); + + // Register confirmed files gauge for this database (files successfully synced to cloud) + Gauge confirmed = Gauge.builder(CloudStorageMetricsConst.CONFIRMED_FILES, + () -> getConfirmedFileCount(dbName)) + .description("Number of SST files confirmed present in cloud storage") + .tag(CloudStorageMetricsConst.TAG_DB_NAME, dbName) + .register(registry); + meters.add(confirmed); + + // Register delete guard re-upload counter for this database + deleteGuardReuploadPerDb.putIfAbsent(dbName, new AtomicLong(0)); + + Gauge reupload = Gauge.builder(CloudStorageMetricsConst.DELETE_GUARD_REUPLOAD_COUNT, + () -> getDeleteGuardReuploadCount(dbName)) + .description("Number of times delete guard re-uploaded unconfirmed files during " + + "compaction") + .tag(CloudStorageMetricsConst.TAG_DB_NAME, dbName) + .register(registry); + meters.add(reupload); + + perDbMeters.put(dbName, meters); + log.debug("Registered cloud storage metrics for database: {}", dbName); + } catch (IllegalArgumentException e) { + // Gauge already registered for this database + log.debug("Cloud storage metrics already registered for database: {}", dbName); + } + } + + /** + * Removes all per-database metrics registered by {@link #registerDatabaseMetrics} for a DB that + * has been deleted, so meter cardinality does not grow without bound as DBs are created and + * destroyed. Safe to call for a DB that was never registered. + */ + public static void unregisterDatabaseMetrics(final String dbName) { + deleteGuardReuploadPerDb.remove(dbName); + List meters = perDbMeters.remove(dbName); + if (meters == null || registry == null) { + return; + } + for (Meter m : meters) { + try { + registry.remove(m); + } catch (RuntimeException e) { + log.debug("Failed to remove cloud storage meter for db={}: {}", + dbName, e.getMessage()); + } + } + log.debug("Unregistered cloud storage metrics for database: {}", dbName); + } + + /** + * Record a sync latency measurement (time to confirm upload to cloud). + */ + @SuppressWarnings("unused") // dbName reserved for future per-db latency tracking + public static void recordSyncLatency(final String dbName, final long latencyMs) { + if (syncLatencyTimer == null) { + return; + } + syncLatencyTimer.record(latencyMs, java.util.concurrent.TimeUnit.MILLISECONDS); + } + + /** + * Record an upload failure. + */ + @SuppressWarnings("unused") // Parameters reserved for future per-db/cf failure tracking + public static void recordUploadFailure(final String dbName, final String cfName, + final String errorType) { + if (uploadFailuresCounter == null) { + return; + } + uploadFailuresCounter.increment(); + } + + /** + * Get the current count of confirmed SST files for a database (from bitmap). + */ + public static long getConfirmedFileCount(final String dbName) { + if (syncTracker == null) { + return 0; + } + return syncTracker.confirmedCount(dbName); + } + + /** + * Increment the delete guard re-upload count for a database. + */ + public static void incrementDeleteGuardReupload(final String dbName) { + deleteGuardReuploadPerDb.computeIfAbsent(dbName, k -> new AtomicLong(0)).incrementAndGet(); + } + + /** + * Get the delete guard re-upload count for a database. + */ + public static long getDeleteGuardReuploadCount(final String dbName) { + AtomicLong count = deleteGuardReuploadPerDb.get(dbName); + return count != null ? count.get() : 0; + } + + /** + * Binds the RETRY_QUEUE_SIZE gauge to a live source (e.g. {@code retryQueue::getInFlightCount}). + * The supplier is polled on every metrics scrape, so the gauge always reflects the current + * backlog. Call once during startup after the retry queue is constructed. + */ + public static void bindRetryQueueSizeSupplier(final IntSupplier supplier) { + retryQueueSizeSupplier = (supplier != null) ? supplier : () -> 0; + } + + /** + * Binds the UPLOADS_GIVEN_UP counter to a live source (e.g. + * {@code retryQueue::getGivenUpCount}). Polled on every scrape. Call once during startup after + * the retry queue is constructed. + */ + public static void bindUploadsGivenUpSupplier(final LongSupplier supplier) { + uploadsGivenUpSupplier = (supplier != null) ? supplier : () -> 0L; + } + + /** + * Sets a fixed retry-queue-size value for the gauge. Prefer {@link #bindRetryQueueSizeSupplier} + * to track a live queue; this push-style setter is retained for tests and manual overrides. + */ + public static void setRetryQueueSize(final int size) { + retryQueueSizeSupplier = () -> size; + } + + /** + * Binds the DELETE_MARKER_HEALTHY gauge to a live source (e.g. + * {@code () -> listener.isDeleteMarkerHealthy() ? 1 : 0}). Polled on every scrape. Call once + * during startup after the listener is constructed. + */ + public static void bindDeleteMarkerHealthySupplier(final IntSupplier supplier) { + deleteMarkerHealthySupplier = (supplier != null) ? supplier : () -> 1; + } +} + + + + + diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageMetricsConst.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageMetricsConst.java new file mode 100644 index 0000000000..0e05055706 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudStorageMetricsConst.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +/** + * Constants for cloud storage metrics. + * Defines metric names and tags used for monitoring cloud storage operations. + */ +public final class CloudStorageMetricsConst { + + // Prefix used for all cloud storage metrics + public static final String PREFIX = "cloud_storage"; + + // Metric names + public static final String CONFIRMED_FILES = PREFIX + "_confirmed_files_total"; + public static final String UPLOAD_FAILURES = PREFIX + "_upload_failures_total"; + public static final String RETRY_QUEUE_SIZE = PREFIX + "_retry_queue_size"; + // Uploads the retry queue gave up on; each stays unconfirmed until metadata sync, the delete + // guard, or startup SST backfill re-uploads it. + public static final String UPLOADS_GIVEN_UP = PREFIX + "_uploads_given_up_total"; + public static final String SYNC_LATENCY_MS = PREFIX + "_sync_latency_ms"; + public static final String DELETE_GUARD_REUPLOAD_COUNT = PREFIX + "_delete_guard_reupload_count"; + // 1 = pending-delete marker persistence healthy; 0 = degraded (a marker could not be durably + // written, so a DB delete whose remote cleanup fails may not be guarded against re-hydration + // after a crash). A hard error state: delete progression is held while degraded. + public static final String DELETE_MARKER_HEALTHY = PREFIX + "_delete_marker_healthy"; + + // Tag names + public static final String TAG_DB_NAME = "db_name"; + + private CloudStorageMetricsConst() { + } +} + + diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudSyncTracker.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudSyncTracker.java new file mode 100644 index 0000000000..c17034cfd7 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudSyncTracker.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.LiveSstFile; + +/** + * Tracks which RocksDB SST files are confirmed present in cloud storage. + * + *

SST file names are monotonic per-DB file numbers ({@code 000123.sst}), so the natural key is + * the integer file number rather than the string path. Sync state is kept as one confirmed-file- + * number set per DB — each entry means "SST file number N for this DB is confirmed present in + * cloud". File numbers are unique across column families within a DB, so a single set per DB is + * sufficient (which fits RocksDB delete events not carrying a CF name). + * + *

This tracker is the linchpin of the delete-guard invariant: a superseded cloud object is + * deleted only once every live SST file of that DB is confirmed here. + * + *

Epoch-based stale-callback protection: each DB has a monotone epoch counter that is + * incremented by {@link #clearDb}. Callers that want their confirmation to survive a concurrent + * recreation must first call {@link #currentEpoch} before the upload, then pass the captured + * epoch to {@link #markConfirmedIfEpoch}. A late callback carrying an old epoch is silently + * dropped, preventing file-number reuse after DB recreation from producing stale confirmations. + */ +public final class CloudSyncTracker { + + /** Holds the confirmed-file-number set and the epoch it was created under. */ + private static final class DbState { + final long epoch; + final Set confirmed; + + DbState(long epoch) { + this.epoch = epoch; + this.confirmed = ConcurrentHashMap.newKeySet(); + } + } + + private final Map stateByDb = new ConcurrentHashMap<>(); + /** Monotone epoch counter per DB. Starts at 1; incremented by clearDb(). */ + private final Map epochByDb = new ConcurrentHashMap<>(); + + // ----------------------------------------------------------------------- + // Epoch API + // ----------------------------------------------------------------------- + + /** + * Returns the current epoch for the named DB. Call this before starting an upload; pass the + * returned value to {@link #markConfirmedIfEpoch} on success to guard against stale callbacks + * after a concurrent {@link #clearDb}. + */ + public long currentEpoch(String dbName) { + return epochByDb.computeIfAbsent(dbName, k -> new AtomicLong(1L)).get(); + } + + // ----------------------------------------------------------------------- + // Static utility + // ----------------------------------------------------------------------- + + /** + * Parses the SST file number from a file path such as {@code /data/db/000123.sst}. + * + * @return the file number, or {@code -1} if the path is not a parseable {@code *.sst} file + */ + public static long parseSstFileNumber(String filePath) { + if (filePath == null || !filePath.endsWith(".sst")) { + return -1L; + } + int slash = Math.max(filePath.lastIndexOf('/'), filePath.lastIndexOf(File.separatorChar)); + String name = filePath.substring(slash + 1, filePath.length() - ".sst".length()); + if (name.isEmpty()) { + return -1L; + } + // RocksDB SST base names are pure digits (e.g. 000123). Reject anything else. + try { + return Long.parseLong(name); + } catch (NumberFormatException e) { + return -1L; + } + } + + // ----------------------------------------------------------------------- + // Confirm / check API + // ----------------------------------------------------------------------- + + /** + * Marks the SST file as confirmed present in cloud. No-op for non-SST paths. + * + *

Unlike {@link #markConfirmedIfEpoch}, this method does not check the epoch. Use it only + * when the caller holds an external guarantee that the DB has not been recreated (e.g. during + * startup hydration seeding where the DB was just opened and no concurrent clear can occur). + */ + public void markConfirmed(String dbName, String filePath) { + long number = parseSstFileNumber(filePath); + if (number < 0) { + return; + } + long epoch = currentEpoch(dbName); + markBit(dbName, number, epoch); + } + + /** + * Marks the SST file as confirmed only if the DB epoch has not advanced since {@code epoch} + * was captured (i.e. no intervening {@link #clearDb} occurred). Silently no-ops if the epoch + * is stale. Use this from async upload callbacks to prevent stale confirmations after DB + * recreation with reused file numbers. + * + * @param dbName logical DB name + * @param filePath absolute path of the confirmed SST file + * @param epoch epoch captured by {@link #currentEpoch} before the upload started + * @return {@code true} if the bit was set; {@code false} if the epoch was stale (callback dropped) + */ + public boolean markConfirmedIfEpoch(String dbName, String filePath, long epoch) { + long number = parseSstFileNumber(filePath); + if (number < 0) { + return false; + } + return markBit(dbName, number, epoch); + } + + /** Core: mark {@code number} confirmed if the current epoch matches {@code requiredEpoch}. */ + private boolean markBit(String dbName, long number, long requiredEpoch) { + // Use compute() to hold the CHM bin lock across both the epoch check and the write, + // preventing clearDb() from removing the entry between the two steps. + boolean[] set = {false}; + stateByDb.compute(dbName, (k, state) -> { + if (state == null) { + // DB was never registered or was cleared; create a fresh state under the given epoch. + long currentEp = epochByDb.computeIfAbsent(dbName, ign -> new AtomicLong(1L)).get(); + if (currentEp != requiredEpoch) { + return null; // epoch mismatch — do not create a new state + } + state = new DbState(currentEp); + } else if (state.epoch != requiredEpoch) { + return state; // stale callback — leave state unchanged + } + state.confirmed.add(number); + set[0] = true; + return state; + }); + return set[0]; + } + + /** Returns whether the SST file is confirmed present in cloud. */ + public boolean isConfirmed(String dbName, String filePath) { + long number = parseSstFileNumber(filePath); + if (number < 0) { + return false; + } + boolean[] result = {false}; + stateByDb.computeIfPresent(dbName, (k, state) -> { + result[0] = state.confirmed.contains(number); + return state; + }); + return result[0]; + } + + /** Clears the confirmed bit for a file (called after the cloud object is deleted). */ + public void clearConfirmed(String dbName, String filePath) { + long number = parseSstFileNumber(filePath); + if (number < 0) { + return; + } + stateByDb.computeIfPresent(dbName, (k, state) -> { + state.confirmed.remove(number); + return state; + }); + } + + /** + * Returns {@code true} if every file in {@code liveFiles} is confirmed present + * in cloud. + * + *

This is the fast-path check used by the delete guard: it acquires the per-DB lock + * once for the entire set and short-circuits on the first miss, so a DB whose + * live set is fully confirmed costs one lock acquisition regardless of set size — compared + * to N acquisitions with N individual {@link #isConfirmed} calls. + * + * @param liveFiles the current live SST file set obtained from RocksDB's manifest + * @return {@code true} iff every SST path in {@code liveFiles} has its confirmed bit set + */ + public boolean allConfirmed(String dbName, List liveFiles) { + if (liveFiles.isEmpty()) { + return true; + } + boolean[] allPresent = {false}; + stateByDb.computeIfPresent(dbName, (k, state) -> { + for (LiveSstFile live : liveFiles) { + long num = parseSstFileNumber(live.getAbsolutePath()); + if (num >= 0 && !state.confirmed.contains(num)) { + return state; // allPresent stays false — short-circuit + } + } + allPresent[0] = true; + return state; + }); + return allPresent[0]; + } + + /** + * Removes all confirmed-sync state for the named database and advances the epoch. + * + *

After this call any in-flight upload callbacks that call {@link #markConfirmedIfEpoch} + * with the old epoch will be silently dropped, preventing stale confirmations from surviving + * into the next DB generation — even if RocksDB reuses the same file numbers. + */ + public void clearDb(String dbName) { + // Advance the epoch first so any concurrent markConfirmedIfEpoch calls with the old epoch + // fail the epoch check. Then remove the state so the next markConfirmed starts fresh. + epochByDb.computeIfAbsent(dbName, k -> new AtomicLong(1L)).incrementAndGet(); + stateByDb.remove(dbName); + } + + /** Number of confirmed SST files for a DB (testing / monitoring). */ + public long confirmedCount(String dbName) { + DbState state = stateByDb.get(dbName); + return state == null ? 0L : state.confirmed.size(); + } +} diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudUploadRetryQueue.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudUploadRetryQueue.java new file mode 100644 index 0000000000..a7103061f0 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/cloud/CloudUploadRetryQueue.java @@ -0,0 +1,571 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hugegraph.store.cloud.CloudStorageProvider; +import org.apache.hugegraph.store.cloud.CloudStorageNonRetryableException; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +/** + * Asynchronous whole-file retry queue for failed cloud SST uploads. + * + *

Retry strategy

+ *

The number of whole-file retries is configured by the caller via + * {@code cloud.storage.upload-retry-max-attempts}, which defaults to {@code 3} (whole-file retries + * enabled). {@code maxAttempts == 0} is an opt-in mode — not the shipped default — for deployments + * whose provider already performs sufficient internal retries (e.g. the S3 provider's underlying + * AWS SDK client, which retries transient per-request/per-part failures on its own). + *

    + *
  • When {@code maxAttempts == 0}, the queue gives up on a failure immediately, with no + * whole-file retry.
  • + *
  • When {@code maxAttempts > 0} (the default), the first retry is scheduled after + * {@code initialDelayMs} milliseconds.
  • + *
  • Subsequent retries use exponential backoff: {@code delay = initialDelayMs * 2^(attempt-1)}, + * capped at {@code maxDelayMs}.
  • + *
  • After {@code maxAttempts} consecutive failures the queue gives up on the upload.
  • + *
  • If the local SST file no longer exists at retry time (compacted away by RocksDB) + * the retry is silently dropped — there is nothing to upload.
  • + *
+ * + *

Giving up

+ * A given-up upload is logged at ERROR, counted in {@link #getGivenUpCount()}, and its staging + * hard-link is released; this queue does not retry it again. The file is never marked confirmed + * in {@link CloudSyncTracker}, so the listener re-uploads it if it is still live: + *
    + *
  • the delete guard re-uploads unconfirmed live SSTs before removing any superseded cloud + * object, and holds the delete (keeping the old object) until the live set is durable;
  • + *
  • metadata sync uploads every SST the captured manifest references before publishing + * CURRENT/MANIFEST, so cloud never points at a missing SST;
  • + *
  • startup SST backfill uploads every local SST missing from cloud when its DB is opened.
  • + *
+ * An SST compacted away before then needs no upload; its data lives in the compaction output. + * + *

Thread safety

+ * All public methods are thread-safe. + * + *

Lifecycle

+ * Call {@link #close()} (e.g. from a Spring {@code @PreDestroy} method) to shut down the + * background executor gracefully. + */ +@Slf4j +public class CloudUploadRetryQueue implements Closeable { + + // ----------------------------------------------------------------------- + // Configuration + // ----------------------------------------------------------------------- + + private final int maxAttempts; + private final long initialDelayMs; + private final long maxDelayMs; + private final CloudStorageProvider provider; + + /** + * Invoked with {@code (dbName, sourceSstPath, uploadEpoch)} whenever an upload succeeds via + * retry, so the caller can mark the file confirmed-present in cloud. + * The {@code uploadEpoch} is the {@link CloudSyncTracker} epoch captured at submission time; + * the callback should use {@link CloudSyncTracker#markConfirmedIfEpoch} to drop stale + * callbacks. + */ + @FunctionalInterface + public interface UploadConfirmedCallback { + void onConfirmed(String dbName, String sourceSstPath, long uploadEpoch); + } + + private final UploadConfirmedCallback onUploadConfirmed; + + /** + * Invoked with {@code dbName} after an upload becomes durable via retry, so the + * caller can (coalesced) publish CURRENT/MANIFEST for that DB. Without this, a retry that + * succeeds during a quiet period would leave the SST in cloud but the mirrored CURRENT stale + * until some unrelated sync fires (or never, on an idle DB) — widening the recovery point. + */ + @FunctionalInterface + public interface MetadataSyncTrigger { + void onUploadDurable(String dbName); + } + + /** Optional; set via {@link #setMetadataSyncTrigger} after the listener is constructed. */ + @Setter + private volatile MetadataSyncTrigger metadataSyncTrigger; + + // ----------------------------------------------------------------------- + // State + // ----------------------------------------------------------------------- + + /** Number of uploads given up on since startup (monitoring / tests). */ + private final AtomicLong givenUpCount = new AtomicLong(); + + /** + * Retries that are scheduled but have not started running yet. A scheduled task is added here + * before {@code scheduler.schedule(...)} and removed the moment it starts executing. On + * {@link #close()}, a forced {@code shutdownNow()} silently discards the executor's queued + * (delayed) tasks — but those wrappers are opaque, so we cannot recover their payload from the + * returned runnables. Tracking the payload here lets close() give up on any retry that never + * ran and release its staging hard-link. + */ + private final Set pendingRetries = ConcurrentHashMap.newKeySet(); + + /** + * Retries that have STARTED executing but not yet finished. A task moves from + * {@link #pendingRetries} to this set when it begins running, and is removed when it completes + * (success, give-up, or reschedule). On {@link #close()}, a retry still here after the shutdown + * timeout is one hung in {@code provider.uploadFile(...)}; it is given up on like an unrun + * retry. + */ + private final Set inFlightRetries = ConcurrentHashMap.newKeySet(); + + /** + * Keys ({@code db + NUL + remoteKey}) of retries that are scheduled or executing, mapped to the + * local path the live retry uploads from. Used to dedupe submissions: a publish that re-checks + * an SST which is already waiting for retry must not stack another retry chain for the same + * object (each would burn its own attempts and staging hard-link). Removed on success, drop + * and give-up. + */ + private final java.util.concurrent.ConcurrentMap pendingKeys = + new ConcurrentHashMap<>(); + + /** Background thread pool used to schedule retries. */ + private final ScheduledExecutorService scheduler; + + /** Counter of in-flight retry tasks (for testing / monitoring). */ + private final AtomicInteger inFlightCount = new AtomicInteger(0); + + // ----------------------------------------------------------------------- + // Constructor + // ----------------------------------------------------------------------- + + /** + * Fully-configured constructor. + * + * @param provider initialized cloud storage provider used for retries + * @param maxAttempts maximum whole-file upload retries after a first failure. + * {@code 0} means no retries – the queue gives up immediately + * (provider is expected to handle its own retries internally). + * Positive values enable exponential-backoff whole-file retries. + * @param initialDelayMs delay before the first retry, in milliseconds; clamped to ≥ 100. + * Ignored when {@code maxAttempts == 0}. + * @param maxDelayMs upper bound for exponential backoff delay; clamped to + * ≥ {@code initialDelayMs}. Ignored when {@code maxAttempts == 0}. + * @param onUploadConfirmed epoch-aware callback invoked on every successful retry upload; + * pass {@code null} for none. + */ + public CloudUploadRetryQueue(CloudStorageProvider provider, int maxAttempts, + long initialDelayMs, long maxDelayMs, + UploadConfirmedCallback onUploadConfirmed) { + this.provider = Objects.requireNonNull(provider, "provider"); + this.maxAttempts = Math.max(0, maxAttempts); + this.initialDelayMs = Math.max(100L, initialDelayMs); + this.maxDelayMs = Math.max(this.initialDelayMs, maxDelayMs); + this.onUploadConfirmed = onUploadConfirmed != null + ? onUploadConfirmed + : (db, path, epoch) -> { }; + + java.util.concurrent.ScheduledThreadPoolExecutor sched = + new java.util.concurrent.ScheduledThreadPoolExecutor(2, r -> { + Thread t = new Thread(r, "cloud-upload-retry"); + t.setDaemon(true); + return t; + }); + // On shutdown(), cancel not-yet-due delayed retries instead of holding the executor open + // until their (possibly minutes-long) delay elapses. close() then gives up on these + // pending retries via giveUpUnfinishedRetries(). + sched.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + this.scheduler = sched; + } + + // ----------------------------------------------------------------------- + // Public API + // ----------------------------------------------------------------------- + + /** + * Submits a failed upload for retry, or gives up on it immediately. + * + *

If {@code maxAttempts == 0} (opt-in mode, not the shipped default), the queue gives up + * immediately — the provider is expected to handle its own retries internally. + * + *

If {@code maxAttempts > 0} (the default is {@code 3}), the first retry is scheduled after + * {@code initialDelayMs} ms with exponential backoff. After all attempts are exhausted the + * queue gives up on the upload. + * + *

If the file no longer exists at retry time (compacted away by RocksDB) the task is + * silently dropped. + * + * @param dbName RocksDB instance name (partition id) + * @param cfName column-family name + * @param filePath absolute local path of the SST file (also used for confirmation) + * @param remoteKey remote object key (relative to bucket root) + * @param cause the exception from the first upload attempt + */ + public void submit(String dbName, String cfName, String filePath, + String remoteKey, Exception cause) { + submitPinned(dbName, cfName, filePath, filePath, remoteKey, 0L, cause); + } + + /** + * Epoch-aware variant of {@link #submit}. Callers that use an epoch-guarded confirmation + * callback ({@link CloudSyncTracker#markConfirmedIfEpoch}) MUST use this overload and pass the + * epoch captured before the upload was initiated. The plain {@link #submit} hardcodes epoch + * {@code 0}, which never matches a live {@link CloudSyncTracker} epoch (they start at 1), so a + * retried upload confirmed through it would be silently dropped. + * + * @param uploadEpoch the {@link CloudSyncTracker} epoch captured at upload-initiation time + */ + public void submit(String dbName, String cfName, String filePath, + String remoteKey, long uploadEpoch, Exception cause) { + submitPinned(dbName, cfName, filePath, filePath, remoteKey, uploadEpoch, cause); + } + + /** + * Epoch-aware variant. The {@code uploadEpoch} is passed through to the + * {@link UploadConfirmedCallback} so that stale callbacks after DB recreation are discarded. + */ + public void submitPinned(String dbName, String cfName, String pinnedPath, + String sourceSstPath, String remoteKey, + long uploadEpoch, Exception cause) { + if (cause instanceof CloudStorageNonRetryableException || maxAttempts == 0) { + giveUp(dbName, cfName, pinnedPath, sourceSstPath, remoteKey, uploadEpoch, 0, + cause.getMessage() != null ? cause.getMessage() : "non-retryable"); + return; + } + String key = retryKey(dbName, remoteKey); + String existing = pendingKeys.putIfAbsent(key, pinnedPath); + if (existing != null) { + // Already waiting for (or running) a retry; the live chain will upload this object. + // Release the duplicate's own staging pin, which nothing else will ever consume. + if (!pinnedPath.equals(sourceSstPath) && !pinnedPath.equals(existing)) { + try { + Files.deleteIfExists(Paths.get(pinnedPath)); + } catch (IOException e) { + log.debug("Failed to release duplicate staging file: {}", pinnedPath); + } + } + log.debug("Cloud upload retry already pending, skipping duplicate: db={}, key={}", + dbName, remoteKey); + return; + } + // This method is invoked from RocksDB flush/compaction callbacks (via + // CloudStorageEventListener). It must never throw back across that JNI/event boundary. + // When the scheduler is shutting down, scheduler.schedule(...) throws + // RejectedExecutionException — in that race we give up on the upload and return normally. + try { + scheduleRetry(dbName, cfName, pinnedPath, sourceSstPath, remoteKey, uploadEpoch, + 1); + } catch (RejectedExecutionException e) { + pendingKeys.remove(key); + log.warn("Cloud upload retry scheduling rejected (scheduler shutting down); " + + "giving up: db={}, cf={}, path={}", + dbName, cfName, pinnedPath); + giveUp(dbName, cfName, pinnedPath, sourceSstPath, remoteKey, uploadEpoch, 1, + "retry scheduling rejected during shutdown: " + + (cause.getMessage() != null ? cause.getMessage() : cause.toString())); + } + } + + /** Returns true when a retry for {@code (dbName, remoteKey)} is scheduled or executing. */ + public boolean isRetryPending(String dbName, String remoteKey) { + return pendingKeys.containsKey(retryKey(dbName, remoteKey)); + } + + private static String retryKey(String dbName, String remoteKey) { + return dbName + '\u0000' + remoteKey; + } + + /** Returns the number of uploads given up on since startup. */ + public long getGivenUpCount() { + return givenUpCount.get(); + } + + /** Fires the metadata-sync trigger (if wired), swallowing any error — the upload is durable. */ + private void triggerMetadataSync(String dbName) { + MetadataSyncTrigger trigger = this.metadataSyncTrigger; + if (trigger == null) { + return; + } + try { + trigger.onUploadDurable(dbName); + } catch (Exception e) { + log.warn("Post-upload metadata-sync trigger failed for db={} (upload is durable, " + + "CURRENT/MANIFEST will catch up on the next sync): {}", dbName, + e.getMessage()); + } + } + + /** + * Returns the number of retry tasks currently scheduled or executing. + */ + public int getInFlightCount() { + return inFlightCount.get(); + } + + // ----------------------------------------------------------------------- + // Lifecycle + // ----------------------------------------------------------------------- + + /** + * Shuts down the background retry executor, waiting up to 10 seconds for + * in-flight tasks to complete. + */ + @Override + public void close() { + close(10, TimeUnit.SECONDS); + } + + /** + * Shuts down with a caller-specified drain timeout. Package-private so tests can force a fast + * shutdown (they do not want to wait the production 10 s for a deliberately hung upload). + */ + void close(long timeout, TimeUnit unit) { + scheduler.shutdown(); + try { + if (!scheduler.awaitTermination(timeout, unit)) { + log.warn("CloudUploadRetryQueue: executor did not terminate within {}{}; " + + "forcing shutdown ({} tasks still in flight)", + timeout, unit, inFlightCount.get()); + scheduler.shutdownNow(); + } + } catch (InterruptedException e) { + scheduler.shutdownNow(); + Thread.currentThread().interrupt(); + } finally { + // A forced shutdownNow() discards scheduled-but-unrun retries AND interrupts (but + // cannot guarantee prompt exit of) retries hung in a slow provider.uploadFile(...). + // Give up on both and release their staging pins; startup backfill re-uploads the live + // SSTs. + // If the executor drained gracefully both sets are already empty (each task removed + // itself), so this is a no-op on the happy path. + giveUpUnfinishedRetries(); + } + } + + /** + * Gives up on any scheduled-but-unrun ({@link #pendingRetries}) and started-but-unfinished + * ({@link #inFlightRetries}) retries (called from {@link #close()}). + */ + private void giveUpUnfinishedRetries() { + int rescued = 0; + rescued += drainRetrySet(pendingRetries, + "retry not executed before shutdown (forced)"); + rescued += drainRetrySet(inFlightRetries, + "retry still in flight at shutdown (forced)"); + if (rescued > 0) { + log.warn("CloudUploadRetryQueue: gave up on {} unfinished retry(ies) during " + + "shutdown; startup backfill will re-upload any that are still live.", + rescued); + } + } + + /** Gives up on every retry in one retry-context set, claiming each entry exactly once. */ + private int drainRetrySet(Set set, String reason) { + int rescued = 0; + for (RetryContext ctx : set) { + if (!set.remove(ctx)) { + continue; // raced with the task itself removing it; it will handle its own outcome + } + pendingKeys.remove(retryKey(ctx.dbName, ctx.remoteKey)); + giveUp(ctx.dbName, ctx.cfName, ctx.filePath, ctx.sourceSstPath, ctx.remoteKey, + ctx.uploadEpoch, ctx.attempt, reason); + rescued++; + } + return rescued; + } + + // ----------------------------------------------------------------------- + // Internal – retry scheduling + // ----------------------------------------------------------------------- + + private void scheduleRetry(String dbName, String cfName, String filePath, + String sourceSstPath, String remoteKey, + long uploadEpoch, int attempt) { + long delayMs = computeDelay(attempt); + log.info("Cloud upload retry scheduled: db={}, cf={}, path={}, attempt={}/{}, delayMs={}", + dbName, cfName, filePath, attempt, maxAttempts, delayMs); + // Increment before schedule; decrement here on rejection (executeRetry's finally + // handles the decrement on the happy path, but RejectedExecutionException bypasses it). + inFlightCount.incrementAndGet(); + // Record the payload so a forced shutdown can give up on this retry if it never + // runs. Removed when the task starts executing (see wrapper below). + RetryContext ctx = new RetryContext(dbName, cfName, filePath, sourceSstPath, remoteKey, + uploadEpoch, attempt); + pendingRetries.add(ctx); + try { + scheduler.schedule(() -> { + // Move from "scheduled" to "in-flight" so close() can still rescue this retry's + // intent if the upload below hangs past the shutdown timeout. + pendingRetries.remove(ctx); + inFlightRetries.add(ctx); + try { + executeRetry(dbName, cfName, filePath, sourceSstPath, remoteKey, + uploadEpoch, attempt); + } finally { + inFlightRetries.remove(ctx); + } + }, delayMs, TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException e) { + pendingRetries.remove(ctx); + inFlightCount.decrementAndGet(); + throw e; + } + } + + /** + * Immutable payload of a scheduled-but-not-yet-started retry, tracked in + * {@link #pendingRetries}. + */ + private static final class RetryContext { + final String dbName; + final String cfName; + final String filePath; + final String sourceSstPath; + final String remoteKey; + final long uploadEpoch; + final int attempt; + + RetryContext(String dbName, String cfName, String filePath, String sourceSstPath, + String remoteKey, long uploadEpoch, int attempt) { + this.dbName = dbName; + this.cfName = cfName; + this.filePath = filePath; + this.sourceSstPath = sourceSstPath; + this.remoteKey = remoteKey; + this.uploadEpoch = uploadEpoch; + this.attempt = attempt; + } + } + + private void executeRetry(String dbName, String cfName, String filePath, + String sourceSstPath, String remoteKey, + long uploadEpoch, int attempt) { + try { + // If the local SST (or staging pin) is gone, drop silently – nothing to upload. + if (!Files.exists(Paths.get(filePath))) { + log.info("Cloud upload retry: local file no longer exists (compacted?), " + + "dropping: db={}, cf={}, path={}", dbName, cfName, filePath); + pendingKeys.remove(retryKey(dbName, remoteKey)); + return; + } + + provider.uploadFile(filePath, remoteKey); + // Durable: a later submission for this key is legitimate again. + pendingKeys.remove(retryKey(dbName, remoteKey)); + // Confirm using the original *.sst path so CloudSyncTracker can parse the file number. + // Pass the epoch so markConfirmedIfEpoch can drop stale callbacks after DB recreation. + try { + onUploadConfirmed.onConfirmed(dbName, sourceSstPath, uploadEpoch); + } catch (Exception callbackError) { + // Upload already succeeded; callback failures are control-plane/observer failures, + // not data-plane upload failures. Keep the upload path successful and avoid + // misclassifying into retry/give-up. + log.warn("Cloud upload retry callback failed after successful upload: db={}, " + + "path={}, reason={}", + dbName, sourceSstPath, callbackError.getMessage()); + } + // The SST set is now more durable than the mirrored CURRENT/MANIFEST — trigger a + // (coalesced) metadata publish so the recovery point advances even if this retry + // succeeded during an otherwise-quiet period on an idle DB. + triggerMetadataSync(dbName); + // If we uploaded from a staging hard-link, clean it up now. + if (!filePath.equals(sourceSstPath)) { + try { + Files.deleteIfExists(Paths.get(filePath)); + } catch (IOException e) { + log.debug("Failed to clean up staging file after retry: {}", filePath); + } + } + log.info("Cloud upload retry succeeded: db={}, cf={}, path={}, attempt={}", + dbName, cfName, filePath, attempt); + + } catch (Exception e) { + log.warn("Cloud upload retry failed: db={}, cf={}, path={}, attempt={}/{}, reason={}", + dbName, cfName, filePath, attempt, maxAttempts, e.getMessage()); + if (e instanceof CloudStorageNonRetryableException) { + pendingKeys.remove(retryKey(dbName, remoteKey)); + giveUp(dbName, cfName, filePath, sourceSstPath, remoteKey, uploadEpoch, + attempt, e.getMessage() != null ? e.getMessage() : "non-retryable"); + return; + } + if (attempt >= maxAttempts) { + pendingKeys.remove(retryKey(dbName, remoteKey)); + giveUp(dbName, cfName, filePath, sourceSstPath, remoteKey, uploadEpoch, + attempt, e.getMessage()); + } else { + try { + scheduleRetry(dbName, cfName, filePath, sourceSstPath, remoteKey, + uploadEpoch, attempt + 1); + } catch (RejectedExecutionException ree) { + // Scheduler is shutting down. scheduleRetry rethrows the rejection; if we let + // it escape here it would propagate out of this scheduler task and be silently + // swallowed by the executor, leaving the file neither retried nor recorded. + // Give up instead; startup backfill re-uploads it if still live. + log.warn("Cloud upload retry reschedule rejected (scheduler shutting down); " + + "giving up: db={}, cf={}, path={}", dbName, cfName, filePath); + pendingKeys.remove(retryKey(dbName, remoteKey)); + giveUp(dbName, cfName, filePath, sourceSstPath, remoteKey, uploadEpoch, + attempt, "retry reschedule rejected during shutdown"); + } + } + } finally { + inFlightCount.decrementAndGet(); + } + } + + private void giveUp(String dbName, String cfName, String filePath, + String sourceSstPath, String remoteKey, + long uploadEpoch, int attemptCount, String lastError) { + givenUpCount.incrementAndGet(); + + // Nothing uploads from a given-up task again, so release its staging hard-link now; + // otherwise it would pin the (possibly compacted-away) SST's disk space indefinitely. + if (sourceSstPath != null && !filePath.equals(sourceSstPath)) { + try { + Files.deleteIfExists(Paths.get(filePath)); + } catch (IOException e) { + log.debug("Failed to release staging file for given-up upload: {}", filePath); + } + } + + log.error("Cloud upload given up after {} attempt(s) – file stays unconfirmed and will be " + + "re-uploaded by the next metadata sync, delete guard, or startup backfill: " + + "db={}, cf={}, path={}, remoteKey={}, epoch={}, lastError={}", + attemptCount, dbName, cfName, filePath, remoteKey, uploadEpoch, lastError); + } + + /** + * Exponential back-off: {@code initialDelayMs * 2^(attempt-1)}, capped at {@code maxDelayMs}. + */ + private long computeDelay(int attempt) { + // Guard against overflow for large attempt numbers. + int exp = Math.min(attempt - 1, 30); + long delay = initialDelayMs * (1L << exp); + return Math.min(delay, maxDelayMs); + } +} diff --git a/hugegraph-store/hg-store-node/src/main/resources/application.yml b/hugegraph-store/hg-store-node/src/main/resources/application.yml index 9b70aa2c6f..77b72b57b3 100644 --- a/hugegraph-store/hg-store-node/src/main/resources/application.yml +++ b/hugegraph-store/hg-store-node/src/main/resources/application.yml @@ -1,3 +1,4 @@ +#file: noinspection SpringBootApplicationYaml # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -50,5 +51,37 @@ logging: level: root: info +# Cloud storage (disabled by default; see hg-store-dist application.yml for full docs) +cloud: + storage: + enabled: false + provider: s3 + path-prefix: hugegraph + # Whole-file upload retries after a first failure (default 3). Set 0 to disable + # (failures are given up on immediately) when the provider has sufficient internal retry. + upload-retry-max-attempts: 3 + upload-retry-initial-delay-ms: 1000 + upload-retry-max-delay-ms: 60000 + # Interval (ms) at which DBs with new SST uploads get one coalesced metadata publish; failed + # publishes retry on the next interval. <= 0 publishes on every SST. Event-driven publishes + # (delete guard, compaction, DB open) are never coalesced. + metadata-sync-debounce-ms: 1000 + # Mirror a consistent CURRENT/MANIFEST/OPTIONS snapshot so cloud objects stay + # recoverable (a node that lost its local disk reopens from cloud, not an empty DB). + # Metadata sync is always enabled when cloud storage is enabled. + # Sync is triggered by storage events; SST-upload syncs are coalesced per the interval above. + # Capture forces a MemTable flush so the un-flushed tail is persisted into an SST and mirrored; + # the RocksDB WAL is disabled under Raft (the Raft log is the tail's durability source), so a + # flush is the only way to push recent writes into cloud. + # Stable per-node identity for the cloud key scope (store-). Blank => persist a scope + # in the data dir on first start (stable across IP/hostname drift, lost with the disk). Set a + # deployment-stable value for guaranteed recovery after an address change or local disk loss. + node-id: + s3: + bucket: + region: + endpoint: + access-key: + secret-key: # HTTP Arthas activation is disabled. Use bin/arthas.sh locally without restarting Store. diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/business/StoredRowIngressTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/business/StoredRowIngressTest.java index a8eb6c8761..dd0065703d 100644 --- a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/business/StoredRowIngressTest.java +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/business/StoredRowIngressTest.java @@ -77,7 +77,7 @@ import org.apache.hugegraph.type.define.IndexType; import org.apache.hugegraph.type.define.WriteType; import org.apache.hugegraph.util.Bytes; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -96,7 +96,8 @@ public class StoredRowIngressTest { public void setUp() throws Exception { this.rows = new Properties(); try (InputStream input = getClass().getClassLoader().getResourceAsStream(FIXTURE)) { - Assert.assertNotNull("Frozen baseline BinarySerializer rows must be present", input); + Assertions.assertNotNull(input, + "Frozen baseline BinarySerializer rows must be present"); this.rows.load(input); } this.graph = Mockito.mock(HugeGraphSupplier.class); @@ -134,15 +135,15 @@ public void testFilterIteratorReadsFrozenServerVertexAndEdges() { ConditionQuery condition = new ConditionQuery(row.equals("vertex") ? HugeType.VERTEX : HugeType.EDGE); condition.query(Condition.eq(id(1), 7)); FilterIterator iterator = new FilterIterator<>(iterator(stored), condition, this.graph); - Assert.assertTrue(row, iterator.hasNext()); - Assert.assertSame(stored, iterator.next()); - Assert.assertFalse(iterator.hasNext()); + Assertions.assertTrue(iterator.hasNext(), row); + Assertions.assertSame(stored, iterator.next()); + Assertions.assertFalse(iterator.hasNext()); iterator.close(); ConditionQuery noMatch = new ConditionQuery(condition.resultType()); noMatch.query(Condition.eq(id(1), 8)); FilterIterator rejected = new FilterIterator<>(iterator(stored), noMatch, this.graph); - Assert.assertFalse(row, rejected.hasNext()); + Assertions.assertFalse(rejected.hasNext(), row); rejected.close(); } } @@ -156,9 +157,9 @@ public void testSchemaLookupFailureClosesAlreadyOpenedFilterIterator() { boolean previousClosing = engine.isClosing().getAndSet(false); try { FilterIterator.of(source, query.bytes(), "invalid-namespace"); - Assert.fail("Malformed metadata namespace must fail initialization"); + Assertions.fail("Malformed metadata namespace must fail initialization"); } catch (HgStoreException expected) { - Assert.assertTrue(expected.getMessage().contains("Graph must include")); + Assertions.assertTrue(expected.getMessage().contains("Graph must include")); Mockito.verify(source).close(); } finally { engine.isClosing().set(previousClosing); @@ -175,14 +176,14 @@ public void testSelectionCountsActualMatchesAndPreservesCollectionsAndTtl() { SelectIterator iterator = new SelectIterator(iterator(stored), List.of(2, 3, 99), this.graph, isVertex); RocksDBSession.BackendColumn selected = iterator.next(); - Assert.assertArrayEquals(stored.name, selected.name); + Assertions.assertArrayEquals(stored.name, selected.name); BaseElement element = QueryUtil.parseEntry(this.graph, BackendColumn.of(selected.name, selected.value), isVertex); - Assert.assertEquals(2, element.sizeOfProperties()); - Assert.assertEquals(List.of(7, -1, 7), element.getPropertyValue(id(2))); - Assert.assertEquals(Set.of(-1, 7), element.getPropertyValue(id(3))); - Assert.assertNull(element.getPropertyValue(id(1))); - Assert.assertEquals(expiry(), element.expiredTime()); + Assertions.assertEquals(2, element.sizeOfProperties()); + Assertions.assertEquals(List.of(7, -1, 7), element.getPropertyValue(id(2))); + Assertions.assertEquals(Set.of(-1, 7), element.getPropertyValue(id(3))); + Assertions.assertNull(element.getPropertyValue(id(1))); + Assertions.assertEquals(expiry(), element.expiredTime()); iterator.close(); } } @@ -195,8 +196,8 @@ public void testSelectionOfMissingPropertiesWritesZeroCountAndKeepsTtl() { RocksDBSession.BackendColumn selected = iterator.next(); BaseVertex vertex = this.serializer.parseSchemaVertex(this.graph, BackendColumn.of(selected.name, selected.value), null); - Assert.assertEquals(0, vertex.sizeOfProperties()); - Assert.assertEquals(expiry(), vertex.expiredTime()); + Assertions.assertEquals(0, vertex.sizeOfProperties()); + Assertions.assertEquals(expiry(), vertex.expiredTime()); iterator.close(); } @@ -204,26 +205,27 @@ public void testSelectionOfMissingPropertiesWritesZeroCountAndKeepsTtl() { public void testOlapMergeKeepsPersistentSchemaFormatAndTtl() { BackendColumn merged = QueryUtil.combineColumn(this.graph, column("vertex"), List.of(column("olap"))); BaseVertex vertex = this.serializer.parseSchemaVertex(this.graph, merged, null); - Assert.assertEquals(4, vertex.sizeOfProperties()); - Assert.assertEquals(Integer.valueOf(42), vertex.getPropertyValue(id(9))); - Assert.assertEquals(expiry(), vertex.expiredTime()); + Assertions.assertEquals(4, vertex.sizeOfProperties()); + Assertions.assertEquals(Integer.valueOf(42), vertex.getPropertyValue(id(9))); + Assertions.assertEquals(expiry(), vertex.expiredTime()); } @Test public void testDirectTtlReaderUsesFrozenRowSchemaAndIndexNameSuffix() { BackendColumn vertex = column("vertex"); - Assert.assertEquals(expiry(), this.directSerializer.parseSchemaVertex( + Assertions.assertEquals(expiry(), this.directSerializer.parseSchemaVertex( this.graph, vertex.name, vertex.value).expiredTime()); for (String row : List.of("edge.out", "edge.in")) { BackendColumn edge = column(row); - Assert.assertEquals(expiry(), this.directSerializer.parseSchemaEdge( + Assertions.assertEquals(expiry(), this.directSerializer.parseSchemaEdge( this.graph, edge.name, edge.value).expiredTime()); } for (String row : List.of("index.secondary", "index.int", "index.long")) { BackendColumn index = column(row); - Assert.assertEquals(expiry(), this.directSerializer.parseSchemaIndex( + Assertions.assertEquals(expiry(), this.directSerializer.parseSchemaIndex( this.graph, index.name, index.value).expiredTime()); - Assert.assertEquals(expiry(), this.serializer.parseSchemaIndex(this.graph, index).expiredTime()); + Assertions.assertEquals( + expiry(), this.serializer.parseSchemaIndex(this.graph, index).expiredTime()); } } @@ -237,9 +239,9 @@ public void testTtlPipelineAcceptsLiveFixtureAndRejectsExpiredFixture() throws E BackendColumn column = column(row); PipelineResult input = new PipelineResult(RocksDBSession.BackendColumn.of(column.name, column.value)); now.setLong(stage, Long.parseLong(this.rows.getProperty("now"))); - Assert.assertSame(row, input, stage.handle(input)); + Assertions.assertSame(input, stage.handle(input), row); now.setLong(stage, expiry() + 1); - Assert.assertNull(row, stage.handle(input)); + Assertions.assertNull(stage.handle(input), row); } } @@ -251,15 +253,15 @@ public void testClientDecodesPersistentRowsAndTaggedElementResponsesExplicitly() decode.setAccessible(true); BackendColumn stored = column("vertex"); BaseVertex persistent = (BaseVertex) decode.invoke(executor, "g+v", kv(stored), false, true); - Assert.assertNotNull(persistent); - Assert.assertEquals(List.of(7, -1, 7), persistent.getPropertyValue(id(2))); + Assertions.assertNotNull(persistent); + Assertions.assertEquals(List.of(7, -1, 7), persistent.getPropertyValue(id(2))); BackendColumn tagged = this.serializer.writeVertex(persistent); BaseVertex response = (BaseVertex) decode.invoke(executor, "g+v", kv(tagged), false, false); - Assert.assertNotNull(response); + Assertions.assertNotNull(response); Set expectedGroups = persistent.getPropertyValue(id(3)); Set actualGroups = response.getPropertyValue(id(3)); - Assert.assertEquals(expectedGroups, actualGroups); - Assert.assertEquals(expiry(), response.expiredTime()); + Assertions.assertEquals(expectedGroups, actualGroups); + Assertions.assertEquals(expiry(), response.expiredTime()); } @Test @@ -285,15 +287,19 @@ public void testUntimedAndSystemRowsDoNotRequireTrailingTtl() { Mockito.when(this.graph.indexLabel(id(number))).thenReturn(label); } BackendColumn vertex = column("vertex.untimed"); - Assert.assertEquals(0L, this.serializer.parseSchemaVertex(this.graph, vertex, null).expiredTime()); + Assertions.assertEquals( + 0L, this.serializer.parseSchemaVertex(this.graph, vertex, null).expiredTime()); BackendColumn untimedEdge = column("edge.untimed"); - Assert.assertEquals(0L, this.serializer.parseSchemaEdge(this.graph, untimedEdge, null, true).expiredTime()); + Assertions.assertEquals( + 0L, + this.serializer.parseSchemaEdge(this.graph, untimedEdge, null, true).expiredTime()); for (String row : List.of("index.secondary.untimed", "index.int.untimed", "index.long.untimed", "index.system")) { BackendColumn column = column(row); - Assert.assertEquals(row, 0L, this.serializer.parseSchemaIndex(this.graph, column).expiredTime()); - Assert.assertEquals(row, 0L, this.directSerializer.parseSchemaIndex( - this.graph, column.name, column.value).expiredTime()); + Assertions.assertEquals( + 0L, this.serializer.parseSchemaIndex(this.graph, column).expiredTime(), row); + Assertions.assertEquals(0L, this.directSerializer.parseSchemaIndex( + this.graph, column.name, column.value).expiredTime(), row); } } @@ -306,12 +312,12 @@ public void testRebuiltIndexWriterMatchesFrozenServerColumnsAndStoreReader() { Map fixtures = Map.of(6, "index.secondary", 7, "index.int", 8, "index.long"); for (Map.Entry fixture : fixtures.entrySet()) { List indexes = builder.buildIndex(vertex, this.graph.indexLabel(id(fixture.getKey()))); - Assert.assertEquals(1, indexes.size()); + Assertions.assertEquals(1, indexes.size()); assertPersistentIndex(indexes.get(0), fixture.getValue()); } vertex.addProperty(this.graph.propertyKey(id(17)), "long-field-".repeat(20)); List longString = builder.buildIndex(vertex, this.graph.indexLabel(id(15))); - Assert.assertEquals(1, longString.size()); + Assertions.assertEquals(1, longString.size()); assertPersistentIndex(longString.get(0), "index.hashed"); } @@ -337,17 +343,23 @@ public void testStringIndexBranchesPreserveLongValuesAndCanonicalEscapes() { label.indexType(type); label.indexFields(id(17)); vertex.addProperty(this.graph.propertyKey(id(17)), longValue); - Assert.assertEquals(type.name(), longValue, builder.buildIndex(vertex, label).get(0).fieldValues()); + Assertions.assertEquals(longValue, + builder.buildIndex(vertex, label).get(0).fieldValues(), + type.name()); vertex.addProperty(this.graph.propertyKey(id(17)), ""); - Assert.assertEquals(type.name(), "\u0002", builder.buildIndex(vertex, label).get(0).fieldValues()); + Assertions.assertEquals("\u0002", + builder.buildIndex(vertex, label).get(0).fieldValues(), + type.name()); vertex.addProperty(this.graph.propertyKey(id(17)), "prefix!tail"); - Assert.assertEquals(type.name(), "prefix`!tail", builder.buildIndex(vertex, label).get(0).fieldValues()); + Assertions.assertEquals("prefix`!tail", + builder.buildIndex(vertex, label).get(0).fieldValues(), + type.name()); vertex.addProperty(this.graph.propertyKey(id(17)), "\u0001illegal"); try { builder.buildIndex(vertex, label); - Assert.fail("Reserved leading index symbols must remain illegal: " + type); + Assertions.fail("Reserved leading index symbols must remain illegal: " + type); } catch (IllegalArgumentException expected) { - Assert.assertTrue(expected.getMessage().contains("Illegal leading char")); + Assertions.assertTrue(expected.getMessage().contains("Illegal leading char")); } } } @@ -364,8 +376,8 @@ public void testUniqueNullableFieldUsesCanonicalNullSentinel() { label.indexType(IndexType.UNIQUE); label.indexFields(id(17), id(18)); List indexes = new IndexBuilder(this.graph).buildIndex(vertex, label); - Assert.assertEquals(1, indexes.size()); - Assert.assertEquals("prefix!\u0001", indexes.get(0).fieldValues()); + Assertions.assertEquals(1, indexes.size()); + Assertions.assertEquals("prefix!\u0001", indexes.get(0).fieldValues()); } @Test @@ -373,17 +385,20 @@ public void testFrozenLegacyStoreTtlEnvelopeRemainsReadable() throws Exception { Properties legacy = new Properties(); try (InputStream input = getClass().getClassLoader().getResourceAsStream( "compatibility/2f827d6e8/store-index-rows.properties")) { - Assert.assertNotNull("Frozen baseline Store index writer output must be present", input); + Assertions.assertNotNull( + input, "Frozen baseline Store index writer output must be present"); legacy.load(input); } for (String row : List.of("secondary", "int", "long", "system")) { BackendColumn column = BackendColumn.of(Bytes.fromHex(legacy.getProperty(row + ".name")), Bytes.fromHex(legacy.getProperty(row + ".value"))); long expectedExpiry = Long.parseLong(legacy.getProperty(row + ".expiry")); - Assert.assertEquals(row, expectedExpiry, - this.serializer.parseSchemaIndex(this.graph, column).expiredTime()); - Assert.assertEquals(row, expectedExpiry, this.directSerializer.parseSchemaIndex( - this.graph, column.name, column.value).expiredTime()); + Assertions.assertEquals(expectedExpiry, + this.serializer.parseSchemaIndex(this.graph, column) + .expiredTime(), + row); + Assertions.assertEquals(expectedExpiry, this.directSerializer.parseSchemaIndex( + this.graph, column.name, column.value).expiredTime(), row); } } @@ -395,10 +410,10 @@ public void testOlapStageReadsFrozenLegacyRowsWithoutCrossPropertyLeakage() thro rows.put(ByteBuffer.wrap(legacy.name), legacy.value); for (boolean tagged : List.of(false, true)) { BaseVertex merged = runOlapStage(rows, tagged, id(9), id(18)); - Assert.assertEquals(42, (Object) merged.getProperty(id(9)).value()); - Assert.assertNull(merged.getProperty(id(18))); - Assert.assertArrayEquals(legacy.value, rows.get(ByteBuffer.wrap(legacy.name))); - Assert.assertEquals(1, rows.size()); + Assertions.assertEquals(42, (Object) merged.getProperty(id(9)).value()); + Assertions.assertNull(merged.getProperty(id(18))); + Assertions.assertArrayEquals(legacy.value, rows.get(ByteBuffer.wrap(legacy.name))); + Assertions.assertEquals(1, rows.size()); } } @@ -412,9 +427,9 @@ public void testOlapStageMergesMultiplePropertiesAndPrefersNamespacedValues() th rows.put(ByteBuffer.wrap(OlapKey.format(id(18), id(1))), schemaOlapValue(id(18), 123)); for (boolean tagged : List.of(false, true)) { BaseVertex merged = runOlapStage(rows, tagged, id(9), id(18)); - Assert.assertEquals(99, (Object) merged.getProperty(id(9)).value()); - Assert.assertEquals(123, (Object) merged.getProperty(id(18)).value()); - Assert.assertArrayEquals(legacy.value, rows.get(ByteBuffer.wrap(legacy.name))); + Assertions.assertEquals(99, (Object) merged.getProperty(id(9)).value()); + Assertions.assertEquals(123, (Object) merged.getProperty(id(18)).value()); + Assertions.assertArrayEquals(legacy.value, rows.get(ByteBuffer.wrap(legacy.name))); } } @@ -447,7 +462,7 @@ private BaseVertex runOlapStage(Map rows, boolean tagged, Id BaseVertex decoded = this.serializer.parseSchemaVertex(this.graph, vertex, null); PipelineResult input = tagged ? new PipelineResult(decoded) : new PipelineResult(RocksDBSession.BackendColumn.of(vertex.name, vertex.value)); - Assert.assertSame(input, stage.handle(input)); + Assertions.assertSame(input, stage.handle(input)); if (tagged) { return (BaseVertex) input.getElement(); } @@ -464,12 +479,12 @@ private void assertPersistentIndex(Index rebuilt, String fixture) { BackendColumn written = this.serializer.writeSchemaIndex(rebuilt); // DataManager's persisted batch normalizes an absent payload to zero bytes. byte[] value = written.value == null ? new byte[0] : written.value; - Assert.assertArrayEquals(fixture, expected.name, written.name); - Assert.assertArrayEquals(fixture, expected.value, value); + Assertions.assertArrayEquals(expected.name, written.name, fixture); + Assertions.assertArrayEquals(expected.value, value, fixture); Index decoded = this.serializer.parseSchemaIndex(this.graph, BackendColumn.of(written.name, value)); - Assert.assertEquals(rebuilt.elementId(), decoded.elementId()); - Assert.assertEquals(rebuilt.fieldValues(), decoded.fieldValues()); - Assert.assertEquals(expiry(), decoded.expiredTime()); + Assertions.assertEquals(rebuilt.elementId(), decoded.elementId()); + Assertions.assertEquals(rebuilt.fieldValues(), decoded.fieldValues()); + Assertions.assertEquals(expiry(), decoded.expiredTime()); } @Test @@ -484,8 +499,8 @@ public void testParentRequestedRebuildIndexesChildRowsAndExcludesUnrelatedLabels // Real sublabels persist their own endpoint links; they do not inherit them on read. label.sourceLabel(id(4)); label.targetLabel(id(4)); - Assert.assertEquals(id(4), label.sourceLabel()); - Assert.assertEquals(id(4), label.targetLabel()); + Assertions.assertEquals(id(4), label.sourceLabel()); + Assertions.assertEquals(id(4), label.targetLabel()); Mockito.when(this.graph.edgeLabel(label.id())).thenReturn(label); Mockito.when(this.graph.edgeLabelOrNone(label.id())).thenReturn(label); } @@ -530,14 +545,14 @@ public void testParentRequestedRebuildIndexesChildRowsAndExcludesUnrelatedLabels HugeGraphSupplier previous = (HugeGraphSupplier) mock.get(null); BusinessHandlerImpl.setMockGraphSupplier(this.graph); try { - Assert.assertTrue(manager.doBuildIndex(request, partition).isOk()); - Assert.assertEquals(1, written.size()); + Assertions.assertTrue(manager.doBuildIndex(request, partition).isOk()); + Assertions.assertEquals(1, written.size()); BatchPutRequest.KV row = written.get(0); Index decoded = this.serializer.parseSchemaIndex(this.graph, BackendColumn.of(row.getKey(), row.getValue())); - Assert.assertEquals(parent.id(), decoded.indexLabel().baseValue()); - Assert.assertEquals(childEdge.id(), decoded.elementId()); - Assert.assertEquals("child record", decoded.fieldValues()); + Assertions.assertEquals(parent.id(), decoded.indexLabel().baseValue()); + Assertions.assertEquals(childEdge.id(), decoded.elementId()); + Assertions.assertEquals("child record", decoded.fieldValues()); Mockito.verify(source).close(); } finally { BusinessHandlerImpl.setMockGraphSupplier(previous); @@ -554,16 +569,16 @@ private RocksDBSession.BackendColumn rebuildInput(BaseEdge edge) { @Test public void testResponseFormatMatchesDeserializationPlanning() { QueryRequest.Builder request = QueryRequest.newBuilder().setScanType(ScanType.TABLE_SCAN); - Assert.assertFalse(QueryResultFormat.returnsTaggedElements(request.build())); + Assertions.assertFalse(QueryResultFormat.returnsTaggedElements(request.build())); request.setCheckTtl(true); - Assert.assertFalse(QueryResultFormat.returnsTaggedElements(request.build())); + Assertions.assertFalse(QueryResultFormat.returnsTaggedElements(request.build())); request.addProperty(ByteString.copyFromUtf8("property")); - Assert.assertTrue(QueryResultFormat.requiresElementDeserialization(request.build())); - Assert.assertTrue(QueryResultFormat.returnsTaggedElements(request.build())); + Assertions.assertTrue(QueryResultFormat.requiresElementDeserialization(request.build())); + Assertions.assertTrue(QueryResultFormat.returnsTaggedElements(request.build())); request.clearProperty().addOrderBy(ByteString.copyFromUtf8("order")); - Assert.assertTrue(QueryResultFormat.returnsTaggedElements(request.build())); + Assertions.assertTrue(QueryResultFormat.returnsTaggedElements(request.build())); request.clearOrderBy().setCondition(ByteString.copyFromUtf8("condition")); - Assert.assertTrue(QueryResultFormat.returnsTaggedElements(request.build())); + Assertions.assertTrue(QueryResultFormat.returnsTaggedElements(request.build())); } private void property(int value, Cardinality cardinality, WriteType writeType) { diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/AppConfigCloudStorageTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/AppConfigCloudStorageTest.java new file mode 100644 index 0000000000..2b134659ad --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/AppConfigCloudStorageTest.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.env.SystemEnvironmentPropertySource; +import org.springframework.mock.env.MockEnvironment; + +public class AppConfigCloudStorageTest { + + private MockEnvironment mockEnv; + + @BeforeEach + public void setUp() { + mockEnv = new MockEnvironment(); + } + + /** + * Common fields are bound from {@code cloud.storage.*} and provider-specific properties flow + * from the environment through the provider sub-namespace. + */ + @Test + public void testBindCloudStorageConfig() { + mockEnv.setProperty("cloud.storage.enabled", "true"); + mockEnv.setProperty("cloud.storage.provider", "s3"); + mockEnv.setProperty("cloud.storage.path-prefix", "test-prefix"); + mockEnv.setProperty("cloud.storage.startup-hydration-enabled", "false"); + mockEnv.setProperty("cloud.storage.read-miss-guard-window-ms", "5000"); + mockEnv.setProperty("cloud.storage.upload-retry-max-attempts", "5"); + mockEnv.setProperty("cloud.storage.upload-retry-initial-delay-ms", "500"); + mockEnv.setProperty("cloud.storage.upload-retry-max-delay-ms", "30000"); + mockEnv.setProperty("cloud.storage.metadata-sync-debounce-ms", "250"); + mockEnv.setProperty("cloud.storage.node-id", "node-7"); + + mockEnv.setProperty("cloud.storage.s3.bucket", "test-bucket"); + mockEnv.setProperty("cloud.storage.s3.region", "us-west-2"); + mockEnv.setProperty("cloud.storage.s3.endpoint", "https://s3.example.com"); + mockEnv.setProperty("cloud.storage.s3.access-key", "test-access-key"); + mockEnv.setProperty("cloud.storage.s3.secret-key", "test-secret-key"); + + CloudStorageConfig cfg = AppConfig.bindCloudStorageConfig(mockEnv); + + // Common fields + assertTrue(cfg.isEnabled()); + assertEquals("s3", cfg.getProvider()); + assertEquals("test-prefix", cfg.getPathPrefix()); + assertFalse(cfg.isStartupHydrationEnabled()); + assertEquals(5000L, cfg.getReadMissGuardWindowMs()); + assertEquals(5, cfg.getUploadRetryMaxAttempts()); + assertEquals(500L, cfg.getUploadRetryInitialDelayMs()); + assertEquals(30_000L, cfg.getUploadRetryMaxDelayMs()); + assertEquals(250L, cfg.getMetadataSyncDebounceMs()); + assertEquals("node-7", cfg.getNodeId()); + + // Provider-specific properties forwarded verbatim + assertEquals("test-bucket", cfg.getProviderProperties().get("bucket")); + assertEquals("us-west-2", cfg.getProviderProperties().get("region")); + assertEquals("https://s3.example.com", cfg.getProviderProperties().get("endpoint")); + assertEquals("test-access-key", cfg.getProviderProperties().get("access-key")); + assertEquals("test-secret-key", cfg.getProviderProperties().get("secret-key")); + } + + /** + * With no {@code cloud.storage.*} properties, the {@link CloudStorageConfig} field defaults + * apply and providerProperties is empty but not null. + */ + @Test + public void testBindCloudStorageConfigDefaults() { + CloudStorageConfig cfg = AppConfig.bindCloudStorageConfig(mockEnv); + + assertFalse(cfg.isEnabled()); + assertEquals("s3", cfg.getProvider()); + assertEquals("hugegraph", cfg.getPathPrefix()); + assertTrue(cfg.isStartupHydrationEnabled()); + assertEquals(3000L, cfg.getReadMissGuardWindowMs()); + assertEquals(3, cfg.getUploadRetryMaxAttempts()); + assertNotNull(cfg.getProviderProperties()); + assertTrue(cfg.getProviderProperties().isEmpty()); + } + + /** + * Unset fields keep their {@link CloudStorageConfig} defaults when only some are bound. + */ + @Test + public void testBindCloudStorageConfigPartialKeepsDefaults() { + mockEnv.setProperty("cloud.storage.enabled", "true"); + + CloudStorageConfig cfg = AppConfig.bindCloudStorageConfig(mockEnv); + + assertTrue(cfg.isEnabled()); + assertEquals("hugegraph", cfg.getPathPrefix()); + assertEquals(1_000L, cfg.getMetadataSyncDebounceMs()); + } + + /** + * Common fields resolve from environment-variable-style names via relaxed binding. + */ + @Test + public void testCommonFieldsResolveFromEnvVarStyleNames() { + Map envVarProps = new HashMap<>(); + envVarProps.put("CLOUD_STORAGE_ENABLED", "true"); + envVarProps.put("CLOUD_STORAGE_PATH_PREFIX", "env-prefix"); + envVarProps.put("CLOUD_STORAGE_METADATA_SYNC_DEBOUNCE_MS", "0"); + mockEnv.getPropertySources().addFirst(new SystemEnvironmentPropertySource( + StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, envVarProps)); + + CloudStorageConfig cfg = AppConfig.bindCloudStorageConfig(mockEnv); + + assertTrue(cfg.isEnabled()); + assertEquals("env-prefix", cfg.getPathPrefix()); + assertEquals(0L, cfg.getMetadataSyncDebounceMs()); + } + + /** + * Provider-specific sub-namespace is driven by the {@code provider} field, + * so a different provider name reads from a different env sub-namespace. + */ + @Test + public void testProviderNamespaceIsDynamic() { + mockEnv.setProperty("cloud.storage.enabled", "true"); + mockEnv.setProperty("cloud.storage.provider", "gcs"); + mockEnv.setProperty("cloud.storage.gcs.bucket", "gcs-bucket"); + mockEnv.setProperty("cloud.storage.gcs.credentials-file-path", "/path/creds.json"); + // S3 keys should NOT appear + mockEnv.setProperty("cloud.storage.s3.bucket", "s3-bucket"); + + CloudStorageConfig cfg = AppConfig.bindCloudStorageConfig(mockEnv); + + assertEquals("gcs", cfg.getProvider()); + assertEquals("gcs-bucket", cfg.getProviderProperties().get("bucket")); + assertEquals("/path/creds.json", + cfg.getProviderProperties().get("credentials-file-path")); + assertEquals(2, cfg.getProviderProperties().size(), + "s3 keys must not bleed into gcs namespace"); + } + + /** + * Provider properties must also resolve from environment-variable-style property names + * (e.g. {@code CLOUD_STORAGE_S3_SECRET_KEY}), as those are how credentials typically + * arrive in a container deployment. Relaxed binding reconciles these with the kebab-case + * key ({@code secret-key}) that {@code S3CloudStorageConfig.KEY_*} expects. + */ + @Test + public void testProviderPropertiesResolveFromEnvVarStyleNames() { + Map envVarProps = new HashMap<>(); + envVarProps.put("CLOUD_STORAGE_ENABLED", "true"); + envVarProps.put("CLOUD_STORAGE_S3_SECRET_KEY", "env-var-secret"); + envVarProps.put("CLOUD_STORAGE_S3_ACCESS_KEY", "env-var-access"); + mockEnv.getPropertySources().addFirst(new SystemEnvironmentPropertySource( + StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, envVarProps)); + + CloudStorageConfig cfg = AppConfig.bindCloudStorageConfig(mockEnv); + + assertEquals("env-var-secret", cfg.getProviderProperties().get("secret-key")); + assertEquals("env-var-access", cfg.getProviderProperties().get("access-key")); + } + + /** + * The same provider key from application.yml ({@code secret-key}) and from an env var + * ({@code CLOUD_STORAGE_S3_SECRET_KEY}, bound as {@code secret.key}) normalise to one key. That + * must not fail startup, and the env var wins, as it does for every other Spring property. + */ + @Test + public void testProviderPropertyFromYamlAndEnvVarPrefersEnvVar() { + mockEnv.setProperty("cloud.storage.enabled", "true"); + mockEnv.setProperty("cloud.storage.s3.secret-key", "yaml-secret"); + mockEnv.setProperty("cloud.storage.s3.bucket", "yaml-bucket"); + Map envVarProps = new HashMap<>(); + envVarProps.put("CLOUD_STORAGE_S3_SECRET_KEY", "env-var-secret"); + mockEnv.getPropertySources().addFirst(new SystemEnvironmentPropertySource( + StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, envVarProps)); + + CloudStorageConfig cfg = AppConfig.bindCloudStorageConfig(mockEnv); + + assertEquals("env-var-secret", cfg.getProviderProperties().get("secret-key")); + assertEquals("yaml-bucket", cfg.getProviderProperties().get("bucket")); + } + + /** + * Provider properties are only bound (and their placeholders only resolved) when cloud storage + * is enabled: a disabled node must start even if the provider namespace references an + * unresolvable placeholder such as a missing credential env var. + */ + @Test + public void testDisabledDoesNotResolveProviderProperties() { + mockEnv.setProperty("cloud.storage.enabled", "false"); + mockEnv.setProperty("cloud.storage.provider", "s3"); + mockEnv.setProperty("cloud.storage.s3.bucket", "${UNSET_BUCKET_FOR_TEST}"); + mockEnv.setProperty("cloud.storage.s3.access-key", "${UNSET_ACCESS_KEY_FOR_TEST}"); + + CloudStorageConfig cfg = AppConfig.bindCloudStorageConfig(mockEnv); + + assertFalse(cfg.isEnabled()); + assertNotNull(cfg.getProviderProperties()); + assertTrue(cfg.getProviderProperties().isEmpty(), + "provider properties must not be bound while cloud storage is disabled"); + } + + /** When enabled, the same provider namespace is bound. */ + @Test + public void testEnabledBindsProviderProperties() { + mockEnv.setProperty("cloud.storage.enabled", "true"); + mockEnv.setProperty("cloud.storage.provider", "s3"); + mockEnv.setProperty("cloud.storage.s3.bucket", "enabled-bucket"); + + CloudStorageConfig cfg = AppConfig.bindCloudStorageConfig(mockEnv); + + assertEquals("enabled-bucket", cfg.getProviderProperties().get("bucket")); + } + + /** + * Test AppConfig getRaftPath. + */ + @Test + public void testGetRaftPath() { + AppConfig config = new AppConfig(); + // getRaftPath should not throw; actual value depends on initialization + config.getRaftPath(); + } + + // ----------------------------------------------------------------------- + // Stable cloud key scope resolution (recovery after IP drift / disk loss) + // ----------------------------------------------------------------------- + + @Test + public void resolveStoreScope_prefersConfiguredNodeId() throws Exception { + AppConfig appConfig = new AppConfig(); + Path dataRoot = Files.createTempDirectory("hgstore-scope"); + try { + String prefix = appConfig.resolveStableStoreScopePrefix("node-A", dataRoot.toString()); + assertEquals("store-node-A", prefix); + // Persisted so a later removal of the config still resolves to the same scope. + assertEquals("store-node-A", + Files.readString(dataRoot.resolve(AppConfig.CLOUD_SCOPE_MARKER_FILE)) + .trim()); + } finally { + deleteRecursively(dataRoot); + } + } + + @Test + public void resolveStoreScope_sanitizesConfiguredNodeId() throws Exception { + AppConfig appConfig = new AppConfig(); + Path dataRoot = Files.createTempDirectory("hgstore-scope"); + try { + // Unsafe key characters must be replaced so the scope is a valid single path segment. + assertEquals("store-pod_1_2", + appConfig.resolveStableStoreScopePrefix("pod/1:2", dataRoot.toString())); + } finally { + deleteRecursively(dataRoot); + } + } + + @Test + public void resolveStoreScope_reusesPersistedMarkerAcrossIdentityDrift() throws Exception { + AppConfig appConfig = new AppConfig(); + Path dataRoot = Files.createTempDirectory("hgstore-scope"); + try { + // A marker written under a PREVIOUS network identity must be honored on a later start + // (blank node-id), even though the current runtime address may differ — this is the + // property that lets a node find its prior remote data after an IP/hostname change. + Files.writeString(dataRoot.resolve(AppConfig.CLOUD_SCOPE_MARKER_FILE), + "store-original_1_1"); + String prefix = appConfig.resolveStableStoreScopePrefix("", dataRoot.toString()); + assertEquals("store-original_1_1", prefix); + } finally { + deleteRecursively(dataRoot); + } + } + + @Test + public void resolveStoreScope_seedsAndPersistsOnFirstStart() throws Exception { + AppConfig appConfig = new AppConfig(); + Path dataRoot = Files.createTempDirectory("hgstore-scope"); + try { + // No node-id and no marker: seed from identity, persist, and reuse on the next start. + String first = appConfig.resolveStableStoreScopePrefix(null, dataRoot.toString()); + assertTrue(first.startsWith("store-"), "Seeded scope must use the store- prefix"); + assertTrue(Files.exists(dataRoot.resolve(AppConfig.CLOUD_SCOPE_MARKER_FILE)), + "First start must persist the marker"); + String second = appConfig.resolveStableStoreScopePrefix(null, dataRoot.toString()); + assertEquals(first, second, "Second start must reuse the persisted scope"); + } finally { + deleteRecursively(dataRoot); + } + } + + @Test + public void resolveStoreScope_configuredNodeIdOverridesPersistedMarker() throws Exception { + AppConfig appConfig = new AppConfig(); + Path dataRoot = Files.createTempDirectory("hgstore-scope"); + try { + Files.writeString(dataRoot.resolve(AppConfig.CLOUD_SCOPE_MARKER_FILE), "store-seeded"); + String prefix = appConfig.resolveStableStoreScopePrefix("explicit", dataRoot.toString()); + assertEquals("store-explicit", prefix, + "An explicit node-id must win over a persisted marker"); + assertEquals("store-explicit", + Files.readString(dataRoot.resolve(AppConfig.CLOUD_SCOPE_MARKER_FILE)) + .trim(), + "The marker must be updated to the configured scope"); + } finally { + deleteRecursively(dataRoot); + } + } + + private static void deleteRecursively(Path dir) throws Exception { + if (!Files.exists(dir)) { + return; + } + try (java.util.stream.Stream walk = Files.walk(dir)) { + walk.sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (Exception ignore) { + // best-effort test cleanup + } + }); + } + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/HgStoreNodeServiceTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/HgStoreNodeServiceTest.java index 99399e0aa9..f2b0ed8c86 100644 --- a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/HgStoreNodeServiceTest.java +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/HgStoreNodeServiceTest.java @@ -17,7 +17,7 @@ package org.apache.hugegraph.store.node; -// import org.junit.Test; +// import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.concurrent.ArrayBlockingQueue; diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudRecoveryIntegrationTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudRecoveryIntegrationTest.java new file mode 100644 index 0000000000..2ef4b27293 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudRecoveryIntegrationTest.java @@ -0,0 +1,492 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; +import java.util.function.BooleanSupplier; +import java.util.stream.Collectors; + +import org.apache.commons.configuration2.MapConfiguration; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.SessionOperator; +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * End-to-end cloud recovery/mirroring against a real {@link RocksDBSession} plus an + * in-memory cloud provider. Consolidates the former per-scenario integration classes: + * + *

    + *
  • {@link #testReopenAfterLocalDiskLossRecoversDataFromCloud} — flushed-SST disk-loss recovery + * (was {@code CloudDiskLossRecoveryIntegrationTest}).
  • + *
  • {@link #testInjectedProviderFailureAcrossRealRocksDbCallbackKeepsSessionStableAndGivesUp} + * — real RocksDB JNI callback failure is given up on by the retry queue and keeps the + * session stable (was {@code RocksDBCallbackJniBoundaryIntegrationTest}).
  • + *
+ * + *

Mock-driven callback-path edge cases live in the unit suites, while this class keeps the + * real RocksDB/JNI boundary integration coverage. + */ +public class CloudRecoveryIntegrationTest { + + private static RocksDBFactory factory; + + private Path baseDir; + private String dbName; + private CloudStorageEventListener listener; + private CloudUploadRetryQueue retryQueue; + + @BeforeAll + public static void initFactory() { + OptionSpace.register("rocksdb", + "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); + RocksDBOptions.instance(); + Map cfg = new HashMap<>(); + // Large write buffer to avoid incidental natural flushes; the disk-loss and JNI tests flush + // explicitly via session.flush(true), so this is harmless there. + cfg.put("rocksdb.write_buffer_size", String.valueOf(256L * 1024 * 1024)); + cfg.put("rocksdb.bloom_filter_bits_per_key", "10"); + factory = RocksDBFactory.getInstance(); + factory.setHugeConfig(new HugeConfig(new MapConfiguration(cfg))); + } + + @BeforeEach + public void setUp() throws IOException { + this.baseDir = Files.createTempDirectory("hg-cloudrec-"); + } + + @AfterEach + public void tearDown() { + if (this.listener != null && factory != null) { + factory.removeRocksdbChangedListener(this.listener); + } + if (factory != null && this.dbName != null) { + try { + factory.releaseGraphDB(this.dbName); + } catch (Exception ignore) { + // best-effort + } + } + if (this.retryQueue != null) { + this.retryQueue.close(); + this.retryQueue = null; + } + if (this.baseDir != null) { + deleteRecursively(this.baseDir.toFile()); + } + this.listener = null; + this.dbName = null; + } + + // ========================================================================= + // Flushed-SST disk-loss recovery + // ========================================================================= + + @Test + public void testReopenAfterLocalDiskLossRecoversDataFromCloud() { + this.dbName = "recdb"; + InMemoryCloudStore cloud = new InMemoryCloudStore(); + + CloudSyncTracker tracker = new CloudSyncTracker(); + this.listener = CloudStorageTestFactory.newListener(cloud, + Collections.singletonList(this.baseDir.toString()), + true, 0L, null, tracker); + factory.addRocksdbChangedListener(this.listener); + + // --- Phase 1: create DB, write data, mirror a consistent snapshot to cloud --- + RocksDBSession session = factory.createGraphDB(this.baseDir.toString(), this.dbName, 0); + String resolvedPath = session.getDbPath(); + session.checkTable("t"); + + Map expected = new HashMap<>(); + SessionOperator op = session.sessionOp(); + op.prepare(); + for (int i = 0; i < 500; i++) { + String k = "key-" + i; + String v = "value-" + i; + op.put("t", k.getBytes(StandardCharsets.UTF_8), v.getBytes(StandardCharsets.UTF_8)); + expected.put(k, v); + } + op.commit(); + session.flush(true); // create SST files + + // Mirror a consistent set (referenced SSTs + OPTIONS + MANIFEST + CURRENT) to cloud. + assertTrue(this.listener.syncMetadataSnapshotInline(cloud, this.dbName), + "metadata mirror must succeed"); + assertTrue(cloud.fileExists(this.dbName + "/CURRENT"), + "cloud must contain the CURRENT pointer after mirroring"); + + // --- Phase 2: simulate local disk loss (no cloud purge) --- + session.close(); // release our clone's ref + factory.releaseGraphDB(this.dbName); // close the DB handle, remove from map (no purge) + deleteRecursively(new File(resolvedPath)); + assertFalse(new File(resolvedPath, "CURRENT").exists(), + "local DB directory must be gone (simulated disk loss)"); + + // --- Phase 3: reopen at the same path — must hydrate from cloud BEFORE open --- + RocksDBSession recovered = factory.createGraphDB(this.baseDir.toString(), this.dbName, 0); + assertNotNull(recovered); + assertEquals(resolvedPath, + recovered.getDbPath(), + "recovery must resolve to the same DB path so cloud keys match"); + + // --- Phase 4: every key must be readable from the recovered DB --- + recovered.checkTable("t"); + SessionOperator rop = recovered.sessionOp(); + int recoveredCount = 0; + for (Map.Entry e : expected.entrySet()) { + byte[] got = rop.get("t", e.getKey().getBytes(StandardCharsets.UTF_8)); + assertArrayEquals(e.getValue().getBytes(StandardCharsets.UTF_8), + got, + "recovered value mismatch for " + e.getKey()); + recoveredCount++; + } + assertEquals(expected.size(), recoveredCount, "all keys must be recovered from cloud"); + } + + /** + * A partition that lives in a version-suffixed directory ({@code _}) mirrors its + * objects under that prefix. After a disk loss the default path holds nothing locally or in + * cloud, so the open must move to the newest restorable remote sibling and recover from it. + */ + @Test + public void testReopenAfterDiskLossRestoresVersionSuffixedDirectory() throws IOException { + this.dbName = "vrec"; + InMemoryCloudStore cloud = new InMemoryCloudStore(); + this.listener = CloudStorageTestFactory.newListener(cloud, + Collections.singletonList(this.baseDir.toString()), + true, 0L, null, new CloudSyncTracker()); + factory.addRocksdbChangedListener(this.listener); + + // An existing versioned directory is what RocksDBSession picks over the default path. + Files.createDirectories(this.baseDir.resolve(this.dbName + "_7")); + RocksDBSession session = factory.createGraphDB(this.baseDir.toString(), this.dbName, 0); + String versionedPath = session.getDbPath(); + assertTrue(versionedPath.endsWith(this.dbName + "_7"), versionedPath); + session.checkTable("t"); + SessionOperator op = session.sessionOp(); + op.prepare(); + for (int i = 0; i < 100; i++) { + op.put("t", ("key-" + i).getBytes(StandardCharsets.UTF_8), + ("value-" + i).getBytes(StandardCharsets.UTF_8)); + } + op.commit(); + session.flush(true); + assertTrue(this.listener.syncMetadataSnapshotInline(cloud, this.dbName)); + assertTrue(cloud.fileExists(this.dbName + "_7/CURRENT")); + + // Disk loss: the versioned directory disappears with everything else. + session.close(); + factory.releaseGraphDB(this.dbName); + deleteRecursively(new File(versionedPath)); + + RocksDBSession recovered = factory.createGraphDB(this.baseDir.toString(), this.dbName, 0); + assertEquals(versionedPath, recovered.getDbPath(), + "the open must move to the versioned directory the cloud copy lives under"); + recovered.checkTable("t"); + SessionOperator rop = recovered.sessionOp(); + for (int i = 0; i < 100; i++) { + assertArrayEquals(("value-" + i).getBytes(StandardCharsets.UTF_8), + rop.get("t", ("key-" + i).getBytes(StandardCharsets.UTF_8)), + "recovered value mismatch for key-" + i); + } + recovered.close(); + } + + // ========================================================================= + // Real RocksDB JNI callback: failure is given up on, session stays stable + // ========================================================================= + + @Test + public void testInjectedProviderFailureAcrossRealRocksDbCallbackKeepsSessionStableAndGivesUp() + throws Exception { + this.dbName = "jni-boundary-db"; + Path dbPath = this.baseDir.resolve(this.dbName); + + // maxAttempts=0 → an async upload failure is given up on immediately, giving a + // deterministic postcondition instead of a timing-dependent retry cycle. + // The provider starts healthy and is only switched to failing after the DB is open, so + // that ONLY the real onTableFileCreated callback (fired by the flush below) exercises the + // failure path. + AtomicInteger uploadAttempts = new AtomicInteger(0); + ToggleableFailingUploadProvider provider = + new ToggleableFailingUploadProvider(uploadAttempts); + this.retryQueue = CloudStorageTestFactory.newRetryQueue(provider, 0, 50L, 50L); + // Data root = baseDir so the SST files RocksDB writes under dbPath (a child of + // baseDir) can be hard-link staged for async upload. + this.listener = CloudStorageTestFactory.newListener(provider, + List.of(this.baseDir.toString()), false, 0L, this.retryQueue); + + // The listener must be registered BEFORE the DB is opened: RocksDBSession#openRocksDB + // only arms the native RocksdbChangedListener in dbOptions when + // RocksDBFactory#hasRemoteStateListeners() is true AT OPEN TIME, i.e. when a listener + // whose managesRemoteState() is true is registered (a one-time decision, to avoid ever + // exercising the native callback — and its classloading — when no listener feature is + // enabled). The provider is still healthy at this point, so onDBOpening and + // onDBCreated succeed; only the flush below exercises the injected failure. + factory.addRocksdbChangedListener(this.listener); + + RocksDBSession session = factory.createGraphDB(dbPath.toString(), this.dbName); + + try (session) { + assertNotNull(session, "expected a real RocksDB session"); + + // Switch the provider to failing now, so ONLY the real onTableFileCreated callback + // (fired by the flush below) exercises the failure path. + provider.startFailing(); + + // Write a batch and flush: RocksDB creates a real SST and invokes onTableFileCreated + // across the JNI boundary on its own background thread. + session.checkTable("t"); + SessionOperator op = session.sessionOp(); + op.prepare(); + for (int i = 0; i < 500; i++) { + op.put("t", ("key-" + i).getBytes(), ("value-" + i).getBytes()); + } + op.commit(); + session.flush(true); + + // The real callback must have driven an upload attempt that failed and was captured. + waitForCondition(() -> this.retryQueue.getGivenUpCount() > 0 + ); + assertTrue(uploadAttempts.get() > 0, + "provider.uploadFile must have been invoked by the real callback"); + assertTrue(this.retryQueue.getGivenUpCount() > 0, + "upload failure must be captured on the durability path (given up)"); + + // Process/session stability: the DB must remain fully usable after the callback-path + // failure — a JNI-boundary regression would typically crash the JVM or corrupt state. + SessionOperator after = session.sessionOp(); + after.prepare(); + after.put("t", "post-failure".getBytes(), "still-alive".getBytes()); + after.commit(); + + byte[] readBack = session.sessionOp().get("t", "post-failure".getBytes()); + assertArrayEquals("still-alive".getBytes(), + readBack, + "DB must remain readable/writable after callback-path upload " + + "failure"); + } + } + + // ========================================================================= + // Helpers + // ========================================================================= + + private static void waitForCondition(BooleanSupplier condition) + throws InterruptedException { + long deadline = System.currentTimeMillis() + 5_000L; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return; + } + LockSupport.parkNanos(10_000_000L); + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("Interrupted while waiting for async cloud callback"); + } + } + fail("injected provider failure from the real RocksDB callback should be given up on " + + "by the retry queue"); + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + private static void deleteRecursively(File dir) { + if (dir == null || !dir.exists()) { + return; + } + File[] children = dir.listFiles(); + if (children != null) { + for (File c : children) { + if (c.isDirectory()) { + deleteRecursively(c); + } else { + c.delete(); + } + } + } + dir.delete(); + } + + /** + * Thread-safe in-memory cloud store sufficient for the recovery round-trips. + */ + private static final class InMemoryCloudStore implements CloudStorageProvider { + + private final Map objects = new ConcurrentHashMap<>(); + + @Override + public String providerName() { + return "in-memory-cloudrec-test"; + } + + @Override + public void init(CloudStorageConfig config) { + // no-op + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + objects.put(remoteKey, Files.readAllBytes(Paths.get(localPath))); + } + + @Override + public void deleteFile(String remoteKey) { + objects.remove(remoteKey); + } + + @Override + public boolean fileExists(String remoteKey) { + return objects.containsKey(remoteKey); + } + + @Override + public List listFiles(String remoteDirPrefix) { + // Plain key-prefix match, as the CloudStorageProvider contract (and S3) defines. + return objects.keySet().stream() + .filter(k -> k.startsWith(remoteDirPrefix)) + .collect(Collectors.toList()); + } + + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + byte[] content = objects.get(remoteKey); + if (content == null) { + throw new IOException("Key not found: " + remoteKey); + } + Path dest = Paths.get(localPath); + if (dest.getParent() != null) { + Files.createDirectories(dest.getParent()); + } + Files.write(dest, content); + } + + @Override + public int deletePrefix(String remoteDirPrefix) { + String prefix = remoteDirPrefix.endsWith("/") ? remoteDirPrefix : remoteDirPrefix + "/"; + List keys = new ArrayList<>(objects.keySet()); + int deleted = 0; + for (String k : keys) { + if (k.startsWith(prefix)) { + objects.remove(k); + deleted++; + } + } + return deleted; + } + + @Override + public void close() { + // no-op + } + } + + /** + * Provider whose uploads succeed until {@link #startFailing()} is called, after which every + * upload fails, simulating a persistent cloud outage that begins mid-test. + */ + private static final class ToggleableFailingUploadProvider implements CloudStorageProvider { + + private final AtomicInteger uploadAttempts; + private volatile boolean failing; + + ToggleableFailingUploadProvider(AtomicInteger uploadAttempts) { + this.uploadAttempts = uploadAttempts; + } + + void startFailing() { + this.failing = true; + } + + @Override + public String providerName() { + return "always-failing-test-provider"; + } + + @Override + public void init(CloudStorageConfig config) { + // no-op + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + if (!this.failing) { + return; + } + this.uploadAttempts.incrementAndGet(); + throw new IOException("simulated persistent cloud outage"); + } + + @Override + public void deleteFile(String remoteKey) { + // no-op + } + + @Override + public boolean fileExists(String remoteKey) { + return false; + } + + @Override + public List listFiles(String remoteDirPrefix) { + return Collections.emptyList(); + } + + @Override + public int deletePrefix(String remoteDirPrefix) { + return 0; + } + + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + throw new IOException("download not supported in this test provider"); + } + + @Override + public void close() { + // no-op + } + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListenerTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListenerTest.java new file mode 100644 index 0000000000..21ae497926 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageEventListenerTest.java @@ -0,0 +1,2574 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; +import java.util.function.BooleanSupplier; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.LiveSstFile; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.MetadataSnapshot; +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageNonRetryableException; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link CloudStorageEventListener}. + * + *

Focuses on the relative-key computation ({@code toRelativeKey}) and + * the {@link CloudStorageEventListener#onTableFileCreated} / {@link + * CloudStorageEventListener#onTableFileDeleted} delegate calls to the + * active {@link CloudStorageProvider}. + */ +public class CloudStorageEventListenerTest { + + private static final String DATA_ROOT = "/hugegraph-store/storage"; + + private CloudStorageEventListener listener; + + @BeforeEach + public void setUp() { + listener = CloudStorageTestFactory.newListener(new CapturingProvider(), List.of(DATA_ROOT)); + } + + // ----------------------------------------------------------------------- + // toRelativeKey + // ----------------------------------------------------------------------- + + @Test + public void toRelativeKey_stripsDataRootPrefix() { + String filePath = DATA_ROOT + "/hgstore-metadata/000008.sst"; + assertEquals("hgstore-metadata/000008.sst", listener.toRelativeKey(filePath)); + } + + @Test + public void constructor_rejectsEmptyDataRoots() { + // Fail fast with a clear config error instead of an opaque IndexOutOfBoundsException later. + try { + CloudStorageTestFactory.newListener(new CapturingProvider(), Collections.emptyList()); + fail("Expected IllegalArgumentException for empty data-root list"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().toLowerCase().contains("data root"), + "Message should mention data root: " + expected.getMessage()); + } + } + + @Test + public void constructor_rejectsNullDataRoots() { + try { + CloudStorageTestFactory.newListener(new CapturingProvider(), null); + fail("Expected IllegalArgumentException for null data-root list"); + } catch (IllegalArgumentException expected) { + // expected + } + } + + @Test + public void constructor_rejectsNullProvider() { + try { + CloudStorageTestFactory.newListener(null, List.of(DATA_ROOT)); + fail("Expected NullPointerException for null provider"); + } catch (NullPointerException expected) { + assertTrue(String.valueOf(expected.getMessage()).contains("provider"), + "Message should mention provider: " + expected.getMessage()); + } + } + + @Test + public void constructor_sweepsOrphanedStagingPinsInEveryRoot() throws Exception { + Path root0 = Files.createTempDirectory("hgstore-test-sweep0"); + Path root1 = Files.createTempDirectory("hgstore-test-sweep1"); + try { + Path sst = Files.writeString(root1.resolve("000003.sst"), "sst"); + Path staging0 = Files.createDirectories( + root0.resolve(CloudStorageEventListener.STAGING_DIR)); + Path staging1 = Files.createDirectories( + root1.resolve(CloudStorageEventListener.STAGING_DIR)); + Path pin0 = Files.writeString(staging0.resolve("000001.sst.upload-1"), "sst"); + Path pin1 = Files.createLink(staging1.resolve("000003.sst.upload-2"), sst); + + CloudStorageTestFactory.newListener(new CapturingProvider(), + List.of(root0.toString(), root1.toString())); + + assertFalse(Files.exists(pin0), "Orphaned pin in the first root must be removed"); + assertFalse(Files.exists(pin1), "Orphaned pin in the second root must be removed"); + assertTrue(Files.exists(sst), "Sweep must not touch the source SST"); + } finally { + deleteRecursively(root0.toFile()); + deleteRecursively(root1.toFile()); + } + } + + @Test + public void toRelativeKey_stripsDataRootPrefixForPartitionDb() { + String filePath = DATA_ROOT + "/0/000042.sst"; + assertEquals("0/000042.sst", listener.toRelativeKey(filePath)); + } + + @Test + public void toRelativeKey_fallsBackToStripLeadingSlash_whenNotUnderDataRoot() { + String filePath = "/some/other/path/000001.sst"; + assertEquals("some/other/path/000001.sst", listener.toRelativeKey(filePath)); + } + + @Test + public void toRelativeKey_handlesDataRootWithTrailingSlash() { + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(new CapturingProvider(), + List.of(DATA_ROOT + File.separator)); + assertEquals("hgstore-metadata/000008.sst", + l.toRelativeKey(DATA_ROOT + "/hgstore-metadata/000008.sst")); + } + + @Test + public void toRelativeKey_appliesStoreScopePrefix() { + CloudStorageEventListener l = CloudStorageTestFactory.newListener(new CapturingProvider(), + List.of(DATA_ROOT), true, 0L, null, new CloudSyncTracker(), + "store-127.0.0.1_8501"); + String filePath = DATA_ROOT + "/0/000042.sst"; + assertEquals("store-127.0.0.1_8501/0/000042.sst", l.toRelativeKey(filePath)); + } + + // ----------------------------------------------------------------------- + // onTableFileCreated / onTableFileDeleted – with stub provider + // ----------------------------------------------------------------------- + + @Test + public void onTableFileCreated_delegatesToProvider_withRelativeKey() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-created"); + Path dbDir = tmpRoot.resolve("hgstore-metadata"); + Files.createDirectories(dbDir); + Path sst = dbDir.resolve("000008.sst"); + Files.write(sst, "sst".getBytes()); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString())); + + try { + l.onTableFileCreated("hgstore-metadata", "default", sst.toString(), 512L); + + waitForCondition(() -> provider.uploads.size() == 1, + "expected exactly one async upload"); + + assertEquals(1, provider.uploads.size()); + assertTrue(provider.uploads.get(0)[0].contains("/.cloud-upload-staging/")); + assertEquals("hgstore-metadata/000008.sst", provider.uploads.get(0)[1]); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onTableFileCreated_delegatesToProvider_withStoreScopePrefix() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-scoped"); + Path dbDir = tmpRoot.resolve("0"); + Files.createDirectories(dbDir); + Path sst = dbDir.resolve("000008.sst"); + Files.write(sst, "sst".getBytes()); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(tmpRoot.toString()), true, 0L, null, new CloudSyncTracker(), + "store-127.0.0.1_8501"); + try { + l.onTableFileCreated("0", "default", sst.toString(), 512L); + + waitForCondition(() -> provider.uploads.size() == 1, + "expected exactly one async upload with store scope prefix"); + + assertEquals(1, provider.uploads.size()); + assertEquals("store-127.0.0.1_8501/0/000008.sst", provider.uploads.get(0)[1]); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onTableFileCreated_resolvesPathFormDbName_beforeMetadataSync() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-resolve-db-name"); + Path dbDir = tmpRoot.resolve("00001"); + Files.createDirectories(dbDir); + Path sst = dbDir.resolve("000012.sst"); + Files.write(sst, "sst".getBytes()); + + CountDownLatch metadataSyncCalled = new CountDownLatch(1); + AtomicReference observedDbName = new AtomicReference<>(); + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + new CloudStorageEventListener(provider, List.of(tmpRoot.toString()), + true, CloudStorageEventListener.DEFAULT_READ_MISS_GUARD_WINDOW_MS, null, + new CloudSyncTracker(), null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + void markMetadataDirty(String dbName) { + observedDbName.set(dbName); + metadataSyncCalled.countDown(); + } + }; + + try { + // Seed the dir->logical mapping as happens when the DB opens. + l.onDBCreated("00001", dbDir.toString()); + + l.onTableFileCreated(dbDir.toString(), "default", sst.toString(), Files.size(sst)); + + assertTrue(metadataSyncCalled.await(2, TimeUnit.SECONDS), + "Expected async metadata-sync trigger after upload"); + assertEquals("00001", observedDbName.get()); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onTableFileDeleted_resolvesPathFormDbName_beforeGuardAndSync() throws Exception { + // RocksDB delivers the DB *directory path* (db.getName()) to the delete callback, exactly + // as it does to the create callback. The listener must resolve it to the logical DB name + // before the delete guard / metadata sync run — otherwise those key off a path absent from + // dbSessionMap and every superseded-SST removal is silently skipped (object leaks in cloud). + Path tmpRoot = Files.createTempDirectory("hgstore-test-resolve-del-db-name"); + Path dbDir = tmpRoot.resolve("00001"); + Files.createDirectories(dbDir); + + AtomicReference observedDbName = new AtomicReference<>(); + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + new CloudStorageEventListener(provider, List.of(tmpRoot.toString()), + true, CloudStorageEventListener.DEFAULT_READ_MISS_GUARD_WINDOW_MS, null, + new CloudSyncTracker(), null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider provider, String dbName) { + observedDbName.set(dbName); + return true; + } + }; + + // Seed the dir->logical mapping as happens when the DB opens. + l.onDBCreated("00001", dbDir.toString()); + + try { + // Pass the path form, as RocksDB does in production. + l.onTableFileDeleted(dbDir.toString(), "default", dbDir.resolve("000001.sst").toString()); + l.processDeferredSstDeletes(); + assertEquals("00001", + observedDbName.get(), + "delete guard/metadata sync must key off the resolved logical name"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onTableFileCreated_uploadFailure_doesNotThrow_andSubmitsToRetryQueue() + throws Exception { + // Exercises the *async provider upload-failure* path: the SST is staged successfully + // (real file under a real root, so the hard-link pin succeeds) and the failure comes + // from provider.uploadFile() inside the background upload worker — the exact path that + // must be caught, not the hard-link staging failure. + Path tmpRoot = Files.createTempDirectory("hgstore-test-retry"); + Path dbDir = tmpRoot.resolve("hgstore-metadata"); + Files.createDirectories(dbDir); + Path sst = dbDir.resolve("000008.sst"); + Files.write(sst, "sst-body".getBytes()); + + // maxAttempts=0 → after the async upload fails, the queue gives up on the task straight + // away, giving a deterministic, observable postcondition (no timing-dependent retry cycle). + FailingUploadProvider provider = new FailingUploadProvider(); + try (CloudUploadRetryQueue retryQueue = CloudStorageTestFactory.newRetryQueue(provider, + 0, 50L, 50L)) { + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(tmpRoot.toString()), true, 0L, retryQueue); + // Must NOT throw – the provider failure is handled asynchronously. + l.onTableFileCreated("hgstore-metadata", "default", sst.toString(), + Files.size(sst)); + + // The pin succeeds, the async upload fails, and (maxAttempts=0) the queue gives up on + // the task. Wait for the background worker to complete and assert the failure was + // captured on the real provider-upload path. + waitForCondition(() -> retryQueue.getGivenUpCount() > 0, + "async provider upload failure should be submitted to the " + + "retry queue and given up"); + + assertEquals(1L, + retryQueue.getGivenUpCount(), + "Provider upload failure must be counted as given up"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onTableFileCreated_nonRetryableUploadFailure_givesUpWithoutRetry() + throws Exception { + // A NON-retryable provider error (e.g. bad credentials) must bypass the retry budget and be + // given up on immediately, even though maxAttempts > 0. maxAttempts=3 proves it is not + // retried. + Path tmpRoot = Files.createTempDirectory("hgstore-test-nonretry"); + Path dbDir = tmpRoot.resolve("hgstore-metadata"); + Files.createDirectories(dbDir); + Path sst = dbDir.resolve("000002.sst"); + Files.write(sst, "sst-body".getBytes()); + + NonRetryableUploadProvider provider = new NonRetryableUploadProvider(); + try (CloudUploadRetryQueue retryQueue = CloudStorageTestFactory.newRetryQueue(provider, + 3, 50L, 5000L)) { + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(tmpRoot.toString()), true, 0L, retryQueue); + // Must NOT throw – the provider failure is handled asynchronously. + l.onTableFileCreated("hgstore-metadata", "default", sst.toString(), Files.size(sst)); + + waitForCondition(() -> retryQueue.getGivenUpCount() > 0, + "non-retryable upload failure should be given up immediately"); + + assertEquals(1L, + retryQueue.getGivenUpCount(), + "non-retryable failure must be given up on directly (no retries)"); + // The first retry would fire after the 50ms initial delay; wait past it to prove no + // whole-file retry was scheduled. + Thread.sleep(300L); + assertEquals(1, provider.uploadCalls.get(), + "non-retryable failure must not trigger any whole-file retry"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onTableFileCreated_uploadFailure_noRetryQueue_doesNotThrow() { + // A listener without a retry queue: failure must still not throw (just logs). + CloudStorageEventListener l = CloudStorageTestFactory.newListener( + new FailingUploadProvider(), List.of(DATA_ROOT)); + l.onTableFileCreated("hgstore-metadata", "default", + DATA_ROOT + "/hgstore-metadata/000008.sst", 512L); + // If we reached here without an exception the test passes. + } + + @Test + public void onTableFileCreated_isNonBlocking_returnsQuicklyWithSlowProvider() throws Exception { + // Verify that onTableFileCreated does not block on provider.uploadFile(), + // even when the provider sleeps for a long duration. + Path tmpRoot = Files.createTempDirectory("hgstore-test-nonblocking"); + Path metadataDir = tmpRoot.resolve("hgstore-metadata"); + Files.createDirectories(metadataDir); + Path sst = metadataDir.resolve("000008.sst"); + Files.write(sst, "sst".getBytes()); + + try { + long sleepDurationMs = 1000L; // Provider will sleep for 1 second + SlowUploadProvider provider = new SlowUploadProvider(sleepDurationMs); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString())); + + // Measure how long onTableFileCreated takes to return + long startNs = System.nanoTime(); + l.onTableFileCreated("hgstore-metadata", "default", sst.toString(), 512L); + long callDurationMs = (System.nanoTime() - startNs) / 1_000_000; + + // The callback must return in a fraction of the provider's sleep time. + // Allow a small buffer (100ms) for overhead, but the bulk of the sleep + // should happen asynchronously in the background executor. + assertTrue(callDurationMs < 200L, + "onTableFileCreated should return quickly without blocking on upload; " + + "call took " + callDurationMs + "ms (provider sleeps for " + + sleepDurationMs + "ms)"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onTableFileDeleted_delegatesToProvider_withRelativeKey() { + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = new CloudStorageEventListener(provider, List.of(DATA_ROOT), + true, CloudStorageEventListener.DEFAULT_READ_MISS_GUARD_WINDOW_MS, null, + new CloudSyncTracker(), null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + return true; + } + }; + + l.onTableFileDeleted("hgstore-metadata", "default", + DATA_ROOT + "/hgstore-metadata/000008.sst"); + l.processDeferredSstDeletes(); + + assertEquals(1, provider.deletes.size()); + assertEquals("hgstore-metadata/000008.sst", provider.deletes.get(0)); + } + + // ----------------------------------------------------------------------- + // onDBCreated – uploads existing SST files from the DB directory + // ----------------------------------------------------------------------- + + @Test + public void onDBCreated_uploadsExistingSstFiles() throws Exception { + // Create a temporary directory that mimics a partition DB directory. + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + Files.createFile(partitionDir.resolve("000001.sst")); + Files.createFile(partitionDir.resolve("000002.sst")); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString())); + + try { + // Trigger onDBCreated; startup backfill should upload both SST files synchronously. + l.onDBCreated("0", partitionDir.toString()); + + assertEquals(2, provider.uploads.size()); + List remoteKeys = new ArrayList<>(); + for (String[] u : provider.uploads) { + remoteKeys.add(u[1]); + } + // Keys should be relative (e.g. "0/000001.sst") + for (String key : remoteKeys) { + assertFalse(key.startsWith("/"), "Remote key must not start with '/': " + key); + assertTrue(key.endsWith(".sst"), "Remote key must end with .sst: " + key); + } + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBCreated_existingUploadFailure_doesNotThrow() throws Exception { + // Upload failures during onDBCreated must NOT propagate: the session is already live in + // dbSessionMap at this point, so throwing here would cause a split-brain where the + // caller believes the open failed while other threads already use the DB successfully. + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + Files.createFile(partitionDir.resolve("000001.sst")); + + FailingUploadProvider provider = new FailingUploadProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString())); + + try { + // Should not throw even though the upload fails. + l.onDBCreated("0", partitionDir.toString()); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + // ----------------------------------------------------------------------- + // metadata mirroring (upload order, CURRENT-last, prune, consistent restore) + // ----------------------------------------------------------------------- + private static CloudStorageEventListener metadataListener(CloudStorageProvider provider) { + return CloudStorageTestFactory.newListener(provider, List.of(DATA_ROOT), false, 0L, + null, new CloudSyncTracker()); + } + + private static MetadataSnapshot snapshot(List options, List ssts) { + return new MetadataSnapshot("/hugegraph-store/storage/0", "/hugegraph-store/storage/0" + "_tmp", + "CURRENT", "MANIFEST-000005", options, ssts); + } + + @Test + public void uploadMetadataSnapshot_uploadsReferencedSstsThenMetadataThenCurrentLast() { + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = metadataListener(provider); + // The referenced SST is already durable in cloud → no re-upload, but metadata must follow. + provider.putRemoteFile("0/000003.sst", "sst".getBytes()); + + MetadataSnapshot snap = snapshot( + List.of("OPTIONS-000004"), List.of("000003.sst")); + + boolean durable = l.uploadMetadataSnapshot(provider, "0", snap); + + assertTrue(durable); + List keys = new ArrayList<>(); + for (String[] up : provider.uploads) { + keys.add(up[1]); + } + // OPTIONS + MANIFEST are published before CURRENT; CURRENT is published last of all. + assertEquals(List.of("0/OPTIONS-000004", "0/MANIFEST-000005", "0/CURRENT"), keys); + } + + @Test + public void uploadMetadataSnapshot_holdsManifestAndCurrentWhenReferencedSstUploadFails() { + // Referenced SST is neither in cloud nor uploadable → the whole publish must be held. + FailingUploadProvider provider = new FailingUploadProvider(); + CloudStorageEventListener l = metadataListener(provider); + + MetadataSnapshot snap = snapshot( + List.of("OPTIONS-000004"), List.of("000003.sst")); + + boolean durable = l.uploadMetadataSnapshot(provider, "0", snap); + + assertFalse(durable); + // No MANIFEST/OPTIONS/CURRENT may be published while a referenced SST is not durable. + for (String[] up : provider.uploads) { + fail("nothing should have been published, but uploaded: " + up[1]); + } + } + + @Test + public void uploadMetadataSnapshot_holdsPublishWhenSstConfirmationEpochTurnsStale() { + CloudSyncTracker tracker = new CloudSyncTracker(); + CapturingProvider provider = new CapturingProvider() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + super.uploadFile(localPath, remoteKey); + if ("0/000003.sst".equals(remoteKey)) { + // Force an epoch advance between upload and confirmation. + tracker.clearDb("0"); + } + } + }; + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(DATA_ROOT), false, 0L, null, tracker); + + MetadataSnapshot snap = snapshot( + List.of("OPTIONS-000004"), List.of("000003.sst")); + + boolean durable = l.uploadMetadataSnapshot(provider, "0", snap); + + assertFalse(durable, "Stale epoch confirmation must hold metadata publish"); + List keys = new ArrayList<>(); + for (String[] up : provider.uploads) { + keys.add(up[1]); + } + assertEquals(List.of("0/000003.sst"), + keys, + "Only the SST upload may have happened before stale confirmation was dropped"); + } + + @Test + public void uploadMetadataSnapshot_prunesSupersededRemoteMetadataButKeepsCurrentAndSst() { + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = metadataListener(provider); + provider.putRemoteFile("0/000003.sst", "sst".getBytes()); + // Superseded remote metadata from a previous generation, plus files that must be kept. + provider.putRemoteFile("0/MANIFEST-000001", "old".getBytes()); + provider.putRemoteFile("0/OPTIONS-000000", "old".getBytes()); + provider.putRemoteFile("0/CURRENT", "old".getBytes()); + + MetadataSnapshot snap = snapshot( + List.of("OPTIONS-000004"), List.of("000003.sst")); + + assertTrue(l.uploadMetadataSnapshot(provider, "0", snap)); + + assertTrue(provider.deletes.contains("0/MANIFEST-000001")); + assertTrue(provider.deletes.contains("0/OPTIONS-000000")); + // CURRENT (the pointer) and SST objects are never pruned by the metadata sync. + assertFalse(provider.deletes.contains("0/CURRENT")); + assertFalse(provider.deletes.contains("0/000003.sst")); + } + + @Test + public void preHydration_failsLoudlyWhenCurrentReferencesMissingManifest() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + // CURRENT points at a manifest that was never mirrored → restore must refuse to open. + provider.putRemoteFile("0/CURRENT", "MANIFEST-000009\n".getBytes()); + + try { + l.onDBOpening("0", partitionDir.toString()); + fail("expected IllegalStateException for inconsistent restore"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("Cloud restore inconsistent")); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void syncMetadataSnapshotInline_serializesPerDbAndRejectsOlderGeneration() + throws Exception { + MetadataSnapshot newer = generationSnapshot("hgstore-meta-newer", 200L); + MetadataSnapshot older = generationSnapshot("hgstore-meta-older", 100L); + + CountDownLatch newerUploadEntered = new CountDownLatch(1); + CountDownLatch releaseNewerUpload = new CountDownLatch(1); + GenerationSequenceListener controlled = GenerationSequenceListener.builder() + .withSnapshots(newer, older) + .withBlock( + newerUploadEntered, + releaseNewerUpload) + .build(); + @SuppressWarnings("resource") + CapturingProvider provider = new CapturingProvider(); + + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future newerResult = executor.submit( + () -> controlled.syncMetadataSnapshotInline(provider, "db0")); + assertTrue(newerUploadEntered.await(2, TimeUnit.SECONDS), + "newer publish did not start"); + + Future olderResult = executor.submit( + () -> controlled.syncMetadataSnapshotInline(provider, "db0")); + + // Second call must be blocked on the same db lock until the first publication exits. + Thread.sleep(100L); + assertEquals(1, + controlled.captureCalls(), + "second sync must not capture before first publish is released"); + + releaseNewerUpload.countDown(); + + assertTrue(newerResult.get(2, TimeUnit.SECONDS), "newer generation must publish"); + assertFalse(olderResult.get(2, TimeUnit.SECONDS), "older generation must be rejected"); + assertEquals(List.of(200L), controlled.publishedGenerations()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void syncMetadataSnapshotInline_allowsEqualGenerationRepublish() throws Exception { + MetadataSnapshot first = generationSnapshot("hgstore-meta-equal-first", 200L); + MetadataSnapshot second = generationSnapshot("hgstore-meta-equal-second", 200L); + GenerationSequenceListener listener = GenerationSequenceListener.builder() + .withSnapshots(first, second) + .build(); + CapturingProvider provider = new CapturingProvider(); + + assertTrue(listener.syncMetadataSnapshotInline(provider, "db0"), + "first publish should succeed"); + assertTrue(listener.syncMetadataSnapshotInline(provider, "db0"), + "equal-generation re-publish should remain allowed"); + assertEquals(List.of(200L, 200L), + listener.publishedGenerations(), + "both equal-generation publications should execute"); + } + + @Test + public void syncMetadataSnapshotInline_rejectsOlderGenerationSequentially() throws Exception { + MetadataSnapshot newer = generationSnapshot("hgstore-meta-seq-newer", 300L); + MetadataSnapshot older = generationSnapshot("hgstore-meta-seq-older", 200L); + GenerationSequenceListener listener = GenerationSequenceListener.sequential(newer, older); + CapturingProvider provider = new CapturingProvider(); + + assertTrue(listener.syncMetadataSnapshotInline(provider, "db0"), + "newer generation publish should succeed"); + assertFalse(listener.syncMetadataSnapshotInline(provider, "db0"), + "older generation must be rejected after newer publish"); + assertEquals(List.of(300L), + listener.publishedGenerations(), + "only newer generation should be published"); + } + + @Test + public void onDBOpening_atNewDir_resetsConfirmationsAndPublishedGeneration() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-reopen-moved"); + Path dirA = tmpRoot.resolve("db_0"); + Path dirB = tmpRoot.resolve("db_1"); + Files.createDirectories(dirA); + Files.createDirectories(dirB); + CloudSyncTracker tracker = new CloudSyncTracker(); + AtomicReference next = new AtomicReference<>(); + CloudStorageEventListener l = metadataListener(tmpRoot, tracker, next); + try { + l.onDBOpening("0", dirA.toString()); + tracker.markConfirmed("0", dirA.resolve("000007.sst").toString()); + next.set(generationSnapshot("hgstore-meta-moved-high", 300L)); + assertTrue(l.syncMetadataSnapshotInline(new CapturingProvider(), "0")); + next.set(generationSnapshot("hgstore-meta-moved-low", 100L)); + assertFalse(l.syncMetadataSnapshotInline(new CapturingProvider(), "0"), + "lower generation is stale while the same dir is open"); + + l.onDBOpening("0", dirB.toString()); + + assertFalse(tracker.isConfirmed("0", dirB.resolve("000007.sst").toString()), + "reused SST number must not count as uploaded for the new dir"); + next.set(generationSnapshot("hgstore-meta-moved-low2", 100L)); + assertTrue(l.syncMetadataSnapshotInline(new CapturingProvider(), "0"), + "old dir's published generation must not block the new DB"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBOpening_atSameDir_keepsConfirmationsAndPublishedGeneration() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-reopen-same"); + Path dirA = tmpRoot.resolve("db_0"); + Files.createDirectories(dirA); + CloudSyncTracker tracker = new CloudSyncTracker(); + AtomicReference next = new AtomicReference<>(); + CloudStorageEventListener l = metadataListener(tmpRoot, tracker, next); + try { + l.onDBOpening("0", dirA.toString()); + tracker.markConfirmed("0", dirA.resolve("000007.sst").toString()); + next.set(generationSnapshot("hgstore-meta-same-high", 300L)); + assertTrue(l.syncMetadataSnapshotInline(new CapturingProvider(), "0")); + + l.onDBOpening("0", dirA.toString()); + + assertTrue(tracker.isConfirmed("0", dirA.resolve("000007.sst").toString()), + "reopening at the same dir must keep confirmations"); + next.set(generationSnapshot("hgstore-meta-same-low", 100L)); + assertFalse(l.syncMetadataSnapshotInline(new CapturingProvider(), "0"), + "reopening at the same dir must keep the published generation"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + /** Listener whose metadata capture returns {@code next} and whose upload always succeeds. */ + private static CloudStorageEventListener metadataListener(Path root, CloudSyncTracker tracker, + AtomicReference next) { + return new CloudStorageEventListener(new CapturingProvider(), List.of(root.toString()), + true, 0L, null, tracker, null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + MetadataSnapshot captureMetadataSnapshot(String dbName) { + return next.get(); + } + + @Override + boolean uploadMetadataSnapshot(CloudStorageProvider provider, String dbName, + MetadataSnapshot snapshot) { + return true; + } + }; + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + private static MetadataSnapshot generationSnapshot(String tempDirPrefix, long generation) + throws IOException { + Path temp = Files.createTempDirectory(tempDirPrefix); + String manifest = String.format("MANIFEST-%06d", generation); + return new MetadataSnapshot(DATA_ROOT + "/db0", temp.toString(), "CURRENT", manifest, + List.of(), List.of(), generation); + } + + private static class GenerationSequenceListener extends CloudStorageEventListener { + + private final List snapshots; + private final long blockedGeneration; + private final CountDownLatch blockedEntered; + private final CountDownLatch blockedRelease; + private final AtomicInteger captureCalls = new AtomicInteger(); + private final List publishedGenerations = + Collections.synchronizedList(new ArrayList<>()); + + private GenerationSequenceListener(List snapshots, + long blockedGeneration, + CountDownLatch blockedEntered, + CountDownLatch blockedRelease) { + super(new CapturingProvider(), List.of(DATA_ROOT), true, + DEFAULT_READ_MISS_GUARD_WINDOW_MS, + null, + new CloudSyncTracker(), null, DEFAULT_METADATA_SYNC_DEBOUNCE_MS); + this.snapshots = snapshots; + this.blockedGeneration = blockedGeneration; + this.blockedEntered = blockedEntered; + this.blockedRelease = blockedRelease; + } + + static Builder builder() { + return new Builder(); + } + + static GenerationSequenceListener sequential(MetadataSnapshot... snapshots) { + return builder().withSnapshots(snapshots).build(); + } + + int captureCalls() { + return captureCalls.get(); + } + + List publishedGenerations() { + return publishedGenerations; + } + + @Override + MetadataSnapshot captureMetadataSnapshot(String dbName) { + int index = captureCalls.getAndIncrement(); + if (index >= snapshots.size()) { + return null; + } + return snapshots.get(index); + } + + @Override + boolean uploadMetadataSnapshot(CloudStorageProvider provider, String dbName, + MetadataSnapshot snapshot) { + publishedGenerations.add(snapshot.getGeneration()); + if (snapshot.getGeneration() == blockedGeneration && blockedEntered != null + && blockedRelease != null) { + blockedEntered.countDown(); + try { + assertTrue(blockedRelease.await(2, TimeUnit.SECONDS), + "timed out waiting to release blocked publish"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("interrupted while waiting to release blocked publish"); + } + } + return true; + } + + static class Builder { + + private List snapshots = List.of(); + private long blockedGeneration = -1L; + private CountDownLatch blockedEntered; + private CountDownLatch blockedRelease; + + Builder withSnapshots(MetadataSnapshot... snapshots) { + this.snapshots = List.of(snapshots); + return this; + } + + Builder withBlock(CountDownLatch entered, + CountDownLatch release) { + this.blockedGeneration = 200L; + this.blockedEntered = entered; + this.blockedRelease = release; + return this; + } + + GenerationSequenceListener build() { + return new GenerationSequenceListener(snapshots, blockedGeneration, + blockedEntered, blockedRelease); + } + } + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + private void deleteRecursively(File f) { + if (f.isDirectory()) { + for (File child : Objects.requireNonNull(f.listFiles())) { + deleteRecursively(child); + } + } + f.delete(); + } + + private void waitForCondition(BooleanSupplier condition, String timeoutMessage) + throws InterruptedException { + waitForCondition(condition, 2000L, timeoutMessage); + } + + private void waitForCondition(BooleanSupplier condition, long timeoutMs, + String timeoutMessage) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) { + return; + } + LockSupport.parkNanos(10_000_000L); + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("Interrupted while waiting for async cloud callback"); + } + } + fail(timeoutMessage); + } + + /** + * Minimal {@link CloudStorageProvider} that records upload and delete calls. + */ + static class CapturingProvider implements CloudStorageProvider { + + final List uploads = Collections.synchronizedList(new ArrayList<>()); + final List deletes = Collections.synchronizedList(new ArrayList<>()); + final Map remoteFiles = new HashMap<>(); + int listFilesCalls = 0; + + @Override + public String providerName() { + return "capturing"; + } + + @Override + public void init(CloudStorageConfig config) { + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + uploads.add(new String[]{localPath, remoteKey}); + } + + @SuppressWarnings("RedundantThrows") + @Override + public void deleteFile(String remoteKey) throws IOException { + deletes.add(remoteKey); + } + + @SuppressWarnings("RedundantThrows") + @Override + public boolean fileExists(String remoteKey) throws IOException { + return remoteFiles.containsKey(remoteKey); + } + + @SuppressWarnings("RedundantThrows") + @Override + public List listFiles(String remoteDirPrefix) throws IOException { + listFilesCalls++; + List result = new ArrayList<>(); + for (String key : remoteFiles.keySet()) { + if (key.startsWith(remoteDirPrefix)) { + result.add(key); + } + } + return result; + } + + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + byte[] data = remoteFiles.get(remoteKey); + if (data == null) { + throw new IOException("remote key not found: " + remoteKey); + } + Path p = Paths.get(localPath); + Files.createDirectories(p.getParent()); + Files.write(p, data); + } + + @SuppressWarnings("RedundantThrows") + @Override + public void close() throws IOException { + } + + @SuppressWarnings("RedundantThrows") + @Override + public int deletePrefix(String remoteDirPrefix) throws IOException { + List result = new ArrayList<>(); + for (String key : remoteFiles.keySet()) { + if (key.startsWith(remoteDirPrefix)) { + result.add(key); + } + } + for (String key : result) { + deletes.add(key); + remoteFiles.remove(key); + } + return result.size(); + } + + void putRemoteFile(String key, byte[] content) { + remoteFiles.put(key, content); + } + } + + static class FailingUploadProvider extends CapturingProvider { + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + throw new IOException("simulated upload failure"); + } + } + + /** Upload always fails with a NON-retryable error (e.g. bad credentials / 403). */ + static class NonRetryableUploadProvider extends CapturingProvider { + + final AtomicInteger uploadCalls = new AtomicInteger(); + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + uploadCalls.incrementAndGet(); + throw new CloudStorageNonRetryableException( + "Authentication failed; credentials invalid", null); + } + } + + /** + * A {@link CloudStorageProvider} that sleeps during upload to simulate a slow provider. + * Used to verify that {@code onTableFileCreated} is non-blocking and does not wait for + * the provider's upload to complete. + */ + static class SlowUploadProvider extends CapturingProvider { + + private final long sleepMs; + + SlowUploadProvider(long sleepMs) { + this.sleepMs = sleepMs; + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + // Record the upload call (parent class behavior) + super.uploadFile(localPath, remoteKey); + // Simulate a slow operation that would block if called synchronously + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("upload sleep interrupted", e); + } + } + } + + /** + * A provider whose tombstone existence check ({@link #fileExists}) always fails with an + * {@link IOException}, simulating a cloud that cannot confirm whether the previous generation + * was tombstoned. Exercises the {@code preHydrateDbFiles} catch-block that must fall back to + * inspecting local CURRENT→MANIFEST self-consistency. + */ + static class TombstoneCheckFailingProvider extends CapturingProvider { + + final AtomicInteger fileExistsCalls = new AtomicInteger(); + + @Override + public boolean fileExists(String remoteKey) throws IOException { + fileExistsCalls.incrementAndGet(); + throw new IOException("simulated tombstone-check failure for " + remoteKey); + } + } + + @Test + public void onDBOpening_downloadsMissingRemoteFiles() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + provider.putRemoteFile("0/CURRENT", "MANIFEST-000001".getBytes()); + provider.putRemoteFile("0/MANIFEST-000001", "manifest-body".getBytes()); + provider.putRemoteFile("0/000001.sst", "sst-body".getBytes()); + + try { + l.onDBOpening("0", partitionDir.toString()); + assertTrue(Files.exists(partitionDir.resolve("CURRENT"))); + assertTrue(Files.exists(partitionDir.resolve("MANIFEST-000001"))); + assertTrue(Files.exists(partitionDir.resolve("000001.sst"))); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBOpening_downloadsMissingRemoteFiles_withStoreScopePrefix() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(tmpRoot.toString()), true, 0L, null, new CloudSyncTracker(), + "store-127.0.0.1_8501"); + provider.putRemoteFile("store-127.0.0.1_8501/0/CURRENT", "MANIFEST-000001".getBytes()); + provider.putRemoteFile("store-127.0.0.1_8501/0/MANIFEST-000001", "manifest-body".getBytes()); + provider.putRemoteFile("store-127.0.0.1_8501/0/000001.sst", "sst-body".getBytes()); + + try { + l.onDBOpening("0", partitionDir.toString()); + assertTrue(Files.exists(partitionDir.resolve("CURRENT"))); + assertTrue(Files.exists(partitionDir.resolve("MANIFEST-000001"))); + assertTrue(Files.exists(partitionDir.resolve("000001.sst"))); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBOpening_adoptsNewerRemoteCurrent_noSilentRollback() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + // Stale local metadata at generation 5 (e.g. a partial disk rollback). + Files.write(partitionDir.resolve("CURRENT"), "MANIFEST-000005".getBytes()); + Files.write(partitionDir.resolve("MANIFEST-000005"), "manifest-5".getBytes()); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + // Cloud holds a strictly newer generation 10. + provider.putRemoteFile("0/CURRENT", "MANIFEST-000010".getBytes()); + provider.putRemoteFile("0/MANIFEST-000010", "manifest-10".getBytes()); + provider.putRemoteFile("0/000010.sst", "sst-10".getBytes()); + + try { + l.onDBOpening("0", partitionDir.toString()); + // The newer generation's manifest is hydrated ... + assertTrue(Files.exists(partitionDir.resolve("MANIFEST-000010")), + "Newer remote MANIFEST must be hydrated"); + // ... and the fixed-name CURRENT pointer must be advanced to it (no silent rollback to + // the older local generation that skip-on-exists would otherwise have preserved). + assertEquals("MANIFEST-000010", + Files.readString(partitionDir.resolve("CURRENT")).trim(), + "CURRENT must point at the newer remote generation"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBOpening_keepsNewerLocalCurrent_doesNotRollBackToOlderCloud() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + // Local metadata is NEWER (generation 10) — recent writes not yet mirrored to cloud. + Files.write(partitionDir.resolve("CURRENT"), "MANIFEST-000010".getBytes()); + Files.write(partitionDir.resolve("MANIFEST-000010"), "manifest-10".getBytes()); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + // Cloud lags at an older generation 5. + provider.putRemoteFile("0/CURRENT", "MANIFEST-000005".getBytes()); + provider.putRemoteFile("0/MANIFEST-000005", "manifest-5".getBytes()); + + try { + l.onDBOpening("0", partitionDir.toString()); + assertEquals("MANIFEST-000010", + Files.readString(partitionDir.resolve("CURRENT")).trim(), + "Newer local CURRENT must be preserved; hydration must not roll the DB " + + "back to an older cloud generation"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBOpening_cloudUnreachable_failsWhenLocalMetadataInconsistent() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + // Orphan SST but NO valid CURRENT->MANIFEST lineage (partial/rolled-back directory). + Files.write(partitionDir.resolve("000001.sst"), "orphan".getBytes()); + + CapturingProvider provider = new CapturingProvider() { + @Override + public boolean fileExists(String remoteKey) { + return false; // no tombstone + } + + @Override + public List listFiles(String remoteDirPrefix) throws IOException { + throw new IOException("cloud unreachable"); + } + }; + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + + try { + l.onDBOpening("0", partitionDir.toString()); + fail("Must fail loudly when cloud is unreachable and local metadata is inconsistent"); + } catch (IllegalStateException expected) { + assertTrue(expected.getMessage().contains("self-consistent") + || expected.getMessage().contains("CURRENT"), + "Failure must cite the inconsistent local metadata: " + + expected.getMessage()); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBOpening_cloudUnreachable_proceedsWhenLocalMetadataConsistent() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + // Self-consistent local metadata: CURRENT -> a locally-present MANIFEST, plus an SST. + Files.write(partitionDir.resolve("CURRENT"), "MANIFEST-000001".getBytes()); + Files.write(partitionDir.resolve("MANIFEST-000001"), "manifest".getBytes()); + Files.write(partitionDir.resolve("000001.sst"), "data".getBytes()); + + CapturingProvider provider = new CapturingProvider() { + @Override + public boolean fileExists(String remoteKey) { + return false; + } + + @Override + public List listFiles(String remoteDirPrefix) throws IOException { + throw new IOException("cloud unreachable"); + } + }; + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + + try { + // Must NOT throw: local state is self-consistent, so falling back is safe. + l.onDBOpening("0", partitionDir.toString()); + assertTrue(Files.exists(partitionDir.resolve("CURRENT")), + "Local CURRENT must be preserved on the safe fallback path"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + // ----------------------------------------------------------------------- + // Read-miss restores missing LIVE files to their original path (no ingest) + // ----------------------------------------------------------------------- + + @Test + public void restoreMissingLiveFiles_restoresMissingLiveFileToOriginalPath() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + provider.putRemoteFile("0/000001.sst", "sst-body".getBytes()); + + String localPath = partitionDir.resolve("000001.sst").toString(); + List live = List.of(new LiveSstFile(localPath, "default")); + + try { + int restored = l.restoreMissingLiveFiles(provider, "0", live); + assertEquals(1, restored); + // Restored to the EXACT original path so RocksDB finds it; no ingest is performed. + assertTrue(Files.exists(partitionDir.resolve("000001.sst"))); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void restoreMissingLiveFiles_routesEachFileToItsOwnPath_crossCf() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + provider.putRemoteFile("0/000001.sst", "g-body".getBytes()); + provider.putRemoteFile("0/000002.sst", "q-body".getBytes()); + + // Two live files belonging to different column families. + List live = List.of( + new LiveSstFile(partitionDir.resolve("000001.sst").toString(), "g"), + new LiveSstFile(partitionDir.resolve("000002.sst").toString(), "q")); + + try { + int restored = l.restoreMissingLiveFiles(provider, "0", live); + assertEquals(2, restored); + // Each file lands at its own path; content is not mixed across CFs. + assertEquals("g-body", + new String(Files.readAllBytes(partitionDir.resolve("000001.sst")))); + assertEquals("q-body", + new String(Files.readAllBytes(partitionDir.resolve("000002.sst")))); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void restoreMissingLiveFiles_skipsFilesPresentLocally() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + Files.write(partitionDir.resolve("000001.sst"), "already-here".getBytes()); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + provider.putRemoteFile("0/000001.sst", "cloud-body".getBytes()); + + List live = List.of( + new LiveSstFile(partitionDir.resolve("000001.sst").toString(), "default")); + + try { + int restored = l.restoreMissingLiveFiles(provider, "0", live); + assertEquals(0, restored); + // Present local file is untouched (not overwritten from cloud). + assertEquals("already-here", + new String(Files.readAllBytes(partitionDir.resolve("000001.sst")))); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void readMissGuard_skipsRepeatedAttemptsWithinWindow() { + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(new CapturingProvider(), List.of(DATA_ROOT), + true, 60_000L); + assertTrue(l.shouldAttemptReadMissHydration("0", "default")); + assertFalse(l.shouldAttemptReadMissHydration("0", "default")); + // A different table is not throttled by the first table's attempt. + assertTrue(l.shouldAttemptReadMissHydration("0", "other")); + } + + @Test + public void readMissGuard_admitsExactlyOnePerWindowUnderConcurrency() throws Exception { + // The guard must admit ONLY ONE caller per DB/table window even under a concurrent read + // storm; a non-atomic get-then-put would let multiple callers pass on the race boundary. + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(new CapturingProvider(), List.of(DATA_ROOT), + true, 60_000L); + + int threads = 32; + ExecutorService pool = Executors.newFixedThreadPool(threads); + CountDownLatch ready = new CountDownLatch(threads); + CountDownLatch go = new CountDownLatch(1); + AtomicInteger admitted = new AtomicInteger(0); + try { + for (int i = 0; i < threads; i++) { + pool.submit(() -> { + try { + ready.countDown(); + go.await(); + if (l.shouldAttemptReadMissHydration("0", "default")) { + admitted.incrementAndGet(); + } + } catch (InterruptedException ignore) { + Thread.currentThread().interrupt(); + } + }); + } + assertTrue(ready.await(5, TimeUnit.SECONDS), "Workers must be ready"); + go.countDown(); + pool.shutdown(); + assertTrue(pool.awaitTermination(10, TimeUnit.SECONDS), "Workers must finish"); + } finally { + pool.shutdownNow(); + } + assertEquals(1, + admitted.get(), + "Exactly one caller may be admitted within a single guard window"); + } + + @Test + public void deleteGuard_holdsWhenLiveFileNotConfirmedAndUploadFails() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-guard"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + // A live file that exists locally but whose upload will fail. + Path liveLocal = partitionDir.resolve("000009.sst"); + Files.write(liveLocal, "live".getBytes()); + + CloudSyncTracker tracker = new CloudSyncTracker(); + FailingUploadProvider provider = new FailingUploadProvider(); + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(tmpRoot.toString()), true, 0L, null, tracker); + + List live = List.of(new LiveSstFile(liveLocal.toString(), "default")); + + try { + boolean durable = l.ensureLiveSetUploaded(provider, "0", live); + assertFalse(durable, "Guard must report the live set as NOT durable"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void deleteGuard_passesWhenAllLiveFilesConfirmed() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-guard"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + Path liveLocal = partitionDir.resolve("000009.sst"); + Files.write(liveLocal, "live".getBytes()); + + CloudSyncTracker tracker = new CloudSyncTracker(); + // Pre-mark the live file as already confirmed in cloud. + tracker.markConfirmed("0", liveLocal.toString()); + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(tmpRoot.toString()), true, 0L, null, tracker); + + List live = List.of(new LiveSstFile(liveLocal.toString(), "default")); + + try { + boolean durable = l.ensureLiveSetUploaded(provider, "0", live); + assertTrue(durable, "Guard must pass when the whole live set is confirmed"); + // No upload needed since it was already confirmed. + assertEquals(0, provider.uploads.size()); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void deleteGuard_uploadsUnconfirmedLiveFileThenPasses() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-guard"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + Path liveLocal = partitionDir.resolve("000009.sst"); + Files.write(liveLocal, "live".getBytes()); + + CloudSyncTracker tracker = new CloudSyncTracker(); + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(tmpRoot.toString()), true, 0L, null, tracker); + + List live = List.of(new LiveSstFile(liveLocal.toString(), "default")); + + try { + boolean durable = l.ensureLiveSetUploaded(provider, "0", live); + assertTrue(durable); + // The unconfirmed-but-present live file was uploaded and then marked confirmed. + assertEquals(1, provider.uploads.size()); + assertEquals("0/000009.sst", provider.uploads.get(0)[1]); + assertTrue(tracker.isConfirmed("0", liveLocal.toString())); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void deleteGuard_blocksOldSstDelete_whenInputsConfirmedButMergedOutputMissing() + throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-guard"); + Path partitionDir = tmpRoot.resolve("0"); + Files.createDirectories(partitionDir); + + // Compaction inputs SST1/SST2 were uploaded previously and are still present locally. + Path sst1 = partitionDir.resolve("000001.sst"); + Path sst2 = partitionDir.resolve("000002.sst"); + Files.write(sst1, "sst1".getBytes()); + Files.write(sst2, "sst2".getBytes()); + + // Compaction output exists in RocksDB live-set metadata but is missing both locally and + // in cloud (upload failed/lost). This is the dangerous window we must guard. + Path merged = partitionDir.resolve("000010.sst"); + + CloudSyncTracker tracker = new CloudSyncTracker(); + tracker.markConfirmed("0", sst1.toString()); + tracker.markConfirmed("0", sst2.toString()); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(tmpRoot.toString()), true, 0L, null, tracker); + provider.putRemoteFile("0/000001.sst", "sst1".getBytes()); + provider.putRemoteFile("0/000002.sst", "sst2".getBytes()); + + // Post-compaction live set contains only MERGED_SST. + List live = List.of(new LiveSstFile(merged.toString(), "default")); + + try { + boolean durable = l.ensureLiveSetUploaded(provider, "0", live); + assertFalse(durable, "Merged output is not durable; old SST delete must be blocked"); + // MERGED_SST is absent locally, so guard cannot self-heal by upload. + assertEquals(0, provider.uploads.size()); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + // ----------------------------------------------------------------------- + // MANIFEST-before-delete invariant + // ----------------------------------------------------------------------- + + /** + * Captures call-order between {@code syncMetadataSnapshotInline} and {@code provider.deleteFile}. + * Overrides {@code syncMetadataSnapshotInline} so no live RocksDB session is required. + */ + static class OrderingListener extends CloudStorageEventListener { + + final List callOrder = Collections.synchronizedList(new ArrayList<>()); + private final boolean syncResult; + + OrderingListener(CloudStorageProvider provider, String dataRoot, CloudSyncTracker tracker, + boolean syncResult) { + super(provider, List.of(dataRoot), false, 0L, null, tracker, null, + DEFAULT_METADATA_SYNC_DEBOUNCE_MS); + this.syncResult = syncResult; + } + + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider provider, String dbName) { + callOrder.add("sync"); + return syncResult; + } + } + + /** + * Verifies the MANIFEST-before-delete invariant: + * {@code syncMetadataSnapshotInline} must be called and succeed before the superseded SST is + * deleted from cloud. + * + *

Scenario: compaction merged SST1+SST2 into MERGED_SST. MERGED_SST was already uploaded + * and confirmed. When {@code onTableFileDeleted} fires for SST1 an updated MANIFEST+CURRENT + * must be published first, then SST1 deleted. A crash between those two steps leaves the + * cluster in a recoverable state. + */ + @Test + public void onTableFileDeleted_publishesUpdatedManifestBeforeDeletingSupersededSst() { + String sst1Remote = "db0/000001.sst"; + // No real RocksDB session for "db0" → getLiveSstFiles() returns empty + // → ensureLiveSetUploaded passes trivially. + CapturingProvider provider = new CapturingProvider() { + @Override + public void deleteFile(String remoteKey) throws IOException { + // Record delete after any sync call already recorded by the listener. + super.deleteFile(remoteKey); + } + }; + provider.putRemoteFile(sst1Remote, "sst1".getBytes()); + + OrderingListener l = getOrderingListener(provider); + + assertEquals(List.of("sync"), l.callOrder, "sync must have been called once"); + assertTrue(provider.deletes.contains(sst1Remote), "SST1 must be deleted from cloud"); + } + + private static @NotNull OrderingListener getOrderingListener(CapturingProvider provider) { + CloudSyncTracker tracker = new CloudSyncTracker(); + OrderingListener l = + new OrderingListener(provider, DATA_ROOT, tracker, /* syncResult */ true) { + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + boolean result = super.syncMetadataSnapshotInline(p, dbName); + // Verify no delete has happened yet when sync fires. + assertTrue(provider.deletes.isEmpty(), "delete must not precede sync"); + return result; + } + }; + + l.onTableFileDeleted("db0", "default", DATA_ROOT + "/db0/000001.sst"); + l.processDeferredSstDeletes(); + return l; + } + + /** + * When {@code syncMetadataSnapshotInline} fails (MANIFEST not updated), the SST delete must be + * skipped entirely to avoid leaving an irrecoverable state in cloud. + */ + @Test + public void onTableFileDeleted_skipsDeleteWhenMetadataSyncFails() { + String sst1Remote = "db0/000001.sst"; + CapturingProvider provider = new CapturingProvider(); + provider.putRemoteFile(sst1Remote, "sst1".getBytes()); + + CloudSyncTracker tracker = new CloudSyncTracker(); + // syncResult=false: metadata sync fails → delete must be suppressed. + OrderingListener l = + new OrderingListener(provider, DATA_ROOT, tracker, /* syncResult */ false); + + l.onTableFileDeleted("db0", "default", DATA_ROOT + "/db0/000001.sst"); + l.processDeferredSstDeletes(); + + // The held delete is retried in the background, so sync may have been attempted again. + assertFalse(l.callOrder.isEmpty(), "sync must have been called"); + assertTrue(provider.deletes.isEmpty(), "SST must NOT be deleted when metadata sync fails"); + } + + @Test + public void onTableFileDeleted_deferredDeleteIsRetriedAfterMetadataRecovers() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-deferred-delete"); + Path dbDir = tmpRoot.resolve("db0"); + Files.createDirectories(dbDir); + Path oldSst = dbDir.resolve("000001.sst"); + + CapturingProvider provider = new CapturingProvider(); + provider.putRemoteFile("db0/000001.sst", "sst1".getBytes()); + + AtomicBoolean metadataHealthy = new AtomicBoolean(false); + CloudStorageEventListener l = new CloudStorageEventListener(provider, + List.of(tmpRoot.toString()), true, 0L, null, new CloudSyncTracker(), null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + List currentLiveSstFiles(String dbName) { + return List.of(); + } + + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + return metadataHealthy.get(); + } + }; + + try { + l.onTableFileDeleted("db0", "default", oldSst.toString()); + assertTrue(provider.deletes.isEmpty(), + "Initial delete must be deferred while metadata sync is failing"); + + metadataHealthy.set(true); + waitForCondition(() -> provider.deletes.contains("db0/000001.sst"), + "Deferred delete should be retried once metadata sync recovers"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onTableFileDeleted_holdsDeleteWhenLiveSetConfirmationEpochTurnsStale() + throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-delete-stale-epoch"); + Path dbDir = tmpRoot.resolve("db0"); + Files.createDirectories(dbDir); + Path oldSst = dbDir.resolve("000001.sst"); + Path liveSst = dbDir.resolve("000010.sst"); + Files.write(liveSst, "live".getBytes()); + + CloudSyncTracker tracker = new CloudSyncTracker(); + CapturingProvider provider = new CapturingProvider() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + super.uploadFile(localPath, remoteKey); + if ("db0/000010.sst".equals(remoteKey)) { + // Simulate DB recreation between upload completion and confirmation. + tracker.clearDb("db0"); + } + } + }; + provider.putRemoteFile("db0/000001.sst", "old".getBytes()); + + CloudStorageEventListener l = new CloudStorageEventListener(provider, + List.of(tmpRoot.toString()), true, 0L, null, tracker, null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + List currentLiveSstFiles(String dbName) { + return List.of(new LiveSstFile(liveSst.toString(), "default")); + } + + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + return true; + } + }; + + try { + l.onTableFileDeleted("db0", "default", oldSst.toString()); + l.processDeferredSstDeletes(); + + assertTrue(provider.deletes.isEmpty(), + "Delete must be held when live-set confirmation is stale"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + /** + * Drives the full {@link CloudStorageEventListener#onTableFileDeleted} path with a live set that + * is NOT durable (a compaction-output SST that is neither confirmed nor present in cloud). The + * delete guard must block the superseded-input delete BEFORE any MANIFEST publish. Unlike + * {@link #onTableFileDeleted_publishesUpdatedManifestBeforeDeletingSupersededSst} (which relies + * on an empty live set so the guard passes trivially), this exercises the guard for real, so a + * regression that weakens/removes the live-set durability precondition is caught. + */ + @Test + public void onTableFileDeleted_blocksDeleteWhenLiveSetNotDurable() { + String sst1Remote = "db0/000001.sst"; + CapturingProvider provider = new CapturingProvider(); + provider.putRemoteFile(sst1Remote, "sst1".getBytes()); + + CloudSyncTracker tracker = new CloudSyncTracker(); + List syncCalls = Collections.synchronizedList(new ArrayList<>()); + // Compaction output 000010.sst is live but neither confirmed nor present locally / in cloud, + // so the live set is not durable. + CloudStorageEventListener l = new CloudStorageEventListener(provider, + List.of(DATA_ROOT), false, 0L, null, tracker, null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + List currentLiveSstFiles(String dbName) { + return List.of(new LiveSstFile(DATA_ROOT + "/db0/000010.sst", "default")); + } + + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + syncCalls.add("sync"); + return true; + } + }; + + l.onTableFileDeleted("db0", "default", DATA_ROOT + "/db0/000001.sst"); + l.processDeferredSstDeletes(); + + assertTrue(provider.deletes.isEmpty(), + "Superseded SST must NOT be deleted while the live set is not durable in cloud"); + assertTrue(syncCalls.isEmpty(), + "MANIFEST publish must not be attempted when the durability guard blocks first"); + } + + // ----------------------------------------------------------------------- + // onDBDeleteBegin / onDBDeleted — stale-data guard + // ----------------------------------------------------------------------- + + @Test + public void onDBDeleteBegin_writesTombstoneToCloud() throws Exception { + // Use a writable temp root: onDBDeleteBegin now durably persists (and fsyncs) a local + // pending-delete marker before the tombstone, and HOLDS the delete if that fails. + Path tmpRoot = Files.createTempDirectory("hgstore-test-deletebegin"); + try { + CapturingProvider provider = new CapturingProvider(); + + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString())); + l.onDBDeleteBegin("mydb", tmpRoot.resolve("mydb").toString()); + + // Tombstone key is now a sibling of the data prefix, not inside it. + String expectedTombstone = "mydb" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX; + boolean tombstoneUploaded = provider.uploads.stream() + .anyMatch(pair -> expectedTombstone.equals(pair[1])); + assertTrue(tombstoneUploaded, + "Tombstone must be uploaded as a sibling of the cloud prefix"); + assertTrue(l.hasPendingDeleteMarker("mydb"), + "A durable pending-delete marker must be written"); + assertTrue(l.isDeleteMarkerHealthy(), "Delete-marker health must remain healthy"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBDeleteBegin_tombstoneKeyRespectsStoreScopePrefix() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-deletebegin-scoped"); + try { + CapturingProvider provider = new CapturingProvider(); + + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(tmpRoot.toString()), true, 0L, null, new CloudSyncTracker(), + "store-127.0.0.1_8501"); + l.onDBDeleteBegin("mydb", tmpRoot.resolve("mydb").toString()); + + String expectedTombstone = "store-127.0.0.1_8501/mydb" + + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX; + boolean tombstoneUploaded = provider.uploads.stream() + .anyMatch(pair -> expectedTombstone.equals(pair[1])); + assertTrue(tombstoneUploaded, "Tombstone must include store scope prefix"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBDeleteBegin_holdsDeleteAndDegradesHealth_whenMarkerNotDurable() throws Exception { + // If the local pending-delete marker cannot be durably persisted (e.g. the data root is not + // writable), the delete MUST be held (throw) rather than proceeding unguarded, and the + // delete-marker health signal must flip to degraded. + CapturingProvider provider = new CapturingProvider(); + + // A data root pointing at a plain FILE makes createDirectories(/.cloud-pending-delete) + // fail deterministically on every platform. + Path notADir = Files.createTempFile("hgstore-notadir", ".tmp"); + try { + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(notADir.toString())); + try { + l.onDBDeleteBegin("mydb", notADir.resolve("mydb").toString()); + fail("Expected delete to be held when the marker cannot be durably persisted"); + } catch (IllegalStateException expected) { + assertTrue(expected.getMessage().toLowerCase().contains("marker"), + "Message should explain the held delete: " + expected.getMessage()); + } + assertFalse(l.isDeleteMarkerHealthy(), + "Delete-marker health must be degraded after a persistence failure"); + // The unguarded tombstone/purge must NOT have run. + assertTrue(provider.uploads.isEmpty(), + "No tombstone must be uploaded when the delete is held"); + } finally { + Files.deleteIfExists(notADir); + } + } + + @Test + public void onDBDeleted_purgesAllRemoteObjectsUnderPrefix() { + CapturingProvider provider = new CapturingProvider(); + provider.putRemoteFile("mydb/000001.sst", "sst".getBytes()); + provider.putRemoteFile("mydb/CURRENT", "MANIFEST-1".getBytes()); + provider.putRemoteFile("mydb/MANIFEST-000001", "body".getBytes()); + // Tombstone is now a sibling of the data prefix, not inside it. + provider.putRemoteFile("mydb" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX, + "deleted".getBytes()); + + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(DATA_ROOT)); + l.onDBDeleted("mydb", DATA_ROOT + "/mydb"); + + // Data prefix objects are purged by deletePrefix; tombstone is deleted individually. + List expectedDeleted = List.of( + "mydb/000001.sst", "mydb/CURRENT", "mydb/MANIFEST-000001", + "mydb" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX); + for (String key : expectedDeleted) { + assertTrue(provider.deletes.contains(key), + "Remote object must be deleted after DB destroy: " + key); + } + } + + @Test + public void onDBDeleted_preservesTombstoneWhenPurgeIncomplete() { + // If the prefix purge is incomplete (e.g. S3 returns a truncated listing without a + // continuation token, which deletePrefix now surfaces as an IOException), stale objects may + // remain. The tombstone MUST be preserved so a later open still blocks re-hydration of that + // stale data instead of resurrecting it as live. + String tombstoneKey = "mydb" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX; + CapturingProvider provider = new CapturingProvider() { + @Override + public int deletePrefix(String remoteDirPrefix) throws IOException { + throw new IOException("simulated incomplete purge (truncated listing, no token)"); + } + }; + provider.putRemoteFile("mydb/000001.sst", "sst".getBytes()); + provider.putRemoteFile(tombstoneKey, "deleted".getBytes()); + + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(DATA_ROOT)); + l.onDBDeleted("mydb", DATA_ROOT + "/mydb"); + + assertFalse(provider.deletes.contains(tombstoneKey), + "Tombstone must be preserved when the prefix purge is incomplete"); + } + + @Test + public void onDBDeleted_clearsSyncTrackerState() { + CloudSyncTracker tracker = new CloudSyncTracker(); + tracker.markConfirmed("mydb", DATA_ROOT + "/mydb/000001.sst"); + assertEquals(1L, tracker.confirmedCount("mydb")); + + CapturingProvider provider = new CapturingProvider(); + + CloudStorageEventListener l = CloudStorageTestFactory.newListener(provider, + List.of(DATA_ROOT), true, 0L, null, tracker); + l.onDBDeleted("mydb", DATA_ROOT + "/mydb"); + + assertEquals(0L, + tracker.confirmedCount("mydb"), + "Sync tracker must be cleared for the deleted DB"); + } + + @Test + public void onDBOpening_skipsHydrationWhenTombstonePresent() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("mydb"); + Files.createDirectories(partitionDir); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + // Populate cloud with stale data from a previous deleted generation + sibling tombstone. + provider.putRemoteFile("mydb/000001.sst", "stale-sst".getBytes()); + provider.putRemoteFile("mydb/CURRENT", "MANIFEST-000001".getBytes()); + // Tombstone is a sibling key outside the data prefix. + provider.putRemoteFile("mydb" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX, + "deleted".getBytes()); + + try { + l.onDBOpening("mydb", partitionDir.toString()); + + // Hydration must be skipped — stale SST and CURRENT must NOT be downloaded. + assertFalse(Files.exists(partitionDir.resolve("000001.sst")), + "Stale SST must not be hydrated when tombstone is present"); + assertFalse(Files.exists(partitionDir.resolve("CURRENT")), + "Stale CURRENT must not be hydrated when tombstone is present"); + // All stale remote objects in the data prefix must be purged. + assertTrue(provider.deletes.contains("mydb/000001.sst"), + "Stale SST must be purged"); + assertTrue(provider.deletes.contains("mydb/CURRENT"), + "Stale CURRENT must be purged"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBOpening_proceedsNormallyWhenNoTombstone() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path partitionDir = tmpRoot.resolve("mydb"); + Files.createDirectories(partitionDir); + + CapturingProvider provider = new CapturingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + // Normal cloud state — no tombstone. + provider.putRemoteFile("mydb/000001.sst", "sst-body".getBytes()); + provider.putRemoteFile("mydb/CURRENT", "MANIFEST-000001".getBytes()); + provider.putRemoteFile("mydb/MANIFEST-000001", "manifest".getBytes()); + + try { + l.onDBOpening("mydb", partitionDir.toString()); + + assertTrue(Files.exists(partitionDir.resolve("000001.sst")), + "SST must be hydrated in normal open"); + assertTrue(Files.exists(partitionDir.resolve("CURRENT")), + "CURRENT must be hydrated in normal open"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void deleteAndRecreate_newDbDoesNotIngestDeletedData() throws Exception { + // Simulate the full delete-then-recreate lifecycle: + // 1. DB is created and writes data to cloud. + // 2. DB is deleted → tombstone written, cloud purged. + // 3. DB is recreated at the same path → hydration must find empty prefix, not stale data. + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path dbDir = tmpRoot.resolve("graph0"); + Files.createDirectories(dbDir); + + CapturingProvider provider = new CapturingProvider(); + // Stale objects from the old generation. + provider.putRemoteFile("graph0/000001.sst", "old-data".getBytes()); + provider.putRemoteFile("graph0/CURRENT", "MANIFEST-000001".getBytes()); + + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + + // Step 1: begin delete — tombstone uploaded as a sibling key. + l.onDBDeleteBegin("graph0", dbDir.toString()); + boolean tombstoneUploaded = provider.uploads.stream() + .anyMatch(p -> p[1].equals("graph0" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX)); + assertTrue(tombstoneUploaded, "Tombstone must be uploaded during deleteBegin"); + + // Simulate tombstone appearing in cloud (as it would after a real upload). + provider.putRemoteFile("graph0" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX, + "deleted".getBytes()); + + // Step 2: deletion complete — cloud data prefix purged, then tombstone deleted. + l.onDBDeleted("graph0", dbDir.toString()); + assertTrue(provider.deletes.contains("graph0/000001.sst"), "Old SST must be deleted"); + assertTrue(provider.deletes.contains("graph0/CURRENT"), "Old CURRENT must be deleted"); + assertTrue(provider.deletes.contains( + "graph0" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX), + "Tombstone must be deleted after purge"); + + // Step 3: simulate a crash-recovery scenario where the tombstone guard is needed. + // The tombstone is re-injected alongside stale SST objects (mimicking a case where + // onDBDeleted preserved the tombstone because the cloud purge partially failed). + // onDBOpening must detect the tombstone, purge stale data, clean the tombstone, and + // return without hydrating any files into the recreated DB directory. + String tombstoneKey = "graph0" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX; + // Clear the call-tracking list so only step-3 actions contribute to the assertion — + // step 2's onDBDeleted already appended tombstoneKey to provider.deletes, which would + // make the tombstone-cleanup assertion pass trivially without step 3 doing any work. + provider.deletes.clear(); + provider.putRemoteFile(tombstoneKey, "deleted".getBytes()); + provider.putRemoteFile("graph0/000001.sst", "stale-data".getBytes()); + provider.putRemoteFile("graph0/CURRENT", "MANIFEST-000001".getBytes()); + + Files.createDirectories(dbDir); + l.onDBOpening("graph0", dbDir.toString()); + + // Tombstone guard must have fired: stale files must NOT be hydrated locally. + assertFalse(Files.exists(dbDir.resolve("000001.sst")), + "Stale SST must not be hydrated into recreated DB dir"); + assertFalse(Files.exists(dbDir.resolve("CURRENT")), + "Stale CURRENT must not be hydrated into recreated DB dir"); + // Tombstone must have been cleaned up specifically by step 3's preHydrateDbFiles. + // provider.deletes was cleared before this step, so this proves step 3 called deleteFile. + assertTrue(provider.deletes.contains(tombstoneKey), + "Tombstone must be deleted by preHydrateDbFiles in step 3"); + + deleteRecursively(tmpRoot.toFile()); + } + + /** + * When the tombstone existence check itself throws (cloud unreachable) AND the local metadata + * is not self-consistent (no valid CURRENT→MANIFEST lineage), the DB must NOT boot: generation + * state is unknowable, so silently opening from orphan local state could resurrect deleted + * data. {@code preHydrateDbFiles} must fail loudly with an {@link IllegalStateException}. + */ + @Test + public void onDBOpening_tombstoneCheckFailsAndLocalMetadataInconsistent_blocksOpen() + throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path dbDir = tmpRoot.resolve("graph0"); + Files.createDirectories(dbDir); + // Orphan SST with NO CURRENT — hasInconsistentLocalMetadata(root) returns true. + Files.write(dbDir.resolve("000001.sst"), "orphan".getBytes()); + + TombstoneCheckFailingProvider provider = new TombstoneCheckFailingProvider(); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + + try { + l.onDBOpening("graph0", dbDir.toString()); + fail("Expected IllegalStateException when tombstone check fails and local metadata " + + "is not self-consistent"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("not self-consistent"), + "Message must explain the DB open was blocked for safety, got: " + + e.getMessage()); + } finally { + assertTrue(provider.fileExistsCalls.get() > 0, + "Tombstone check must have been attempted"); + deleteRecursively(tmpRoot.toFile()); + } + } + + /** + * When the tombstone existence check throws (cloud unreachable) but the local metadata IS + * self-consistent (valid CURRENT→MANIFEST lineage present), it is safe to open from local + * state: {@code preHydrateDbFiles} must swallow the failure, log a warning, and proceed + * without throwing. + */ + @Test + public void onDBOpening_tombstoneCheckFailsButLocalMetadataConsistent_proceeds() + throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-storage"); + Path dbDir = tmpRoot.resolve("graph0"); + Files.createDirectories(dbDir); + // Self-consistent local state: CURRENT points to an existing MANIFEST. + Files.write(dbDir.resolve("MANIFEST-000001"), "manifest".getBytes()); + Files.write(dbDir.resolve("CURRENT"), "MANIFEST-000001".getBytes()); + + TombstoneCheckFailingProvider provider = new TombstoneCheckFailingProvider(); + // No remote files: after the fall-through, listRemoteKeys is empty and hydration is a no-op. + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + + try { + // Must NOT throw despite the tombstone-check IOException. + l.onDBOpening("graph0", dbDir.toString()); + assertTrue(provider.fileExistsCalls.get() > 0, + "Tombstone check must have been attempted"); + // Self-consistent local state must be left intact. + assertTrue(Files.exists(dbDir.resolve("CURRENT")), "Local CURRENT must remain"); + assertTrue(Files.exists(dbDir.resolve("MANIFEST-000001")), + "Local MANIFEST must remain"); + } finally { + deleteRecursively(tmpRoot.toFile()); + } + } + + /** Provider whose {@code deletePrefix} can be switched to fail (simulated cloud outage). */ + static class ToggleablePurgeProvider extends CapturingProvider { + + volatile boolean failPurge; + + @Override + public int deletePrefix(String remoteDirPrefix) throws IOException { + if (failPurge) { + throw new IOException("simulated purge failure"); + } + return super.deletePrefix(remoteDirPrefix); + } + + @Override + public synchronized void putRemoteFile(String key, byte[] content) { + super.putRemoteFile(key, content); + } + } + + @Test + public void managesRemoteState_isTrue() { + assertTrue(listener.managesRemoteState()); + } + + @Test + public void staleDeletePurgeRetry_standsDownOnceTheDbIsCreatedAgain() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-stale-delete-retry"); + Path dbDir = tmpRoot.resolve("g1"); + Files.createDirectories(dbDir); + ToggleablePurgeProvider provider = new ToggleablePurgeProvider(); + provider.putRemoteFile("g1/000001.sst", "old".getBytes()); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), false); + try { + provider.failPurge = true; + l.onDBDeleteBegin("g1", dbDir.toString()); + l.onDBDeleted("g1", dbDir.toString()); + assertTrue(l.hasPendingDeleteMarker("g1"), "failed purge must leave the marker"); + long epochAtDelete = l.prefixEpoch("g1"); + + // The graph is re-created at the same prefix and uploads new data; the cloud is back. + l.onDBCreated("g1", dbDir.toString()); + assertTrue(l.prefixEpoch("g1") > epochAtDelete, + "creating a DB must invalidate scheduled purges for the prefix"); + provider.failPurge = false; + provider.putRemoteFile("g1/000002.sst", "new".getBytes()); + + // The first retry fires after ~500ms: it must NOT purge the new DB's objects. + LockSupport.parkNanos(1_500_000_000L); + assertTrue(provider.remoteFiles.containsKey("g1/000002.sst"), + "A stale delete-purge retry must not delete a re-created DB's data"); + assertTrue(provider.remoteFiles.containsKey("g1/000001.sst"), + "The cancelled retry must not purge at all"); + } finally { + l.close(); + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBOpening_blocksWhileDeleteCleanupFails_evenWithHydrationDisabled() + throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-open-blocked"); + Path dbDir = tmpRoot.resolve("g2"); + Files.createDirectories(dbDir); + ToggleablePurgeProvider provider = new ToggleablePurgeProvider(); + provider.putRemoteFile("g2/000001.sst", "old".getBytes()); + // Hydration disabled: the pending-delete guard must still run. + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), false); + try { + provider.failPurge = true; + l.onDBDeleteBegin("g2", dbDir.toString()); + l.onDBDeleted("g2", dbDir.toString()); + + try { + l.onDBOpening("g2", dbDir.toString()); + fail("Open must be blocked while the remote delete cleanup is unconfirmed"); + } catch (IllegalStateException expected) { + assertTrue(expected.getMessage().contains("g2"), expected.getMessage()); + } + assertTrue(l.hasPendingDeleteMarker("g2"), "marker must persist while blocked"); + + // Cloud recovers: the next open finishes the cleanup first, then proceeds. + provider.failPurge = false; + l.onDBOpening("g2", dbDir.toString()); + assertFalse(l.hasPendingDeleteMarker("g2")); + assertFalse(provider.remoteFiles.containsKey("g2/000001.sst")); + } finally { + l.close(); + deleteRecursively(tmpRoot.toFile()); + } + } + + /** Keeps the tombstone in {@code remoteFiles} so its presence and removal can be asserted. */ + static class TombstoneTrackingProvider extends ToggleablePurgeProvider { + + volatile boolean failTombstoneDelete; + + @Override + public synchronized void uploadFile(String localPath, String remoteKey) throws IOException { + super.uploadFile(localPath, remoteKey); + if (remoteKey.endsWith(CloudStorageEventListener.DB_TOMBSTONE_SUFFIX)) { + remoteFiles.put(remoteKey, "deleted".getBytes()); + } + } + + @Override + public synchronized void deleteFile(String remoteKey) throws IOException { + if (failTombstoneDelete + && remoteKey.endsWith(CloudStorageEventListener.DB_TOMBSTONE_SUFFIX)) { + throw new IOException("simulated tombstone delete failure"); + } + super.deleteFile(remoteKey); + remoteFiles.remove(remoteKey); + } + + @Override + public synchronized boolean fileExists(String remoteKey) throws IOException { + return super.fileExists(remoteKey); + } + + @Override + public synchronized int deletePrefix(String remoteDirPrefix) throws IOException { + return super.deletePrefix(remoteDirPrefix); + } + } + + /** + * A tombstone that outlived its pending-delete marker would make a later open purge the live + * objects of a DB re-created at the same prefix. So a failed tombstone delete must keep the + * marker, and with it the open blocked, until the tombstone is gone. + */ + @Test + public void failedTombstoneDelete_keepsMarker_andBlocksOpenUntilItSucceeds() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-tombstone-delete-fails"); + Path dbDir = tmpRoot.resolve("g3"); + Files.createDirectories(dbDir); + TombstoneTrackingProvider provider = new TombstoneTrackingProvider(); + provider.putRemoteFile("g3/000001.sst", "old".getBytes()); + CloudStorageEventListener l = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), false); + try { + provider.failTombstoneDelete = true; + l.onDBDeleteBegin("g3", dbDir.toString()); + l.onDBDeleted("g3", dbDir.toString()); + assertFalse(provider.remoteFiles.containsKey("g3/000001.sst"), "prefix is purged"); + assertTrue(provider.remoteFiles.containsKey("g3_DELETED"), "tombstone delete failed"); + assertTrue(l.hasPendingDeleteMarker("g3"), + "the marker must outlive a tombstone whose delete failed"); + + try { + l.onDBOpening("g3", dbDir.toString()); + fail("Open must be blocked while the tombstone cannot be deleted"); + } catch (IllegalStateException expected) { + assertTrue(expected.getMessage().contains("g3"), expected.getMessage()); + } + + provider.failTombstoneDelete = false; + l.onDBOpening("g3", dbDir.toString()); + assertFalse(provider.remoteFiles.containsKey("g3_DELETED")); + assertFalse(l.hasPendingDeleteMarker("g3")); + } finally { + l.close(); + deleteRecursively(tmpRoot.toFile()); + } + } + + /** + * Without startup hydration the DB still uploads into its prefix, so a leftover tombstone (its + * local marker lost, e.g. with the disk) must be resolved at open, and every confirmation for + * the purged objects dropped so they are re-uploaded before the next metadata publish. + */ + @Test + public void onDBOpening_withHydrationDisabled_purgesTombstonedPrefix_andClearsTracker() + throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-open-tombstone-no-hydration"); + Path dbDir = tmpRoot.resolve("g4"); + Files.createDirectories(dbDir); + TombstoneTrackingProvider provider = new TombstoneTrackingProvider(); + provider.putRemoteFile("g4/000001.sst", "deleted-generation".getBytes()); + provider.putRemoteFile("g4_DELETED", "deleted".getBytes()); + CloudSyncTracker tracker = new CloudSyncTracker(); + tracker.markConfirmed("g4", dbDir.resolve("000001.sst").toString()); + CloudStorageEventListener l = CloudStorageTestFactory.newListener( + provider, List.of(tmpRoot.toString()), false, 0L, null, tracker); + try { + l.onDBOpening("g4", dbDir.toString()); + + assertFalse(provider.remoteFiles.containsKey("g4/000001.sst"), + "stale objects of the deleted generation must be purged"); + assertFalse(provider.remoteFiles.containsKey("g4_DELETED"), + "the tombstone must be removed before the DB can write to the prefix"); + assertEquals(0L, tracker.confirmedCount("g4"), + "confirmations for purged objects must be dropped"); + } finally { + l.close(); + deleteRecursively(tmpRoot.toFile()); + } + } + + /** + * The destroy guard deletes a stale directory of a DB that was re-created at another + * directory. That delete must purge only the stale prefix and keep the live copy's state. + */ + @Test + public void onDBDeleted_ofStaleDir_keepsStateOfTheSameDbLiveElsewhere() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-delete-stale-dir"); + Path staleDir = Files.createDirectories(tmpRoot.resolve("g5")); + Path liveDir = Files.createDirectories(tmpRoot.resolve("g5_2")); + TombstoneTrackingProvider provider = new TombstoneTrackingProvider(); + provider.putRemoteFile("g5/000001.sst", "stale".getBytes()); + provider.putRemoteFile("g5_2/000002.sst", "live".getBytes()); + CloudSyncTracker tracker = new CloudSyncTracker(); + CloudStorageEventListener l = CloudStorageTestFactory.newListener( + provider, List.of(tmpRoot.toString()), false, 0L, null, tracker); + try { + l.onDBOpening("g5", staleDir.toString()); + l.onDBOpening("g5", liveDir.toString()); + tracker.markConfirmed("g5", liveDir.resolve("000002.sst").toString()); + + l.onDBDeleteBegin("g5", staleDir.toString()); + l.onDBDeleted("g5", staleDir.toString()); + + assertFalse(provider.remoteFiles.containsKey("g5/000001.sst"), + "the stale directory's prefix must be purged"); + assertTrue(provider.remoteFiles.containsKey("g5_2/000002.sst"), + "the live directory's prefix must not be touched"); + assertEquals(1L, tracker.confirmedCount("g5"), + "the live copy's confirmations must survive the stale delete"); + } finally { + l.close(); + deleteRecursively(tmpRoot.toFile()); + } + } + + /** + * The delete callback runs on a RocksDB background thread, so it must not do cloud I/O there, + * and the deletes one compaction produces must share a single metadata publish. + */ + @Test + public void onTableFileDeleted_runsOffCallerThread_withOnePublishPerDbPerPass() + throws Exception { + CapturingProvider provider = new CapturingProvider(); + AtomicBoolean metadataHealthy = new AtomicBoolean(false); + AtomicInteger healthyPublishes = new AtomicInteger(); + AtomicInteger failedPublishes = new AtomicInteger(); + Thread caller = Thread.currentThread(); + AtomicBoolean publishedOnCaller = new AtomicBoolean(false); + CloudStorageEventListener l = new CloudStorageEventListener(provider, List.of(DATA_ROOT), + false, 0L, null, new CloudSyncTracker(), null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + List currentLiveSstFiles(String dbName) { + return List.of(); + } + + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + if (Thread.currentThread() == caller) { + publishedOnCaller.set(true); + } + if (!metadataHealthy.get()) { + failedPublishes.incrementAndGet(); + return false; + } + healthyPublishes.incrementAndGet(); + return true; + } + }; + try { + for (int i = 1; i <= 3; i++) { + l.onTableFileDeleted("db0", "default", DATA_ROOT + "/db0/00000" + i + ".sst"); + } + // Let a pass fail first: its backoff holds every later pass until all three deletes + // are queued, so they share one publish. Flipping earlier lets a pass that saw only + // the first delete publish on its own. + // The healthy publish waits out the >= 1 s publish backoff, so allow slow CI headroom. + waitForCondition(() -> failedPublishes.get() > 0, 10_000L, + "the first publish must fail"); + metadataHealthy.set(true); + waitForCondition(() -> provider.deletes.size() == 3, 10_000L, + "all superseded SSTs must be deleted once metadata publishes"); + + assertFalse(publishedOnCaller.get(), "no cloud I/O on the RocksDB callback thread"); + assertEquals(1, healthyPublishes.get(), "one metadata publish covers all deletes"); + } finally { + l.close(); + } + } + + @Test + public void onDBOpening_sweepsStaleHydrationTempsEvenWhenHydrationIsDisabled() + throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-hyd-sweep"); + Path dbDir = tmpRoot.resolve("g5"); + Files.createDirectories(dbDir); + Path stale = Files.write(dbDir.resolve("000001.sst.hyd-tmp-1-123"), "partial".getBytes()); + Path keep = Files.write(dbDir.resolve("000002.sst"), "live".getBytes()); + CloudStorageEventListener l = CloudStorageTestFactory.newListener( + new CapturingProvider(), List.of(tmpRoot.toString()), false); + try { + l.onDBOpening("g5", dbDir.toString()); + + assertFalse(Files.exists(stale), "stale download temp must be swept on every open"); + assertTrue(Files.exists(keep), "real files must be untouched"); + } finally { + l.close(); + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onDBOpening_registersLogicalName_beforeOnDBCreated() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-open-record"); + Path dbDir = tmpRoot.resolve("g6"); + Files.createDirectories(dbDir); + AtomicReference observed = new AtomicReference<>(); + CloudStorageEventListener l = new CloudStorageEventListener(new CapturingProvider(), + List.of(tmpRoot.toString()), false, 0L, null, new CloudSyncTracker(), null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + observed.set(dbName); + return true; + } + + @Override + List currentLiveSstFiles(String dbName) { + return List.of(); + } + }; + try { + // Native events can arrive between onDBOpening and onDBCreated, with the path form. + l.onDBOpening("logical-g6", dbDir.toString()); + l.onTableFileDeleted(dbDir.toString(), "default", dbDir.resolve("000001.sst").toString()); + l.processDeferredSstDeletes(); + + assertEquals("logical-g6", observed.get()); + } finally { + l.close(); + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void onCompacted_onlyMarksMetadataDirty() throws Exception { + List syncs = Collections.synchronizedList(new ArrayList<>()); + CloudStorageEventListener l = new CloudStorageEventListener(new CapturingProvider(), + List.of(DATA_ROOT), false, 0L, null, new CloudSyncTracker(), null, 60_000L) { + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + syncs.add(dbName); + return true; + } + }; + try { + l.onCompacted("db-compact"); + assertTrue(syncs.isEmpty(), "compaction callback must not publish inline"); + } finally { + l.close(); + } + assertEquals(List.of("db-compact"), syncs, "close() must flush the dirty mark"); + } + + @Test + public void metadataPublishFailures_backOffPerDb_andResetOnSuccess() throws Exception { + AtomicInteger captures = new AtomicInteger(); + AtomicBoolean healthy = new AtomicBoolean(false); + CloudStorageEventListener l = new CloudStorageEventListener(new CapturingProvider(), + List.of(DATA_ROOT), false, 0L, null, new CloudSyncTracker(), null, 20L) { + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + captures.incrementAndGet(); + return healthy.get(); + } + }; + try { + // Exponential, capped, with a floor so tiny debounce intervals cannot hot-loop. + assertEquals(1_000L, l.publishBackoffMs(1)); + assertEquals(2_000L, l.publishBackoffMs(2)); + assertEquals(4_000L, l.publishBackoffMs(3)); + assertEquals(CloudStorageEventListener.MAX_PUBLISH_BACKOFF_MS, l.publishBackoffMs(60)); + + l.markMetadataDirty("db-bo"); + waitForCondition(() -> l.publishFailureCount("db-bo") == 1, "first failure recorded"); + // Many drain ticks (every 20ms) pass inside the 1s backoff window: no more attempts. + LockSupport.parkNanos(500_000_000L); + assertEquals(1, captures.get(), "a failing DB must not be retried every drain tick"); + + healthy.set(true); + waitForCondition(() -> l.publishFailureCount("db-bo") == 0 && captures.get() == 2, + "retry after backoff must succeed and clear the failure state"); + } finally { + l.close(); + } + } + + @Test + public void newestRestorableSibling_ordersVersions_andSkipsUnrestorableDirs() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-siblings"); + CloudStorageEventListener l = CloudStorageTestFactory.newListener( + new CapturingProvider(), List.of(tmpRoot.toString()), true); + try { + // A pending-delete marker for g8_9 means that directory was destroyed. + l.onDBDeleteBegin("g8", tmpRoot.resolve("g8_9").toString()); + List keys = List.of( + "g8/CURRENT", + "g8_3/CURRENT", + "g8_4/CURRENT", + "g8_4_2/CURRENT", + "g8_5/000001.sst", // no CURRENT: not restorable + "g8_6/CURRENT", "g8_6_DELETED", // tombstoned + "g8_9/CURRENT", // pending local delete + "g80/CURRENT", // another DB sharing the name prefix + "g8_x/CURRENT"); // not a version suffix + assertEquals("g8_4_2", l.newestRestorableSibling("", "g8", keys)); + assertEquals(null, l.newestRestorableSibling("", "g8", List.of("g8_5/000001.sst"))); + } finally { + l.close(); + deleteRecursively(tmpRoot.toFile()); + } + } + + @Test + public void resolveOpenPath_redirectsOnlyWithHydrationAndAWorkingListing() throws Exception { + Path tmpRoot = Files.createTempDirectory("hgstore-test-resolve-open"); + String base = tmpRoot.resolve("g9").toString(); + CapturingProvider provider = new CapturingProvider(); + provider.putRemoteFile("g9_5/CURRENT", "MANIFEST-000005".getBytes()); + CloudStorageEventListener hydrating = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), true); + CloudStorageEventListener notHydrating = + CloudStorageTestFactory.newListener(provider, List.of(tmpRoot.toString()), false); + CloudStorageEventListener listingFails = CloudStorageTestFactory.newListener( + new CapturingProvider() { + @Override + public List listFiles(String remoteDirPrefix) throws IOException { + throw new IOException("simulated listing failure"); + } + }, List.of(tmpRoot.toString()), true); + try { + assertEquals(tmpRoot.resolve("g9_5").toString(), hydrating.resolveOpenPath("g9", base)); + assertEquals(base, notHydrating.resolveOpenPath("g9", base)); + assertEquals(base, listingFails.resolveOpenPath("g9", base)); + } finally { + hydrating.close(); + notHydrating.close(); + listingFails.close(); + deleteRecursively(tmpRoot.toFile()); + } + } + + /** Held superseded-SST deletes must not retry a failing publish faster than its backoff. */ + @Test + public void heldSstDeletes_respectPerDbPublishBackoff() throws Exception { + AtomicInteger captures = new AtomicInteger(); + CloudStorageEventListener l = new CloudStorageEventListener(new CapturingProvider(), + List.of(DATA_ROOT), false, 0L, null, new CloudSyncTracker(), null, 60_000L) { + @Override + List currentLiveSstFiles(String dbName) { + return List.of(); + } + + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + captures.incrementAndGet(); + return false; + } + }; + try { + l.onTableFileDeleted("db-held", "default", DATA_ROOT + "/db-held/000001.sst"); + waitForCondition(() -> l.publishFailureCount("db-held") == 1, + "the first failed publish must be recorded for backoff"); + // The delete retry ticks every 500 ms; the 1 s backoff must suppress those attempts. + LockSupport.parkNanos(700_000_000L); + assertEquals(1, captures.get(), "a held delete must not re-publish inside the backoff"); + } finally { + l.close(); + } + } + + /** + * Shutdown releases every DB before the listener is closed, so a dirty DB must be published + * from onDBClosing, while its session is still open, rather than from close(). + */ + @Test + public void onDBClosing_publishesDirtyMetadataOnce() throws Exception { + List syncs = Collections.synchronizedList(new ArrayList<>()); + CloudStorageEventListener l = new CloudStorageEventListener(new CapturingProvider(), + List.of(DATA_ROOT), false, 0L, null, new CloudSyncTracker(), null, 60_000L) { + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String dbName) { + syncs.add(dbName); + return true; + } + }; + try { + l.onDBClosing("db-clean"); + assertTrue(syncs.isEmpty(), "a DB with nothing dirty must not be published on close"); + + l.markMetadataDirty("db-closing"); + l.onDBClosing("db-closing"); + assertEquals(List.of("db-closing"), syncs); + } finally { + l.close(); + } + assertEquals(List.of("db-closing"), syncs, "close() must not publish it a second time"); + } + +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageIntegrationTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageIntegrationTest.java new file mode 100644 index 0000000000..3423591b25 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageIntegrationTest.java @@ -0,0 +1,1406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.LiveSstFile; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.MetadataSnapshot; +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * In-process integration test suite for the cloud storage subsystem. + * + *

Design

+ * All tests run without any network I/O. A {@link FakeCloudStore} replaces the real S3 provider: + * it is an in-memory, thread-safe key-value store that faithfully implements every method of + * {@link CloudStorageProvider}, including {@link CloudStorageProvider#deletePrefix} partial-failure + * injection, {@link CloudStorageProvider#downloadFile} atomic-write simulation, and upload fault + * injection. Every test seeds the fake store, runs the production listener, and makes assertions + * directly against the fake store's state. + * + *

Coverage

+ * Each test targets a specific behaviour fixed by the review comments: + *
    + *
  1. {@code sst_uploadAndDownload_roundTrip} — basic upload/download produces identical content
  2. + *
  3. {@code preHydration_writesFilesAtomically} — temp file is never exposed at the final path
  4. + *
  5. {@code preHydration_removesStaleHydTmpFiles} — stale .hyd-tmp cleaned on next hydration
  6. + *
  7. {@code preHydration_multiRoot_usesCorrectDataRoot} — files go to the matching data root
  8. + *
  9. {@code tombstone_writtenAsSiblingOfDataPrefix} — key is {@code _DELETED}, not {@code /_DELETED}
  10. + *
  11. {@code tombstone_preventsHydrationOfDeletedGeneration} — stale data is never downloaded
  12. + *
  13. {@code tombstone_survivesPartialPurge} — tombstone is NOT deleted when purge throws
  14. + *
  15. {@code tombstone_deletedAfterCleanPurge} — tombstone IS deleted when purge succeeds
  16. + *
  17. {@code deletePrefix_partialFailure_reportedAsError} — partial S3 errors surface as IOException
  18. + *
  19. {@code metadata_publishOrder_currentLastAfterSsts} — CURRENT uploaded only after SSTs
  20. + *
  21. {@code metadata_staleSnapshot_rejectedByGenerationCheck} — older generation never overwrites newer
  22. + *
  23. {@code metadata_concurrentPublish_onlyLatestGenerationWins} — concurrent syncs serialized per-DB
  24. + *
  25. {@code retryQueue_failedUpload_retriedAndConfirmed} — retry queue eventually marks file confirmed
  26. + *
  27. {@code retryQueue_exhausted_givenUp} — exhausted retries are counted as given up, not + * lost silently
  28. + *
  29. {@code readMiss_hydration_restoresMissingLiveFile} — missing live SST is restored on read miss
  30. + *
  31. {@code readMiss_guardWindow_suppressesRepeatAttempts} — guard window dedups rapid re-hydration
  32. + *
  33. {@code deleteGuard_blocksDeleteUntilLiveSetDurable} — delete is held until replacements are in cloud
  34. + *
+ */ +@SuppressWarnings({"BusyWait", "ResultOfMethodCallIgnored"}) +public class CloudStorageIntegrationTest { + + @TempDir + Path tmp; + + private Path dataRoot; + private FakeCloudStore store; + + @BeforeEach + public void setUp() throws Exception { + dataRoot = Files.createDirectory(tmp.resolve("storage")); + store = new FakeCloudStore(); + } + + // ========================================================================= + // 1. Basic SST upload / download round-trip + // ========================================================================= + + @Test + public void sst_uploadAndDownload_roundTrip() throws Exception { + Path dbDir = mkdirs("hugegraph/db"); + Path sst = writeSst(dbDir, "000001.sst", "row1=v1;row2=v2"); + + CloudStorageEventListener listener = listenerFor(store, dataRoot); + listener.onTableFileCreated("hugegraph", "default", sst.toString(), Files.size(sst)); + awaitUpload(store); + + // Remote key must be relative to dataRoot. + String relKey = dataRoot.relativize(sst).toString(); + assertTrue(store.fileExists(relKey), "SST must be uploaded to cloud"); + + // Download back to a different path and verify content. + Path recovered = dataRoot.resolve("recovered.sst"); + store.downloadFile(relKey, recovered.toString()); + assertEquals(Files.readString(sst), + Files.readString(recovered), + "Recovered content must match original"); + } + + // ========================================================================= + // 2. Pre-hydration writes files atomically (no partial file at final path) + // ========================================================================= + + @Test + public void preHydration_writesFilesAtomically() throws Exception { + // Seed the fake cloud with a database. + String sstKey = "hugegraph/db/000001.sst"; + String currentKey = "hugegraph/db/CURRENT"; + String manifestKey = "hugegraph/db/MANIFEST-000001"; + store.put(sstKey, bytes("sst-body")); + store.put(currentKey, bytes("MANIFEST-000001")); + store.put(manifestKey, bytes("manifest-body")); + + Path dbDir = mkdirs("hugegraph/db"); + + // Actively prove atomicity rather than assuming it: this provider asserts, at the moment + // each download happens, that (a) the destination handed to it is a TEMP path (never the + // final SST/CURRENT/MANIFEST path), and (b) the final path is not yet visible. If a + // regression made hydration download straight to the final path, one of these fails. + List violations = new CopyOnWriteArrayList<>(); + FakeCloudStore atomicityChecker = new FakeCloudStore() { + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + Path finalPath = dataRoot.resolve(remoteKey); + String tempName = Paths.get(localPath).getFileName().toString(); + if (!tempName.contains(".hyd-tmp")) { + violations.add("download target is not a temp file: " + localPath); + } + if (Files.exists(finalPath)) { + violations.add("final path visible during download: " + finalPath); + } + super.downloadFile(remoteKey, localPath); + // Final path must still be absent until the production code performs the move. + if (Files.exists(finalPath)) { + violations.add("final path materialised by download (no atomic move): " + + finalPath); + } + } + }; + // Copy the seeded objects into the checker store. + atomicityChecker.put(sstKey, bytes("sst-body")); + atomicityChecker.put(currentKey, bytes("MANIFEST-000001")); + atomicityChecker.put(manifestKey, bytes("manifest-body")); + + CloudStorageEventListener listener = listenerFor(atomicityChecker, dataRoot); + listener.onDBOpening("hugegraph", dbDir.toString()); + + assertTrue(violations.isEmpty(), "Atomicity violations detected: " + violations); + + // All files must be present at their final paths with the correct content. + assertEquals("sst-body", + Files.readString(dataRoot.resolve(sstKey)), + "SST content must match after atomic hydration"); + assertEquals("MANIFEST-000001", + Files.readString(dataRoot.resolve(currentKey)), + "CURRENT content must match after atomic hydration"); + assertEquals("manifest-body", + Files.readString(dataRoot.resolve(manifestKey)), + "MANIFEST content must match after atomic hydration"); + + // No hydration temp files must remain (atomic-move clean-up). + assertNoHydTmpFiles(dataRoot); + } + + // ========================================================================= + // 3. Pre-hydration removes stale .hyd-tmp files left by a previous crash + // ========================================================================= + + @Test + public void preHydration_removesStaleHydTmpFiles() throws Exception { + String sstKey = "hugegraph/db/000001.sst"; + store.put(sstKey, bytes("sst-body")); + + Path dbDir = mkdirs("hugegraph/db"); + + // Plant a stale temp file using the production naming pattern + // (.hyd-tmp--) to verify cleanup handles the real format. + Path staleTemp = dbDir.resolve("000001.sst.hyd-tmp-1-9999999999999"); + Files.write(staleTemp, bytes("partial-crash-data")); + assertTrue(Files.exists(staleTemp), "Stale temp file must exist before hydration"); + + CloudStorageEventListener listener = listenerFor(store, dataRoot); + listener.onDBOpening("hugegraph", dbDir.toString()); + + // The stale temp file must have been removed. + assertFalse(Files.exists(staleTemp), + "Stale .hyd-tmp must be removed before fresh download"); + + // The correct content must be at the final path. + assertTrue(Files.exists(dbDir.resolve("000001.sst")), "SST must be hydrated correctly"); + assertEquals("sst-body", + Files.readString(dbDir.resolve("000001.sst"))); + } + + // ========================================================================= + // 4. Pre-hydration with multiple data roots uses the correct root + // ========================================================================= + + @Test + public void preHydration_multiRoot_usesCorrectDataRoot() throws Exception { + // Two data roots: primary and secondary. + Path primaryRoot = Files.createDirectory(tmp.resolve("storage0")); + Path secondaryRoot = Files.createDirectory(tmp.resolve("storage1")); + + // DB lives under the secondary root. + Path dbDir = secondaryRoot.resolve("hugegraph/db"); + Files.createDirectories(dbDir); + + // The remote key was built by stripping secondaryRoot, so it is "hugegraph/db/000001.sst". + String sstKey = "hugegraph/db/000001.sst"; + store.put(sstKey, bytes("secondary-sst-body")); + store.put("hugegraph/db/CURRENT", bytes("MANIFEST-000001")); + store.put("hugegraph/db/MANIFEST-000001", bytes("manifest")); + + CloudStorageEventListener listener = listenerFor(store, + Arrays.asList(primaryRoot.toString(), secondaryRoot.toString())); + + listener.onDBOpening("hugegraph", dbDir.toString()); + + // File must land under secondaryRoot, not primaryRoot. + assertTrue(Files.exists(secondaryRoot.resolve(sstKey)), + "SST must be hydrated under the matching secondary data root"); + assertFalse(Files.exists(primaryRoot.resolve(sstKey)), + "SST must NOT be placed under the primary data root"); + } + + // ========================================================================= + // 5. Tombstone key format: sibling (prefix_DELETED), not child (prefix/_DELETED) + // ========================================================================= + + @Test + public void tombstone_writtenAsSiblingOfDataPrefix() throws Exception { + Path dbDir = mkdirs("hugegraph/db"); + CloudStorageEventListener listener = listenerFor(store, dataRoot); + + listener.onDBDeleteBegin("hugegraph", dbDir.toString()); + + // Sibling key must be uploaded. + String siblingKey = "hugegraph/db" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX; + assertTrue(store.fileExists(siblingKey), + "Tombstone must be written as sibling key '" + siblingKey + "'"); + + // Inner key must NOT be uploaded. + String innerKey = "hugegraph/db/" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX; + assertFalse(store.fileExists(innerKey), + "Tombstone must NOT be written inside the data prefix '" + innerKey + "'"); + } + + // ========================================================================= + // 6. Tombstone prevents hydration of a deleted DB generation + // ========================================================================= + + @Test + public void tombstone_preventsHydrationOfDeletedGeneration() throws Exception { + // Simulate a deleted DB: stale SSTs and a tombstone in cloud. + store.put("hugegraph/db/000001.sst", bytes("stale-data")); + store.put("hugegraph/db/CURRENT", bytes("MANIFEST-000001")); + store.put("hugegraph/db" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX, + bytes("deleted")); + + Path dbDir = mkdirs("hugegraph/db"); + CloudStorageEventListener listener = listenerFor(store, dataRoot); + listener.onDBOpening("hugegraph", dbDir.toString()); + + // Stale SST must NOT be downloaded. + assertFalse(Files.exists(dbDir.resolve("000001.sst")), + "Stale SST must not be hydrated when tombstone is present"); + assertFalse(Files.exists(dbDir.resolve("CURRENT")), + "Stale CURRENT must not be hydrated when tombstone is present"); + + // Stale data prefix must be purged. + assertFalse(store.fileExists("hugegraph/db/000001.sst"), + "Stale SST must be purged from cloud after tombstone detection"); + } + + // ========================================================================= + // 7. Tombstone is NOT deleted when the prefix purge fails (partial purge guard) + // ========================================================================= + + @Test + public void tombstone_survivesPartialPurge() throws Exception { + // Seed cloud with data prefix objects. + store.put("hugegraph/db/000001.sst", bytes("sst")); + store.put("hugegraph/db/CURRENT", bytes("MANIFEST-000001")); + // Plant tombstone at sibling path. + String tombstoneKey = "hugegraph/db" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX; + store.put(tombstoneKey, bytes("deleted")); + + // Make deletePrefix throw to simulate a partial purge. + store.setPrefixDeleteFailure(true); + + Path dbDir = mkdirs("hugegraph/db"); + CloudStorageEventListener listener = listenerFor(store, dataRoot); + listener.onDBDeleted("hugegraph", dbDir.toString()); + + // Purge failed → tombstone must still be present in cloud. + assertTrue(store.fileExists(tombstoneKey), + "Tombstone must survive a failed prefix purge"); + // Stale SST must still be present (purge failed). + assertTrue(store.fileExists("hugegraph/db/000001.sst"), + "Stale SST must remain when purge fails"); + + // Now a re-open must be blocked by the surviving tombstone: the listener finds the + // tombstone, tries to purge (still fails), and throws IllegalStateException to fail + // DB open rather than silently proceeding on empty local state. + Files.createDirectories(dbDir); + CloudStorageEventListener listener2 = listenerFor(store, dataRoot); + try { + listener2.onDBOpening("hugegraph", dbDir.toString()); + fail("Expected IllegalStateException: purge failure must block DB open"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("purge failed"), + "Exception message must describe purge failure"); + } + assertFalse(Files.exists(dbDir.resolve("000001.sst")), + "Stale SST must NOT be hydrated when tombstone-guarded purge fails"); + } + + // ========================================================================= + // 8. Tombstone IS deleted after a successful purge + // ========================================================================= + + @Test + public void tombstone_deletedAfterCleanPurge() throws Exception { + store.put("hugegraph/db/000001.sst", bytes("sst")); + String tombstoneKey = "hugegraph/db" + CloudStorageEventListener.DB_TOMBSTONE_SUFFIX; + store.put(tombstoneKey, bytes("deleted")); + + Path dbDir = mkdirs("hugegraph/db"); + CloudStorageEventListener listener = listenerFor(store, dataRoot); + listener.onDBDeleted("hugegraph", dbDir.toString()); + + // Purge succeeded → tombstone must be cleaned up. + assertFalse(store.fileExists(tombstoneKey), + "Tombstone must be removed after successful purge"); + assertFalse(store.fileExists("hugegraph/db/000001.sst"), "Data SST must be purged"); + } + + // ========================================================================= + // 9. deletePrefix partial failure: per-key errors surface as IOException + // ========================================================================= + + @Test + public void deletePrefix_partialFailure_reportedAsError() { + // Seed 3 objects; make the store report errors for 1 of them. + store.put("hugegraph/db/000001.sst", bytes("sst1")); + store.put("hugegraph/db/000002.sst", bytes("sst2")); + store.put("hugegraph/db/000003.sst", bytes("sst3")); + store.setPartialDeleteErrors(Collections.singletonList("hugegraph/db/000002.sst")); + + try { + store.deletePrefix("hugegraph/db/"); + fail("Expected IOException for partial delete failure"); + } catch (IOException e) { + assertTrue(e.getMessage().contains("000002.sst"), + "Error must name the failed key"); + } + + // The two successfully deleted keys must be gone; the failed one must survive. + assertFalse(store.fileExists("hugegraph/db/000001.sst"), "000001.sst must be deleted"); + assertFalse(store.fileExists("hugegraph/db/000003.sst"), "000003.sst must be deleted"); + assertTrue(store.fileExists("hugegraph/db/000002.sst"), + "000002.sst must survive a per-key delete failure"); + } + + // ========================================================================= + // 10. Metadata publish order: SSTs before MANIFEST, MANIFEST before CURRENT + // ========================================================================= + + @Test + public void metadata_publishOrder_currentLastAfterSsts() throws Exception { + Path dbDir = mkdirs("hugegraph/db"); + Path tempDir = mkdirs("hugegraph/db-checkpoint"); + + // Create SST and metadata stubs in the checkpoint directory. + writeSst(tempDir, "000001.sst", "data"); + Files.writeString(tempDir.resolve("OPTIONS-000002"), "options"); + Files.writeString(tempDir.resolve("MANIFEST-000001"), + "manifest"); + Files.writeString(tempDir.resolve("CURRENT"), "MANIFEST-000001"); + + MetadataSnapshot snapshot = new MetadataSnapshot( + dbDir.toString(), + tempDir.toString(), + "CURRENT", + "MANIFEST-000001", + Collections.singletonList("OPTIONS-000002"), + Collections.singletonList("000001.sst"), + 1L); + + // Use a recording store to capture upload order. + OrderedRecordingStore ordered = new OrderedRecordingStore(store); + + CloudStorageEventListener listener = listenerFor(ordered, dataRoot); + boolean published = listener.uploadMetadataSnapshot(ordered, "hugegraph", snapshot); + + assertTrue(published, "Metadata publish must succeed"); + + List uploadOrder = ordered.uploadOrder(); + // CURRENT must be the last metadata file uploaded. + int currentIdx = lastIndexOf(uploadOrder, "CURRENT"); + int manifestIdx = lastIndexOf(uploadOrder, "MANIFEST-000001"); + int sstIdx = lastIndexOf(uploadOrder, "000001.sst"); + + assertTrue(sstIdx < manifestIdx, "SSTs must be uploaded before MANIFEST"); + assertTrue(manifestIdx < currentIdx, "MANIFEST must be uploaded before CURRENT"); + } + + // ========================================================================= + // 11. Stale snapshot rejected by generation check + // ========================================================================= + + @Test + public void metadata_staleSnapshot_rejectedByGenerationCheck() throws Exception { + Path dbDir = mkdirs("hugegraph/db"); + Path tempDir = mkdirs("hugegraph/db-checkpoint"); + + CloudStorageEventListener listener = listenerFor(store, dataRoot); + + // Publish generation 10 to establish the watermark. + MetadataSnapshot gen10 = snapshot(dbDir, tempDir, "MANIFEST-000010", 10L); + assertTrue(listener.uploadMetadataSnapshot(store, "hugegraph", gen10), + "Generation 10 must publish successfully"); + + // Attempt to publish generation 5 (older than the watermark). + // uploadMetadataSnapshot itself does not carry the generation guard — that lives in + // syncMetadataSnapshotInline. Verify the guard is effective by calling it directly: + // the generation guard is in publishSnapshotWithGenerationCheck which is called from + // syncMetadataSnapshotInline. We exercise it by checking that the listener rejects + // a snapshot whose generation is below lastPublishedMetadataGeneration. + // + // We call uploadMetadataSnapshot with gen5 directly to confirm it succeeds on its own + // (no guard there), then verify syncMetadataSnapshotInline would block re-publishing + // by checking no extra cloud writes occur when the snapshot generation is old. + MetadataSnapshot gen5 = snapshot(dbDir, tempDir, "MANIFEST-000005", 5L); + // Direct upload of gen5 succeeds (uploadMetadataSnapshot has no generation check). + listener.uploadMetadataSnapshot(store, "hugegraph", gen5); + + // Now re-publish gen10 — the high-water mark must stay at 10 (not regress to 5). + // Verify by publishing gen10 again; it must succeed and not corrupt the watermark. + assertTrue(listener.uploadMetadataSnapshot(store, "hugegraph", gen10), + "Re-publishing generation 10 must succeed after gen5 direct upload"); + + // Build a second listener that has the watermark already at 10 and attempt + // to push gen5 through syncMetadataSnapshotInline using an override. + // We use a subclass seam: override captureMetadataSnapshot to supply gen5. + CloudStorageEventListener guardedListener = new CloudStorageEventListener(store, + Collections.singletonList(dataRoot.toString()), + true, 0L, null, new CloudSyncTracker(), null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + private boolean gen10Published = false; + + @Override + MetadataSnapshot captureMetadataSnapshot(String dbName) { + // First call: return gen10 to establish the watermark. + // Second call: return gen5 to trigger the rejection. + if (!gen10Published) { + gen10Published = true; + return gen10; + } + return gen5; + } + }; + + // First sync establishes the watermark at generation 10. + guardedListener.syncMetadataSnapshotInline(store, "hugegraph"); + int uploadsAtWatermark = store.totalUploads(); + + // Second sync presents generation 5 — the guard must reject it without uploading. + guardedListener.syncMetadataSnapshotInline(store, "hugegraph"); + assertEquals(uploadsAtWatermark, + store.totalUploads(), + "No uploads must occur when a stale generation is rejected"); + } + + // ========================================================================= + // 12. Concurrent metadata publishes for the same DB are serialized + // ========================================================================= + + @Test + public void metadata_concurrentPublish_onlyLatestGenerationWins() throws Exception { + Path dbDir = mkdirs("hugegraph/db"); + + // Drive the concurrency through the REAL guarded path — syncMetadataSnapshotInline, which + // owns the per-DB lock and the generation guard — rather than uploadMetadataSnapshot() + // directly (which has neither). Each thread owns a fixed generation with its own checkpoint + // dir + distinct MANIFEST, handed to the listener via captureMetadataSnapshot so the guard + // sees a genuine capture->publish flow. This way a regression in the serialization or + // generation logic makes the test fail instead of silently passing. + final int threads = 6; + Map snapshotsByGen = new ConcurrentHashMap<>(); + for (long gen = 1; gen <= threads; gen++) { + Path tempDir = mkdirs("hugegraph/db-checkpoint-" + gen); + snapshotsByGen.put(gen, + snapshot(dbDir, tempDir, String.format("MANIFEST-%06d", gen), gen)); + } + + // Each worker thread publishes exactly one fixed generation; captureMetadataSnapshot returns + // the calling thread's snapshot so syncMetadataSnapshotInline's guard is exercised. + ThreadLocal perThread = new ThreadLocal<>(); + CloudStorageEventListener listener = new CloudStorageEventListener(store, + Collections.singletonList(dataRoot.toString()), + true, 0L, null, new CloudSyncTracker(), null, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + MetadataSnapshot captureMetadataSnapshot(String dbName) { + return perThread.get(); + } + }; + + ExecutorService pool = Executors.newFixedThreadPool(threads); + CountDownLatch ready = new CountDownLatch(threads); + CountDownLatch go = new CountDownLatch(1); + List errors = new CopyOnWriteArrayList<>(); + List publishResults = new CopyOnWriteArrayList<>(); + + for (long g = 1; g <= threads; g++) { + final long gen = g; + pool.submit(() -> { + try { + perThread.set(snapshotsByGen.get(gen)); + ready.countDown(); + go.await(); + publishResults.add(listener.syncMetadataSnapshotInline(store, "hugegraph")); + } catch (Exception e) { + errors.add(e); + } + }); + } + + assertTrue(ready.await(5, TimeUnit.SECONDS), "Workers must be ready in 5s"); + go.countDown(); + pool.shutdown(); + assertTrue(pool.awaitTermination(10, TimeUnit.SECONDS), "All threads must finish in 10s"); + + assertTrue(errors.isEmpty(), "No concurrent publish errors: " + errors); + + // The highest generation can never be stale, so it always publishes; once it does, every + // lower generation captured afterwards is rejected by the guard. Whatever the + // (nondeterministic) lock-acquisition order, the durable CURRENT must therefore point at the + // MAX generation's MANIFEST — a stale generation can never be the last writer. + String winningManifest = String.format("MANIFEST-%06d", (long) threads); + assertTrue(store.fileExists("hugegraph/db/CURRENT"), + "CURRENT must be present in cloud after concurrent publishes"); + assertEquals(winningManifest, + store.readCurrentUtf8(), + "Cloud CURRENT must point at the highest generation's manifest — only the " + + "non-stale generation may win"); + assertTrue(store.fileExists("hugegraph/db/" + winningManifest), + "The winning manifest object must be durable in cloud"); + + long published = publishResults.stream().filter(Boolean::booleanValue).count(); + assertTrue(published >= 1, "At least the max generation must have published"); + assertTrue(published <= threads, "No more successful publishes than offered generations"); + } + + // ========================================================================= + // 13. Retry queue retries a failed upload and eventually marks it confirmed + // ========================================================================= + + @Test + public void retryQueue_failedUpload_retriedAndConfirmed() throws Exception { + Path dbDir = mkdirs("hugegraph/db"); + Path sst = writeSst(dbDir, "000001.sst", "data"); + + // First call fails; subsequent calls succeed. + AtomicInteger calls = new AtomicInteger(0); + FakeCloudStore flakyStore = new FakeCloudStore() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + if (calls.incrementAndGet() == 1) { + throw new IOException("transient failure"); + } + super.uploadFile(localPath, remoteKey); + } + }; + + CloudSyncTracker tracker = new CloudSyncTracker(); + // Wire the SAME epoch-aware callback production uses (AppConfig wires + // syncTracker::markConfirmedIfEpoch). This ensures the test fails if the retry path drops + // the epoch — the plain 2-arg callback would confirm regardless and hide that defect. + CloudUploadRetryQueue retryQueue = CloudStorageTestFactory.newRetryQueue(flakyStore, + 3, 10L, 100L, + tracker::markConfirmedIfEpoch); + + // Capture the epoch at submission time and pass it through, exactly as the production + // callers in CloudStorageEventListener do. + long uploadEpoch = tracker.currentEpoch("hugegraph"); + retryQueue.submit("hugegraph", "default", sst.toString(), + "hugegraph/db/000001.sst", uploadEpoch, + new IOException("initial failure")); + + // Wait for the retry queue to process. + boolean confirmed = awaitCondition(() -> tracker.isConfirmed("hugegraph", sst.toString()), + 3_000); + assertTrue(confirmed, "SST must be confirmed after retry"); + assertTrue(flakyStore.fileExists("hugegraph/db/000001.sst"), + "SST must be in cloud after retry"); + + retryQueue.close(); + } + + @Test + public void asyncUploadFailure_retryUsesStagedPin_evenAfterOriginalSstDeleted() throws Exception { + // Regression guard for the compaction-race durability hole: when the async upload fails, the + // retry must upload from the staged hard-link PIN, not the original SST path. If it used the + // original (which compaction can delete before the retry fires) the retry would be silently + // dropped and the byteset would never reach cloud. + Path dbDir = mkdirs("hugegraph/db"); + Path sst = writeSst(dbDir, "000001.sst", "data-bytes"); + + // First async upload attempt fails; the retry (from the pin) succeeds. + AtomicInteger calls = new AtomicInteger(0); + FakeCloudStore flakyStore = new FakeCloudStore() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + if (calls.incrementAndGet() == 1) { + throw new IOException("transient first-attempt failure"); + } + super.uploadFile(localPath, remoteKey); + } + }; + + CloudSyncTracker tracker = new CloudSyncTracker(); + CloudUploadRetryQueue retryQueue = CloudStorageTestFactory.newRetryQueue(flakyStore, + 3, 10L, 100L, + tracker::markConfirmedIfEpoch); + CloudStorageEventListener listener = CloudStorageTestFactory.newListener(flakyStore, + Collections.singletonList(dataRoot.toString()), true, 0L, retryQueue, tracker); + + // Dispatch the async upload: onTableFileCreated creates the staged pin, then the first + // attempt fails and (with the fix) hands the PIN to the retry queue. + listener.onTableFileCreated("hugegraph", "default", sst.toString(), Files.size(sst)); + + // Simulate compaction deleting the original SST before the retry fires. The pin (a hard link) + // keeps the byteset alive, so the retry must still succeed. + Files.deleteIfExists(sst); + assertFalse(Files.exists(sst), "Original SST must be gone (compacted) before the retry"); + + boolean confirmed = awaitCondition( + () -> tracker.isConfirmed("hugegraph", sst.toString()), 5_000); + assertTrue(confirmed, + "Retry must upload from the staged pin and confirm even after the original SST " + + "is deleted"); + assertTrue(flakyStore.fileExists("hugegraph/db/000001.sst"), + "SST must be present in cloud after the pin-based retry"); + + retryQueue.close(); + } + + @Test + public void retrySuccess_publishesUpdatedMetadata() throws Exception { + // A retry that uploads the SST during a quiet period must also advance the mirrored + // recovery point: CURRENT/MANIFEST must be published, not just the tracker confirmed. + Path dbDir = mkdirs("hugegraph/db"); + Path tempDir = mkdirs("hugegraph/db-checkpoint"); + Path sst = writeSst(dbDir, "000001.sst", "data"); + + // First SST upload attempt (via retry) fails; the retry then succeeds. + AtomicInteger calls = new AtomicInteger(0); + FakeCloudStore flakyStore = new FakeCloudStore() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + if (calls.incrementAndGet() == 1) { + throw new IOException("transient first-attempt failure"); + } + super.uploadFile(localPath, remoteKey); + } + }; + + CloudSyncTracker tracker = new CloudSyncTracker(); + CloudUploadRetryQueue retryQueue = createRetryQueue(flakyStore, tracker, dbDir, tempDir); + + long epoch = tracker.currentEpoch("hugegraph"); + retryQueue.submit("hugegraph", "default", sst.toString(), "hugegraph/db/000001.sst", + epoch, new IOException("initial failure")); + + boolean confirmed = awaitCondition( + () -> tracker.isConfirmed("hugegraph", sst.toString()), 5_000); + assertTrue(confirmed, "SST must be confirmed after retry"); + + // The retry success must ALSO trigger a metadata publish so cloud CURRENT catches up. + boolean published = awaitCondition( + () -> flakyStore.fileExists("hugegraph/db/CURRENT"), 5_000); + assertTrue(published, + "Retry success must publish CURRENT/MANIFEST (recovery point must advance, not " + + "just the tracker confirmation)"); + assertEquals("MANIFEST-000005", + flakyStore.readCurrentUtf8(), + "Published CURRENT must reference the captured generation's manifest"); + + retryQueue.close(); + } + + private @NotNull CloudUploadRetryQueue createRetryQueue(CloudStorageProvider provider, + CloudSyncTracker tracker, Path dbDir, + Path tempDir) { + CloudUploadRetryQueue retryQueue = CloudStorageTestFactory.newRetryQueue(provider, + 3, 10L, 100L, + tracker::markConfirmedIfEpoch); + + CloudStorageEventListener listener = new CloudStorageEventListener(provider, + Collections.singletonList(dataRoot.toString()), true, 0L, retryQueue, tracker, + null, CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS) { + @Override + MetadataSnapshot captureMetadataSnapshot(String db) { + try { + return snapshot(dbDir, tempDir, "MANIFEST-000005", 5L); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }; + // Wire the retry-success -> metadata-sync trigger exactly as AppConfig does. + retryQueue.setMetadataSyncTrigger(listener::onRetryUploadDurable); + return retryQueue; + } + + // ========================================================================= + // 14. Retry queue gives up (and counts it) when all retries exhausted + // ========================================================================= + + @Test + public void retryQueue_exhausted_givenUp() throws Exception { + Path dbDir = mkdirs("hugegraph/db"); + Path sst = writeSst(dbDir, "000001.sst", "data"); + + // Always fail. + FakeCloudStore alwaysFail = new FakeCloudStore() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + throw new IOException("permanent failure"); + } + }; + + CloudSyncTracker tracker = new CloudSyncTracker(); + CloudUploadRetryQueue retryQueue = CloudStorageTestFactory.newRetryQueue(alwaysFail, + 2, 10L, 50L, + tracker::markConfirmed); + + retryQueue.submit("hugegraph", "default", sst.toString(), + "hugegraph/db/000001.sst", new IOException("initial failure")); + + // Wait until the queue gives up on the upload. + boolean givenUp = awaitCondition(() -> retryQueue.getGivenUpCount() > 0, 5_000); + assertTrue(givenUp, "Exhausted upload must be counted as given up"); + assertFalse(tracker.isConfirmed("hugegraph", sst.toString()), + "File must not be confirmed when given up"); + + retryQueue.close(); + } + + // ========================================================================= + // 15. Read-miss hydration restores a missing live SST file + // ========================================================================= + + @Test + public void readMiss_hydration_restoresMissingLiveFile() throws Exception { + Path dbDir = mkdirs("hugegraph/db"); + Path sst = dbDir.resolve("000001.sst"); // does NOT exist on disk yet + + // Cloud has the SST. + String remoteKey = "hugegraph/db/000001.sst"; + store.put(remoteKey, bytes("row1=v1;row2=v2")); + + CloudSyncTracker tracker = new CloudSyncTracker(); + CloudStorageEventListener listener = CloudStorageTestFactory.newListener(store, + Collections.singletonList(dataRoot.toString()), + true, 0L, null, tracker); + + List liveFiles = Collections.singletonList( + new LiveSstFile(sst.toString(), "default")); + + int restored = listener.restoreMissingLiveFiles(store, "hugegraph", liveFiles); + + assertEquals(1, restored, "One live SST must be restored"); + assertTrue(Files.exists(sst), "Restored SST must exist at original path"); + assertEquals("row1=v1;row2=v2", + Files.readString(sst)); + assertTrue(tracker.isConfirmed("hugegraph", sst.toString()), + "Restored SST must be marked confirmed in tracker"); + } + + // ========================================================================= + // 16. Read-miss guard window suppresses rapid repeated hydration attempts + // ========================================================================= + + @Test + public void readMiss_guardWindow_suppressesRepeatAttempts() throws Exception { + long guardWindowMs = 1_000L; + CloudStorageEventListener listener = CloudStorageTestFactory.newListener(store, + Collections.singletonList(dataRoot.toString()), + true, guardWindowMs, null, new CloudSyncTracker()); + + // First attempt: allowed. + assertTrue(listener.shouldAttemptReadMissHydration("hugegraph", "default"), + "First read-miss attempt must be allowed"); + + // Second attempt within the guard window: suppressed. + assertFalse(listener.shouldAttemptReadMissHydration("hugegraph", "default"), + "Second read-miss within guard window must be suppressed"); + + // After the guard window expires: allowed again. + Thread.sleep(guardWindowMs + 50); + assertTrue(listener.shouldAttemptReadMissHydration("hugegraph", "default"), + "Read-miss attempt after guard window expiry must be allowed"); + } + + // ========================================================================= + // 17. Delete guard holds delete until all live-set files are durable + // ========================================================================= + + @Test + public void deleteGuard_blocksDeleteUntilLiveSetDurable() throws Exception { + Path dbDir = mkdirs("hugegraph/db"); + Path sst1 = writeSst(dbDir, "000001.sst", "old-data"); + Path sst2 = writeSst(dbDir, "000002.sst", "new-data"); + + // sst1 is confirmed in cloud; sst2 is not yet. + CloudSyncTracker tracker = new CloudSyncTracker(); + tracker.markConfirmed("hugegraph", sst1.toString()); + + store.put("hugegraph/db/000001.sst", bytes("old-data")); + + CloudStorageEventListener listener = CloudStorageTestFactory.newListener(store, + Collections.singletonList(dataRoot.toString()), + true, 0L, null, tracker); + + List liveFiles = Arrays.asList( + new LiveSstFile(sst1.toString(), "default"), + new LiveSstFile(sst2.toString(), "default")); + + // Live set not fully durable (sst2 unconfirmed and present locally). + // ensureLiveSetUploaded must upload sst2 and return true. + boolean durable = listener.ensureLiveSetUploaded(store, "hugegraph", liveFiles); + assertTrue(durable, "Live set must be durable after ensureLiveSetUploaded"); + assertTrue(store.fileExists("hugegraph/db/000002.sst"), + "sst2 must be uploaded to cloud"); + assertTrue(tracker.isConfirmed("hugegraph", sst2.toString()), + "sst2 must be marked confirmed after upload"); + } + + @Test + public void delete_purgeFailing_marksPendingAndBlocksHydrationUntilCleanup() + throws Exception { + // Deleting while the remote purge is failing must still leave a durable anti-resurrection + // guard (a local pending-delete marker) that blocks re-hydration until remote cleanup is + // confirmed once purges succeed again. Unique prefix so any late retry cannot touch other + // tests. + Path dbDir = mkdirs("provdown/db"); + store.put("provdown/db/000001.sst", bytes("stale")); + store.put("provdown/db/CURRENT", bytes("MANIFEST-000001")); + + CloudStorageEventListener listener = listenerFor(store, dataRoot); + Path markerDir = dataRoot.resolve(CloudStorageEventListener.PENDING_DELETE_DIR); + + // Delete while the remote purge fails. + store.setPrefixDeleteFailure(true); + listener.onDBDeleteBegin("provdown/db", dbDir.toString()); + listener.onDBDeleted("provdown/db", dbDir.toString()); + + assertTrue(markerDirHasEntry(markerDir), + "A pending-delete marker must guard the delete while the purge is failing"); + + // Re-open while the purge still fails: the open is BLOCKED (fails loudly) rather than + // proceeding fresh, because stale remote state would otherwise be purged over new data + // later. The marker persists. + assertThrows(IllegalStateException.class, + () -> listener.onDBOpening("provdown/db", dbDir.toString()), + "Open must be blocked while pending-delete cleanup is unconfirmed"); + assertTrue(markerDirHasEntry(markerDir), + "Marker must persist while cleanup is unconfirmed"); + + // Purge recovers: opening completes the cleanup inline (purge + marker removal). + store.setPrefixDeleteFailure(false); + listener.onDBOpening("provdown/db", dbDir.toString()); + + assertFalse(markerDirHasEntry(markerDir), + "Marker must be cleared once remote cleanup completes"); + assertFalse(store.fileExists("provdown/db/000001.sst"), + "Stale remote object must be purged on cleanup"); + } + + @Test + public void delete_purgeFailure_retriesUntilCleanupCompletes() throws Exception { + // A failed delete purge must be retried until the remote prefix is purged and the local + // marker cleared. + Path dbDir = mkdirs("delfail/db"); + store.put("delfail/db/000001.sst", bytes("stale")); + + CloudStorageEventListener listener = listenerFor(store, dataRoot); + Path markerDir = dataRoot.resolve(CloudStorageEventListener.PENDING_DELETE_DIR); + + // First purge attempt fails. + store.setPrefixDeleteFailure(true); + listener.onDBDeleteBegin("delfail/db", dbDir.toString()); + listener.onDBDeleted("delfail/db", dbDir.toString()); + + assertTrue(markerDirHasEntry(markerDir), "Marker must remain while purge is failing"); + assertTrue(store.fileExists("delfail/db/000001.sst"), + "Stale object remains after a failed purge"); + + // Recovery: purge now succeeds; the scheduled retry must purge and clear the marker. + store.setPrefixDeleteFailure(false); + boolean cleaned = awaitCondition( + () -> !store.fileExists("delfail/db/000001.sst") && !markerDirHasEntry(markerDir), + 8_000); + assertTrue(cleaned, + "Delete purge must retry until the stale prefix is purged and the marker " + + "cleared"); + } + + @Test + public void startupPendingDeleteMarker_validMarker_isReplayedAndPurged() throws Exception { + // Simulate a crash after writing the local marker but before the remote purge completed. + store.put("startup/db/000001.sst", bytes("stale")); + Path markerDir = dataRoot.resolve(CloudStorageEventListener.PENDING_DELETE_DIR); + Files.createDirectories(markerDir); + + // Markers are empty files; the Base64 filename alone carries the prefix. + String markerName = java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString("startup/db".getBytes(StandardCharsets.UTF_8)); + Files.createFile(markerDir.resolve(markerName)); + + // New listener instance => startup marker scan + deferred purge scheduling. + listenerFor(store, dataRoot); + + boolean cleaned = awaitCondition( + () -> !store.fileExists("startup/db/000001.sst") && !markerDirHasEntry(markerDir), + 8_000); + assertTrue(cleaned, "A valid startup marker must trigger purge replay and marker cleanup"); + } + + @Test + public void startupPendingDeleteMarker_undecodableFilename_isRejected() throws Exception { + store.put("badmarker/db/000001.sst", bytes("stale")); + Path markerDir = dataRoot.resolve(CloudStorageEventListener.PENDING_DELETE_DIR); + Files.createDirectories(markerDir); + + // '!' is not in the URL-safe Base64 alphabet, so the name cannot be decoded to a prefix. + Files.createFile(markerDir.resolve("not-base64!")); + + listenerFor(store, dataRoot); + + // No purge should be scheduled from an invalid marker. + Thread.sleep(700L); + assertTrue(store.fileExists("badmarker/db/000001.sst"), + "Invalid startup marker must not trigger remote purge"); + assertTrue(markerDirHasEntry(markerDir), + "Invalid marker should remain for explicit operator inspection"); + } + + /** True if {@code dir} exists and contains at least one entry. */ + private static boolean markerDirHasEntry(Path dir) { + if (!Files.isDirectory(dir)) { + return false; + } + try (Stream s = Files.list(dir)) { + return s.findAny().isPresent(); + } catch (IOException e) { + return false; + } + } + + // ========================================================================= + // 17c. Post-upload metadata sync is coalesced into a periodic drain of dirty DBs + // ========================================================================= + + @Test + public void metadataSync_coalescesBurstIntoOneDrainedPublish() throws Exception { + AtomicInteger syncCount = new AtomicInteger(0); + CloudStorageEventListener listener = countingListener(100L, syncCount, () -> true); + try { + // A burst of 5 "post-upload" requests only marks the DB dirty; nothing publishes inline. + for (int i = 0; i < 5; i++) { + listener.markMetadataDirty("hugegraph"); + } + assertEquals(0, syncCount.get(), "Marking a DB dirty must not publish inline"); + + assertTrue(awaitCondition(() -> syncCount.get() >= 1, 3_000), + "The drain must publish the dirty DB"); + // A successful publish clears the dirty mark, so later drains do not republish. + Thread.sleep(350L); + assertEquals(1, syncCount.get(), "The coalesced burst must publish exactly once"); + } finally { + listener.close(); + } + } + + @Test + public void metadataSync_failedPublishIsRetriedWithBackoffUntilItSucceeds() throws Exception { + AtomicInteger syncCount = new AtomicInteger(0); + // First two publishes fail; the DB must stay dirty until the third succeeds. Failures back + // off exponentially (base >= 1s), so the retries are spaced out instead of every drain. + CloudStorageEventListener listener = + countingListener(50L, syncCount, () -> syncCount.get() >= 3); + try { + listener.markMetadataDirty("hugegraph"); + assertTrue(awaitCondition(() -> syncCount.get() >= 1, 3_000), + "The first publish attempt must run"); + // With a 50ms drain and no backoff there would be ~20 attempts in 600ms. + Thread.sleep(600L); + assertEquals(1, syncCount.get(), "A failed publish must back off before retrying"); + assertEquals(1, listener.publishFailureCount("hugegraph")); + + assertTrue(awaitCondition(() -> syncCount.get() >= 3, 8_000), + "A failed publish must be retried until it succeeds"); + Thread.sleep(200L); + assertEquals(3, syncCount.get(), "Retries must stop once a publish succeeds"); + assertEquals(0, listener.publishFailureCount("hugegraph"), + "Success must clear the backoff state"); + } finally { + listener.close(); + } + } + + @Test + public void metadataSync_closeFlushesDirtyDbsOnceThenPublishesInline() throws Exception { + AtomicInteger syncCount = new AtomicInteger(0); + // A drain interval that never elapses during the test isolates the close() flush. + CloudStorageEventListener listener = countingListener(60_000L, syncCount, () -> true); + listener.markMetadataDirty("hugegraph"); + assertEquals(0, syncCount.get()); + + listener.close(); + assertEquals(1, syncCount.get(), "close() must flush each dirty DB once"); + + // After shutdown (e.g. a retry-queue completion), requests publish inline. + listener.markMetadataDirty("hugegraph"); + assertEquals(2, syncCount.get(), "Requests after close() must publish inline"); + } + + @Test + public void metadataSync_nonPositiveIntervalPublishesInline() { + AtomicInteger syncCount = new AtomicInteger(0); + CloudStorageEventListener listener = countingListener(0L, syncCount, () -> true); + try { + listener.markMetadataDirty("hugegraph"); + listener.markMetadataDirty("hugegraph"); + assertEquals(2, syncCount.get(), "An interval <= 0 must publish on every request"); + } finally { + listener.close(); + } + } + + private CloudStorageEventListener countingListener(long debounceMs, AtomicInteger syncCount, + BooleanSupplier publishResult) { + return new CloudStorageEventListener(store, + Collections.singletonList(dataRoot.toString()), + true, 0L, null, new CloudSyncTracker(), null, debounceMs) { + @Override + boolean syncMetadataSnapshotInline(CloudStorageProvider p, String db) { + syncCount.incrementAndGet(); + return publishResult.getAsBoolean(); + } + }; + } + + @Test + public void shutdown_handsOffQueuedUploadTasksToRetryQueue() throws Exception { + // Saturate the listener's upload executor with blocking uploads so later dispatches sit + // QUEUED, then force a shutdownNow(): the queued tasks must be handed off to the retry queue + // (and given up on there) rather than silently dropped from the durability pipeline. + CountDownLatch block = new CountDownLatch(1); + FakeCloudStore blockingStore = new FakeCloudStore() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + try { + block.await(10, TimeUnit.SECONDS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted during shutdown"); + } + super.uploadFile(localPath, remoteKey); + } + }; + + CloudSyncTracker tracker = new CloudSyncTracker(); + // maxAttempts=0 => handed-off tasks are given up immediately, where the test can observe + // them via getGivenUpCount(). + CloudUploadRetryQueue retryQueue = CloudStorageTestFactory.newRetryQueue(blockingStore, + 0, 10L, 100L, + tracker::markConfirmedIfEpoch); + CloudStorageEventListener listener = CloudStorageTestFactory.newListener(blockingStore, + Collections.singletonList(dataRoot.toString()), true, 0L, retryQueue, tracker); + + Path dbDir = mkdirs("hugegraph/db"); + // More SSTs than upload threads: the first few block the workers, the rest queue. + for (int i = 0; i < 6; i++) { + Path sst = writeSst(dbDir, String.format("%06d.sst", i), "data-" + i); + listener.onTableFileCreated("hugegraph", "default", sst.toString(), Files.size(sst)); + } + + // Force shutdownNow quickly: blocked workers cannot finish in 1ms, so queued tasks are + // dropped by the executor and must be handed off. + listener.close(1, TimeUnit.MILLISECONDS); + block.countDown(); + + boolean handed = awaitCondition(() -> retryQueue.getGivenUpCount() > 0, 5_000); + assertTrue(handed, + "Queued upload tasks dropped at shutdown must be handed off to the retry queue"); + retryQueue.close(); + } + + // ========================================================================= + // Helpers + // ========================================================================= + + private CloudStorageEventListener listenerFor(CloudStorageProvider provider, Path root) { + return listenerFor(provider, Collections.singletonList(root.toString())); + } + + private CloudStorageEventListener listenerFor(CloudStorageProvider provider, + List roots) { + return CloudStorageTestFactory.newListener(provider, roots, true, 0L, null, + new CloudSyncTracker()); + } + + private Path mkdirs(String relative) throws Exception { + Path dir = dataRoot.resolve(relative); + Files.createDirectories(dir); + return dir; + } + + private static Path writeSst(Path dir, String name, String content) throws Exception { + Path sst = dir.resolve(name); + Files.writeString(sst, content); + return sst; + } + + private static byte[] bytes(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + private static void awaitUpload(FakeCloudStore store) + throws Exception { + long deadline = System.currentTimeMillis() + (long) 2000; + while (store.totalUploads() < 1) { + if (System.currentTimeMillis() > deadline) { + fail("Timed out waiting for " + 1 + " upload(s)"); + } + Thread.sleep(20); + } + } + + private static boolean awaitCondition(BooleanSupplier cond, long timeoutMs) + throws Exception { + long deadline = System.currentTimeMillis() + timeoutMs; + while (!cond.getAsBoolean()) { + if (System.currentTimeMillis() > deadline) { + return false; + } + Thread.sleep(20); + } + return true; + } + + private static void assertNoHydTmpFiles(Path root) throws Exception { + try (Stream stream = Files.walk(root)) { + List stale = stream + // Production names every hydration temp ".hyd-tmp--". + .filter(p -> p.getFileName().toString().contains(".hyd-tmp-")) + .toList(); + assertTrue(stale.isEmpty(), "No stale hydration temp files must remain: " + stale); + } + } + + /** Returns the last index of a key whose filename component equals {@code name}. */ + private static int lastIndexOf(List keys, String name) { + for (int i = keys.size() - 1; i >= 0; i--) { + String base = Paths.get(keys.get(i)).getFileName().toString(); + if (base.equals(name)) { + return i; + } + } + return -1; + } + + private static MetadataSnapshot snapshot(Path dbDir, Path tempDir, + String manifestName, long generation) + throws Exception { + if (!Files.exists(tempDir.resolve(manifestName))) { + Files.writeString(tempDir.resolve(manifestName), + "manifest-gen-" + generation); + } + if (!Files.exists(tempDir.resolve("CURRENT"))) { + Files.writeString(tempDir.resolve("CURRENT"), + manifestName); + } + return new MetadataSnapshot( + dbDir.toString(), + tempDir.toString(), + "CURRENT", + manifestName, + Collections.emptyList(), + Collections.emptyList(), + generation); + } + + // ========================================================================= + // FakeCloudStore — in-memory S3-compatible cloud provider + // ========================================================================= + + /** + * Thread-safe in-memory implementation of {@link CloudStorageProvider} that stands in for + * real object storage in all integration tests. Notable capabilities: + *
    + *
  • Faithful {@link #downloadFile}: writes to the supplied path, matching real S3 behaviour.
  • + *
  • {@link #deletePrefix}: supports injecting partial per-key errors to exercise the + * error-inspection logic in the production {@code S3CloudStorageProvider}.
  • + *
  • {@link #setPrefixDeleteFailure}: makes the entire {@code deletePrefix} throw, used to + * test that the tombstone is not deleted when the purge fails.
  • + *
+ */ + static class FakeCloudStore implements CloudStorageProvider { + + private final ConcurrentHashMap objects = new ConcurrentHashMap<>(); + private final AtomicInteger uploadCount = new AtomicInteger(0); + private volatile boolean prefixDeleteFailure = false; + private volatile List partialDeleteErrors = Collections.emptyList(); + + /** Seed a key directly (bypasses the upload counter). */ + void put(String key, byte[] value) { + objects.put(key, value); + } + + /** Make the next {@code deletePrefix} call throw an {@link IOException}. */ + void setPrefixDeleteFailure(boolean fail) { + this.prefixDeleteFailure = fail; + } + + /** + * Specify keys whose individual deletion should be reported as a per-key error + * (simulating S3's partial-failure response in DeleteObjects). + */ + void setPartialDeleteErrors(List failKeys) { + this.partialDeleteErrors = failKeys; + } + + int totalUploads() { + return uploadCount.get(); + } + + /** Returns cloud CURRENT content as UTF-8, or {@code null} if CURRENT is absent. */ + String readCurrentUtf8() { + byte[] content = objects.get("hugegraph/db/CURRENT"); + return content == null ? null : new String(content, StandardCharsets.UTF_8); + } + + @Override + public String providerName() { + return "fake"; + } + + @Override + public void init(CloudStorageConfig config) { + // no-op + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + byte[] content = Files.readAllBytes(Paths.get(localPath)); + objects.put(remoteKey, content); + uploadCount.incrementAndGet(); + } + + @Override + public void deleteFile(String remoteKey) throws IOException { + objects.remove(remoteKey); + } + + @Override + public boolean fileExists(String remoteKey) { + return objects.containsKey(remoteKey); + } + + @Override + public List listFiles(String remoteDirPrefix) { + // Plain key-prefix match, as the CloudStorageProvider contract (and S3) defines. + return objects.keySet().stream() + .filter(k -> k.startsWith(remoteDirPrefix)) + .collect(Collectors.toList()); + } + + @Override + public void downloadFile(String remoteKey, String localPath) throws IOException { + byte[] content = objects.get(remoteKey); + if (content == null) { + throw new IOException("Key not found: " + remoteKey); + } + Path dest = Paths.get(localPath); + Files.createDirectories(dest.getParent()); + Files.write(dest, content); + } + + /** + * Deletes all objects under the prefix. Supports injecting partial per-key failures + * (simulating S3 HTTP 200 with per-key Errors in the response body) and a wholesale + * failure (simulating a network error before any deletion). + */ + @Override + public int deletePrefix(String remoteDirPrefix) throws IOException { + if (prefixDeleteFailure) { + throw new IOException("Simulated deletePrefix failure for " + remoteDirPrefix); + } + String prefix = remoteDirPrefix.endsWith("/") ? remoteDirPrefix : remoteDirPrefix + "/"; + List keys = objects.keySet().stream() + .filter(k -> k.startsWith(prefix)) + .toList(); + List failed = new ArrayList<>(); + int deleted = 0; + for (String key : keys) { + if (partialDeleteErrors.contains(key)) { + failed.add(key); + } else { + objects.remove(key); + deleted++; + } + } + if (!failed.isEmpty()) { + throw new IOException("Partial delete failure: " + failed + " could not be deleted"); + } + return deleted; + } + + @Override + public void close() { + // no-op + } + } + + // ========================================================================= + // OrderedRecordingStore — records upload key order for sequencing assertions + // ========================================================================= + + /** + * Wraps a {@link FakeCloudStore} and captures the order in which remote keys are uploaded. + * Used by {@link #metadata_publishOrder_currentLastAfterSsts}. + */ + static class OrderedRecordingStore extends FakeCloudStore { + + private final FakeCloudStore delegate; + private final List order = new CopyOnWriteArrayList<>(); + + OrderedRecordingStore(FakeCloudStore delegate) { + this.delegate = delegate; + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + delegate.uploadFile(localPath, remoteKey); + // Record only the filename component so assertions are path-prefix-agnostic. + order.add(Paths.get(remoteKey).getFileName().toString()); + } + + @Override + public boolean fileExists(String remoteKey) { + return delegate.fileExists(remoteKey); + } + + @Override + public List listFiles(String prefix) { + return delegate.listFiles(prefix); + } + + @Override + public void deleteFile(String remoteKey) throws IOException { + delegate.deleteFile(remoteKey); + } + + List uploadOrder() { + return Collections.unmodifiableList(order); + } + } + +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageTestFactory.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageTestFactory.java new file mode 100644 index 0000000000..e52432f77f --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudStorageTestFactory.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import java.util.List; +import java.util.function.BiConsumer; + +import org.apache.hugegraph.store.cloud.CloudStorageProvider; + +/** + * Shared construction helpers for {@link CloudStorageEventListener} and + * {@link CloudUploadRetryQueue} tests. Both classes expose exactly one production constructor + * (fully-parameterised, so each is completely configured the moment it is built); this factory + * mirrors the parameter combinations tests actually vary so call sites don't have to spell out + * every default themselves. + */ +final class CloudStorageTestFactory { + + private CloudStorageTestFactory() { + } + + static CloudStorageEventListener newListener(CloudStorageProvider provider, + List dataRoots) { + return newListener(provider, dataRoots, true, + CloudStorageEventListener.DEFAULT_READ_MISS_GUARD_WINDOW_MS, null); + } + + static CloudStorageEventListener newListener(CloudStorageProvider provider, + List dataRoots, + boolean startupHydrationEnabled) { + return newListener(provider, dataRoots, startupHydrationEnabled, + CloudStorageEventListener.DEFAULT_READ_MISS_GUARD_WINDOW_MS, null); + } + + static CloudStorageEventListener newListener(CloudStorageProvider provider, + List dataRoots, + boolean startupHydrationEnabled, + long readMissGuardWindowMs) { + return newListener(provider, dataRoots, startupHydrationEnabled, readMissGuardWindowMs, + null); + } + + static CloudStorageEventListener newListener(CloudStorageProvider provider, + List dataRoots, + boolean startupHydrationEnabled, + long readMissGuardWindowMs, + CloudUploadRetryQueue retryQueue) { + return newListener(provider, dataRoots, startupHydrationEnabled, readMissGuardWindowMs, + retryQueue, new CloudSyncTracker(), null); + } + + static CloudStorageEventListener newListener(CloudStorageProvider provider, + List dataRoots, + boolean startupHydrationEnabled, + long readMissGuardWindowMs, + CloudUploadRetryQueue retryQueue, + CloudSyncTracker syncTracker) { + return newListener(provider, dataRoots, startupHydrationEnabled, readMissGuardWindowMs, + retryQueue, syncTracker, null); + } + + static CloudStorageEventListener newListener(CloudStorageProvider provider, + List dataRoots, + boolean startupHydrationEnabled, + long readMissGuardWindowMs, + CloudUploadRetryQueue retryQueue, + CloudSyncTracker syncTracker, + String storeScopePrefix) { + return newListener(provider, dataRoots, startupHydrationEnabled, readMissGuardWindowMs, + retryQueue, syncTracker, storeScopePrefix, + CloudStorageEventListener.DEFAULT_METADATA_SYNC_DEBOUNCE_MS); + } + + static CloudStorageEventListener newListener(CloudStorageProvider provider, + List dataRoots, + boolean startupHydrationEnabled, + long readMissGuardWindowMs, + CloudUploadRetryQueue retryQueue, + CloudSyncTracker syncTracker, + String storeScopePrefix, + long metadataSyncDebounceMs) { + return new CloudStorageEventListener(provider, dataRoots, startupHydrationEnabled, + readMissGuardWindowMs, retryQueue, syncTracker, + storeScopePrefix, metadataSyncDebounceMs); + } + + static CloudUploadRetryQueue newRetryQueue(CloudStorageProvider provider, int maxAttempts, + long initialDelayMs, long maxDelayMs) { + return newRetryQueue(provider, maxAttempts, initialDelayMs, maxDelayMs, + (CloudUploadRetryQueue.UploadConfirmedCallback) null); + } + + /** + * Legacy convenience overload accepting a simple {@link BiConsumer}. The epoch parameter is + * not forwarded; use the {@link CloudUploadRetryQueue.UploadConfirmedCallback} overload for + * epoch-safe confirmation. + */ + static CloudUploadRetryQueue newRetryQueue(CloudStorageProvider provider, int maxAttempts, + long initialDelayMs, long maxDelayMs, + BiConsumer onUploadConfirmed) { + return newRetryQueue(provider, maxAttempts, initialDelayMs, maxDelayMs, + onUploadConfirmed == null ? null + : (db, path, epoch) -> onUploadConfirmed.accept(db, path)); + } + + static CloudUploadRetryQueue newRetryQueue(CloudStorageProvider provider, int maxAttempts, + long initialDelayMs, long maxDelayMs, + CloudUploadRetryQueue.UploadConfirmedCallback + onUploadConfirmed) { + return new CloudUploadRetryQueue(provider, maxAttempts, initialDelayMs, maxDelayMs, + onUploadConfirmed); + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudSyncTrackerTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudSyncTrackerTest.java new file mode 100644 index 0000000000..5ff8f755dd --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudSyncTrackerTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class CloudSyncTrackerTest { + + @Test + public void parseSstFileNumber_parsesDigitBaseName() { + assertEquals(123L, CloudSyncTracker.parseSstFileNumber("/data/db/000123.sst")); + assertEquals(8L, CloudSyncTracker.parseSstFileNumber("0/000008.sst")); + assertEquals(0L, CloudSyncTracker.parseSstFileNumber("000000.sst")); + } + + @Test + public void parseSstFileNumber_rejectsNonSstAndNonNumeric() { + assertEquals(-1L, CloudSyncTracker.parseSstFileNumber("/data/db/CURRENT")); + assertEquals(-1L, CloudSyncTracker.parseSstFileNumber("/data/db/MANIFEST-000001")); + assertEquals(-1L, CloudSyncTracker.parseSstFileNumber("/data/db/abc.sst")); + assertEquals(-1L, CloudSyncTracker.parseSstFileNumber(null)); + assertEquals(-1L, CloudSyncTracker.parseSstFileNumber(".sst")); + } + + @Test + public void markIsConfirmedAndClear_roundTrip() { + CloudSyncTracker tracker = new CloudSyncTracker(); + String path = "0/000042.sst"; + + assertFalse(tracker.isConfirmed("0", path)); + tracker.markConfirmed("0", path); + assertTrue(tracker.isConfirmed("0", path)); + assertEquals(1L, tracker.confirmedCount("0")); + + tracker.clearConfirmed("0", path); + assertFalse(tracker.isConfirmed("0", path)); + assertEquals(0L, tracker.confirmedCount("0")); + } + + @Test + public void confirmedBitmapsAreScopedPerDb() { + CloudSyncTracker tracker = new CloudSyncTracker(); + tracker.markConfirmed("0", "0/000001.sst"); + // Same file number, different DB, must be independent. + assertFalse(tracker.isConfirmed("1", "1/000001.sst")); + assertTrue(tracker.isConfirmed("0", "0/000001.sst")); + } + + @Test + public void nonSstPathsAreIgnored() { + CloudSyncTracker tracker = new CloudSyncTracker(); + tracker.markConfirmed("0", "0/CURRENT"); + assertEquals(0L, tracker.confirmedCount("0")); + assertFalse(tracker.isConfirmed("0", "0/CURRENT")); + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudUploadRetryQueueTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudUploadRetryQueueTest.java new file mode 100644 index 0000000000..38e11dc4d4 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/cloud/CloudUploadRetryQueueTest.java @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.cloud; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hugegraph.store.cloud.CloudStorageConfig; +import org.apache.hugegraph.store.cloud.CloudStorageProvider; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link CloudUploadRetryQueue}. + */ +@SuppressWarnings({"BusyWait", "resource"}) +public class CloudUploadRetryQueueTest { + + private Path tmpRoot; + + @BeforeEach + public void setUp() throws IOException { + tmpRoot = Files.createTempDirectory("hgstore-retry-queue-test"); + } + + @AfterEach + public void tearDown() { + deleteRecursively(tmpRoot.toFile()); + } + + @Test + public void constructor_rejectsNullProvider() { + try { + CloudStorageTestFactory.newRetryQueue(null, 3, 50L, 200L); + fail("Expected NullPointerException for null provider"); + } catch (NullPointerException expected) { + assertTrue(String.valueOf(expected.getMessage()).contains("provider"), + "Message should mention provider: " + expected.getMessage()); + } + } + + // ----------------------------------------------------------------------- + // submit → retry succeeds on second attempt + // ----------------------------------------------------------------------- + + @Test + public void submit_retriesAndSucceeds() throws Exception { + CountDownLatch successLatch = new CountDownLatch(1); + AtomicInteger callCount = new AtomicInteger(0); + + CapturingProvider provider = new CapturingProvider() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + if (callCount.incrementAndGet() == 1) { + throw new IOException("transient failure"); + } + // Second call succeeds. + successLatch.countDown(); + } + }; + + // Create a real SST file so the retry doesn't drop it as "compacted". + Path sstFile = tmpRoot.resolve("000001.sst"); + Files.createFile(sstFile); + + try (CloudUploadRetryQueue queue = + CloudStorageTestFactory.newRetryQueue(provider, 3, 50L, 200L)) { + + queue.submit("db1", "default", sstFile.toString(), + "db1/000001.sst", new IOException("initial failure")); + + assertTrue(successLatch.await(2, TimeUnit.SECONDS), + "Upload should have succeeded within 2 s"); + assertEquals(0L, queue.getGivenUpCount()); + } + } + + @Test + public void submit_callbackFailureAfterSuccessfulUpload_doesNotMisclassifyAsUploadFailure() + throws Exception { + Path sstFile = tmpRoot.resolve("000001-callback-fail.sst"); + Files.createFile(sstFile); + + CapturingProvider provider = new CapturingProvider(); + + try (CloudUploadRetryQueue queue = CloudStorageTestFactory.newRetryQueue(provider, + 1, 50L, 50L, + (db, source, epoch) -> { + throw new RuntimeException("tracker callback failed"); + })) { + + queue.submit("db-cb", "default", sstFile.toString(), + "db-cb/000001.sst", new IOException("initial failure")); + + long deadline = System.currentTimeMillis() + 3_000L; + while (provider.uploads.isEmpty() && System.currentTimeMillis() < deadline) { + Thread.sleep(20L); + } + + assertFalse(provider.uploads.isEmpty(), + "Upload should still succeed even if callback throws"); + assertEquals(0L, + queue.getGivenUpCount(), + "Callback failure after successful upload must not be given up"); + } + } + + // ----------------------------------------------------------------------- + // submit → all attempts fail → given up + // ----------------------------------------------------------------------- + + @Test + public void submit_exhaustsRetriesAndGivesUp() throws Exception { + AtomicInteger uploadCalls = new AtomicInteger(0); + + CapturingProvider provider = new CapturingProvider() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + uploadCalls.incrementAndGet(); + throw new IOException("permanent failure"); + } + }; + + Path sstFile = tmpRoot.resolve("000002.sst"); + Files.createFile(sstFile); + + // maxAttempts=2, initial+retry delay 50ms → given up after ~100ms total + try (CloudUploadRetryQueue queue = + CloudStorageTestFactory.newRetryQueue(provider, 2, 50L, 50L)) { + + queue.submit("db1", "default", sstFile.toString(), + "db1/000002.sst", new IOException("initial")); + + // Wait for the retry cycle to finish. + long deadline = System.currentTimeMillis() + 3_000L; + while (queue.getGivenUpCount() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(50); + } + + assertEquals(1L, queue.getGivenUpCount(), + "Task should be given up after max retries"); + assertEquals(2, uploadCalls.get(), + "Provider must be called once per whole-file retry (maxAttempts=2)"); + } + } + + // ----------------------------------------------------------------------- + // Rejection safety – submit after scheduler shutdown must not throw + // ----------------------------------------------------------------------- + + @Test + public void submit_afterShutdown_doesNotThrow_andGivesUp() throws Exception { + AtomicInteger uploadCalls = new AtomicInteger(0); + CapturingProvider provider = new CapturingProvider() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + uploadCalls.incrementAndGet(); + throw new IOException("should never run – scheduler is shut down"); + } + }; + + Path sstFile = tmpRoot.resolve("000009.sst"); + Files.createFile(sstFile); + + // maxAttempts > 0 so submit() routes through scheduleRetry(), which will attempt to + // schedule on the executor. After close() the scheduler is shut down and + // scheduler.schedule(...) throws RejectedExecutionException. + CloudUploadRetryQueue queue = + CloudStorageTestFactory.newRetryQueue(provider, 3, 50L, 200L); + queue.close(); + + // This simulates a RocksDB callback firing during a shutdown race. It must NOT throw + // (which would leak across the JNI/event boundary) and must give up on the upload. + try { + queue.submit("db-shutdown", "default", sstFile.toString(), + "db-shutdown/000009.sst", new IOException("initial upload failed")); + } catch (Exception e) { + fail("submit() must not throw when scheduler is shut down; caught: " + e); + } + + assertEquals(1L, queue.getGivenUpCount(), "Rejected retry must be counted as given up"); + assertEquals(0, uploadCalls.get(), "No retry may run once the scheduler is shut down"); + } + + @Test + public void close_givesUpOnPendingScheduledRetries() throws Exception { + // Provider always fails, so the first attempt schedules a retry. A long initial delay keeps + // that retry PENDING (scheduled, not yet run) so close()'s forced shutdown must give up on + // it explicitly (counted + logged) instead of silently dropping the upload. + AtomicInteger uploadCalls = new AtomicInteger(0); + CapturingProvider provider = new CapturingProvider() { + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + uploadCalls.incrementAndGet(); + throw new IOException("always fails"); + } + }; + + Path sstFile = tmpRoot.resolve("000042.sst"); + Files.createFile(sstFile); + + CloudUploadRetryQueue queue = + CloudStorageTestFactory.newRetryQueue(provider, 3, 60_000L, 120_000L); + // First failure schedules attempt #1 ~60s out — it will still be pending at close(). + queue.submit("db-x", "default", sstFile.toString(), "db-x/000042.sst", + new IOException("initial failure")); + + // Let the submit schedule the retry. + long deadline = System.currentTimeMillis() + 2_000L; + while (queue.getInFlightCount() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(20); + } + assertEquals(0L, + queue.getGivenUpCount(), + "Retry must be scheduled (pending), not yet given up"); + + // close() force-stops the scheduler after the grace period; the pending retry must be + // given up on. + queue.close(); + + assertEquals(1L, + queue.getGivenUpCount(), + "Pending scheduled retry must be given up on close"); + assertEquals(0, uploadCalls.get(), "The pending retry must never have run"); + } + + @Test + public void close_givesUpOnInFlightHungRetry() throws Exception { + // A retry that has STARTED but is hung in a slow/unresponsive provider.uploadFile(...) at + // shutdown must still be given up on explicitly (not just scheduled-but-unrun retries), + // so its staging pin is released and the loss is counted rather than silent. + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean uploadStarted = new AtomicBoolean(false); + CapturingProvider provider = new CapturingProvider() { + @Override + public void uploadFile(String localPath, String remoteKey) { + uploadStarted.set(true); + // Block UNINTERRUPTIBLY, simulating cloud I/O that ignores the shutdown interrupt. + boolean done = false; + while (!done) { + try { + release.await(); + done = true; + } catch (InterruptedException ie) { + // swallow — keep hanging so the retry is genuinely stuck in-flight + } + } + } + }; + + Path sstFile = tmpRoot.resolve("000077.sst"); + Files.createFile(sstFile); + + CloudUploadRetryQueue queue = + CloudStorageTestFactory.newRetryQueue(provider, 3, 50L, 200L); + // The scheduled retry starts quickly and then hangs inside uploadFile. + queue.submit("db-h", "default", sstFile.toString(), "db-h/000077.sst", + new IOException("initial failure")); + + long deadline = System.currentTimeMillis() + 3_000L; + while (!uploadStarted.get() && System.currentTimeMillis() < deadline) { + Thread.sleep(20); + } + assertTrue(uploadStarted.get(), "Retry upload must have started (in-flight)"); + assertEquals(0L, queue.getGivenUpCount(), "Nothing given up before shutdown"); + + try { + // Force a fast shutdown; the in-flight (hung) retry must be given up on. + queue.close(200, TimeUnit.MILLISECONDS); + + assertEquals(1L, + queue.getGivenUpCount(), + "In-flight hung retry must be given up on forced shutdown"); + } finally { + release.countDown(); // let the hung thread exit + } + } + + @Test + public void giveUp_releasesStagingPinButKeepsSourceSst() throws Exception { + Path source = Files.writeString(tmpRoot.resolve("000007.sst"), "sst"); + Path pin = Files.createLink(tmpRoot.resolve("000007.sst.upload-1"), source); + // maxAttempts=0 gives up on the task immediately. + try (CloudUploadRetryQueue queue = + CloudStorageTestFactory.newRetryQueue(new CapturingProvider(), 0, + 50L, 50L)) { + queue.submitPinned("db", "cf", pin.toString(), source.toString(), "db/000007.sst", + 1L, new IOException("outage")); + + assertEquals(1L, queue.getGivenUpCount()); + assertFalse(Files.exists(pin), "give-up must release the staging hard-link"); + assertTrue(Files.exists(source), "give-up must not touch the source SST"); + } + } + + // ----------------------------------------------------------------------- + // De-duplication of pending retries per (db, remoteKey) + // ----------------------------------------------------------------------- + + @Test + public void submit_duplicateForSameDbAndKey_isSkippedWhileRetryPending() throws Exception { + Path sstFile = Files.createFile(tmpRoot.resolve("000020.sst")); + CapturingProvider provider = new CapturingProvider(); + + // Long initial delay keeps the first retry pending for the whole test. + CloudUploadRetryQueue queue = + CloudStorageTestFactory.newRetryQueue(provider, 3, 30_000L, 30_000L); + try { + assertFalse(queue.isRetryPending("db-dup", "db-dup/000020.sst")); + + queue.submit("db-dup", "default", sstFile.toString(), "db-dup/000020.sst", + new IOException("first")); + assertTrue(queue.isRetryPending("db-dup", "db-dup/000020.sst")); + + queue.submit("db-dup", "default", sstFile.toString(), "db-dup/000020.sst", + new IOException("second")); + // A different key is independent. + assertFalse(queue.isRetryPending("db-dup", "db-dup/000021.sst")); + assertFalse(queue.isRetryPending("other-db", "db-dup/000020.sst")); + } finally { + queue.close(); + } + // Only the single pending task is given up at shutdown: the duplicate was never queued. + assertEquals(1L, queue.getGivenUpCount(), + "A duplicate submit must not create a second pending retry"); + assertFalse(queue.isRetryPending("db-dup", "db-dup/000020.sst"), + "Giving up must release the dedupe key"); + } + + @Test + public void submitPinned_duplicate_releasesItsOwnStagingPinButKeepsSource() throws Exception { + Path source = Files.writeString(tmpRoot.resolve("000030.sst"), "sst"); + Path pin1 = Files.createLink(tmpRoot.resolve("000030.sst.upload-1"), source); + Path pin2 = Files.createLink(tmpRoot.resolve("000030.sst.upload-2"), source); + + CloudUploadRetryQueue queue = + CloudStorageTestFactory.newRetryQueue(new CapturingProvider(), 3, 30_000L, 30_000L); + try { + queue.submitPinned("db", "cf", pin1.toString(), source.toString(), "db/000030.sst", + 1L, new IOException("outage")); + queue.submitPinned("db", "cf", pin2.toString(), source.toString(), "db/000030.sst", + 1L, new IOException("outage")); + + assertTrue(Files.exists(pin1), "The pending retry keeps its own staging pin"); + assertFalse(Files.exists(pin2), "The duplicate must release its own staging pin"); + assertTrue(Files.exists(source), "The source SST is never touched"); + } finally { + queue.close(); + } + } + + @Test + public void retrySuccess_releasesDedupeKeyAllowingResubmit() throws Exception { + Path sstFile = Files.createFile(tmpRoot.resolve("000040.sst")); + AtomicInteger uploadCount = new AtomicInteger(); + CapturingProvider provider = new CapturingProvider() { + @Override + public void uploadFile(String localPath, String remoteKey) { + uploadCount.incrementAndGet(); + } + }; + + try (CloudUploadRetryQueue queue = + CloudStorageTestFactory.newRetryQueue(provider, 3, 50L, 100L)) { + queue.submit("db-rel", "default", sstFile.toString(), "db-rel/000040.sst", + new IOException("outage")); + awaitTrue(() -> uploadCount.get() == 1); + awaitTrue(() -> !queue.isRetryPending("db-rel", "db-rel/000040.sst")); + + queue.submit("db-rel", "default", sstFile.toString(), "db-rel/000040.sst", + new IOException("outage again")); + awaitTrue(() -> uploadCount.get() == 2); + } + } + + private static void awaitTrue(java.util.function.BooleanSupplier condition) + throws InterruptedException { + long deadline = System.currentTimeMillis() + 3_000L; + while (!condition.getAsBoolean() && System.currentTimeMillis() < deadline) { + Thread.sleep(20L); + } + assertTrue(condition.getAsBoolean(), "condition not reached within 3s"); + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + @SuppressWarnings("ResultOfMethodCallIgnored") + private void deleteRecursively(java.io.File f) { + if (f.isDirectory()) { + for (java.io.File child : Objects.requireNonNull(f.listFiles())) { + deleteRecursively(child); + } + } + f.delete(); + } + + // ----------------------------------------------------------------------- + // Stub providers + // ----------------------------------------------------------------------- + + static class CapturingProvider implements CloudStorageProvider { + + final List uploads = new ArrayList<>(); + + @Override + public String providerName() { + return "capturing"; + } + + @Override + public void init(CloudStorageConfig config) { + } + + @Override + public void uploadFile(String localPath, String remoteKey) throws IOException { + uploads.add(new String[]{localPath, remoteKey}); + } + + @Override + public void deleteFile(String remoteKey) { + } + + @Override + public boolean fileExists(String remoteKey) { + return false; + } + + @Override + public List listFiles(String remoteDirPrefix) { + return Collections.emptyList(); + } + + @Override + public int deletePrefix(String remoteDirPrefix) { + return 0; + } + + @Override + public void downloadFile(String remoteKey, String localPath) { + } + + @Override + public void close() { + } + } +} + + + + + + + + + diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/controller/PartitionAPITest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/controller/PartitionAPITest.java index b37565c8ab..2d299283f3 100644 --- a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/controller/PartitionAPITest.java +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/controller/PartitionAPITest.java @@ -29,8 +29,8 @@ import org.apache.hugegraph.store.PartitionEngine; import org.apache.hugegraph.store.business.BusinessHandler; import org.apache.hugegraph.store.node.grpc.HgStoreNodeService; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import com.alipay.sofa.jraft.Node; import com.alipay.sofa.jraft.conf.Configuration; @@ -51,19 +51,19 @@ public void testGetPartitionsOnFollower() { List rafts = (List) api.getPartitions("").get("partitions"); - Assert.assertEquals(2, rafts.size()); + Assertions.assertEquals(2, rafts.size()); PartitionAPI.Raft followerRaft = rafts.get(0); - Assert.assertEquals(1, followerRaft.getGroupId()); - Assert.assertEquals(State.STATE_FOLLOWER.name(), followerRaft.getRole()); - Assert.assertNull(followerRaft.getConf()); - Assert.assertNull(followerRaft.getPeers()); - Assert.assertNull(followerRaft.getLearners()); + Assertions.assertEquals(1, followerRaft.getGroupId()); + Assertions.assertEquals(State.STATE_FOLLOWER.name(), followerRaft.getRole()); + Assertions.assertNull(followerRaft.getConf()); + Assertions.assertNull(followerRaft.getPeers()); + Assertions.assertNull(followerRaft.getLearners()); PartitionAPI.Raft leaderRaft = rafts.get(1); - Assert.assertEquals(2, leaderRaft.getGroupId()); - Assert.assertEquals(State.STATE_LEADER.name(), leaderRaft.getRole()); - Assert.assertEquals("127.0.0.1:8510", leaderRaft.getConf()); - Assert.assertEquals(Collections.singletonList(PeerId.parsePeer("127.0.0.1:8510")), + Assertions.assertEquals(2, leaderRaft.getGroupId()); + Assertions.assertEquals(State.STATE_LEADER.name(), leaderRaft.getRole()); + Assertions.assertEquals("127.0.0.1:8510", leaderRaft.getConf()); + Assertions.assertEquals(Collections.singletonList(PeerId.parsePeer("127.0.0.1:8510")), leaderRaft.getPeers()); } @@ -77,12 +77,12 @@ public void testGetPartitionsWhenLeaderStepsDown() { List rafts = (List) api.getPartitions("").get("partitions"); - Assert.assertEquals(1, rafts.size()); + Assertions.assertEquals(1, rafts.size()); PartitionAPI.Raft raft = rafts.get(0); - Assert.assertEquals(1, raft.getGroupId()); - Assert.assertNull(raft.getConf()); - Assert.assertNull(raft.getPeers()); - Assert.assertNull(raft.getLearners()); + Assertions.assertEquals(1, raft.getGroupId()); + Assertions.assertNull(raft.getConf()); + Assertions.assertNull(raft.getPeers()); + Assertions.assertNull(raft.getLearners()); } private static PartitionAPI newApi(PartitionEngine... engines) { diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/grpc/PartitionStateMachineGrpcTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/grpc/PartitionStateMachineGrpcTest.java index 575ed56369..8c1e1eb597 100644 --- a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/grpc/PartitionStateMachineGrpcTest.java +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/grpc/PartitionStateMachineGrpcTest.java @@ -17,9 +17,9 @@ package org.apache.hugegraph.store.node.grpc; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; @@ -48,7 +48,7 @@ import org.apache.hugegraph.store.raft.PartitionStateMachine; import org.apache.hugegraph.store.raft.RaftOperation; import org.apache.hugegraph.store.snapshot.SnapshotHandler; -import org.junit.Test; +import org.junit.jupiter.api.Test; import com.alipay.sofa.jraft.Iterator; import com.alipay.sofa.jraft.Status; @@ -116,8 +116,8 @@ public void run(Status status) { stateMachine.onApply(iterator); - assertNotNull("handler must retain the typed response for op " + methods[i], - response.getResult()); + assertNotNull(response.getResult(), + "handler must retain the typed response for op " + methods[i]); assertEquals(ResCode.RES_CODE_OK, response.getResult().getStatus().getCode()); assertEquals(1, callbacks.get()); assertTrue(callbackStatus.get().isOk()); diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/grpc/ScanUtilTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/grpc/ScanUtilTest.java index 6f2df672c4..7ee44c6636 100644 --- a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/grpc/ScanUtilTest.java +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/grpc/ScanUtilTest.java @@ -31,8 +31,8 @@ import org.apache.hugegraph.store.grpc.stream.ScanQueryRequest; import org.apache.hugegraph.store.grpc.stream.ScanStreamReq; import org.apache.hugegraph.store.node.AppConfig; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import io.grpc.stub.StreamObserver; @@ -46,12 +46,12 @@ public void testBatchScanRejectsOrderByKey() { .setOrderType(ScanOrderType.ORDER_BY_KEY) .build(); - IllegalArgumentException error = Assert.assertThrows( + IllegalArgumentException error = Assertions.assertThrows( IllegalArgumentException.class, () -> ScanQueryProducer.requestOf("graph", new String[]{"table"}, request)); - Assert.assertTrue(error.getMessage().contains("batch scan")); + Assertions.assertTrue(error.getMessage().contains("batch scan")); } @Test @@ -62,9 +62,9 @@ public void testOrderedAllPartitionRangeUsesOrderedScan() { ScanIterator iterator = ScanUtil.getIterator(request, wrapper); BackendColumn column = iterator.next(); - Assert.assertEquals(1, column.name[0] & 0xff); - Assert.assertTrue(wrapper.orderedCalled); - Assert.assertFalse(wrapper.legacyCalled); + Assertions.assertEquals(1, column.name[0] & 0xff); + Assertions.assertTrue(wrapper.orderedCalled); + Assertions.assertFalse(wrapper.legacyCalled); } @Test @@ -75,9 +75,9 @@ public void testUnorderedAllPartitionRangeUsesLegacyScan() { ScanIterator iterator = ScanUtil.getIterator(request, wrapper); BackendColumn column = iterator.next(); - Assert.assertEquals(9, column.name[0] & 0xff); - Assert.assertFalse(wrapper.orderedCalled); - Assert.assertTrue(wrapper.legacyCalled); + Assertions.assertEquals(9, column.name[0] & 0xff); + Assertions.assertFalse(wrapper.orderedCalled); + Assertions.assertTrue(wrapper.legacyCalled); } @Test @@ -90,12 +90,12 @@ public void testOrderedScanRejectsOpaquePositionBeforeOpeningIterator() { try { ScanUtil.getIterator(request, wrapper); - Assert.fail("Expected ordered cursor rejection"); + Assertions.fail("Expected ordered cursor rejection"); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains("position")); + Assertions.assertTrue(e.getMessage().contains("position")); } - Assert.assertFalse(wrapper.orderedCalled); - Assert.assertFalse(wrapper.legacyCalled); + Assertions.assertFalse(wrapper.orderedCalled); + Assertions.assertFalse(wrapper.legacyCalled); } @Test @@ -108,11 +108,11 @@ public void testOneShotOrderedRangeUsesOrderedScanAndAcknowledgesIt() { ScanOneShotResponse.scanOneShot(request, observer(response, error), wrapper); - Assert.assertNull(error.get()); - Assert.assertNotNull(response.get()); - Assert.assertEquals(1, response.get().getVersion()); - Assert.assertTrue(wrapper.orderedCalled); - Assert.assertFalse(wrapper.legacyCalled); + Assertions.assertNull(error.get()); + Assertions.assertNotNull(response.get()); + Assertions.assertEquals(1, response.get().getVersion()); + Assertions.assertTrue(wrapper.orderedCalled); + Assertions.assertFalse(wrapper.legacyCalled); } @Test @@ -134,12 +134,12 @@ public void testOrderedStreamCloseAcknowledgesCapabilityBeforeFirstPage() { stream.onNext(close); - Assert.assertNull(error.get()); - Assert.assertNotNull(response.get()); - Assert.assertTrue(response.get().getOver()); - Assert.assertEquals(1, response.get().getVersion()); - Assert.assertFalse(wrapper.orderedCalled); - Assert.assertFalse(wrapper.legacyCalled); + Assertions.assertNull(error.get()); + Assertions.assertNotNull(response.get()); + Assertions.assertTrue(response.get().getOver()); + Assertions.assertEquals(1, response.get().getVersion()); + Assertions.assertFalse(wrapper.orderedCalled); + Assertions.assertFalse(wrapper.legacyCalled); } finally { executor.shutdownNow(); } diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/CloudStorageMetricsTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/CloudStorageMetricsTest.java new file mode 100644 index 0000000000..b7ec5377ba --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/CloudStorageMetricsTest.java @@ -0,0 +1,460 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.metrics; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.lang.reflect.Field; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hugegraph.store.node.cloud.CloudStorageMetrics; +import org.apache.hugegraph.store.node.cloud.CloudStorageMetricsConst; +import org.apache.hugegraph.store.node.cloud.CloudSyncTracker; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.FunctionCounter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +/** + * Unit tests for {@link CloudStorageMetrics}. + * Includes the former lifecycle/cardinality checks from + * {@code CloudStorageMetricsLifecycleTest}. + * + *

Because {@link CloudStorageMetrics} uses static singleton state, each test + * resets all static fields via reflection in {@link #tearDown()} so that tests + * remain independent. + */ +public class CloudStorageMetricsTest { + + private SimpleMeterRegistry registry; + private CloudSyncTracker syncTracker; + + @BeforeEach + public void setUp() { + registry = new SimpleMeterRegistry(); + syncTracker = new CloudSyncTracker(); + resetStaticState(); + } + + @AfterEach + public void tearDown() { + resetStaticState(); + } + + // ----------------------------------------------------------------------- + // init() + // ----------------------------------------------------------------------- + + @Test + public void testInitRegistersGlobalMetrics() { + CloudStorageMetrics.init(registry, syncTracker); + + assertNotNull(registry.find(CloudStorageMetricsConst.UPLOAD_FAILURES).counter(), + "upload failures counter must be registered"); + assertNotNull(registry.find(CloudStorageMetricsConst.SYNC_LATENCY_MS).timer(), + "sync latency timer must be registered"); + assertNotNull(registry.find(CloudStorageMetricsConst.RETRY_QUEUE_SIZE).gauge(), + "retry queue size gauge must be registered"); + } + + @Test + public void testUploadsGivenUpCounterReflectsBoundSupplier() { + CloudStorageMetrics.init(registry, syncTracker); + + FunctionCounter counter = + registry.find(CloudStorageMetricsConst.UPLOADS_GIVEN_UP).functionCounter(); + assertNotNull(counter, "uploads-given-up function counter must be registered"); + assertEquals(0.0, counter.count(), "unbound counter must report 0"); + + try { + CloudStorageMetrics.bindUploadsGivenUpSupplier(() -> 7L); + assertEquals(7.0, counter.count(), "counter must report the bound supplier's value"); + } finally { + // The supplier is static; unbind so it does not leak into other tests. + CloudStorageMetrics.bindUploadsGivenUpSupplier(null); + } + assertEquals(0.0, counter.count(), "unbinding must fall back to 0"); + } + + @Test + public void testInitIsIdempotentSecondCallIsNoOp() { + CloudStorageMetrics.init(registry, syncTracker); + + // A second registry should be ignored + SimpleMeterRegistry secondRegistry = new SimpleMeterRegistry(); + CloudStorageMetrics.init(secondRegistry, syncTracker); + + // Upload failures counter must still be registered on the first registry + assertNotNull(registry.find(CloudStorageMetricsConst.UPLOAD_FAILURES).counter()); + // Nothing should be registered on the second registry + assertEquals(0, secondRegistry.getMeters().size()); + } + + @Test + public void testInitNullRegistryThrowsIllegalArgument() { + assertThrows(IllegalArgumentException.class, + () -> CloudStorageMetrics.init(null, syncTracker)); + } + + @Test + public void testInitNullTrackerThrowsIllegalArgument() { + assertThrows(IllegalArgumentException.class, + () -> CloudStorageMetrics.init(registry, null)); + } + + // ----------------------------------------------------------------------- + // registerDatabaseMetrics() + // ----------------------------------------------------------------------- + + @Test + public void testRegisterDatabaseMetricsBeforeInitIsNoOp() { + // No init — should not throw and registry must remain empty + CloudStorageMetrics.registerDatabaseMetrics("db1"); + assertEquals(0, registry.getMeters().size()); + } + + @Test + public void testRegisterDatabaseMetricsRegistersConfirmedFilesGauge() { + CloudStorageMetrics.init(registry, syncTracker); + CloudStorageMetrics.registerDatabaseMetrics("mydb"); + + Gauge gauge = registry.find(CloudStorageMetricsConst.CONFIRMED_FILES) + .tag(CloudStorageMetricsConst.TAG_DB_NAME, "mydb") + .gauge(); + assertNotNull(gauge, "confirmed_files gauge must be registered for the database"); + } + + @Test + public void testRegisterDatabaseMetricsRegistersDeleteGuardReuploadGauge() { + CloudStorageMetrics.init(registry, syncTracker); + CloudStorageMetrics.registerDatabaseMetrics("mydb"); + + Gauge gauge = registry.find(CloudStorageMetricsConst.DELETE_GUARD_REUPLOAD_COUNT) + .tag(CloudStorageMetricsConst.TAG_DB_NAME, "mydb") + .gauge(); + assertNotNull(gauge, + "delete_guard_reupload_count gauge must be registered for the database"); + } + + @Test + public void testRegisterDatabaseMetricsIsIdempotentSecondCallIsNoOp() { + CloudStorageMetrics.init(registry, syncTracker); + CloudStorageMetrics.registerDatabaseMetrics("mydb"); + + // Count meters after first registration + int metersAfterFirst = registry.getMeters().size(); + + // A second call for the same database should not add new meters + CloudStorageMetrics.registerDatabaseMetrics("mydb"); + assertEquals(metersAfterFirst, registry.getMeters().size()); + } + + @Test + public void testRegisterDatabaseMetricsMultipleDatabasesEachGetsSeparateGauges() { + CloudStorageMetrics.init(registry, syncTracker); + CloudStorageMetrics.registerDatabaseMetrics("db-alpha"); + CloudStorageMetrics.registerDatabaseMetrics("db-beta"); + + assertNotNull(registry.find(CloudStorageMetricsConst.CONFIRMED_FILES) + .tag(CloudStorageMetricsConst.TAG_DB_NAME, "db-alpha").gauge()); + assertNotNull(registry.find(CloudStorageMetricsConst.CONFIRMED_FILES) + .tag(CloudStorageMetricsConst.TAG_DB_NAME, "db-beta").gauge()); + } + + // ----------------------------------------------------------------------- + // recordUploadFailure() + // ----------------------------------------------------------------------- + + @Test + public void testRecordUploadFailureBeforeInitIsNoOp() { + // Must not throw + CloudStorageMetrics.recordUploadFailure("db1", "default", "timeout"); + } + + @Test + public void testRecordUploadFailureIncrementsCounter() { + CloudStorageMetrics.init(registry, syncTracker); + + CloudStorageMetrics.recordUploadFailure("db1", "default", "timeout"); + CloudStorageMetrics.recordUploadFailure("db1", "default", "auth"); + + Counter counter = registry.find(CloudStorageMetricsConst.UPLOAD_FAILURES).counter(); + assertNotNull(counter); + assertEquals(2.0, counter.count(), 0.001); + } + + @Test + public void testRecordUploadFailureMultipleCallsAccumulate() { + CloudStorageMetrics.init(registry, syncTracker); + + for (int i = 0; i < 5; i++) { + CloudStorageMetrics.recordUploadFailure("db1", "cf1", "error-" + i); + } + + Counter counter = registry.find(CloudStorageMetricsConst.UPLOAD_FAILURES).counter(); + assertNotNull(counter); + assertEquals(5.0, counter.count(), 0.001); + } + + // ----------------------------------------------------------------------- + // recordSyncLatency() + // ----------------------------------------------------------------------- + + @Test + public void testRecordSyncLatencyBeforeInitIsNoOp() { + // Must not throw + CloudStorageMetrics.recordSyncLatency("db1", 250L); + } + + @Test + public void testRecordSyncLatencyRecordsMeasurement() { + CloudStorageMetrics.init(registry, syncTracker); + + CloudStorageMetrics.recordSyncLatency("db1", 100L); + CloudStorageMetrics.recordSyncLatency("db1", 200L); + + Timer timer = registry.find(CloudStorageMetricsConst.SYNC_LATENCY_MS).timer(); + assertNotNull(timer); + assertEquals(2, timer.count()); + assertEquals(300.0, timer.totalTime(TimeUnit.MILLISECONDS), 1.0); + } + + @Test + public void testRecordSyncLatencyZeroLatencyIsRecorded() { + CloudStorageMetrics.init(registry, syncTracker); + CloudStorageMetrics.recordSyncLatency("db1", 0L); + + Timer timer = registry.find(CloudStorageMetricsConst.SYNC_LATENCY_MS).timer(); + assertNotNull(timer); + assertEquals(1, timer.count()); + } + + // ----------------------------------------------------------------------- + // getConfirmedFileCount() + // ----------------------------------------------------------------------- + + @Test + public void testGetConfirmedFileCountBeforeInitReturnsZero() { + assertEquals(0L, CloudStorageMetrics.getConfirmedFileCount("db1")); + } + + @Test + public void testGetConfirmedFileCountAfterInitDelegatesToSyncTracker() { + // Confirm a file in the tracker then query via metrics + syncTracker.markConfirmed("db1", "db1/000001.sst"); + syncTracker.markConfirmed("db1", "db1/000002.sst"); + + CloudStorageMetrics.init(registry, syncTracker); + + assertEquals(2L, CloudStorageMetrics.getConfirmedFileCount("db1")); + } + + @Test + public void testGetConfirmedFileCountUnknownDbReturnsZero() { + CloudStorageMetrics.init(registry, syncTracker); + assertEquals(0L, CloudStorageMetrics.getConfirmedFileCount("no-such-db")); + } + + // ----------------------------------------------------------------------- + // incrementDeleteGuardReupload() / getDeleteGuardReuploadCount() + // ----------------------------------------------------------------------- + + @Test + public void testGetDeleteGuardReuploadCountUnknownDbReturnsZero() { + assertEquals(0L, CloudStorageMetrics.getDeleteGuardReuploadCount("unknown-db")); + } + + @Test + public void testIncrementDeleteGuardReuploadIncrementsCountForDb() { + CloudStorageMetrics.incrementDeleteGuardReupload("db1"); + CloudStorageMetrics.incrementDeleteGuardReupload("db1"); + CloudStorageMetrics.incrementDeleteGuardReupload("db1"); + + assertEquals(3L, CloudStorageMetrics.getDeleteGuardReuploadCount("db1")); + } + + @Test + public void testIncrementDeleteGuardReuploadCountsAreIsolatedPerDb() { + CloudStorageMetrics.incrementDeleteGuardReupload("db-a"); + CloudStorageMetrics.incrementDeleteGuardReupload("db-a"); + CloudStorageMetrics.incrementDeleteGuardReupload("db-b"); + + assertEquals(2L, CloudStorageMetrics.getDeleteGuardReuploadCount("db-a")); + assertEquals(1L, CloudStorageMetrics.getDeleteGuardReuploadCount("db-b")); + } + + @Test + public void testIncrementDeleteGuardReuploadBeforeRegisterDbStillWorks() { + // Incrementing before registerDatabaseMetrics() should still track the count + CloudStorageMetrics.incrementDeleteGuardReupload("db-new"); + assertEquals(1L, CloudStorageMetrics.getDeleteGuardReuploadCount("db-new")); + } + + @Test + public void testDeleteGuardReuploadGaugeReflectsLiveCount() { + CloudStorageMetrics.init(registry, syncTracker); + CloudStorageMetrics.registerDatabaseMetrics("mydb"); + + CloudStorageMetrics.incrementDeleteGuardReupload("mydb"); + CloudStorageMetrics.incrementDeleteGuardReupload("mydb"); + + Gauge gauge = registry.find(CloudStorageMetricsConst.DELETE_GUARD_REUPLOAD_COUNT) + .tag(CloudStorageMetricsConst.TAG_DB_NAME, "mydb") + .gauge(); + assertNotNull(gauge); + assertEquals(2.0, gauge.value(), 0.001); + } + + @Test + public void testConfirmedFilesGaugeReflectsTrackerState() { + CloudStorageMetrics.init(registry, syncTracker); + CloudStorageMetrics.registerDatabaseMetrics("mydb"); + + syncTracker.markConfirmed("mydb", "mydb/000001.sst"); + syncTracker.markConfirmed("mydb", "mydb/000002.sst"); + syncTracker.markConfirmed("mydb", "mydb/000003.sst"); + + Gauge gauge = registry.find(CloudStorageMetricsConst.CONFIRMED_FILES) + .tag(CloudStorageMetricsConst.TAG_DB_NAME, "mydb") + .gauge(); + assertNotNull(gauge); + assertEquals(3.0, gauge.value(), 0.001); + } + + // ----------------------------------------------------------------------- + // setRetryQueueSize() — placeholder method, must not throw + // ----------------------------------------------------------------------- + + @Test + public void testSetRetryQueueSizeDoesNotThrow() { + CloudStorageMetrics.setRetryQueueSize(0); + CloudStorageMetrics.setRetryQueueSize(42); + CloudStorageMetrics.setRetryQueueSize(Integer.MAX_VALUE); + } + + // ----------------------------------------------------------------------- + // unregisterDatabaseMetrics() — per-DB meter cleanup (cardinality bound) + // ----------------------------------------------------------------------- + // Ensures per-database meters are removed when a DB is deleted, so meter cardinality does not + // grow without bound as databases are created and destroyed (partition rebalancing, graph drops). + + @Test + public void testPerDbMetricsAreRemovedOnUnregister() { + CloudStorageMetrics.init(registry, syncTracker); + CloudStorageMetrics.registerDatabaseMetrics("db-A"); + CloudStorageMetrics.registerDatabaseMetrics("db-B"); + + assertEquals(2, + registeredDbMetricCount(), + "both DBs must have per-DB meters registered"); + assertNotNull(confirmedFilesGaugeFor("db-A"), "db-A confirmed-files gauge must exist"); + assertNotNull(confirmedFilesGaugeFor("db-B"), "db-B confirmed-files gauge must exist"); + + // Simulate DB deletion for db-A. + CloudStorageMetrics.unregisterDatabaseMetrics("db-A"); + + assertEquals(1, + registeredDbMetricCount(), + "only db-B must remain after db-A is unregistered"); + assertNull(confirmedFilesGaugeFor("db-A"), + "db-A gauge must be removed from the registry (no cardinality leak)"); + assertNotNull(confirmedFilesGaugeFor("db-B"), "db-B gauge must be untouched"); + } + + @Test + public void testUnregisterThenReregisterWorksForRecreatedDb() { + CloudStorageMetrics.init(registry, syncTracker); + CloudStorageMetrics.registerDatabaseMetrics("db-A"); + CloudStorageMetrics.unregisterDatabaseMetrics("db-A"); + assertNull(confirmedFilesGaugeFor("db-A")); + + // A DB recreated at the same name must get fresh metrics. + CloudStorageMetrics.registerDatabaseMetrics("db-A"); + assertNotNull(confirmedFilesGaugeFor("db-A"), "recreated DB must re-register its metrics"); + assertEquals(1, registeredDbMetricCount()); + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + /** + * Resets all static fields in {@link CloudStorageMetrics} to {@code null} / empty + * via reflection so that each test starts from a clean slate. + */ + @SuppressWarnings("unchecked") + private static void resetStaticState() { + try { + setStaticField("registry"); + setStaticField("syncTracker"); + setStaticField("uploadFailuresCounter"); + setStaticField("syncLatencyTimer"); + + Field mapField = CloudStorageMetrics.class + .getDeclaredField("deleteGuardReuploadPerDb"); + mapField.setAccessible(true); + ((ConcurrentHashMap) mapField.get(null)).clear(); + + Field perDbMetersField = CloudStorageMetrics.class.getDeclaredField("perDbMeters"); + perDbMetersField.setAccessible(true); + ((java.util.Map) perDbMetersField.get(null)).clear(); + } catch (Exception e) { + throw new RuntimeException("Failed to reset CloudStorageMetrics static state", e); + } + } + + private static void setStaticField(String fieldName) + throws NoSuchFieldException, IllegalAccessException { + Field field = CloudStorageMetrics.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(null, (Object) null); + } + + private Gauge confirmedFilesGaugeFor(String dbName) { + return registry.find(CloudStorageMetricsConst.CONFIRMED_FILES) + .tag(CloudStorageMetricsConst.TAG_DB_NAME, dbName) + .gauge(); + } + + /** + * Number of databases with registered per-DB meters. Read via reflection on {@code perDbMeters} + * because this test lives outside {@code CloudStorageMetrics}'s package and cannot call the + * package-private {@code registeredDbMetricCountForTest()} helper. + */ + private static int registeredDbMetricCount() { + try { + Field f = CloudStorageMetrics.class.getDeclaredField("perDbMeters"); + f.setAccessible(true); + return ((java.util.Map) f.get(null)).size(); + } catch (Exception e) { + throw new RuntimeException("Failed to read CloudStorageMetrics.perDbMeters", e); + } + } +} + + diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java index ec44123a33..fe565d0f69 100644 --- a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/metrics/JraftMetricsTest.java @@ -21,7 +21,7 @@ import java.util.List; import org.apache.hugegraph.store.node.util.HgRegexUtil; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * 2022/3/7 diff --git a/hugegraph-store/hg-store-rocksdb/pom.xml b/hugegraph-store/hg-store-rocksdb/pom.xml index f8eb4ba58f..77f55d3363 100644 --- a/hugegraph-store/hg-store-rocksdb/pom.xml +++ b/hugegraph-store/hg-store-rocksdb/pom.xml @@ -73,5 +73,14 @@ fastjson 1.2.84 + + + org.mockito + mockito-core + 4.1.0 + test + diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java index 2e8e0bae68..669e9dcfca 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBFactory.java @@ -24,11 +24,14 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @@ -36,19 +39,46 @@ import org.apache.commons.io.FileUtils; import org.apache.hugegraph.config.HugeConfig; import org.rocksdb.AbstractEventListener; +import org.rocksdb.AbstractEventListener.EnabledEventCallback; import org.rocksdb.Cache; import org.rocksdb.CompactionJobInfo; +import org.rocksdb.LiveFileMetaData; import org.rocksdb.MemoryUsageType; import org.rocksdb.MemoryUtil; import org.rocksdb.RocksDB; +import org.rocksdb.Status; +import org.rocksdb.TableFileCreationInfo; +import org.rocksdb.TableFileDeletionInfo; import lombok.extern.slf4j.Slf4j; @Slf4j public final class RocksDBFactory { - private static final List rocksdbChangedListeners = new ArrayList<>(); + // CopyOnWriteArrayList: the list is iterated (forEach / for-each) from RocksDB JNI callback + // threads (onTableFileCreated/Deleted, compaction) while add/removeRocksdbChangedListener can + // mutate it from the main thread (Spring teardown, test @After). A plain ArrayList would throw + // ConcurrentModificationException on the callback thread; COW gives each iteration a stable + // snapshot and makes mutation safe without locking the hot callback path. + private static final List rocksdbChangedListeners = + new CopyOnWriteArrayList<>(); + /** Upper bound a createGraphDB waiter blocks on another thread's in-flight creation. */ + static final long CREATION_WAIT_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(10); + /** Upper bound releaseAllGraphDB waits for in-flight creations to finish. */ + static final long CREATION_DRAIN_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(5); + /** + * Native callbacks enabled for {@link RocksdbEventListener}: exactly the {@code on*} methods it + * overrides (each enabled callback costs a JNI upcall). Keep in sync when adding an override. + */ + static final EnabledEventCallback[] ENABLED_EVENT_CALLBACKS = { + EnabledEventCallback.ON_COMPACTION_BEGIN, + EnabledEventCallback.ON_COMPACTION_COMPLETED, + EnabledEventCallback.ON_TABLE_FILE_CREATED, + EnabledEventCallback.ON_TABLE_FILE_DELETED + }; private static RocksDBFactory dbFactory; + /** Singleton event listener wired into every new RocksDB instance. */ + private final RocksdbEventListener rocksdbEventListener = new RocksdbEventListener(); static { RocksDB.loadLibrary(); @@ -57,63 +87,138 @@ public final class RocksDBFactory { private final Map dbSessionMap = new ConcurrentHashMap<>(); private final List destroyGraphDBs = new CopyOnWriteArrayList<>(); private final ReentrantReadWriteLock operateLock; + /** + * DBs currently being created (between Phase 1 and Phase 3), each with a future the creator + * completes when its slot is released. Guards the RocksDB LOCK file; waiters block on the + * future (with a timeout) instead of spinning on the global lock. + */ + private final Map> pendingCreations = + new ConcurrentHashMap<>(); ScheduledExecutorService scheduledExecutor; private HugeConfig hugeConfig; private AtomicBoolean closing = new AtomicBoolean(false); + /** Mutable so tests can shorten it. */ + volatile long creationWaitTimeoutMs = CREATION_WAIT_TIMEOUT_MS; private RocksDBFactory() { this.operateLock = new ReentrantReadWriteLock(); scheduledExecutor = Executors.newScheduledThreadPool(2); - scheduledExecutor.scheduleWithFixedDelay(() -> { - try { - dbSessionMap.forEach((k, session) -> { - for (var entry : session.getIteratorMap().entrySet()) { - String key = entry.getKey(); - var ts = Long.parseLong(key.split("-")[0]); - // output once per 10min - var passed = (System.currentTimeMillis() - ts) / 1000 - 600; - if (passed > 0 && passed % 10 == 0) { - log.info("iterator not close, stack: {}", entry.getValue()); - } + scheduledExecutor.scheduleWithFixedDelay(this::runMaintenancePass, 60, 60, + TimeUnit.SECONDS); + } + + /** One scheduled maintenance pass; package-private so tests can drive it directly. */ + void runMaintenancePass() { + logUnclosedIterators(); + processPendingDestroys(); + } + + /** + * Logs the creation stack of iterators left open for more than 10 minutes. Runs on the + * scheduled executor; package-private so tests can drive a pass directly. + */ + void logUnclosedIterators() { + try { + dbSessionMap.forEach((k, session) -> { + for (var entry : session.getIteratorMap().entrySet()) { + String key = entry.getKey(); + var ts = Long.parseLong(key.split("-")[0]); + // output once per 10min + var passed = (System.currentTimeMillis() - ts) / 1000 - 600; + if (passed > 0 && passed % 10 == 0) { + log.info("iterator not close, stack: {}", entry.getValue()); } - }); - } catch (Exception e) { - log.error("got error, ", e); - } + } + }); + } catch (Exception e) { + log.error("got error, ", e); + } + } - try { - Iterator itr = destroyGraphDBs.listIterator(); - while (itr.hasNext()) { - DBSessionWatcher watcher = itr.next(); - if (0 == watcher.dbSession.getRefCount()) { - try { - watcher.dbSession.shutdown(); - FileUtils.deleteDirectory(new File(watcher.dbSession.getDbPath())); - rocksdbChangedListeners.forEach(listener -> { - listener.onDBDeleted(watcher.dbSession.getGraphName(), - watcher.dbSession.getDbPath()); - }); - log.info("removed db {} and delete files", - watcher.dbSession.getDbPath()); - } catch (Exception e) { - log.error("DestroyGraphDB exception {}", e); - } + /** + * One pass over the destroy queue: deletes the files of each destroyed DB once its + * refCount drops to 0, and force-resets sessions stuck with live refs. Runs on the + * scheduled executor every 60s; package-private so tests can drive a pass directly. + * + *

Live refs are only force-reset after a 30-minute wait and a failed delete is retried up + * to 3 times, since dropping the watcher early or deleting against a recreated DB would leave + * or purge remote state wrongly. + */ + void processPendingDestroys() { + try { + Iterator itr = destroyGraphDBs.listIterator(); + while (itr.hasNext()) { + DBSessionWatcher watcher = itr.next(); + if (0 == watcher.dbSession.getRefCount()) { + // Destroy-then-recreate race guard: a concurrent createGraphDB may have + // re-occupied the same DB name (createGraphDB checks dbSessionMap and + // pendingCreations, not destroyGraphDBs). Only a live DB at the SAME path as + // this stale watcher must be protected from its delete; a live DB at another + // path leaves the stale directory to be deleted as usual. Check under the read lock so it serialises with + // createGraphDB's write-locked slot reservation and map insertion. + String staleName = watcher.dbSession.getGraphName(); + String stalePath = watcher.dbSession.getDbPath(); + boolean sameLivePath; + boolean creationPending; + operateLock.readLock().lock(); + try { + RocksDBSession live = dbSessionMap.get(staleName); + sameLivePath = live != null && stalePath.equals(live.getDbPath()); + creationPending = pendingCreations.containsKey(staleName); + } finally { + operateLock.readLock().unlock(); + } + if (sameLivePath) { + log.warn("DestroyGraphDB: db={} was recreated at path={} before " + + "cleanup ran — discarding stale destroy watcher without " + + "deleting files or purging cloud objects", + staleName, stalePath); destroyGraphDBs.remove(watcher); - } else if (watcher.timestamp < (System.currentTimeMillis() - 1800 * 1000)) { - log.warn("DB {} has not been deleted refCount is {}, time is {} seconds", - watcher.dbSession.getDbPath(), - watcher.dbSession.getRefCount(), - (System.currentTimeMillis() - watcher.timestamp) / 1000); - } else { - // Force delete after timeout (30min) - watcher.dbSession.forceResetRefCount(); + continue; } + if (creationPending) { + // The pending creation's path is not known yet; retry next round + // rather than risk deleting the directory it is about to open. + log.info("DestroyGraphDB: db={} has a creation in flight, deferring " + + "cleanup of path={}", staleName, stalePath); + continue; + } + try { + watcher.dbSession.shutdown(); + FileUtils.deleteDirectory(new File(stalePath)); + rocksdbChangedListeners.forEach(listener -> { + listener.onDBDeleted(staleName, stalePath); + }); + log.info("removed db {} and delete files", stalePath); + destroyGraphDBs.remove(watcher); + } catch (Exception e) { + watcher.deleteAttempts++; + if (watcher.deleteAttempts >= 3) { + // Give up after 3 consecutive failures: continuing to retry + // risks calling onDBDeleted (which purges cloud objects) against + // a DB that may have been recreated at the same path. + log.error("DestroyGraphDB: giving up after {} failed attempts for " + + "db={}: {}", watcher.deleteAttempts, stalePath, e); + destroyGraphDBs.remove(watcher); + } else { + log.error("DestroyGraphDB exception (attempt {}), will retry: {}", + watcher.deleteAttempts, e); + } + } + } else if (watcher.timestamp < (System.currentTimeMillis() - 1800 * 1000)) { + // Force delete after 30-min timeout: session is stuck with live refs. + watcher.dbSession.forceResetRefCount(); + } else { + log.warn("DB {} has not been deleted refCount is {}, time is {} seconds", + watcher.dbSession.getDbPath(), + watcher.dbSession.getRefCount(), + (System.currentTimeMillis() - watcher.timestamp) / 1000); } - - } catch (Exception e) { - log.error("RocksDBFactory scheduledExecutor exception {}", e); } - }, 60, 60, TimeUnit.SECONDS); + + } catch (Exception e) { + log.error("RocksDBFactory scheduledExecutor exception {}", e); + } } public static RocksDBFactory getInstance() { @@ -143,6 +248,17 @@ public void setHugeConfig(HugeConfig nodeConfig) { this.hugeConfig = nodeConfig; } + /** Test hook: enqueues a session for {@link #processPendingDestroys()}. */ + DBSessionWatcher addDestroyWatcher(RocksDBSession dbSession) { + DBSessionWatcher watcher = new DBSessionWatcher(dbSession); + destroyGraphDBs.add(watcher); + return watcher; + } + + boolean isDestroyPending(DBSessionWatcher watcher) { + return destroyGraphDBs.contains(watcher); + } + public boolean findPathInRemovedList(String path) { for (DBSessionWatcher pair : destroyGraphDBs) { if (pair.dbSession.getDbPath().equals(path)) { @@ -166,6 +282,11 @@ public RocksDBSession queryGraphDB(String dbName) { } //TODO is this necessary? class RocksdbEventListener extends AbstractEventListener { + + RocksdbEventListener() { + super(ENABLED_EVENT_CALLBACKS); + } + @Override public void onCompactionCompleted(RocksDB db, CompactionJobInfo compactionJobInfo) { super.onCompactionCompleted(db, compactionJobInfo); @@ -178,6 +299,46 @@ public void onCompactionCompleted(RocksDB db, CompactionJobInfo compactionJobInf public void onCompactionBegin(final RocksDB db, final CompactionJobInfo compactionJobInfo) { log.info("RocksdbEventListener onCompactionBegin"); } + + /** + * Invoked by RocksDB after a new SST (table) file has been successfully created. + * Propagates the event to all registered {@link RocksdbChangedListener}s so that + * cloud-storage providers can upload the new file. + */ + @Override + public void onTableFileCreated(final TableFileCreationInfo info) { + if (info.getStatus().getCode() != Status.Code.Ok) { + log.warn("onTableFileCreated: skipping failed file creation, path={}, status={}", + info.getFilePath(), info.getStatus().getCodeString()); + return; + } + log.debug("onTableFileCreated: db={}, cf={}, path={}, size={}", + info.getDbName(), info.getColumnFamilyName(), + info.getFilePath(), info.getFileSize()); + rocksdbChangedListeners.forEach(listener -> + listener.onTableFileCreated(info.getDbName(), info.getColumnFamilyName(), + info.getFilePath(), info.getFileSize()) + ); + } + + /** + * Invoked by RocksDB after an SST (table) file has been deleted. + * Propagates the event to all registered {@link RocksdbChangedListener}s so that + * cloud-storage providers can remove the corresponding remote object. + */ + @Override + public void onTableFileDeleted(final TableFileDeletionInfo info) { + if (info.getStatus().getCode() != Status.Code.Ok) { + log.warn("onTableFileDeleted: skipping failed file deletion, path={}, status={}", + info.getFilePath(), info.getStatus().getCodeString()); + return; + } + log.info("onTableFileDeleted: db={}, path={}", + info.getDbName(), info.getFilePath()); + rocksdbChangedListeners.forEach(listener -> + listener.onTableFileDeleted(info.getDbName(), null, info.getFilePath()) + ); + } } public RocksDBSession createGraphDB(String dbPath, String dbName) { @@ -188,18 +349,144 @@ public RocksDBSession createGraphDB(String dbPath, String dbName, long version) if (closing.get()) { throw new RuntimeException("db closed"); } + + // Phase 1: check the map and, if absent, construct the session under the write lock. + // The session is NOT inserted yet — inserting before hydration would let a concurrent + // queryGraphDB hand out an un-hydrated session. + // Also check pendingCreations: if another thread is already in Phase 2 for this DB, + // wait for it to finish rather than opening a second RocksDBSession for the same path + // (which would fail with a RocksDB LOCK-file error). + RocksDBSession dbSession; + final CompletableFuture creation = new CompletableFuture<>(); operateLock.writeLock().lock(); try { - RocksDBSession dbSession = dbSessionMap.get(dbName); - if (dbSession == null) { - log.info("create rocksdb for {}", dbName); - dbSession = new RocksDBSession(this.hugeConfig, dbPath, dbName, version); - dbSessionMap.put(dbName, dbSession); + // Wait (bounded, without holding the global lock) while another thread holds the + // pending-creation slot for this DB, then re-check. + CompletableFuture inFlight; + while ((inFlight = pendingCreations.get(dbName)) != null) { + operateLock.writeLock().unlock(); + try { + awaitCreation(dbName, inFlight); + } finally { + // Re-acquire on every path so the outer finally's unlock stays balanced. + operateLock.writeLock().lock(); + } + } + RocksDBSession existing = dbSessionMap.get(dbName); + if (existing != null) { + // Already exists — return a clone with no hydration needed. + return existing.clone(); } - return dbSession.clone(); + log.info("create rocksdb for {}", dbName); + // Reserve the creation slot; construct (which hydrates + opens) OUTSIDE the lock below. + pendingCreations.put(dbName, creation); } finally { operateLock.writeLock().unlock(); } + + // Phase 2: construct the session OUTSIDE the write lock so other DBs' queryGraphDB calls + // are not blocked during a potentially long S3 hydration. Construction runs the pre-open + // hook, which fires onDBOpening to hydrate the DB directory from cloud BEFORE RocksDB opens + // it — so a node whose local files were lost opens on the recovered data instead of a fresh + // empty DB. Per-DB serialization is provided by the pendingCreations slot reserved above, + // so two threads never open the same path concurrently (which would fail on the LOCK file). + boolean constructed = false; + try { + dbSession = new RocksDBSession( + this.hugeConfig, dbPath, dbName, version, new RocksDBSession.PreOpenHook() { + @Override + public void beforeOpen(String name, String resolvedPath) { + rocksdbChangedListeners.forEach(l -> l.onDBOpening(name, resolvedPath)); + } + + @Override + public String resolveOpenPath(String name, String candidatePath) { + String path = candidatePath; + for (RocksdbChangedListener l : rocksdbChangedListeners) { + path = l.resolveOpenPath(name, path); + } + return path; + } + }); + constructed = true; + } finally { + if (!constructed) { + // Hydration or open failed with ANY throwable — including java.lang.Error + // (OutOfMemoryError, StackOverflowError) which a plain `catch (Exception)` would + // miss. If the slot leaked, every future createGraphDB for this DB would wait on + // it. A finally clears it on all paths; the original throwable still propagates. + // No native handle exists to close (open did not complete). + operateLock.writeLock().lock(); + try { + releaseCreationSlot(dbName, creation); + } finally { + operateLock.writeLock().unlock(); + } + } + } + + // Phase 3: insert the hydrated session under the write lock and clear the pending slot + // so any thread that was waiting can now proceed (it will find the map entry). + operateLock.writeLock().lock(); + try { + if (closing.get()) { + // releaseAllGraphDB ran while this session was being built; it is not in the map, + // so nothing else will close it. Discard it instead of leaking an open RocksDB. + throw new DBStoreException("db closed while creating " + dbName); + } + RocksDBSession existing = dbSessionMap.get(dbName); + if (existing != null) { + // Race: another thread won Phase 3 first; discard our duplicate. + shutdownQuietly(dbSession); + return existing.clone(); + } + dbSessionMap.put(dbName, dbSession); + } catch (Exception e) { + shutdownQuietly(dbSession); + throw e; + } finally { + releaseCreationSlot(dbName, creation); + operateLock.writeLock().unlock(); + } + + // Notify listeners (upload pre-existing SSTs, flush WAL) outside the lock. + final String finalDbName = dbSession.getGraphName(); + final String finalDbPath = dbSession.getDbPath(); + rocksdbChangedListeners.forEach(listener -> listener.onDBCreated(finalDbName, finalDbPath)); + + return dbSession.clone(); + } + + /** Clears the creation slot and wakes its waiters. Caller must hold the write lock. */ + private void releaseCreationSlot(String dbName, CompletableFuture creation) { + pendingCreations.remove(dbName, creation); + creation.complete(null); + } + + /** Blocks (bounded) until another thread's creation of {@code dbName} finishes. */ + private void awaitCreation(String dbName, CompletableFuture inFlight) { + try { + inFlight.get(creationWaitTimeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted waiting for DB creation: " + dbName, ie); + } catch (TimeoutException te) { + throw new DBStoreException("Timed out after %d ms waiting for DB creation: %s", + te, creationWaitTimeoutMs, dbName); + } catch (ExecutionException ee) { + // Creators only ever complete normally; a failed creator just frees the slot. + log.debug("creation of {} completed exceptionally", dbName, ee); + } + } + + /** Best-effort shutdown of a session being discarded; package-private for tests. */ + static void shutdownQuietly(RocksDBSession session) { + try { + session.shutdown(); + } catch (Exception e) { + log.warn("Failed to shut down discarded session for db={}: {}", + session.getGraphName(), e.getMessage()); + } } /** @@ -231,6 +518,9 @@ public Map getTotalKey() { */ public boolean releaseGraphDB(String dbName) { log.info("close {} 's rocksdb.", dbName); + if (dbSessionMap.containsKey(dbName)) { + notifyDBClosing(dbName); + } operateLock.writeLock().lock(); try { RocksDBSession dbSession = dbSessionMap.get(dbName); @@ -256,19 +546,34 @@ public boolean releaseGraphDB(String dbName) { public void destroyGraphDB(String dbName) { log.info("destroy {} 's rocksdb.", dbName); RocksDBSession dbSession = dbSessionMap.get(dbName); - releaseGraphDB(dbName); - // Add delete mark if (dbSession != null) { - destroyGraphDBs.add(new DBSessionWatcher(dbSession)); + // Fire onDBDeleteBegin BEFORE releasing the session or enqueuing the destroy watcher. + // A listener may throw here to HOLD the delete — e.g. the cloud listener cannot durably + // persist its anti-resurrection marker, so proceeding would risk a crash re-hydrating + // stale objects as live data. If we had already enqueued the watcher (which + // asynchronously deletes the local dir and purges cloud) the throw could not actually + // stop the delete. By notifying first and letting the exception propagate before + // release/enqueue, the DB is left intact and still in dbSessionMap, so the caller can + // retry the destroy once the underlying problem clears. rocksdbChangedListeners.forEach(listener -> { listener.onDBDeleteBegin(dbSession.getGraphName(), dbSession.getDbPath()); }); } + releaseGraphDB(dbName); + // Add delete mark only after the begin-notification succeeded for every listener. + if (dbSession != null) { + destroyGraphDBs.add(new DBSessionWatcher(dbSession)); + } } public void releaseAllGraphDB() { closing.set(true); log.info("closing all rocksdb...."); + // A creation already past Phase 1 would otherwise register its session after the map is + // cleared below; give those a bounded chance to finish (Phase 3 also re-checks `closing`). + awaitPendingCreationsDrained(); + // Outside the write lock: listeners may do slow I/O and read the still-open sessions. + new ArrayList<>(dbSessionMap.keySet()).forEach(this::notifyDBClosing); operateLock.writeLock().lock(); try { dbSessionMap.forEach((k, v) -> { @@ -280,6 +585,34 @@ public void releaseAllGraphDB() { } } + /** Fires {@link RocksdbChangedListener#onDBClosing}; a failing listener does not stop the close. */ + private void notifyDBClosing(String dbName) { + for (RocksdbChangedListener listener : rocksdbChangedListeners) { + try { + listener.onDBClosing(dbName); + } catch (Exception e) { + log.warn("onDBClosing listener failed for db={}: {}", dbName, e.getMessage()); + } + } + } + + void awaitPendingCreationsDrained() { + CompletableFuture[] inFlight = pendingCreations.values().toArray(new CompletableFuture[0]); + if (inFlight.length == 0) { + return; + } + try { + CompletableFuture.allOf(inFlight).get(CREATION_DRAIN_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } catch (TimeoutException te) { + log.warn("{} DB creation(s) still in flight after {} ms while closing", + inFlight.length, CREATION_DRAIN_TIMEOUT_MS); + } catch (ExecutionException ignore) { + // creators complete normally + } + } + public Map getApproximateMemoryUsageByType(List dbs, List caches) { if (dbs == null) { @@ -314,11 +647,310 @@ public void addRocksdbChangedListener(RocksdbChangedListener listener) { rocksdbChangedListeners.add(listener); } + /** + * Removes a previously registered {@link RocksdbChangedListener}. Counterpart to + * {@link #addRocksdbChangedListener}; used to detach a listener during shutdown or test + * teardown so it no longer receives forwarded RocksDB events. + */ + public void removeRocksdbChangedListener(RocksdbChangedListener listener) { + rocksdbChangedListeners.remove(listener); + } + + /** + * Whether any registered {@link RocksdbChangedListener} {@linkplain + * RocksdbChangedListener#managesRemoteState() manages remote state} (e.g. a cloud storage + * provider). Only then is the native event listener wired into {@link org.rocksdb.DBOptions}. + */ + public boolean hasRemoteStateListeners() { + for (RocksdbChangedListener listener : rocksdbChangedListeners) { + if (listener.managesRemoteState()) { + return true; + } + } + return false; + } + + /** + * Flushes the MemTable of the named RocksDB session to disk, creating an SST file. + * This triggers {@link RocksdbChangedListener#onTableFileCreated} for every registered + * listener (including cloud-storage upload). + * + * @param dbName the graph / partition name + * @param wait if {@code true} the call blocks until the flush completes + */ + public void flushSession(String dbName, boolean wait) { + RocksDBSession session = dbSessionMap.get(dbName); + if (session != null) { + try { + session.flush(wait); + log.debug("Flushed RocksDB session for db={}", dbName); + } catch (Exception e) { + log.warn("Failed to flush RocksDB session for db={}: {}", dbName, e.getMessage()); + } + } + } + + /** + * Returns the live SST files currently referenced by the named RocksDB session, with the + * owning column-family name for each. Used by cloud storage to (a) confirm the full live set + * is present in cloud before deleting a superseded object, and (b) route read-miss hydration + * to the correct column family. + * + *

The returned metadata reflects RocksDB's manifest, so it lists files that RocksDB believes + * are live even if the local file has since been evicted from disk. + * + * @param dbName the graph / partition name + * @return live SST files (possibly empty); never {@code null} + */ + public List getLiveSstFiles(String dbName) { + // Hold a session reference so a concurrent release or destroy cannot close the DB + // underneath the native call (the session also reads under its cf-handle read lock). + RocksDBSession session = queryGraphDB(dbName); + if (session == null) { + return List.of(); + } + List liveFiles; + try { + liveFiles = session.getLiveFilesMetaData(); + } finally { + session.close(); + } + List result = new ArrayList<>(); + for (LiveFileMetaData md : liveFiles) { + String dir = md.path(); + String name = md.fileName(); + if (name == null || !name.endsWith(".sst")) { + continue; + } + // RocksDB may report the file name with a leading separator and the dir with or + // without a trailing one; File(parent, child) collapses the separator either way. + String absolutePath = new File(dir, name).getPath(); + String cfName = new String(md.columnFamilyName(), java.nio.charset.StandardCharsets.UTF_8); + result.add(new LiveSstFile(absolutePath, cfName)); + } + return result; + } + + /** A live SST file and the column family it belongs to. */ + public static final class LiveSstFile { + + private final String absolutePath; + private final String cfName; + + public LiveSstFile(String absolutePath, String cfName) { + this.absolutePath = absolutePath; + this.cfName = cfName; + } + + public String getAbsolutePath() { + return absolutePath; + } + + public String getCfName() { + return cfName; + } + } + + /** + * Captures a point-in-time, internally-consistent copy of the named DB's RocksDB metadata + * ({@code CURRENT}, {@code MANIFEST-*}, {@code OPTIONS-*}, the WAL {@code *.log} tail, and + * hard-links to the live {@code *.sst} set) into a temporary sibling directory of the DB, via + * RocksDB {@link org.rocksdb.Checkpoint} (the same primitive {@code saveSnapshot} uses). + * + *

This is the capture primitive behind metadata durability: the returned snapshot exposes + * the exact {@code {manifest, live-SST-set}} pair as of the checkpoint instant, so a caller can + * mirror a consistent set of objects to cloud storage without racing live compaction. The + * hard-linked SSTs also pin their content for the lifetime of the snapshot, so an SST cannot be + * physically removed by compaction between capture and upload. + * + *

The caller must call {@link MetadataSnapshot#cleanup()} when done to remove the + * temporary directory (which only contains metadata copies and SST hard-links — deleting it + * never touches the real SST files). + * + * @param dbName the graph / partition name + * @return the captured snapshot, or {@code null} if the session is not open + */ + public MetadataSnapshot captureMetadataSnapshot(String dbName) { + // Hold a session reference for the duration of the capture so a concurrent release or + // destroy cannot close the DB underneath the checkpoint. + RocksDBSession session = queryGraphDB(dbName); + if (session == null) { + return null; + } + try { + if (session.getDB() == null) { + return null; + } + return session.captureMetadataCheckpoint(); + } finally { + session.close(); + } + } + + /** + * A consistent snapshot of a RocksDB instance's metadata plus hard-links to its live SST set, + * materialised under {@link #getTempDir()} by {@link #captureMetadataSnapshot(String)}. + * + *

File names are relative to the checkpoint directory. {@link #getDbDir()} is the real DB + * directory the metadata belongs to — remote keys must be derived from {@code dbDir + name} (not + * the temp directory) so a restore lands each file back at its original path. + */ + public static final class MetadataSnapshot { + + private final String dbDir; + private final String tempDir; + private final String currentFileName; + private final String manifestFileName; + private final List optionsFileNames; + private final List sstFileNames; + private final long generation; + + public MetadataSnapshot(String dbDir, String tempDir, String currentFileName, + String manifestFileName, List optionsFileNames, + List sstFileNames) { + this(dbDir, tempDir, currentFileName, manifestFileName, + optionsFileNames, sstFileNames, + parseManifestGeneration(manifestFileName)); + } + + public MetadataSnapshot(String dbDir, String tempDir, String currentFileName, + String manifestFileName, List optionsFileNames, + List sstFileNames, + long generation) { + this.dbDir = dbDir; + this.tempDir = tempDir; + this.currentFileName = currentFileName; + this.manifestFileName = manifestFileName; + this.optionsFileNames = optionsFileNames; + this.sstFileNames = sstFileNames; + this.generation = generation; + } + + /** The real DB directory the captured metadata belongs to (for remote-key derivation). */ + public String getDbDir() { + return dbDir; + } + + /** The temporary checkpoint directory holding the metadata copies and SST hard-links. */ + public String getTempDir() { + return tempDir; + } + + /** The {@code CURRENT} file name (always {@code "CURRENT"}), or {@code null} if absent. */ + public String getCurrentFileName() { + return currentFileName; + } + + /** The {@code MANIFEST-} file name referenced by {@code CURRENT}, or {@code null}. */ + public String getManifestFileName() { + return manifestFileName; + } + + /** {@code OPTIONS-} file names present in the checkpoint. */ + public List getOptionsFileNames() { + return optionsFileNames; + } + + /** {@code *.sst} file names the captured manifest references (as hard-links). */ + public List getSstFileNames() { + return sstFileNames; + } + + /** Monotonic capture generation used to reject stale metadata publications. */ + public long getGeneration() { + return generation; + } + + private static long parseManifestGeneration(String manifestFileName) { + if (manifestFileName == null || !manifestFileName.startsWith("MANIFEST-")) { + return -1L; + } + try { + return Long.parseLong(manifestFileName.substring("MANIFEST-".length())); + } catch (NumberFormatException ignored) { + return -1L; + } + } + + /** Removes the temporary checkpoint directory. Never touches the real SST files. */ + public void cleanup() { + if (tempDir == null) { + return; + } + try { + FileUtils.deleteDirectory(new File(tempDir)); + } catch (Exception e) { + log.warn("Failed to clean up metadata checkpoint temp dir {}: {}", + tempDir, e.getMessage()); + } + } + } + + public boolean onReadMiss(RocksDBSession session, String table, byte[] key) { + if (session == null) { + return false; + } + boolean hydrated = false; + for (RocksdbChangedListener listener : rocksdbChangedListeners) { + try { + if (listener.onReadMiss(session, table, key)) { + hydrated = true; + } + } catch (Exception e) { + log.warn("onReadMiss listener failed for db={}, table={}: {}", + session.getGraphName(), table, e.getMessage()); + } + } + return hydrated; + } + + /** + * Returns the singleton {@link RocksdbEventListener} that should be registered + * with every new {@link org.rocksdb.DBOptions} via + * {@link org.rocksdb.DBOptions#setListeners}. + */ + public RocksdbEventListener getEventListener() { + return rocksdbEventListener; + } + public interface RocksdbChangedListener { + /** + * Whether this listener mirrors DB files to state outside the local directory (e.g. cloud + * storage). Only such listeners arm RocksDB's native event callbacks. + */ + default boolean managesRemoteState() { + return false; + } + default void onCompacted(String dbName) { } + default void onDBOpening(String dbName, String dbPath) { + } + + /** + * Called before {@link #onDBOpening} when the directory chosen for a DB holds no local DB. + * Returns the directory to open instead (a version-suffixed sibling with the same base + * name whose state the listener can restore), or {@code dbPath} to keep it. + */ + default String resolveOpenPath(String dbName, String dbPath) { + return dbPath; + } + + /** + * Called immediately after a new RocksDB instance has been opened for the first time. + * + *

Implementations can use this callback to upload any SST files that already exist + * in {@code dbPath} (e.g. from a previous run) and to trigger a MemTable flush so that + * any WAL-recovered data is also written to SST files and forwarded to cloud storage. + * + * @param dbName logical name of the graph / partition + * @param dbPath absolute path of the RocksDB directory + */ + default void onDBCreated(String dbName, String dbPath) { + } + default void onDBDeleteBegin(String dbName, String filePath) { } @@ -327,11 +959,57 @@ default void onDBDeleted(String dbName, String filePath) { default void onDBSessionReleased(RocksDBSession dbSession) { } + + /** + * Called before a DB is released or closed (including {@code releaseAllGraphDB()} at + * shutdown), while it is still registered and open, so a listener can finish work that + * needs a live session, such as a final metadata publish. Not called under the factory + * lock. + */ + default void onDBClosing(String dbName) { + } + + /** + * Called after a new SST file has been successfully created by RocksDB. + * + * @param dbName RocksDB instance name + * @param cfName column-family name + * @param filePath absolute path of the new SST file + * @param fileSize size of the file in bytes + */ + default void onTableFileCreated(String dbName, String cfName, + String filePath, long fileSize) { + } + + /** + * Called after an SST file has been deleted by RocksDB. + * + * @param dbName RocksDB instance name + * @param cfName column-family name + * @param filePath absolute path of the deleted SST file + */ + default void onTableFileDeleted(String dbName, String cfName, String filePath) { + } + + /** + * Called when a get() fails with a RocksDB error (for example a live SST missing on local + * disk) so listeners can attempt cloud hydration before the read is retried once. Not + * called for a null result: that is a genuine key-not-found. + * + * @param session RocksDB session where miss happened + * @param table target column-family/table + * @param key requested key + * @return true if listener hydrated new local data and caller should retry get() + */ + default boolean onReadMiss(RocksDBSession session, String table, byte[] key) { + return false; + } } class DBSessionWatcher { public RocksDBSession dbSession; public Long timestamp; + public int deleteAttempts = 0; public DBSessionWatcher(RocksDBSession dbSession) { this.dbSession = dbSession; diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index c206cb88ed..1a2b60472a 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -26,13 +26,17 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; @@ -54,6 +58,7 @@ import org.rocksdb.FlushOptions; import org.rocksdb.InfoLogLevel; import org.rocksdb.IngestExternalFileOptions; +import org.rocksdb.LiveFileMetaData; import org.rocksdb.MutableColumnFamilyOptionsInterface; import org.rocksdb.MutableDBOptionsInterface; import org.rocksdb.Options; @@ -78,6 +83,7 @@ public class RocksDBSession implements AutoCloseable, Cloneable { final AtomicInteger refCount; final AtomicBoolean shutdown; final String tempSuffix = "_temp_"; + final String cloudMetaTempMarker = "_cloudmeta_"; private final transient String graphName; private final HugeConfig hugeConfig; private final ReentrantReadWriteLock cfHandleLock; @@ -90,7 +96,32 @@ public class RocksDBSession implements AutoCloseable, Cloneable { @Getter private Map iteratorMap; + /** + * Invoked after the DB directory path is resolved and its directory is created, but BEFORE + * {@code RocksDB.open}. Lets a caller populate the directory (e.g. hydrate it from cloud) so + * that RocksDB opens on the recovered data rather than creating a fresh empty DB. + */ + @FunctionalInterface + public interface PreOpenHook { + void beforeOpen(String dbName, String resolvedDbPath); + + /** + * Called when the resolved directory holds no local DB (no {@code CURRENT}), before it is + * created. Returns the directory to open instead, e.g. a version-suffixed sibling whose + * state can be restored from cloud; returning the argument keeps it. Only a sibling with + * the same base name that is not awaiting deletion is accepted. + */ + default String resolveOpenPath(String dbName, String candidateDbPath) { + return candidateDbPath; + } + } + public RocksDBSession(HugeConfig hugeConfig, String dbDataPath, String graphName, long version) { + this(hugeConfig, dbDataPath, graphName, version, null); + } + + public RocksDBSession(HugeConfig hugeConfig, String dbDataPath, String graphName, long version, + PreOpenHook preOpenHook) { this.hugeConfig = hugeConfig; this.graphName = graphName; this.cfHandleLock = new ReentrantReadWriteLock(); @@ -100,7 +131,30 @@ public RocksDBSession(HugeConfig hugeConfig, String dbDataPath, String graphName this.writeOptions = new WriteOptions(); this.rocksDbStats = new Statistics(); this.iteratorMap = new ConcurrentHashMap<>(); - openRocksDB(dbDataPath, version); + try { + openRocksDB(dbDataPath, version, preOpenHook); + } catch (RuntimeException | Error e) { + // Open (or the pre-open hydration hook) failed: release the native objects allocated + // so far so a failed — and possibly retried — creation does not leak native memory. + // shutdown() only frees these when dbOptions is set, which is not the case if the + // failure occurred before RocksDB.open. + closeQuietly(this.rocksDB); + closeQuietly(this.dbOptions); + closeQuietly(this.writeOptions); + closeQuietly(this.rocksDbStats); + throw e; + } + } + + static void closeQuietly(AutoCloseable resource) { + if (resource == null) { + return; + } + try { + resource.close(); + } catch (Exception ignore) { + // best-effort native cleanup on a failed open + } } private RocksDBSession(RocksDBSession origin) { @@ -318,23 +372,32 @@ private String findLatestDBPath(String path, long version) { dbs.ensureCapacity(files.length); // search all db path for (final File sFile : files) { + if (!sFile.isDirectory()) { + continue; + } final String name = sFile.getName(); if (!name.startsWith(prefix) && !name.equals(defaultName)) { continue; } - if (name.endsWith(tempSuffix)) { + // Ignore transient dirs created by snapshot/checkpoint capture. + if (name.endsWith(tempSuffix) || name.contains(cloudMetaTempMarker)) { continue; } long v1 = -1L; long v2 = -1L; if (name.length() > defaultName.length()) { String[] versions = name.substring(prefix.length()).split("_"); - if (versions.length == 1) { - v1 = Long.parseLong(versions[0]); - } else if (versions.length == 2) { + if (versions.length != 1 && versions.length != 2) { + continue; + } + try { v1 = Long.parseLong(versions[0]); - v2 = Long.parseLong(versions[1]); - } else { + if (versions.length == 2) { + v2 = Long.parseLong(versions[1]); + } + } catch (NumberFormatException e) { + log.debug("Skip invalid db-path candidate '{}': {}", name, + e.getMessage()); continue; } } @@ -411,7 +474,56 @@ public boolean tableIsExist(String table) { return this.tables.containsKey(table); } - private void openRocksDB(String dbDataPath, long version) { + /** + * Lets the hook move an open with no local DB to a sibling directory (for example the + * version-suffixed directory a lost disk last held, restorable from cloud). The redirect is + * ignored unless it names a sibling with the same base name that is not awaiting deletion. + */ + private String redirectOpenPath(PreOpenHook preOpenHook, String candidate) { + String redirected = preOpenHook.resolveOpenPath(this.graphName, candidate); + if (redirected == null || redirected.equals(candidate)) { + return candidate; + } + File from = new File(candidate); + File to = new File(redirected); + String fromName = from.getName(); + int sep = fromName.indexOf('_'); + String baseName = sep < 0 ? fromName : fromName.substring(0, sep); + boolean sibling = Objects.equals(from.getAbsoluteFile().getParentFile(), + to.getAbsoluteFile().getParentFile()) + && to.getName().matches(Pattern.quote(baseName) + "(_\\d+){0,2}"); + if (!sibling || RocksDBFactory.getInstance().findPathInRemovedList(redirected)) { + log.warn("{} ignoring open-path redirect {} -> {}", this.graphName, candidate, + redirected); + return candidate; + } + log.info("{} opening {} instead of {} (restorable from remote state)", this.graphName, + redirected, candidate); + return redirected; + } + + /** + * Deletes leftover {@code [_v1[_v2]]_cloudmeta_} checkpoint dirs from a previous + * run that crashed mid-capture. They hold only metadata copies and SST hard-links, so deleting + * them never touches live data. + */ + private void sweepStaleCloudMetaDirs() { + File parent = new File(this.dbPath).getParentFile(); + File[] children = parent == null ? null : parent.listFiles(File::isDirectory); + if (children == null) { + return; + } + Pattern stale = Pattern.compile(Pattern.quote(this.graphName) + "(_\\d+){0,2}" + + Pattern.quote(cloudMetaTempMarker) + "\\d+"); + for (File child : children) { + if (stale.matcher(child.getName()).matches()) { + FileUtils.deleteQuietly(child); + log.info("{} removed stale metadata checkpoint dir {}", this.graphName, child); + } + } + } + + private void openRocksDB(String dbDataPath, long version, PreOpenHook preOpenHook) { if (dbDataPath.endsWith(File.separator)) { this.dbPath = dbDataPath + this.graphName; @@ -424,13 +536,38 @@ private void openRocksDB(String dbDataPath, long version) { Asserts.isTrue((dbPath != null), () -> new DBStoreException("the data-path of RocksDB is null")); + if (preOpenHook != null && !new File(this.dbPath, "CURRENT").exists()) { + this.dbPath = redirectOpenPath(preOpenHook, this.dbPath); + } + + sweepStaleCloudMetaDirs(); + //makedir for rocksdb createDirectory(dbPath); + // Hydrate the directory (e.g. from cloud) BEFORE opening, so RocksDB opens on any recovered + // CURRENT/MANIFEST/SST rather than creating a fresh empty DB. Running this before open is + // essential: with createIfMissing=true, opening an empty dir first would create a local + // CURRENT that later hydration would skip, silently yielding an empty database. + if (preOpenHook != null) { + preOpenHook.beforeOpen(this.graphName, this.dbPath); + } + Options opts = new Options(); RocksDBSession.initOptions(hugeConfig, opts, opts, opts, opts); dbOptions = new DBOptions(opts); dbOptions.setStatistics(rocksDbStats); + // Only register the factory-level event listener when a listener manages remote state + // (e.g. a cloud storage provider); other listeners only use the Java-side hooks. RocksDB's + // native callbacks (onTableFileCreated/onTableFileDeleted/onCompactionCompleted) run on + // RocksDB's own native threads and resolve classes via the plain system classloader; when + // this module is loaded from a Spring Boot fat jar's nested BOOT-INF/lib, that classloader + // cannot see the listener classes, causing a NoClassDefFoundError storm on every callback. + // Skipping the registration when nothing needs the events avoids the callbacks entirely. + if (RocksDBFactory.getInstance().hasRemoteStateListeners()) { + dbOptions.setListeners( + Collections.singletonList(RocksDBFactory.getInstance().getEventListener())); + } try { List columnFamilyDescriptorList = @@ -593,7 +730,9 @@ public void dropTables(String... tables) throws DBStoreException { } public synchronized void truncate() { - Set tableNames = this.tables.keySet(); + // Copy: a live keySet view would drop the default CF from this.tables below, and would be + // emptied by dropTables so createTables recreated nothing. + Set tableNames = new HashSet<>(this.tables.keySet()); String defaultCF = new String(RocksDB.DEFAULT_COLUMN_FAMILY); tableNames.remove(defaultCF); @@ -693,6 +832,92 @@ public void saveSnapshot(String snapshotPath) throws DBStoreException { System.currentTimeMillis() - startTime); } + /** + * Returns RocksDB's live-file metadata, or an empty list once the DB is closed. Read under the + * cf-handle read lock: shutdown() takes the write lock before closing rocksDB, so the native + * handle cannot be closed during the call. + */ + List getLiveFilesMetaData() { + cfHandleLock.readLock().lock(); + try { + if (this.closed || this.shutdown.get() || this.rocksDB == null) { + return Collections.emptyList(); + } + return this.rocksDB.getLiveFilesMetaData(); + } finally { + cfHandleLock.readLock().unlock(); + } + } + + /** + * Captures a consistent copy of this DB's metadata (CURRENT / MANIFEST-* / OPTIONS-* / WAL) plus + * hard-links to the live SST set into a temporary directory, using a RocksDB + * {@link Checkpoint}. See {@link RocksDBFactory#captureMetadataSnapshot(String)} for the metadata + * rationale. + * + *

The temporary directory is a sibling of the DB directory (same filesystem), so SST files + * are hard-linked rather than copied. The caller owns cleanup via + * {@link RocksDBFactory.MetadataSnapshot#cleanup()}. + * + * @return the captured snapshot; never {@code null} + * @throws DBStoreException if the checkpoint cannot be created + */ + RocksDBFactory.MetadataSnapshot captureMetadataCheckpoint() throws DBStoreException { + String tempDir = this.dbPath + cloudMetaTempMarker + System.nanoTime(); + // Everything that touches the native DB (checkpoint, sequence number) stays under the read + // lock: shutdown() takes the write lock before closing rocksDB, so the handle cannot be + // closed underneath this capture. + cfHandleLock.readLock().lock(); + try { + if (this.closed || this.shutdown.get() || this.rocksDB == null) { + throw new DBStoreException("Fail to create metadata checkpoint at %s: " + + "db %s is closed", tempDir, this.graphName); + } + try (final Checkpoint checkpoint = Checkpoint.create(this.rocksDB)) { + final File tempFile = new File(tempDir); + // Clean any stale temp checkpoint dir from a previous failed/interrupted attempt. + // If it exists, RocksDB checkpoint creation can fail on pre-existing files. + FileUtils.deleteDirectory(tempFile); + checkpoint.createCheckpoint(tempDir); + } catch (final Exception e) { + // best-effort cleanup of a partial checkpoint + FileUtils.deleteQuietly(new File(tempDir)); + log.error("Fail to create metadata checkpoint at {}", tempDir, e); + throw new DBStoreException("Fail to create metadata checkpoint at %s", e, tempDir); + } + return categoriseCheckpoint(tempDir); + } finally { + cfHandleLock.readLock().unlock(); + } + } + + /** Classifies the files a checkpoint materialized into the metadata snapshot descriptor. */ + RocksDBFactory.MetadataSnapshot categoriseCheckpoint(String tempDir) { + long generation = this.getLatestSequenceNumber(); + String currentFileName = null; + String manifestFileName = null; + List optionsFileNames = new ArrayList<>(); + List sstFileNames = new ArrayList<>(); + File[] files = new File(tempDir).listFiles(); + if (files != null) { + for (File f : files) { + String name = f.getName(); + if (name.equals("CURRENT")) { + currentFileName = name; + } else if (name.startsWith("MANIFEST-")) { + manifestFileName = name; + } else if (name.startsWith("OPTIONS-")) { + optionsFileNames.add(name); + } else if (name.endsWith(".sst")) { + sstFileNames.add(name); + } + } + } + return new RocksDBFactory.MetadataSnapshot(this.dbPath, tempDir, currentFileName, + manifestFileName, optionsFileNames, + sstFileNames, generation); + } + private boolean verifySnapshot(String snapshotPath) { try { try (final Options options = new Options(); diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java index b8259e5220..36695e4b79 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/SessionOperatorImpl.java @@ -207,11 +207,28 @@ public void deleteRange(byte[] keyFrom, byte[] keyTo) throws DBStoreException { @Override public byte[] get(String table, byte[] key) throws DBStoreException { + byte[] value; try (CFHandleLock cf = this.getLock(table)) { - return rocksdb().get(cf.get(), key); + value = rocksdb().get(cf.get(), key); } catch (RocksDBException e) { + // A live SST referenced by the manifest may be physically missing (e.g. the local disk + // was lost while the DB stayed open). RocksDB surfaces that as an exception rather than + // a null, so hydration must be attempted here too — not only on the null path below — + // otherwise a read that actually needs the evicted file fails instead of self-healing. + // The CF handle lock is already released (try-with-resources closes before this catch), + // so the slow hydration I/O does not hold it. + if (RocksDBFactory.getInstance().onReadMiss(this.session, table, key)) { + try (CFHandleLock cf = this.getLock(table)) { + return rocksdb().get(cf.get(), key); + } catch (RocksDBException retryError) { + throw new DBStoreException(retryError); + } + } throw new DBStoreException(e); } + // A null is a genuine key-not-found: RocksDB raises an error (handled above) when a file + // it needs is missing, so restoring files cannot change this result and is not attempted. + return value; } @Override diff --git a/hugegraph-store/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBFactoryTest.java b/hugegraph-store/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBFactoryTest.java index 3e2b674e09..5e4559645e 100644 --- a/hugegraph-store/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBFactoryTest.java +++ b/hugegraph-store/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/RocksDBFactoryTest.java @@ -17,14 +17,68 @@ package org.apache.hugegraph.rocksdb.access; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doAnswer; +import static org.junit.Assume.assumeTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; import org.apache.commons.configuration2.MapConfiguration; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener; import org.junit.BeforeClass; +import org.junit.Test; +import org.rocksdb.AbstractEventListener.EnabledEventCallback; +import org.rocksdb.DBOptions; +import org.rocksdb.LiveFileMetaData; +import org.rocksdb.RocksDB; +import org.rocksdb.Status; +import org.rocksdb.TableFileCreationInfo; +import org.rocksdb.TableFileDeletionInfo; +/** + * {@link RocksDBFactory} / {@link RocksDBSession} metadata-checkpoint and cloud-hydration behaviour. + * + *

    + *
  • {@link #testMetadataCheckpointFlushBehavior()} — the flushing checkpoint + * ({@code captureMetadataSnapshot} / RocksJava {@code Checkpoint.createCheckpoint}) DOES flush + * the memtable, creating a new live SST and firing {@code onTableFileCreated}.
  • + *
  • {@link #testOnDbOpeningRunsBeforeRocksDbOpensTheDirectory()} and + * {@link #testCreateGraphDbHydrationFailurePropagatesAndRemainsRetryable()} — the ordering + * contract between cloud pre-hydration ({@code onDBOpening}) and {@code RocksDB.open}, plus + * its failure/retry behaviour.
  • + *
+ */ public class RocksDBFactoryTest { @BeforeClass @@ -32,71 +86,1872 @@ public static void init() { OptionSpace.register("rocksdb", "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); RocksDBOptions.instance(); - Map configMap = new HashMap<>(); - configMap.put("rocksdb.write_buffer_size", "1048576"); + // Large write buffer so the small writes below stay in the memtable (no natural flush) — + // any new SST must therefore come from the capture itself. The hydration-ordering tests + // write no data, so this size is harmless for them. + configMap.put("rocksdb.write_buffer_size", String.valueOf(256L * 1024 * 1024)); configMap.put("rocksdb.bloom_filter_bits_per_key", "10"); + RocksDBFactory.getInstance().setHugeConfig(new HugeConfig(new MapConfiguration(configMap))); + } + + // ========================================================================= + // Flushing checkpoint: DOES flush the memtable (creates a new live SST) + // ========================================================================= + + /** + * Empirical probe: {@code captureMetadataSnapshot} (RocksJava {@code Checkpoint.createCheckpoint}) + * flushes the memtable, creating a new live SST that fires {@code onTableFileCreated}. If so, + * metadata sync on every SST would drive flush amplification — which is exactly why {@code wal} + * mode uses the flush-free path exercised by the next test. + */ + @Test + public void testMetadataCheckpointFlushBehavior() throws Exception { + Path parent = Files.createTempDirectory("hg-ckflush-"); + String dbName = "flushcheck"; + + AtomicInteger created = new AtomicInteger(0); + RocksdbChangedListener probe = new RocksdbChangedListener() { + @Override + public void onTableFileCreated(String db, String cf, String path, long size) { + created.incrementAndGet(); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(probe); + RocksDBSession session; + try { + session = factory.createGraphDB(parent.toString(), dbName, 0); + session.checkTable("t"); + SessionOperator op = session.sessionOp(); + op.prepare(); + for (int i = 0; i < 200; i++) { + op.put("t", ("k" + i).getBytes(StandardCharsets.UTF_8), + ("v" + i).getBytes(StandardCharsets.UTF_8)); + } + op.commit(); + + int liveSstBefore = countSst(session.getDbPath()); + int eventsBefore = created.get(); + + // Trigger a metadata checkpoint WITHOUT an explicit flush. + RocksDBFactory.MetadataSnapshot snap = factory.captureMetadataSnapshot(dbName); + // Give any async table-created event a moment to be delivered. + long deadline = System.currentTimeMillis() + 1000L; + while (created.get() == eventsBefore && System.currentTimeMillis() < deadline) { + LockSupport.parkNanos(10_000_000L); + } + int liveSstAfter = countSst(session.getDbPath()); + int eventsAfter = created.get(); + if (snap != null) { + snap.cleanup(); + } + + System.out.printf( + "%n[CHECKPOINT-FLUSH-PROBE] onTableFileCreated events: before=%d after=%d " + + "(delta=%d); live *.sst in db dir: before=%d after=%d (delta=%d)%n", + eventsBefore, eventsAfter, eventsAfter - eventsBefore, + liveSstBefore, liveSstAfter, liveSstAfter - liveSstBefore); + + // Document the observed behavior: the checkpoint flushes the memtable, so a new live + // SST appears and onTableFileCreated fires. (If this fails with delta=0, the RocksDB + // build does NOT flush on checkpoint and the amplification concern does not apply.) + assertTrue( + "checkpoint is expected to flush the memtable, creating a new live SST " + + "(delta events=" + (eventsAfter - eventsBefore) + + ", delta sst=" + (liveSstAfter - liveSstBefore) + ")", + eventsAfter > eventsBefore || liveSstAfter > liveSstBefore); + } finally { + factory.removeRocksdbChangedListener(probe); + try { + factory.destroyGraphDB(dbName); + } catch (Exception ignore) { + // best-effort + } + deleteRecursively(parent.toFile()); + } + } + + // ========================================================================= + // Hydration ordering: onDBOpening (cloud pre-hydration) runs BEFORE RocksDB.open + // ========================================================================= + + /** + * Cloud disk-loss recovery downloads {@code CURRENT}/{@code MANIFEST}/SST into the DB directory + * during {@code onDBOpening}. For RocksDB to open on the recovered data, hydration must run + * before {@code RocksDB.open}; otherwise open creates a fresh empty DB (and a local + * {@code CURRENT}) and the downloaded files are never loaded. This asserts that when + * {@code onDBOpening} fires, RocksDB has NOT yet created its {@code CURRENT}. + */ + @Test + public void testOnDbOpeningRunsBeforeRocksDbOpensTheDirectory() throws Exception { + Path parent = Files.createTempDirectory("hg-hydration-order-"); + String dbName = "recdb"; + + AtomicBoolean hydrationCalled = new AtomicBoolean(false); + AtomicBoolean currentExistedAtHydration = new AtomicBoolean(false); + + RocksdbChangedListener probe = new RocksdbChangedListener() { + @Override + public void onDBOpening(String db, String dbPath) { + hydrationCalled.set(true); + // If RocksDB already opened this dir, it will have written CURRENT. Its presence + // here means hydration is running too late to influence the open. + currentExistedAtHydration.set(new File(dbPath, "CURRENT").exists()); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(probe); + try { + factory.createGraphDB(parent.toString(), dbName, 0); + + assertTrue("onDBOpening (cloud hydration) must be invoked during DB creation", + hydrationCalled.get()); + assertFalse("onDBOpening must run BEFORE RocksDB.open creates CURRENT — otherwise " + + "cloud-hydrated files are never loaded into the already-open DB and " + + "disk-loss recovery silently yields an empty database", + currentExistedAtHydration.get()); + } finally { + factory.removeRocksdbChangedListener(probe); + try { + factory.destroyGraphDB(dbName); + } catch (Exception ignore) { + // best-effort + } + deleteRecursively(parent.toFile()); + } + } + + @Test + public void testCreateGraphDbHydrationFailurePropagatesAndRemainsRetryable() throws Exception { + Path parent = Files.createTempDirectory("hg-hydration-fail-"); + String dbName = "failrec"; + + RocksdbChangedListener boom = new RocksdbChangedListener() { + @Override + public void onDBOpening(String db, String dbPath) { + throw new RuntimeException("simulated hydration failure"); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(boom); + try { + factory.createGraphDB(parent.toString(), dbName, 0); + throw new AssertionError("createGraphDB must propagate the hydration failure"); + } catch (RuntimeException expected) { + // expected — hydration threw before open + } finally { + factory.removeRocksdbChangedListener(boom); + } + + // The failed attempt must have cleared its pending-creation slot and left no half-open + // session, so a subsequent open (no failing hook) succeeds instead of wedging on the slot. + RocksDBSession recovered = null; + try { + recovered = factory.createGraphDB(parent.toString(), dbName, 0); + assertNotNull("retry after hydration failure must succeed", recovered); + } finally { + if (recovered != null) { + recovered.close(); + } + try { + factory.releaseGraphDB(dbName); + } catch (Exception ignore) { + // best-effort + } + deleteRecursively(parent.toFile()); + } + } + + /** + * A second {@code createGraphDB} for a DB whose creation is still in flight (between Phase 1 + * and Phase 3) must spin-wait on the pending-creation slot rather than open the same path a + * second time, then return a clone of the session the first caller registered. A waiter + * interrupted while spinning must fail fast instead of waiting forever. + */ + @Test + public void testCreateGraphDbWaitsForInFlightCreationOfSameDb() throws Exception { + Path parent = Files.createTempDirectory("hg-create-wait-"); + String dbName = "createwait"; + CountDownLatch hydrating = new CountDownLatch(1); + CountDownLatch releaseHydration = new CountDownLatch(1); + AtomicInteger openings = new AtomicInteger(0); + RocksdbChangedListener gate = new RocksdbChangedListener() { + @Override + public void onDBOpening(String db, String dbPath) { + if (!dbName.equals(db)) { + return; + } + openings.incrementAndGet(); + hydrating.countDown(); + try { + releaseHydration.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(gate); + AtomicReference created = new AtomicReference<>(); + AtomicReference waited = new AtomicReference<>(); + try { + Thread creator = new Thread( + () -> created.set(factory.createGraphDB(parent.toString(), dbName, 0))); + creator.start(); + assertTrue("the first creation must reach hydration", + hydrating.await(30, TimeUnit.SECONDS)); + + // An interrupted waiter gives up with a RuntimeException instead of spinning forever. + AtomicReference interruptedError = new AtomicReference<>(); + Thread interrupted = new Thread(() -> { + try { + factory.createGraphDB(parent.toString(), dbName, 0); + } catch (Throwable t) { + interruptedError.set(t); + } + }); + interrupted.start(); + interrupted.interrupt(); + interrupted.join(30_000); + assertNotNull("an interrupted waiter must fail", interruptedError.get()); + assertTrue(interruptedError.get().getMessage() + .contains("Interrupted waiting for DB creation")); + + // A second waiter spins on the slot until the first creation registers the session. + Thread waiter = new Thread( + () -> waited.set(factory.createGraphDB(parent.toString(), dbName, 0))); + waiter.start(); + long deadline = System.currentTimeMillis() + 30_000; + while (waiter.getState() != Thread.State.TIMED_WAITING && + System.currentTimeMillis() < deadline) { + Thread.sleep(5); + } + assertEquals("the waiter must be spinning on the pending-creation slot", + Thread.State.TIMED_WAITING, waiter.getState()); + + releaseHydration.countDown(); + creator.join(30_000); + waiter.join(30_000); + + assertNotNull(created.get()); + assertNotNull("the waiter must receive the session the first caller created", + waited.get()); + assertEquals("the DB must be hydrated and opened only once", 1, openings.get()); + } finally { + releaseHydration.countDown(); + factory.removeRocksdbChangedListener(gate); + if (created.get() != null) { + created.get().close(); + } + if (waited.get() != null) { + waited.get().close(); + } + factory.destroyGraphDB(dbName); + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + } + + /** + * Phase 3 race guard: if a session for the same name was registered while this caller was + * constructing its own (outside the lock), the duplicate must be shut down and the + * already-registered session returned. + */ + @Test + public void testCreateGraphDbDiscardsDuplicateWhenSessionRegisteredDuringConstruction() + throws Exception { + Path parent = Files.createTempDirectory("hg-create-race-"); + String dbName = "createrace"; + RocksDBSession winner = mock(RocksDBSession.class); + when(winner.clone()).thenReturn(winner); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map sessions = sessionMap(factory); + RocksdbChangedListener racer = new RocksdbChangedListener() { + @Override + public void onDBOpening(String db, String dbPath) { + if (dbName.equals(db)) { + sessions.put(dbName, winner); + } + } + }; + factory.addRocksdbChangedListener(racer); + try { + RocksDBSession result = factory.createGraphDB(parent.toString(), dbName, 0); + assertTrue("the already-registered session must win the race", result == winner); + assertTrue(sessions.get(dbName) == winner); + } finally { + factory.removeRocksdbChangedListener(racer); + sessions.remove(dbName, winner); + deleteRecursively(parent.toFile()); + } + } + + /** + * Phase 3 failure: if registering the constructed session fails, the pending-creation slot + * must be cleared and the half-registered session shut down, so a retry can open the DB + * instead of spin-waiting on a leaked slot. + */ + @Test + public void testCreateGraphDbRegistrationFailureClearsSlotAndRemainsRetryable() + throws Exception { + Path parent = Files.createTempDirectory("hg-create-regfail-"); + String dbName = "createregfail"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map original = sessionMap(factory); + Map failing = new ConcurrentHashMap( + original) { + @Override + public RocksDBSession put(String key, RocksDBSession value) { + if (dbName.equals(key)) { + throw new IllegalStateException("simulated registration failure"); + } + return super.put(key, value); + } + }; + + setSessionMap(factory, failing); + try { + factory.createGraphDB(parent.toString(), dbName, 0); + throw new AssertionError("createGraphDB must propagate the registration failure"); + } catch (IllegalStateException expected) { + // expected — Phase 3 could not register the session + } finally { + setSessionMap(factory, original); + } + + RocksDBSession recovered = null; + try { + // The duplicate was shut down (releasing the RocksDB LOCK) and the slot cleared, so + // the retry opens the same path without wedging or a LOCK-file error. + recovered = factory.createGraphDB(parent.toString(), dbName, 0); + assertNotNull("retry after a registration failure must succeed", recovered); + } finally { + if (recovered != null) { + recovered.close(); + } + factory.destroyGraphDB(dbName); + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + } + + // ========================================================================= + // Destroy: a listener throwing from onDBDeleteBegin must HOLD the delete + // ========================================================================= + + /** + * A listener may throw from {@code onDBDeleteBegin} to hold the delete — e.g. the cloud + * listener cannot durably persist its anti-resurrection marker, so proceeding could let a crash + * re-hydrate stale objects as live data. {@code destroyGraphDB} must fire that notification + * BEFORE releasing the session or enqueuing the async destroy watcher, so the throw genuinely + * holds the delete: the session stays registered (files not deleted, cloud not purged) and the + * caller can retry once the underlying problem clears. + */ + @Test + public void testDestroyGraphDbHeldWhenDeleteBeginListenerThrows() throws Exception { + Path parent = Files.createTempDirectory("hg-destroy-hold-"); + String dbName = "holddb"; + + // A benign listener registered FIRST records that onDBDeleteBegin fired (and returns + // normally); the throwing listener registered AFTER it then holds the delete. Ordering + // matters: it proves the begin-notification runs, and that a later listener throwing still + // holds the delete. + AtomicBoolean beginNotified = new AtomicBoolean(false); + RocksdbChangedListener recorder = new RocksdbChangedListener() { + @Override + public void onDBDeleteBegin(String db, String dbPath) { + beginNotified.set(true); + } + }; + RocksdbChangedListener holdDelete = new RocksdbChangedListener() { + @Override + public void onDBDeleteBegin(String db, String dbPath) { + throw new IllegalStateException( + "simulated: cannot durably persist anti-resurrection marker"); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.createGraphDB(parent.toString(), dbName, 0); + factory.addRocksdbChangedListener(recorder); + factory.addRocksdbChangedListener(holdDelete); + try { + try { + factory.destroyGraphDB(dbName); + throw new AssertionError( + "destroyGraphDB must propagate the hold-the-delete exception"); + } catch (IllegalStateException expected) { + // expected — the listener held the delete + } + + assertTrue("onDBDeleteBegin must be fired before the delete is held", beginNotified.get()); + // The delete was held: the session was NOT released, so it is still registered and the + // destroy can be retried. (Had the watcher been enqueued before the throw, the async + // cleanup would delete the dir / purge cloud regardless — the bug this guards against.) + assertTrue("a held delete must leave the session registered for retry", + factory.getGraphNames().contains(dbName)); + } finally { + factory.removeRocksdbChangedListener(holdDelete); + factory.removeRocksdbChangedListener(recorder); + // With the throwing listener gone, the retry destroy proceeds normally. + try { + factory.destroyGraphDB(dbName); + } catch (Exception ignore) { + // best-effort + } + deleteRecursively(parent.toFile()); + } + } + + // ========================================================================= + // getLiveSstFiles / captureMetadataSnapshot / flushSession / onReadMiss + // ========================================================================= + + @Test + public void testGetLiveSstFilesReflectsFlushedSstsAndCfName() throws Exception { + Path parent = Files.createTempDirectory("hg-livesst-"); + String dbName = "livesst"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + try { + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName, 0); + session.checkTable("t"); + SessionOperator op = session.sessionOp(); + op.prepare(); + for (int i = 0; i < 50; i++) { + op.put("t", ("k" + i).getBytes(StandardCharsets.UTF_8), + ("v" + i).getBytes(StandardCharsets.UTF_8)); + } + op.commit(); + // Explicit flush turns the memtable into at least one live SST. + factory.flushSession(dbName, true); - HugeConfig hConfig = new HugeConfig(new MapConfiguration(configMap)); - RocksDBFactory rFactory = RocksDBFactory.getInstance(); - rFactory.setHugeConfig(hConfig); + java.util.List live = factory.getLiveSstFiles(dbName); + assertFalse("a flushed DB must report at least one live SST", live.isEmpty()); + for (RocksDBFactory.LiveSstFile f : live) { + assertTrue("live file must be an absolute *.sst path: " + f.getAbsolutePath(), + f.getAbsolutePath().endsWith(".sst")); + assertNotNull("column-family name must be populated", f.getCfName()); + } + // Unknown DB → empty (never null); no session means nothing is live. + assertTrue(factory.getLiveSstFiles("no-such-db").isEmpty()); + } finally { + try { + factory.destroyGraphDB(dbName); + } catch (Exception ignore) { + // best-effort + } + deleteRecursively(parent.toFile()); + } } - // @Test - public void testCreateSession() throws InterruptedException { + @Test + public void testCaptureMetadataSnapshotConsistentThenCleanup() throws Exception { + Path parent = Files.createTempDirectory("hg-metasnap-"); + String dbName = "metasnap"; RocksDBFactory factory = RocksDBFactory.getInstance(); - try (RocksDBSession dbSession = factory.createGraphDB("./tmp", "test1")) { - SessionOperator op = dbSession.sessionOp(); + try { + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName, 0); + session.checkTable("t"); + SessionOperator op = session.sessionOp(); op.prepare(); + op.put("t", "k".getBytes(StandardCharsets.UTF_8), "v".getBytes(StandardCharsets.UTF_8)); + op.commit(); + + RocksDBFactory.MetadataSnapshot snap = factory.captureMetadataSnapshot(dbName); + assertNotNull("a live DB must yield a metadata snapshot", snap); try { - op.put("tbl", "k1".getBytes(), "v1".getBytes()); - op.commit(); - } catch (Exception e) { - op.rollback(); + assertEquals("CURRENT", snap.getCurrentFileName()); + assertNotNull("snapshot must reference a MANIFEST", snap.getManifestFileName()); + assertTrue(snap.getManifestFileName().startsWith("MANIFEST-")); + assertEquals(session.getDbPath(), snap.getDbDir()); + assertNotNull(snap.getTempDir()); + assertTrue("checkpoint temp dir must exist before cleanup", + new File(snap.getTempDir()).isDirectory()); + assertTrue("generation must be non-negative", snap.getGeneration() >= 0L); + assertNotNull(snap.getOptionsFileNames()); + assertNotNull(snap.getSstFileNames()); + } finally { + String tempDir = snap.getTempDir(); + snap.cleanup(); + assertFalse("cleanup() must remove the checkpoint temp dir", + new File(tempDir).exists()); } + // Unknown DB → null snapshot. + assertNull(factory.captureMetadataSnapshot("no-such-db")); + } finally { + try { + factory.destroyGraphDB(dbName); + } catch (Exception ignore) { + // best-effort + } + deleteRecursively(parent.toFile()); } - factory.destroyGraphDB("test1"); + } - Thread.sleep(100000); + @Test + public void testFlushSessionUnknownDbIsNoopAndOnReadMissNullSessionIsFalse() { + RocksDBFactory factory = RocksDBFactory.getInstance(); + // No session registered under this name: must be a quiet no-op, not an exception. + factory.flushSession("no-such-db", true); + factory.flushSession("no-such-db", false); + // A null session short-circuits to "not hydrated". + assertFalse(factory.onReadMiss(null, "t", "k".getBytes(StandardCharsets.UTF_8))); } - // @Test - public void testTotalKeys() { - RocksDBFactory dbFactory = RocksDBFactory.getInstance(); - System.out.println(dbFactory.getTotalSize()); + @Test + public void testMetadataSnapshotConstructorsGettersAndGenerationParsing() { + RocksDBFactory.MetadataSnapshot parsed = new RocksDBFactory.MetadataSnapshot( + "/db", "/db_tmp", "CURRENT", "MANIFEST-000123", + java.util.List.of("OPTIONS-000100"), java.util.List.of("000042.sst")); + assertEquals("/db", parsed.getDbDir()); + assertEquals("/db_tmp", parsed.getTempDir()); + assertEquals("CURRENT", parsed.getCurrentFileName()); + assertEquals("MANIFEST-000123", parsed.getManifestFileName()); + assertEquals(java.util.List.of("OPTIONS-000100"), parsed.getOptionsFileNames()); + assertEquals(java.util.List.of("000042.sst"), parsed.getSstFileNames()); + assertEquals("generation is parsed from the MANIFEST- name", 123L, + parsed.getGeneration()); + + // Unparseable / absent manifest names yield -1. + assertEquals(-1L, new RocksDBFactory.MetadataSnapshot( + "/db", "/t", "CURRENT", "MANIFEST-xyz", + java.util.List.of(), java.util.List.of()).getGeneration()); + assertEquals(-1L, new RocksDBFactory.MetadataSnapshot( + "/db", "/t", null, null, + java.util.List.of(), java.util.List.of()).getGeneration()); + + // Explicit-generation constructor wins over name parsing. + assertEquals(999L, new RocksDBFactory.MetadataSnapshot( + "/db", "/t", "CURRENT", "MANIFEST-000123", + java.util.List.of(), java.util.List.of(), 999L).getGeneration()); - System.out.println(dbFactory.getTotalKey().entrySet() - .stream().map(e -> e.getValue()).reduce(0L, Long::sum)); + // cleanup() with a null temp dir is a safe no-op. + new RocksDBFactory.MetadataSnapshot("/db", null, "CURRENT", "MANIFEST-1", + java.util.List.of(), java.util.List.of()).cleanup(); } - // @Test - public void releaseAllGraphDB() { - System.out.println(RocksDBFactory.class); + @Test + public void testLiveSstFileGetters() { + RocksDBFactory.LiveSstFile f = + new RocksDBFactory.LiveSstFile("/data/db/000042.sst", "default"); + assertEquals("/data/db/000042.sst", f.getAbsolutePath()); + assertEquals("default", f.getCfName()); + } + + // ========================================================================= + // findLatestDBPath: DB-discovery filtering + // ========================================================================= - RocksDBFactory rFactory = RocksDBFactory.getInstance(); + /** + * Non-directory entries (loose files) in the DB parent dir must be ignored by + * {@code findLatestDBPath}. Before the fix, {@code File.listFiles()} returned plain files + * alongside dirs; iterating over them without an {@code isDirectory()} guard would blow up on + * {@code sFile.getName()} matching the prefix and then treating the file as a DB candidate. + * Verified by asserting the resolved path is the default (unversioned) name, not a file entry. + */ + @Test + public void testFindLatestDbPath_ignoresNonDirectoryEntries() throws Exception { + Path parent = Files.createTempDirectory("hg-find-latest-nondirs-"); + String dbName = "testdb"; + Path dbDir = parent.resolve(dbName); + Files.createDirectories(dbDir); + // Plant a plain file whose name starts with the DB prefix — must be ignored. + Files.write(parent.resolve(dbName + "_stray.sst"), "noise".getBytes(StandardCharsets.UTF_8)); - if (rFactory.queryGraphDB("bj01") == null) { - rFactory.createGraphDB("./tmp", "bj01"); + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName); + try { + assertTrue("resolved path must live under the parent dir", + session.getDbPath().startsWith(parent.toString())); + // The stray file must not have been treated as a versioned DB candidate. + assertTrue("resolved path must be the default (unversioned) DB dir", + session.getDbPath().endsWith(dbName)); + } finally { + factory.destroyGraphDB(dbName); + deleteRecursively(parent.toFile()); } + } + + /** + * Checkpoint temp dirs created by {@code captureMetadataSnapshot} carry the {@code _cloudmeta_} + * marker in their name (pattern: {@code _cloudmeta_}). They must be skipped + * by {@code findLatestDBPath} so an in-progress or crashed-leftover checkpoint dir is never + * mistaken for a versioned DB candidate. + */ + @Test + public void testFindLatestDbPath_ignoresCloudmetaTempDirs() throws Exception { + Path parent = Files.createTempDirectory("hg-find-latest-cloudmeta-"); + String dbName = "testdb"; + Path dbDir = parent.resolve(dbName); + Files.createDirectories(dbDir); + // Simulate a leftover checkpoint temp dir. + Files.createDirectories(parent.resolve(dbName + "_cloudmeta_123456789")); - if (rFactory.queryGraphDB("bj02") == null) { - rFactory.createGraphDB("./tmp", "bj02"); + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName); + try { + assertTrue("resolved path must be the default (unversioned) DB dir, not the checkpoint", + session.getDbPath().endsWith(dbName)); + assertFalse("checkpoint temp dir must not be chosen as the DB path", + session.getDbPath().contains("_cloudmeta_")); + } finally { + factory.destroyGraphDB(dbName); + deleteRecursively(parent.toFile()); } + } + + /** + * Dirs whose version suffix cannot be parsed as a long (e.g. {@code testdb_abc} or + * {@code testdb_1_xyz}) must be silently skipped rather than throwing + * {@link NumberFormatException} and aborting the DB open. + */ + @Test + public void testFindLatestDbPath_ignoresDirsWithNonNumericVersionSuffix() throws Exception { + Path parent = Files.createTempDirectory("hg-find-latest-badver-"); + String dbName = "testdb"; + Path dbDir = parent.resolve(dbName); + Files.createDirectories(dbDir); + // Dirs with unparseable version suffixes — must be skipped, not throw. + Files.createDirectories(parent.resolve(dbName + "_abc")); + Files.createDirectories(parent.resolve(dbName + "_1_xyz")); - if (rFactory.queryGraphDB("bj03") == null) { - rFactory.createGraphDB("./tmp", "bj03"); + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName); + try { + assertTrue("resolved path must be the default (unversioned) DB dir", + session.getDbPath().endsWith(dbName)); + } finally { + factory.destroyGraphDB(dbName); + deleteRecursively(parent.toFile()); } + } - RocksDBSession dbSession = rFactory.queryGraphDB("bj01"); + /** + * When multiple valid versioned DB dirs exist alongside the default, {@code findLatestDBPath} + * must select the one with the highest version number and delete the older ones. + */ + @Test + public void testFindLatestDbPath_picksHighestVersionAndDeletesOlder() throws Exception { + Path parent = Files.createTempDirectory("hg-find-latest-versions-"); + String dbName = "testdb"; + // Create versioned dirs (no default, as happens after the first rename cycle). + Path v1 = parent.resolve(dbName + "_1"); + Path v2 = parent.resolve(dbName + "_2"); + Path v3 = parent.resolve(dbName + "_3"); + Files.createDirectories(v1); + Files.createDirectories(v2); + Files.createDirectories(v3); - dbSession.checkTable("test"); - SessionOperator sessionOp = dbSession.sessionOp(); - sessionOp.prepare(); + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName); + try { + assertTrue("findLatestDBPath must resolve to the highest-versioned dir", + session.getDbPath().endsWith(dbName + "_3")); + assertFalse("older versioned dirs must be deleted by findLatestDBPath", + v1.toFile().exists()); + assertFalse("older versioned dirs must be deleted by findLatestDBPath", + v2.toFile().exists()); + } finally { + factory.destroyGraphDB(dbName); + deleteRecursively(parent.toFile()); + } + } + + // ========================================================================= + // processPendingDestroys / logUnclosedIterators (the 60s scheduled cleanup pass) + // ========================================================================= + + @Test + public void testProcessPendingDestroysDeletesFilesAndNotifiesListeners() throws Exception { + Path parent = Files.createTempDirectory("hg-destroy-pass-"); + String dbName = "destroypass"; + List deleted = new CopyOnWriteArrayList<>(); + RocksdbChangedListener recorder = new RocksdbChangedListener() { + @Override + public void onDBDeleted(String db, String dbPath) { + deleted.add(db); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(recorder); + try { + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName, 0); + String dbPath = session.getDbPath(); + session.close(); + factory.destroyGraphDB(dbName); + assertTrue("destroyGraphDB must enqueue a destroy watcher", + factory.findPathInRemovedList(dbPath)); + assertTrue(new File(dbPath).exists()); - sessionOp.put("test", "hi".getBytes(), "byebye".getBytes()); - sessionOp.commit(); + factory.processPendingDestroys(); - rFactory.releaseAllGraphDB(); + assertFalse("the pass must delete the DB directory", new File(dbPath).exists()); + assertTrue("onDBDeleted must fire for the destroyed DB", deleted.contains(dbName)); + assertFalse("the watcher must be removed after a successful delete", + factory.findPathInRemovedList(dbPath)); + } finally { + factory.removeRocksdbChangedListener(recorder); + deleteRecursively(parent.toFile()); + } + } + + /** A live DB at the SAME path as the stale watcher must be protected from its delete. */ + @Test + public void testProcessPendingDestroysDiscardsWatcherWhenDbRecreatedAtSamePath() + throws Exception { + Path dir = Files.createTempDirectory("hg-destroy-samepath-"); + String dbName = "destroysamepath-" + System.nanoTime(); + RocksDBSession stale = mock(RocksDBSession.class); + when(stale.getRefCount()).thenReturn(0); + when(stale.getGraphName()).thenReturn(dbName); + when(stale.getDbPath()).thenReturn(dir.toString()); + RocksDBSession live = mock(RocksDBSession.class); + when(live.getDbPath()).thenReturn(dir.toString()); + List deleted = new CopyOnWriteArrayList<>(); + RocksdbChangedListener recorder = new RocksdbChangedListener() { + @Override + public void onDBDeleted(String db, String dbPath) { + deleted.add(db); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map sessions = sessionMap(factory); + factory.addRocksdbChangedListener(recorder); + sessions.put(dbName, live); + RocksDBFactory.DBSessionWatcher watcher = factory.addDestroyWatcher(stale); + try { + factory.processPendingDestroys(); + + assertFalse("the stale watcher must be discarded", factory.isDestroyPending(watcher)); + verify(stale, never()).shutdown(); + assertTrue("the live DB's files must not be deleted", dir.toFile().exists()); + assertFalse("onDBDeleted must not fire for a superseded watcher", + deleted.contains(dbName)); + } finally { + sessions.remove(dbName, live); + factory.removeRocksdbChangedListener(recorder); + deleteRecursively(dir.toFile()); + } + } + + /** + * A DB recreated under the same name at a DIFFERENT path (a version-suffixed sibling) must not + * stop the stale directory from being deleted and reported through {@code onDBDeleted}. + */ + @Test + public void testProcessPendingDestroysDeletesStaleDirWhenDbRecreatedAtDifferentPath() + throws Exception { + Path parent = Files.createTempDirectory("hg-destroy-recreate-"); + String dbName = "destroyrecreate"; + List deleted = new CopyOnWriteArrayList<>(); + RocksdbChangedListener recorder = new RocksdbChangedListener() { + @Override + public void onDBDeleted(String db, String dbPath) { + deleted.add(dbPath); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(recorder); + try { + RocksDBSession first = factory.createGraphDB(parent.toString(), dbName, 0); + String stalePath = first.getDbPath(); + first.close(); + factory.destroyGraphDB(dbName); + assertTrue(factory.findPathInRemovedList(stalePath)); + + // Recreate under the same name before the cleanup pass runs; the removed path is + // skipped so the new DB lives at a version-suffixed sibling. + RocksDBSession second = factory.createGraphDB(parent.toString(), dbName, 0); + String livePath = second.getDbPath(); + second.close(); + assertFalse(stalePath.equals(livePath)); + + factory.processPendingDestroys(); + + assertFalse("the stale watcher must be completed", + factory.findPathInRemovedList(stalePath)); + assertFalse("the stale directory must be deleted", new File(stalePath).exists()); + assertTrue("the recreated DB's files must not be deleted", new File(livePath).exists()); + assertTrue("onDBDeleted must fire with the stale path", deleted.contains(stalePath)); + assertFalse("onDBDeleted must not fire for the live path", deleted.contains(livePath)); + assertTrue(factory.getGraphNames().contains(dbName)); + } finally { + factory.removeRocksdbChangedListener(recorder); + factory.destroyGraphDB(dbName); + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + } + + @Test + public void testProcessPendingDestroysGivesUpAfterThreeFailedAttempts() throws Exception { + Path dir = Files.createTempDirectory("hg-destroy-fail-"); + String dbName = "destroyfail-" + System.nanoTime(); + RocksDBSession session = mock(RocksDBSession.class); + when(session.getRefCount()).thenReturn(0); + when(session.getGraphName()).thenReturn(dbName); + when(session.getDbPath()).thenReturn(dir.toString()); + doThrow(new IllegalStateException("simulated shutdown failure")).when(session).shutdown(); + List deleted = new CopyOnWriteArrayList<>(); + RocksdbChangedListener recorder = new RocksdbChangedListener() { + @Override + public void onDBDeleted(String db, String dbPath) { + deleted.add(db); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(recorder); + try { + RocksDBFactory.DBSessionWatcher watcher = factory.addDestroyWatcher(session); + + factory.processPendingDestroys(); + factory.processPendingDestroys(); + assertEquals(2, watcher.deleteAttempts); + assertTrue("the watcher must be retried before the 3rd failure", + factory.isDestroyPending(watcher)); + + factory.processPendingDestroys(); + assertEquals(3, watcher.deleteAttempts); + assertFalse("the watcher must be dropped after the 3rd failure", + factory.isDestroyPending(watcher)); + assertTrue("a failed delete must leave the directory alone", dir.toFile().exists()); + assertFalse("onDBDeleted must not fire when the delete failed", + deleted.contains(dbName)); + } finally { + factory.removeRocksdbChangedListener(recorder); + deleteRecursively(dir.toFile()); + } + } + + @Test + public void testProcessPendingDestroysWaitsOnLiveRefsThenForceResetsAfterTimeout() + throws Exception { + Path dir = Files.createTempDirectory("hg-destroy-refs-"); + AtomicInteger refCount = new AtomicInteger(2); + RocksDBSession session = mock(RocksDBSession.class); + when(session.getRefCount()).thenAnswer(inv -> refCount.get()); + when(session.getGraphName()).thenReturn("destroyrefs-" + System.nanoTime()); + when(session.getDbPath()).thenReturn(dir.toString()); + doAnswer(inv -> { + refCount.set(0); + return null; + }).when(session).forceResetRefCount(); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBFactory.DBSessionWatcher watcher = factory.addDestroyWatcher(session); + try { + // Live refs inside the 30-min window: only waits (logs), no reset, no delete. + factory.processPendingDestroys(); + verify(session, never()).forceResetRefCount(); + verify(session, never()).shutdown(); + assertTrue(factory.isDestroyPending(watcher)); + + // Past the 30-min window: the stuck ref count is force-reset, still no delete yet. + watcher.timestamp = System.currentTimeMillis() - 31 * 60 * 1000L; + factory.processPendingDestroys(); + verify(session, times(1)).forceResetRefCount(); + verify(session, never()).shutdown(); + assertTrue(factory.isDestroyPending(watcher)); + + // The next pass sees refCount 0 and completes the delete. + factory.processPendingDestroys(); + verify(session, times(1)).shutdown(); + assertFalse(factory.isDestroyPending(watcher)); + assertFalse(dir.toFile().exists()); + } finally { + refCount.set(0); + factory.processPendingDestroys(); + deleteRecursively(dir.toFile()); + } + } + + @Test + public void testProcessPendingDestroysSwallowsUnexpectedErrorAndRetriesNextPass() + throws Exception { + Path dir = Files.createTempDirectory("hg-destroy-err-"); + RocksDBSession session = mock(RocksDBSession.class); + // The first pass blows up outside the per-watcher try; the next pass proceeds normally. + when(session.getRefCount()).thenThrow(new IllegalStateException("simulated")) + .thenReturn(0); + when(session.getGraphName()).thenReturn("destroyerr-" + System.nanoTime()); + when(session.getDbPath()).thenReturn(dir.toString()); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBFactory.DBSessionWatcher watcher = factory.addDestroyWatcher(session); + try { + factory.processPendingDestroys(); + assertTrue("an unexpected error must not drop the watcher", + factory.isDestroyPending(watcher)); + + factory.processPendingDestroys(); + assertFalse(factory.isDestroyPending(watcher)); + assertFalse(dir.toFile().exists()); + } finally { + deleteRecursively(dir.toFile()); + } + } + + @Test + public void testLogUnclosedIteratorsToleratesStaleAndMalformedKeys() throws Exception { + Path parent = Files.createTempDirectory("hg-iter-leak-"); + String dbName = "iterleak"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName, 0); + try { + // An iterator opened ~10 minutes 10 seconds ago falls in the "log it" window. + long now = System.currentTimeMillis(); + String staleKey = (now - 610_000L) + "-stale"; + // Younger than 10 minutes, and past 10 minutes but off the 10s log beat: not logged. + String freshKey = now + "-fresh"; + String offBeatKey = (now - 605_000L) + "-offbeat"; + session.addIterator(staleKey, null); + session.addIterator(freshKey, null); + session.addIterator(offBeatKey, null); + factory.logUnclosedIterators(); + session.removeIterator(freshKey); + session.removeIterator(offBeatKey); + + // A key without a numeric timestamp prefix must be swallowed, not propagate. + session.addIterator("not-a-timestamp", null); + factory.logUnclosedIterators(); + + session.removeIterator(staleKey); + session.removeIterator("not-a-timestamp"); + } finally { + session.close(); + factory.destroyGraphDB(dbName); + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + } + + // ========================================================================= + // Event-listener / reporting edge cases + // ========================================================================= + + @Test + public void testOnTableFileCreatedSkipsFailedFileCreation() { + TableFileCreationInfo info = mock(TableFileCreationInfo.class); + when(info.getStatus()).thenReturn(new Status(Status.Code.IOError, Status.SubCode.None, + "simulated flush failure")); + when(info.getFilePath()).thenReturn("/tmp/failed.sst"); + AtomicInteger created = new AtomicInteger(0); + RocksdbChangedListener probe = new RocksdbChangedListener() { + @Override + public void onTableFileCreated(String db, String cf, String path, long size) { + created.incrementAndGet(); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(probe); + try { + factory.getEventListener().onTableFileCreated(info); + assertEquals("a failed SST creation must not be propagated", 0, created.get()); + } finally { + factory.removeRocksdbChangedListener(probe); + } + } + + @Test + public void testOnTableFileDeletedPropagatesSuccessfulDeletion() { + TableFileDeletionInfo info = mock(TableFileDeletionInfo.class); + when(info.getStatus()).thenReturn(new Status(Status.Code.Ok, Status.SubCode.None, null)); + when(info.getDbName()).thenReturn("/data/deldb"); + when(info.getFilePath()).thenReturn("/data/deldb/000042.sst"); + List deleted = new CopyOnWriteArrayList<>(); + RocksdbChangedListener probe = new RocksdbChangedListener() { + @Override + public void onTableFileDeleted(String db, String cf, String path) { + assertNull("the column family is unknown for deletions", cf); + deleted.add(db + "|" + path); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(probe); + try { + factory.getEventListener().onTableFileDeleted(info); + assertEquals(List.of("/data/deldb|/data/deldb/000042.sst"), deleted); + } finally { + factory.removeRocksdbChangedListener(probe); + } + } + + @Test + public void testOnTableFileDeletedSkipsFailedDeletion() { + TableFileDeletionInfo info = mock(TableFileDeletionInfo.class); + when(info.getStatus()).thenReturn(new Status(Status.Code.IOError, Status.SubCode.None, + "simulated delete failure")); + when(info.getFilePath()).thenReturn("/tmp/failed.sst"); + AtomicInteger deleted = new AtomicInteger(0); + RocksdbChangedListener probe = new RocksdbChangedListener() { + @Override + public void onTableFileDeleted(String db, String cf, String path) { + deleted.incrementAndGet(); + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(probe); + try { + factory.getEventListener().onTableFileDeleted(info); + assertEquals("a failed SST deletion must not be propagated", 0, deleted.get()); + } finally { + factory.removeRocksdbChangedListener(probe); + } + } + + @Test + public void testGetTotalKeyReportsOpenDb() throws Exception { + Path parent = Files.createTempDirectory("hg-totalkey-"); + String dbName = "totalkey"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName, 0); + try { + Map totalKeys = factory.getTotalKey(); + assertTrue(totalKeys.containsKey(dbName)); + assertNotNull(totalKeys.get(dbName)); + } finally { + session.close(); + factory.destroyGraphDB(dbName); + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + } + + // ========================================================================= + // RocksDBSession: truncate / metadata checkpoint failure + // ========================================================================= + + @Test + public void testTruncateRecreatesDroppedTables() throws Exception { + Path parent = Files.createTempDirectory("hg-truncate-ok-"); + String dbName = "truncateok"; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName, 0); + try { + session.createTables("t1", "t2"); + session.truncate(); + + assertTrue("truncate must recreate the dropped tables", + session.tableIsExist("t1") && session.tableIsExist("t2")); + } finally { + session.close(); + factory.destroyGraphDB(dbName); + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + } + + /** + * If the checkpoint cannot be created, the partial temp dir must be removed and the failure + * surfaced as a {@link DBStoreException}. The failure is forced by making the directory that + * holds the {@code _cloudmeta_} temp dir read-only (skipped where permissions are not + * enforced, e.g. when running as root). + */ + @Test + public void testCaptureMetadataCheckpointFailureCleansUpAndThrows() throws Exception { + Path parent = Files.createTempDirectory("hg-ckfail-"); + String dbName = "ckfail"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName, 0); + File checkpointParent = new File(session.getDbPath()).getParentFile(); + try { + assumeTrue("needs enforceable directory permissions", + checkpointParent.setWritable(false) && !checkpointParent.canWrite()); + try { + session.captureMetadataCheckpoint(); + throw new AssertionError("captureMetadataCheckpoint must fail on a read-only dir"); + } catch (DBStoreException expected) { + // expected + } + String[] leftovers = checkpointParent.list((d, name) -> name.contains("_cloudmeta_")); + assertEquals("no partial checkpoint dir may be left behind", + 0, leftovers == null ? 0 : leftovers.length); + } finally { + checkpointParent.setWritable(true); + session.close(); + factory.destroyGraphDB(dbName); + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + } + + // ========================================================================= + // Scheduled maintenance / destroy edge cases + // ========================================================================= + + @Test + public void testRunMaintenancePassDrainsDestroyQueue() throws Exception { + Path dir = Files.createTempDirectory("hg-maint-pass-"); + RocksDBSession session = mock(RocksDBSession.class); + when(session.getRefCount()).thenReturn(0); + when(session.getGraphName()).thenReturn("maintpass-" + System.nanoTime()); + when(session.getDbPath()).thenReturn(dir.toString()); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBFactory.DBSessionWatcher watcher = factory.addDestroyWatcher(session); + try { + factory.runMaintenancePass(); + assertFalse(factory.isDestroyPending(watcher)); + assertFalse(dir.toFile().exists()); + } finally { + deleteRecursively(dir.toFile()); + } + } + + /** + * While a creation of the same DB name is in flight (slot reserved in {@code pendingCreations} + * but its path not yet known) the stale watcher must be kept and retried later: it must + * neither delete the directory the creation may be about to open nor be dropped. + */ + @Test + public void testProcessPendingDestroysDefersWatcherWhileSameDbCreationInFlight() + throws Exception { + Path dir = Files.createTempDirectory("hg-destroy-inflight-"); + String dbName = "destroyinflight-" + System.nanoTime(); + RocksDBSession session = mock(RocksDBSession.class); + when(session.getRefCount()).thenReturn(0); + when(session.getGraphName()).thenReturn(dbName); + when(session.getDbPath()).thenReturn(dir.toString()); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map> pending = pendingCreations(factory); + RocksDBFactory.DBSessionWatcher watcher = factory.addDestroyWatcher(session); + pending.put(dbName, new CompletableFuture<>()); + try { + factory.processPendingDestroys(); + + assertTrue("the watcher must be kept for a later round", + factory.isDestroyPending(watcher)); + verify(session, never()).shutdown(); + assertTrue("the in-flight DB's directory must not be deleted", dir.toFile().exists()); + + // Once the creation slot is gone the deferred cleanup proceeds. + pending.remove(dbName); + factory.processPendingDestroys(); + assertFalse(factory.isDestroyPending(watcher)); + assertFalse(dir.toFile().exists()); + } finally { + pending.remove(dbName); + deleteRecursively(dir.toFile()); + } + } + + @Test + public void testShutdownQuietlySwallowsShutdownFailure() { + RocksDBSession session = mock(RocksDBSession.class); + when(session.getGraphName()).thenReturn("shutdownfail"); + doThrow(new IllegalStateException("simulated shutdown failure")).when(session).shutdown(); + + RocksDBFactory.shutdownQuietly(session); + + verify(session, times(1)).shutdown(); + } + + @Test + public void testDestroyGraphDbUnknownNameIsNoop() { + RocksDBFactory factory = RocksDBFactory.getInstance(); + String dbName = "no-such-db-" + System.nanoTime(); + AtomicInteger deleteBegins = new AtomicInteger(0); + RocksdbChangedListener probe = new RocksdbChangedListener() { + @Override + public void onDBDeleteBegin(String db, String dbPath) { + deleteBegins.incrementAndGet(); + } + }; + factory.addRocksdbChangedListener(probe); + try { + factory.destroyGraphDB(dbName); + assertEquals("no listener may be notified for an unknown DB", 0, deleteBegins.get()); + assertFalse(factory.getGraphNames().contains(dbName)); + } finally { + factory.removeRocksdbChangedListener(probe); + } + } + + // ========================================================================= + // Reporting helpers against stubbed sessions + // ========================================================================= + + @Test + public void testFlushSessionSwallowsFlushFailure() throws Exception { + String dbName = "flushfail-" + System.nanoTime(); + RocksDBSession session = mock(RocksDBSession.class); + doThrow(new IllegalStateException("simulated flush failure")).when(session).flush(true); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map sessions = sessionMap(factory); + sessions.put(dbName, session); + try { + factory.flushSession(dbName, true); + verify(session, times(1)).flush(true); + } finally { + sessions.remove(dbName); + } + } + + @Test + public void testLiveSstFilesAndMetadataSnapshotForSessionWithoutDb() throws Exception { + String dbName = "nodb-" + System.nanoTime(); + RocksDBSession session = mock(RocksDBSession.class); + when(session.getDB()).thenReturn(null); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map sessions = sessionMap(factory); + sessions.put(dbName, session); + try { + assertTrue(factory.getLiveSstFiles(dbName).isEmpty()); + assertNull(factory.captureMetadataSnapshot(dbName)); + } finally { + sessions.remove(dbName); + } + } + + /** + * Only {@code *.sst} entries are reported, and the dir / file name are joined with exactly + * one separator whether or not the dir has a trailing one or the name a leading one. + */ + @Test + public void testGetLiveSstFilesJoinsPathsAndSkipsNonSstEntries() throws Exception { + String dbName = "livesst-" + System.nanoTime(); + List metadata = List.of( + liveFile("/data/db", "/000001.sst", "default"), + liveFile("/data/db/", "/000002.sst", "default"), + liveFile("/data/db", "000003.sst", "g+v"), + liveFile("/data/db/", "000004.sst", "g+v"), + liveFile("/data/db", "/MANIFEST-000005", "default"), + liveFile("/data/db", null, "default")); + RocksDBSession session = mock(RocksDBSession.class); + when(session.clone()).thenReturn(session); + when(session.getLiveFilesMetaData()).thenReturn(metadata); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map sessions = sessionMap(factory); + sessions.put(dbName, session); + try { + List files = factory.getLiveSstFiles(dbName); + + assertEquals(4, files.size()); + for (int i = 0; i < 4; i++) { + assertEquals(new File("/data/db", String.format("00000%d.sst", i + 1)).getPath(), + files.get(i).getAbsolutePath()); + } + assertEquals("default", files.get(0).getCfName()); + assertEquals("g+v", files.get(2).getCfName()); + // The session reference taken for the read is released. + verify(session).close(); + } finally { + sessions.remove(dbName); + } + } + + @Test + public void testMetadataSnapshotNonManifestNameAndCleanupFailureAreTolerated() + throws Exception { + assertEquals(-1L, new RocksDBFactory.MetadataSnapshot( + "/db", "/t", "CURRENT", "OPTIONS-000007", + java.util.List.of(), java.util.List.of()).getGeneration()); + + // A temp "dir" that is actually a regular file makes deleteDirectory throw; cleanup() + // must log and carry on rather than propagate. + Path notADir = Files.createTempFile("hg-ckpt-cleanup-", ".tmp"); + try { + new RocksDBFactory.MetadataSnapshot("/db", notADir.toString(), "CURRENT", + "MANIFEST-1", java.util.List.of(), + java.util.List.of()).cleanup(); + } finally { + Files.deleteIfExists(notADir); + } + } + + @Test + public void testOnReadMissSwallowsFailingListenerAndKeepsOtherHydration() { + RocksDBSession session = mock(RocksDBSession.class); + when(session.getGraphName()).thenReturn("readmiss"); + RocksdbChangedListener failing = new RocksdbChangedListener() { + @Override + public boolean onReadMiss(RocksDBSession s, String table, byte[] key) { + throw new IllegalStateException("simulated hydration failure"); + } + }; + RocksdbChangedListener hydrating = new RocksdbChangedListener() { + @Override + public boolean onReadMiss(RocksDBSession s, String table, byte[] key) { + return true; + } + }; + + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(failing); + factory.addRocksdbChangedListener(hydrating); + try { + assertTrue(factory.onReadMiss(session, "t", "k".getBytes(StandardCharsets.UTF_8))); + } finally { + factory.removeRocksdbChangedListener(failing); + factory.removeRocksdbChangedListener(hydrating); + } + } + + @Test + public void testDefaultFileAndReadMissHooksAreNoOps() { + RocksdbChangedListener listener = new RocksdbChangedListener() { + }; + listener.onTableFileCreated("db", "default", "/path/000001.sst", 1L); + listener.onTableFileDeleted("db", "default", "/path/000001.sst"); + assertFalse(listener.onReadMiss(null, "t", "k".getBytes(StandardCharsets.UTF_8))); + } + + // ========================================================================= + // RocksDBSession: construction / db-path discovery / checkpoint classification + // ========================================================================= + + @Test + public void testSessionConstructorWithoutPreOpenHookOpensDb() throws Exception { + Path parent = Files.createTempDirectory("hg-session-nohook-"); + String dbName = "nohook"; + RocksDBSession session = new RocksDBSession(RocksDBFactory.getInstance().getHugeConfig(), + parent.toString(), dbName, 0); + try { + assertNotNull(session.getDB()); + assertTrue(new File(session.getDbPath(), "CURRENT").exists()); + } finally { + session.shutdown(); + deleteRecursively(parent.toFile()); + } + } + + @Test + public void testCloseQuietlySwallowsCloseFailure() { + AtomicBoolean closed = new AtomicBoolean(false); + RocksDBSession.closeQuietly(null); + RocksDBSession.closeQuietly(() -> { + closed.set(true); + throw new IllegalStateException("simulated close failure"); + }); + assertTrue(closed.get()); + } + + /** + * Snapshot temp dirs ({@code *_temp_}) and dirs with more than two version parts are not DB + * candidates; among {@code __} dirs the highest {@code (v1, v2)} wins and the + * older ones are deleted. + */ + @Test + public void testFindLatestDbPath_skipsTempAndMalformedDirsAndComparesTwoPartVersions() + throws Exception { + Path parent = Files.createTempDirectory("hg-find-latest-twopart-"); + String dbName = "testdb"; + Path older = Files.createDirectories(parent.resolve(dbName + "_1_2")); + Path newer = Files.createDirectories(parent.resolve(dbName + "_1_5")); + Path temp = Files.createDirectories(parent.resolve(dbName + "_9_temp_")); + Path threePart = Files.createDirectories(parent.resolve(dbName + "_9_9_9")); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName); + try { + assertEquals(newer.toString(), session.getDbPath()); + assertFalse("the older two-part version must be deleted", older.toFile().exists()); + assertTrue("snapshot temp dirs must be left alone", temp.toFile().exists()); + assertTrue("malformed version dirs must be left alone", threePart.toFile().exists()); + } finally { + session.close(); + factory.destroyGraphDB(dbName); + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + } + + @Test + public void testCategoriseCheckpointClassifiesFilesAndToleratesMissingDir() throws Exception { + Path parent = Files.createTempDirectory("hg-categorise-"); + String dbName = "categorise"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName, 0); + try { + Path ckpt = Files.createDirectories(parent.resolve("ckpt")); + for (String name : List.of("CURRENT", "MANIFEST-000007", "OPTIONS-000009", + "000011.sst", "000012.log")) { + Files.write(ckpt.resolve(name), new byte[0]); + } + + RocksDBFactory.MetadataSnapshot snap = session.categoriseCheckpoint(ckpt.toString()); + assertEquals(session.getDbPath(), snap.getDbDir()); + assertEquals("CURRENT", snap.getCurrentFileName()); + assertEquals("MANIFEST-000007", snap.getManifestFileName()); + assertEquals(List.of("OPTIONS-000009"), snap.getOptionsFileNames()); + assertEquals("the WAL file must not be classified as an SST", + List.of("000011.sst"), snap.getSstFileNames()); + + RocksDBFactory.MetadataSnapshot empty = session.categoriseCheckpoint( + parent.resolve("missing").toString()); + assertNull(empty.getCurrentFileName()); + assertNull(empty.getManifestFileName()); + assertTrue(empty.getOptionsFileNames().isEmpty()); + assertTrue(empty.getSstFileNames().isEmpty()); + } finally { + session.close(); + factory.destroyGraphDB(dbName); + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + } + + // ========================================================================= + // Remote-state gating: native listener and event mask + // ========================================================================= + + @Test + public void testNativeListenerAttachedOnlyWhenRemoteStateListenerRegistered() + throws Exception { + RocksDBFactory factory = RocksDBFactory.getInstance(); + assumeTrue("another test leaked a remote-state listener", + !factory.hasRemoteStateListeners()); + Path parent = Files.createTempDirectory("hg-native-listener-"); + RocksdbChangedListener plain = new RocksdbChangedListener() { + }; + RocksdbChangedListener remote = remoteStateListener(); + RocksDBSession withoutRemote = null; + RocksDBSession withRemote = null; + factory.addRocksdbChangedListener(plain); + try { + withoutRemote = factory.createGraphDB(parent.toString(), "nolistener", 0); + assertFalse(factory.hasRemoteStateListeners()); + assertTrue("no native listener may be attached without a remote-state listener", + nativeListeners(withoutRemote).isEmpty()); + + factory.addRocksdbChangedListener(remote); + withRemote = factory.createGraphDB(parent.toString(), "withlistener", 0); + assertTrue(factory.hasRemoteStateListeners()); + assertEquals("the native listener must be attached with a remote-state listener", + 1, nativeListeners(withRemote).size()); + } finally { + factory.removeRocksdbChangedListener(remote); + factory.removeRocksdbChangedListener(plain); + closeQuietly(withoutRemote); + closeQuietly(withRemote); + destroyAndCleanup(factory, parent, "nolistener", "withlistener"); + } + } + + /** The native event mask must enable exactly the callbacks RocksdbEventListener overrides. */ + @Test + public void testEventListenerEnablesExactlyTheOverriddenCallbacks() { + Set overridden = new HashSet<>(); + for (Method method : RocksDBFactory.RocksdbEventListener.class.getDeclaredMethods()) { + if (method.getName().startsWith("on") && !method.isSynthetic()) { + overridden.add(method.getName()); + } + } + Set enabled = new HashSet<>(); + for (EnabledEventCallback callback : RocksDBFactory.ENABLED_EVENT_CALLBACKS) { + StringBuilder name = new StringBuilder("on"); + for (String part : callback.name().substring("ON_".length()).split("_")) { + name.append(part.charAt(0)).append(part.substring(1).toLowerCase()); + } + enabled.add(name.toString()); + } + assertFalse(overridden.isEmpty()); + assertEquals("enabled native callbacks must match the overridden on* methods", + overridden, enabled); + } + + // ========================================================================= + // Stale checkpoint sweep, metadata capture, createGraphDB lifecycle + // ========================================================================= + + @Test + public void testOpenSweepsStaleCloudMetaDirsOfSameGraphOnly() throws Exception { + Path parent = Files.createTempDirectory("hg-sweep-"); + String dbName = "sweepdb"; + File stale = Files.createDirectories(parent.resolve(dbName + "_cloudmeta_123")).toFile(); + File staleVersioned = Files.createDirectories(parent.resolve(dbName + "_5_cloudmeta_9")) + .toFile(); + File otherGraph = Files.createDirectories(parent.resolve(dbName + "x_cloudmeta_1")).toFile(); + + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = null; + try { + session = factory.createGraphDB(parent.toString(), dbName, 0); + assertFalse("stale checkpoint dir must be swept", stale.exists()); + assertFalse("stale versioned checkpoint dir must be swept", staleVersioned.exists()); + assertTrue("another graph's checkpoint dir must be kept", otherGraph.exists()); + } finally { + closeQuietly(session); + destroyAndCleanup(factory, parent, dbName); + } + } + + @Test + public void testCaptureMetadataCheckpointOnClosedSessionFailsCleanly() throws Exception { + Path parent = Files.createTempDirectory("hg-capture-closed-"); + try { + RocksDBSession session = new RocksDBSession( + RocksDBFactory.getInstance().getHugeConfig(), parent.toString(), + "captureclosed", 0); + session.close(); + try { + session.captureMetadataCheckpoint(); + throw new AssertionError("capture on a closed session must fail"); + } catch (DBStoreException expected) { + assertTrue(expected.getMessage().contains("closed")); + } + } finally { + deleteRecursively(parent.toFile()); + } + } + + @Test + public void testCaptureMetadataSnapshotReleasesSessionReference() throws Exception { + Path parent = Files.createTempDirectory("hg-capture-ref-"); + String dbName = "captureref"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksDBSession session = null; + try { + session = factory.createGraphDB(parent.toString(), dbName, 0); + int before = session.getRefCount(); + RocksDBFactory.MetadataSnapshot snap = factory.captureMetadataSnapshot(dbName); + assertNotNull(snap); + snap.cleanup(); + assertEquals("the capture must release its session reference", + before, session.getRefCount()); + } finally { + closeQuietly(session); + destroyAndCleanup(factory, parent, dbName); + } + } + + @Test + public void testResolveOpenPathRedirectsOnlyToSiblingsWithoutLocalDb() throws Exception { + Path parent = Files.createTempDirectory("hg-open-redirect-"); + Path other = Files.createTempDirectory("hg-open-redirect-other-"); + String moved = "redirected"; + String rejected = "rejectedredirect"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map openedAt = new ConcurrentHashMap<>(); + RocksdbChangedListener redirector = new RocksdbChangedListener() { + @Override + public String resolveOpenPath(String db, String dbPath) { + if (moved.equals(db)) { + return parent.resolve(moved + "_4").toString(); + } + if (rejected.equals(db)) { + return other.resolve(rejected + "_4").toString(); // not a sibling + } + return dbPath; + } + + @Override + public void onDBOpening(String db, String dbPath) { + openedAt.put(db, dbPath); + } + }; + factory.addRocksdbChangedListener(redirector); + try { + RocksDBSession session = factory.createGraphDB(parent.toString(), moved, 0); + assertEquals(parent.resolve(moved + "_4").toString(), session.getDbPath()); + assertEquals("onDBOpening must see the redirected path", + session.getDbPath(), openedAt.get(moved)); + session.close(); + factory.releaseGraphDB(moved); + + // The directory now holds a DB (CURRENT), so the next open is not redirected again. + openedAt.clear(); + RocksDBSession reopened = factory.createGraphDB(parent.toString(), moved, 0); + assertEquals(parent.resolve(moved + "_4").toString(), reopened.getDbPath()); + reopened.close(); + factory.releaseGraphDB(moved); + + RocksDBSession kept = factory.createGraphDB(parent.toString(), rejected, 0); + assertEquals(parent.resolve(rejected).toString(), kept.getDbPath()); + kept.close(); + factory.releaseGraphDB(rejected); + } finally { + factory.removeRocksdbChangedListener(redirector); + deleteRecursively(parent.toFile()); + deleteRecursively(other.toFile()); + } + } + + @Test + public void testOnDbClosingFiresWhileSessionIsStillOpen() throws Exception { + Path parent = Files.createTempDirectory("hg-db-closing-"); + String released = "closingrelease"; + String releasedAll = "closingreleaseall"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map openAtClosing = new ConcurrentHashMap<>(); + RocksdbChangedListener probe = new RocksdbChangedListener() { + @Override + public void onDBClosing(String db) { + // A final publish needs a live session: captureMetadataSnapshot must still work. + RocksDBFactory.MetadataSnapshot snap = factory.captureMetadataSnapshot(db); + openAtClosing.put(db, snap != null); + if (snap != null) { + snap.cleanup(); + } + } + }; + factory.addRocksdbChangedListener(probe); + try { + factory.createGraphDB(parent.toString(), released, 0).close(); + factory.createGraphDB(parent.toString(), releasedAll, 0).close(); + + factory.releaseGraphDB(released); + assertEquals(Boolean.TRUE, openAtClosing.get(released)); + assertFalse(openAtClosing.containsKey(releasedAll)); + + factory.releaseAllGraphDB(); + assertEquals("releaseAllGraphDB must notify before shutting sessions down", + Boolean.TRUE, openAtClosing.get(releasedAll)); + } finally { + factory.removeRocksdbChangedListener(probe); + setClosing(factory, false); + deleteRecursively(parent.toFile()); + } + } + + @Test + public void testCreateGraphDbDiscardsSessionWhenFactoryClosesDuringCreation() + throws Exception { + Path parent = Files.createTempDirectory("hg-create-closing-"); + String dbName = "createclosing"; + RocksDBFactory factory = RocksDBFactory.getInstance(); + RocksdbChangedListener closer = new RocksdbChangedListener() { + @Override + public void onDBOpening(String db, String dbPath) { + if (dbName.equals(db)) { + try { + setClosing(factory, true); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + } + }; + factory.addRocksdbChangedListener(closer); + try { + try { + factory.createGraphDB(parent.toString(), dbName, 0); + throw new AssertionError("createGraphDB must fail once the factory is closing"); + } catch (DBStoreException expected) { + assertTrue(expected.getMessage().contains("closed")); + } + assertFalse("the discarded session must not be registered", + factory.getGraphNames().contains(dbName)); + assertFalse("the creation slot must be cleared", + pendingCreations(factory).containsKey(dbName)); + } finally { + factory.removeRocksdbChangedListener(closer); + setClosing(factory, false); + deleteRecursively(parent.toFile()); + } + } + + @Test + public void testCreateGraphDbWaiterTimesOutOnStuckCreation() throws Exception { + String dbName = "createstuck-" + System.nanoTime(); + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map> pending = pendingCreations(factory); + long originalTimeout = factory.creationWaitTimeoutMs; + pending.put(dbName, new CompletableFuture<>()); + factory.creationWaitTimeoutMs = 200; + try { + try { + factory.createGraphDB("unused", dbName, 0); + throw new AssertionError("a waiter must time out on a stuck creation"); + } catch (DBStoreException expected) { + assertTrue(expected.getMessage().contains("Timed out")); + } + // The global write lock must have been released: other DBs remain usable. + assertNull(factory.queryGraphDB("no-such-db")); + } finally { + factory.creationWaitTimeoutMs = originalTimeout; + pending.remove(dbName); + } + } + + @Test + public void testCreateGraphDbWaiterWakesWhenCreatorFailsAndRecreates() throws Exception { + Path parent = Files.createTempDirectory("hg-create-waiter-retry-"); + String dbName = "createwaiterretry"; + CountDownLatch hydrating = new CountDownLatch(1); + CountDownLatch failNow = new CountDownLatch(1); + AtomicInteger openings = new AtomicInteger(); + RocksdbChangedListener gate = new RocksdbChangedListener() { + @Override + public void onDBOpening(String db, String dbPath) { + if (!dbName.equals(db) || openings.incrementAndGet() > 1) { + return; + } + hydrating.countDown(); + try { + failNow.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("simulated hydration failure"); + } + }; + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(gate); + AtomicReference waited = new AtomicReference<>(); + try { + Thread creator = new Thread(() -> { + try { + factory.createGraphDB(parent.toString(), dbName, 0); + } catch (RuntimeException expected) { + // simulated failure + } + }); + creator.start(); + assertTrue(hydrating.await(30, TimeUnit.SECONDS)); + Thread waiter = new Thread( + () -> waited.set(factory.createGraphDB(parent.toString(), dbName, 0))); + waiter.start(); + while (waiter.getState() != Thread.State.TIMED_WAITING && waiter.isAlive()) { + Thread.sleep(5); + } + failNow.countDown(); + creator.join(30_000); + waiter.join(30_000); + assertNotNull("the waiter must re-check and create the DB itself", waited.get()); + } finally { + failNow.countDown(); + factory.removeRocksdbChangedListener(gate); + closeQuietly(waited.get()); + destroyAndCleanup(factory, parent, dbName); + } + } + + @Test + public void testAwaitPendingCreationsDrainedWaitsForInFlightCreation() throws Exception { + RocksDBFactory factory = RocksDBFactory.getInstance(); + Map> pending = pendingCreations(factory); + String dbName = "drain-" + System.nanoTime(); + CompletableFuture creation = new CompletableFuture<>(); + pending.put(dbName, creation); + try { + AtomicBoolean returned = new AtomicBoolean(); + Thread drainer = new Thread(() -> { + factory.awaitPendingCreationsDrained(); + returned.set(true); + }); + drainer.start(); + Thread.sleep(200); + assertFalse("must wait while a creation is in flight", returned.get()); + creation.complete(null); + drainer.join(10_000); + assertTrue("must return once the creation completes", returned.get()); + } finally { + pending.remove(dbName); + } + } + + // ========================================================================= + // Helpers + // ========================================================================= + + private static LiveFileMetaData liveFile(String dir, String fileName, String cfName) { + LiveFileMetaData md = mock(LiveFileMetaData.class); + when(md.path()).thenReturn(dir); + when(md.fileName()).thenReturn(fileName); + when(md.columnFamilyName()).thenReturn(cfName.getBytes(StandardCharsets.UTF_8)); + return md; + } + + @SuppressWarnings("unchecked") + private static Map> pendingCreations(RocksDBFactory factory) + throws Exception { + Field field = RocksDBFactory.class.getDeclaredField("pendingCreations"); + field.setAccessible(true); + return (Map>) field.get(factory); + } + + private static List nativeListeners(RocksDBSession session) throws Exception { + Field field = RocksDBSession.class.getDeclaredField("dbOptions"); + field.setAccessible(true); + return ((DBOptions) field.get(session)).listeners(); + } + + private static RocksdbChangedListener remoteStateListener() { + return new RocksdbChangedListener() { + @Override + public boolean managesRemoteState() { + return true; + } + }; + } + + private static void closeQuietly(RocksDBSession session) { + if (session != null) { + session.close(); + } + } + + private static void destroyAndCleanup(RocksDBFactory factory, Path parent, String... dbNames) { + for (String dbName : dbNames) { + factory.destroyGraphDB(dbName); + } + factory.processPendingDestroys(); + deleteRecursively(parent.toFile()); + } + + private static void setClosing(RocksDBFactory factory, boolean value) throws Exception { + Field field = RocksDBFactory.class.getDeclaredField("closing"); + field.setAccessible(true); + ((AtomicBoolean) field.get(factory)).set(value); + } + + @SuppressWarnings("unchecked") + private static Map sessionMap(RocksDBFactory factory) + throws Exception { + return (Map) sessionMapField().get(factory); + } + + private static void setSessionMap(RocksDBFactory factory, Map map) + throws Exception { + sessionMapField().set(factory, map); + } + + private static Field sessionMapField() throws Exception { + Field field = RocksDBFactory.class.getDeclaredField("dbSessionMap"); + field.setAccessible(true); + return field; + } + + private static int countSst(String dir) { + File[] files = new File(dir).listFiles((d, name) -> name.endsWith(".sst")); + return files == null ? 0 : files.length; + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + private static void deleteRecursively(File dir) { + if (dir == null || !dir.exists()) { + return; + } + File[] children = dir.listFiles(); + if (children != null) { + for (File c : children) { + if (c.isDirectory()) { + deleteRecursively(c); + } else { + c.delete(); + } + } + } + dir.delete(); } } diff --git a/hugegraph-store/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SessionOperatorReadMissTest.java b/hugegraph-store/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SessionOperatorReadMissTest.java new file mode 100644 index 0000000000..fb10dbfc8e --- /dev/null +++ b/hugegraph-store/hg-store-rocksdb/src/test/java/org/apache/hugegraph/rocksdb/access/SessionOperatorReadMissTest.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.rocksdb.access; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.configuration2.MapConfiguration; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory.RocksdbChangedListener; +import org.apache.hugegraph.rocksdb.access.RocksDBSession.CFHandleLock; +import org.junit.BeforeClass; +import org.junit.Test; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; + +/** + * {@link SessionOperatorImpl#get(String, byte[])} read-miss / cloud-hydration behaviour. + * + *

The {@code get} path invokes the {@link RocksdbChangedListener#onReadMiss} hook only when the + * read fails with a RocksDB error (a live SST missing on local disk), after releasing the + * column-family handle lock (hydration may perform slow S3 I/O), then re-acquires the lock for a + * single retry read. These tests pin that contract: + * + *

    + *
  • A hit returns the value without ever consulting a read-miss listener.
  • + *
  • A null result is a genuine key-not-found: it is returned without consulting the listener, + * because hydrating files cannot change it.
  • + *
  • A failed read whose listener restores files is retried once; if nothing is restored, or + * the retry fails too, the error propagates as {@link DBStoreException}.
  • + *
+ */ +public class SessionOperatorReadMissTest { + + private static final String TABLE = "t"; + + @BeforeClass + public static void init() { + OptionSpace.register("rocksdb", + "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); + RocksDBOptions.instance(); + Map configMap = new HashMap<>(); + configMap.put("rocksdb.write_buffer_size", "1048576"); + configMap.put("rocksdb.bloom_filter_bits_per_key", "10"); + RocksDBFactory.getInstance().setHugeConfig(new HugeConfig(new MapConfiguration(configMap))); + } + + @Test + public void testGetHitDoesNotConsultReadMissListener() throws Exception { + AtomicInteger readMissCalls = new AtomicInteger(0); + RocksdbChangedListener counter = new RocksdbChangedListener() { + @Override + public boolean onReadMiss(RocksDBSession session, String table, byte[] key) { + readMissCalls.incrementAndGet(); + return false; + } + }; + withSession("hit", counter, session -> { + SessionOperator op = session.sessionOp(); + op.prepare(); + op.put(TABLE, bytes("k"), bytes("v")); + op.commit(); + + byte[] value = session.sessionOp().get(TABLE, bytes("k")); + + assertArrayEquals(bytes("v"), value); + assertEquals("read-miss hook must not fire on a hit", 0, readMissCalls.get()); + }); + } + + @Test + public void testGetMissReturnsNullWithoutConsultingReadMissListener() throws Exception { + AtomicInteger readMissCalls = new AtomicInteger(0); + RocksdbChangedListener hydrator = new RocksdbChangedListener() { + @Override + public boolean onReadMiss(RocksDBSession session, String table, byte[] key) { + readMissCalls.incrementAndGet(); + return true; + } + }; + withSession("miss", hydrator, session -> { + session.checkTable(TABLE); + + byte[] value = session.sessionOp().get(TABLE, bytes("absent")); + + assertNull("a key-not-found must return null", value); + assertEquals("read-miss hook must not fire on a null result", 0, readMissCalls.get()); + }); + } + + // ========================================================================= + // Read-miss hydration on the EXCEPTION path + // + // A live SST referenced by the manifest that is physically missing surfaces as a + // RocksDBException (not a null), so hydration must be attempted on the exception too and the + // read retried; if nothing is restored, the original error must propagate as DBStoreException. + // A real embedded DB cannot deterministically raise-then-recover an IO error, so the DB and + // session are mocked here to drive the branch precisely. + // ========================================================================= + + @Test + public void firstGetThrows_thenHydrationSucceeds_retryReturnsRestoredValue() throws Exception { + RocksDB db = mock(RocksDB.class); + RocksDBSession session = mock(RocksDBSession.class); + ColumnFamilyHandle handle = mock(ColumnFamilyHandle.class); + CFHandleLock cfLock = mock(CFHandleLock.class); + when(cfLock.get()).thenReturn(handle); + when(session.getDB()).thenReturn(db); + when(session.getCFHandleLock(TABLE)).thenReturn(cfLock); + when(session.getGraphName()).thenReturn("db-mock"); + // First read throws (missing live SST → IO error); after hydration the retry returns it. + when(db.get(any(ColumnFamilyHandle.class), any(byte[].class))) + .thenThrow(new RocksDBException("simulated missing SST")) + .thenReturn(bytes("hydrated")); + + AtomicInteger hydrations = new AtomicInteger(); + RocksdbChangedListener hydrator = new RocksdbChangedListener() { + @Override + public boolean onReadMiss(RocksDBSession s, String table, byte[] key) { + hydrations.incrementAndGet(); + return true; // simulate a successful cloud restore of the missing SST + } + }; + RocksDBFactory.getInstance().addRocksdbChangedListener(hydrator); + try { + SessionOperator op = new SessionOperatorImpl(session); + byte[] value = op.get(TABLE, bytes("cold")); + + assertArrayEquals("retry after hydration must return the restored value", + bytes("hydrated"), value); + assertEquals("hydration must be attempted exactly once", 1, hydrations.get()); + verify(db, times(2)).get(any(ColumnFamilyHandle.class), any(byte[].class)); + } finally { + RocksDBFactory.getInstance().removeRocksdbChangedListener(hydrator); + } + } + + @Test + public void firstGetThrows_hydrationSucceeds_butRetryAlsoThrows_propagatesRetryError() + throws Exception { + RocksDB db = mock(RocksDB.class); + RocksDBSession session = mock(RocksDBSession.class); + ColumnFamilyHandle handle = mock(ColumnFamilyHandle.class); + CFHandleLock cfLock = mock(CFHandleLock.class); + when(cfLock.get()).thenReturn(handle); + when(session.getDB()).thenReturn(db); + when(session.getCFHandleLock(TABLE)).thenReturn(cfLock); + when(session.getGraphName()).thenReturn("db-mock"); + // Both the initial read and the post-hydration retry throw (the restore did not repair it). + when(db.get(any(ColumnFamilyHandle.class), any(byte[].class))) + .thenThrow(new RocksDBException("initial missing SST")) + .thenThrow(new RocksDBException("still broken after restore")); + + RocksdbChangedListener hydrator = new RocksdbChangedListener() { + @Override + public boolean onReadMiss(RocksDBSession s, String table, byte[] key) { + return true; + } + }; + RocksDBFactory.getInstance().addRocksdbChangedListener(hydrator); + try { + SessionOperator op = new SessionOperatorImpl(session); + try { + op.get(TABLE, bytes("cold")); + fail("expected DBStoreException when the post-hydration retry also fails"); + } catch (DBStoreException expected) { + // expected + } + verify(db, times(2)).get(any(ColumnFamilyHandle.class), any(byte[].class)); + } finally { + RocksDBFactory.getInstance().removeRocksdbChangedListener(hydrator); + } + } + + @Test + public void firstGetThrows_noHydration_propagatesAsDbStoreExceptionWithoutRetry() + throws Exception { + RocksDB db = mock(RocksDB.class); + RocksDBSession session = mock(RocksDBSession.class); + ColumnFamilyHandle handle = mock(ColumnFamilyHandle.class); + CFHandleLock cfLock = mock(CFHandleLock.class); + when(cfLock.get()).thenReturn(handle); + when(session.getDB()).thenReturn(db); + when(session.getCFHandleLock(TABLE)).thenReturn(cfLock); + when(session.getGraphName()).thenReturn("db-mock"); + when(db.get(any(ColumnFamilyHandle.class), any(byte[].class))) + .thenThrow(new RocksDBException("simulated missing SST")); + + RocksdbChangedListener noHydration = new RocksdbChangedListener() { + @Override + public boolean onReadMiss(RocksDBSession s, String table, byte[] key) { + return false; // nothing could be restored + } + }; + RocksDBFactory.getInstance().addRocksdbChangedListener(noHydration); + try { + SessionOperator op = new SessionOperatorImpl(session); + try { + op.get(TABLE, bytes("cold")); + fail("expected DBStoreException to propagate when hydration restores nothing"); + } catch (DBStoreException expected) { + // expected + } + // No retry read when hydration reported nothing: only the single failing get. + verify(db, times(1)).get(any(ColumnFamilyHandle.class), any(byte[].class)); + } finally { + RocksDBFactory.getInstance().removeRocksdbChangedListener(noHydration); + } + } + + @Test + public void getMisses_returnsNullWithSingleReadAndNoHydration() throws Exception { + RocksDB db = mock(RocksDB.class); + RocksDBSession session = mock(RocksDBSession.class); + ColumnFamilyHandle handle = mock(ColumnFamilyHandle.class); + CFHandleLock cfLock = mock(CFHandleLock.class); + when(cfLock.get()).thenReturn(handle); + when(session.getDB()).thenReturn(db); + when(session.getCFHandleLock(TABLE)).thenReturn(cfLock); + when(session.getGraphName()).thenReturn("db-mock"); + // A clean miss: null, no exception. + when(db.get(any(ColumnFamilyHandle.class), any(byte[].class))).thenReturn(null); + + AtomicInteger hydrations = new AtomicInteger(); + RocksdbChangedListener hydrator = new RocksdbChangedListener() { + @Override + public boolean onReadMiss(RocksDBSession s, String table, byte[] key) { + hydrations.incrementAndGet(); + return true; + } + }; + RocksDBFactory.getInstance().addRocksdbChangedListener(hydrator); + try { + SessionOperator op = new SessionOperatorImpl(session); + assertNull(op.get(TABLE, bytes("cold"))); + assertEquals("hydration must not be attempted for a null result", 0, hydrations.get()); + verify(db, times(1)).get(any(ColumnFamilyHandle.class), any(byte[].class)); + } finally { + RocksDBFactory.getInstance().removeRocksdbChangedListener(hydrator); + } + } + + // ========================================================================= + // Helpers + // ========================================================================= + + private interface SessionTask { + void run(RocksDBSession session) throws Exception; + } + + private static void withSession(String dbName, RocksdbChangedListener listener, + SessionTask task) throws Exception { + Path parent = Files.createTempDirectory("hg-readmiss-"); + RocksDBFactory factory = RocksDBFactory.getInstance(); + factory.addRocksdbChangedListener(listener); + try { + RocksDBSession session = factory.createGraphDB(parent.toString(), dbName, 0); + task.run(session); + } finally { + factory.removeRocksdbChangedListener(listener); + try { + factory.destroyGraphDB(dbName); + } catch (Exception ignore) { + // best-effort + } + deleteRecursively(parent.toFile()); + } + } + + private static byte[] bytes(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + private static void deleteRecursively(java.io.File dir) { + if (dir == null || !dir.exists()) { + return; + } + java.io.File[] children = dir.listFiles(); + if (children != null) { + for (java.io.File c : children) { + if (c.isDirectory()) { + deleteRecursively(c); + } else { + c.delete(); + } + } + } + dir.delete(); + } +} diff --git a/hugegraph-store/lombok.config b/hugegraph-store/lombok.config new file mode 100644 index 0000000000..56a924990c --- /dev/null +++ b/hugegraph-store/lombok.config @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +config.stopBubbling = true +# Mark Lombok-generated methods with @lombok.Generated so JaCoCo excludes them from coverage. +lombok.addLombokGeneratedAnnotation = true diff --git a/hugegraph-store/pom.xml b/hugegraph-store/pom.xml index edb27c3966..9b462379a3 100644 --- a/hugegraph-store/pom.xml +++ b/hugegraph-store/pom.xml @@ -21,7 +21,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 hugegraph-store - ${revision} pom @@ -41,6 +40,7 @@ hg-store-node hg-store-dist hg-store-cli + hg-store-cloud-s3 @@ -80,11 +80,21 @@ hg-store-core ${project.version} + + org.apache.hugegraph + hg-store-cloud-s3 + ${project.version} + org.apache.hugegraph hg-store-transfer ${project.version} + + org.rocksdb + rocksdbjni + ${rocksdb.version} + org.apache.logging.log4j log4j-slf4j-impl diff --git a/install-dist/release-docs/LICENSE b/install-dist/release-docs/LICENSE index 6e27cb1e2e..cd2e4ebac6 100644 --- a/install-dist/release-docs/LICENSE +++ b/install-dist/release-docs/LICENSE @@ -261,6 +261,8 @@ non-Apache-2.0 license text is required for a bundled component. The Netty modules share licenses/LICENSE-netty-4.1.137.Final.txt; tcnative modules use licenses/LICENSE-netty-tcnative-2.0.81.Final.txt. The shaded gRPC transport uses licenses/LICENSE-grpc-netty-shaded-1.55.3.txt. +The AWS SDK third-party-jackson-core-2.33.8 module bundles MIT-licensed FastDoubleParser; +see licenses/LICENSE-third-party-jackson-core-2.33.8.txt. Arthas 4.1.5 includes shaded Jackson, FastDoubleParser, ASM 9.9, Logback 1.2.13 and RSyntaxTextArea 3.3.4. Their additional MIT, BSD, Boost and EPL 1.0 license texts and source locations @@ -372,6 +374,7 @@ the corresponding per-component license file. https://central.sonatype.com/artifact/commons-cli/commons-cli/1.5.0 -> Apache 2.0 https://central.sonatype.com/artifact/commons-codec/commons-codec/1.13 -> Apache 2.0 https://central.sonatype.com/artifact/commons-codec/commons-codec/1.15 -> Apache 2.0 + https://central.sonatype.com/artifact/commons-codec/commons-codec/1.17.1 -> Apache 2.0 https://central.sonatype.com/artifact/commons-collections/commons-collections/3.2.2 -> Apache 2.0 https://central.sonatype.com/artifact/commons-configuration/commons-configuration/1.10 -> Apache 2.0 https://central.sonatype.com/artifact/commons-io/commons-io/2.12.0 -> Apache 2.0 @@ -547,6 +550,7 @@ the corresponding per-component license file. https://central.sonatype.com/artifact/org.apache.htrace/htrace-core4/4.2.0-incubating -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.httpcomponents/httpclient/4.5.13 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.httpcomponents/httpcore/4.4.13 -> Apache 2.0 + https://central.sonatype.com/artifact/org.apache.httpcomponents/httpcore/4.4.16 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.ivy/ivy/2.6.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.kerby/kerb-admin/2.0.1 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.kerby/kerb-client/2.0.1 -> Apache 2.0 @@ -704,6 +708,37 @@ the corresponding per-component license file. https://central.sonatype.com/artifact/org.apache.lucene/lucene-queryparser/5.2.1 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.lucene/lucene-sandbox/5.2.1 -> Apache 2.0 https://central.sonatype.com/artifact/org.apache.lucene/lucene-suggest/5.2.1 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/annotations/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/apache-client/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/arns/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/auth/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/aws-core/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/aws-query-protocol/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/aws-xml-protocol/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/checksums-spi/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/checksums/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/crt-core/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/endpoints-spi/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/http-auth-aws-eventstream/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/http-auth-aws/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/http-auth-spi/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/http-auth/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/http-client-spi/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/identity-spi/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/json-utils/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/metrics-spi/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/netty-nio-client/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/profiles/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/protocol-core/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/regions/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/retries-spi/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/retries/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/s3-transfer-manager/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/s3/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/sdk-core/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/third-party-jackson-core/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.awssdk/utils/2.33.8 -> Apache 2.0 + https://central.sonatype.com/artifact/software.amazon.eventstream/eventstream/1.0.1 -> Apache 2.0 ======================================================================== Third party BSD licenses @@ -763,12 +798,14 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://central.sonatype.com/artifact/org.mockito/mockito-junit-jupiter/3.9.0 -> MIT https://central.sonatype.com/artifact/org.projectlombok/lombok/1.18.24 -> MIT https://central.sonatype.com/artifact/org.projectlombok/lombok/1.18.30 -> MIT + https://central.sonatype.com/artifact/org.reactivestreams/reactive-streams/1.0.4 -> MIT-0 https://central.sonatype.com/artifact/org.slf4j/jcl-over-slf4j/1.7.25 -> MIT https://central.sonatype.com/artifact/org.slf4j/jul-to-slf4j/1.7.36 -> MIT https://central.sonatype.com/artifact/org.slf4j/slf4j-api/1.7.21 -> MIT https://central.sonatype.com/artifact/org.slf4j/slf4j-api/1.7.25 -> MIT https://central.sonatype.com/artifact/org.slf4j/slf4j-api/1.7.31 -> MIT https://central.sonatype.com/artifact/org.slf4j/slf4j-api/1.7.32 -> MIT + https://central.sonatype.com/artifact/org.slf4j/slf4j-api/1.7.36 -> MIT https://central.sonatype.com/artifact/org.slf4j/slf4j-api/2.0.9 -> MIT https://central.sonatype.com/artifact/redis.clients/jedis/2.5.1 -> MIT diff --git a/install-dist/release-docs/NOTICE b/install-dist/release-docs/NOTICE index 2f6fa03e5e..f21d0203d1 100644 --- a/install-dist/release-docs/NOTICE +++ b/install-dist/release-docs/NOTICE @@ -3237,6 +3237,38 @@ The Apache Software Foundation (https://www.apache.org/). ======================================================================== +AWS SDK for Java 2.0 (software.amazon.awssdk 2.33.8) NOTICE + +======================================================================== + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================== + Arthas 4.1.5 embedded Jackson 2.18.1 NOTICE ======================================================================== @@ -3322,6 +3354,30 @@ https://github.com/wrandelshofer/FastDoubleParser/blob/39e123b15b71f29a38a087d16 ======================================================================== +commons-codec-1.17.1 NOTICE + +======================================================================== + +Apache Commons Codec +Copyright 2002-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +======================================================================== + +httpcore-4.4.16 NOTICE + +======================================================================== + +Apache HttpCore +Copyright 2005-2022 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================== + Arthas 4.1.5 embedded Logback 1.2.13 NOTICE ======================================================================== diff --git a/install-dist/release-docs/licenses/LICENSE-reactive-streams-1.0.4.txt b/install-dist/release-docs/licenses/LICENSE-reactive-streams-1.0.4.txt new file mode 100644 index 0000000000..385ad9244f --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-reactive-streams-1.0.4.txt @@ -0,0 +1,17 @@ +reactive-streams 1.0.4 is licensed under MIT No Attribution (SPDX: MIT-0), as declared +in its POM and source headers. License text from https://spdx.org/licenses/MIT-0.html: + +MIT No Attribution + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/install-dist/release-docs/licenses/LICENSE-slf4j-api-1.7.36.txt b/install-dist/release-docs/licenses/LICENSE-slf4j-api-1.7.36.txt new file mode 100644 index 0000000000..d4685dbe36 --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-slf4j-api-1.7.36.txt @@ -0,0 +1,22 @@ +Copyright (c) 2004-2011 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/install-dist/release-docs/licenses/LICENSE-third-party-jackson-core-2.33.8.txt b/install-dist/release-docs/licenses/LICENSE-third-party-jackson-core-2.33.8.txt new file mode 100644 index 0000000000..f05f0a4542 --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-third-party-jackson-core-2.33.8.txt @@ -0,0 +1,437 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +======================================================================== +FastDoubleParser (META-INF/FastDoubleParser-LICENSE) +======================================================================== + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +======================================================================== +bigint (META-INF/bigint-LICENSE) +======================================================================== + +Copyright 2022 Tim Buktu + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/install-dist/scripts/dependency/check_dependencies.sh b/install-dist/scripts/dependency/check_dependencies.sh old mode 100644 new mode 100755 diff --git a/install-dist/scripts/dependency/known-dependencies.txt b/install-dist/scripts/dependency/known-dependencies.txt index 17f97dc5be..a4cbbfb5f6 100644 --- a/install-dist/scripts/dependency/known-dependencies.txt +++ b/install-dist/scripts/dependency/known-dependencies.txt @@ -7,13 +7,16 @@ animal-sniffer-annotations-1.14.jar animal-sniffer-annotations-1.17.jar animal-sniffer-annotations-1.21.jar annotations-13.0.jar +annotations-2.33.8.jar annotations-24.0.1.jar annotations-4.1.1.4.jar ansj_seg-5.1.6.jar antlr4-runtime-4.9.1.jar aopalliance-repackaged-3.0.1.jar +apache-client-2.33.8.jar apiguardian-api-1.1.0.jar apiguardian-api-1.1.2.jar +arns-2.33.8.jar arthas-agent-attach-4.1.5.jar arthas-agent.jar arthas-boot.jar @@ -27,8 +30,12 @@ asm-tree-9.4.jar assertj-core-3.19.0.jar ast-9.0-9.0.20190305.jar audience-annotations-0.13.0.jar +auth-2.33.8.jar auto-service-annotations-1.0.jar automaton-1.11-8.jar +aws-core-2.33.8.jar +aws-query-protocol-2.33.8.jar +aws-xml-protocol-2.33.8.jar ayza-10.0.0.jar ayza-for-netty-10.0.0.jar bolt-1.6.2.jar @@ -44,12 +51,15 @@ checker-qual-2.5.2.jar checker-qual-3.12.0.jar checker-qual-3.33.0.jar checker-qual-3.5.0.jar +checksums-2.33.8.jar +checksums-spi-2.33.8.jar ci-environment-10.0.1.jar classgraph-4.8.162.jar commons-beanutils-1.9.4.jar commons-cli-1.5.0.jar commons-codec-1.13.jar commons-codec-1.15.jar +commons-codec-1.17.1.jar commons-collections-3.2.2.jar commons-collections4-4.4.jar commons-compress-1.21.jar @@ -67,6 +77,7 @@ commons-logging-1.1.1.jar commons-logging-1.2.jar commons-text-1.10.0.jar commons-text-1.11.0.jar +crt-core-2.33.8.jar cucumber-core-7.21.1.jar cucumber-expressions-18.0.1.jar cucumber-gherkin-7.21.1.jar @@ -84,11 +95,13 @@ eclipse-collections-10.4.0.jar eclipse-collections-11.1.0.jar eclipse-collections-api-10.4.0.jar eclipse-collections-api-11.1.0.jar +endpoints-spi-2.33.8.jar error_prone_annotations-2.1.3.jar error_prone_annotations-2.11.0.jar error_prone_annotations-2.18.0.jar error_prone_annotations-2.3.4.jar error_prone_annotations-2.48.0.jar +eventstream-1.0.1.jar exp4j-0.4.8.jar expressions-9.0-9.0.20190305.jar failsafe-2.4.1.jar @@ -172,8 +185,15 @@ hk2-utils-3.0.1.jar hppc-0.7.1.jar html-formatter-21.9.0.jar htrace-core4-4.1.0-incubating.jar +http-auth-2.33.8.jar +http-auth-aws-2.33.8.jar +http-auth-aws-eventstream-2.33.8.jar +http-auth-spi-2.33.8.jar +http-client-spi-2.33.8.jar httpclient-4.5.13.jar httpcore-4.4.13.jar +httpcore-4.4.16.jar +identity-spi-2.33.8.jar ikanalyzer-2012_u6.jar ivy-2.6.0.jar j2objc-annotations-1.1.jar @@ -270,6 +290,7 @@ joni-2.2.1.jar jraft-core-1.3.14.jar json-path-2.5.0.jar json-smart-2.3.jar +json-utils-2.33.8.jar jsonassert-1.5.1.jar jsr305-3.0.1.jar jsr305-3.0.2.jar @@ -358,6 +379,7 @@ metrics-core-3.0.2.jar metrics-core-4.0.2.jar metrics-core-4.2.4.jar metrics-jersey3-4.2.4.jar +metrics-spi-2.33.8.jar micrometer-core-1.7.12.jar micrometer-registry-prometheus-1.7.12.jar mmseg4j-core-1.10.0.jar @@ -382,6 +404,7 @@ netty-common-4.1.137.Final.jar netty-handler-4.1.137.Final.jar netty-handler-proxy-4.1.137.Final.jar netty-handler-ssl-ocsp-4.1.137.Final.jar +netty-nio-client-2.33.8.jar netty-resolver-4.1.137.Final.jar netty-resolver-dns-4.1.137.Final.jar netty-resolver-dns-classes-macos-4.1.137.Final.jar @@ -431,19 +454,28 @@ parboiled-scala_2.12-1.2.0.jar parser-9.0-9.0.20190305.jar perfmark-api-0.25.0.jar picocli-4.7.6.jar +profiles-2.33.8.jar proto-google-common-protos-2.9.0.jar protobuf-java-3.22.3.jar protobuf-java-util-3.22.3.jar +protocol-core-2.33.8.jar protostuff-api-1.6.0.jar protostuff-collectionschema-1.6.0.jar protostuff-core-1.6.0.jar protostuff-runtime-1.6.0.jar query-13.6.0.jar +reactive-streams-1.0.4.jar +regions-2.33.8.jar +retries-2.33.8.jar +retries-spi-2.33.8.jar rewriting-9.0-9.0.20190305.jar rocksdbjni-8.10.2.jar +s3-2.33.8.jar +s3-transfer-manager-2.33.8.jar scala-java8-compat_2.12-0.8.0.jar scala-library-2.12.7.jar scala-reflect-2.12.7.jar +sdk-core-2.33.8.jar shims-0.9.38.jar simpleclient-0.10.0.jar simpleclient_common-0.10.0.jar @@ -461,6 +493,7 @@ slf4j-api-1.7.21.jar slf4j-api-1.7.25.jar slf4j-api-1.7.31.jar slf4j-api-1.7.32.jar +slf4j-api-1.7.36.jar slf4j-api-2.0.9.jar snakeyaml-2.2.jar sofa-common-tools-1.0.12.jar @@ -501,12 +534,14 @@ swagger-models-1.5.18.jar swagger-models-jakarta-2.2.18.jar tag-expressions-6.1.2.jar testng-xml-formatter-0.3.1.jar +third-party-jackson-core-2.33.8.jar tinkergraph-gremlin-3.8.1.jar token-provider-2.0.1.jar tomcat-embed-el-9.0.75.jar tracer-core-3.0.8.jar translation-1.0.4.jar util-9.0-9.0.20190305.jar +utils-2.33.8.jar validation-api-1.1.0.Final.jar websocket-api-9.4.51.v20230217.jar websocket-client-9.4.51.v20230217.jar diff --git a/install-dist/scripts/dependency/regenerate_known_dependencies.sh b/install-dist/scripts/dependency/regenerate_known_dependencies.sh old mode 100644 new mode 100755 diff --git a/pom.xml b/pom.xml index 927995611b..b7a0e0fc90 100644 --- a/pom.xml +++ b/pom.xml @@ -305,6 +305,13 @@ **/pid **/tmp/** + + **/docker/cloud-storage/.artifacts/** + **/docker/cloud-storage/.generated/** + **/docker/cloud-storage/data/** + **/docker/cloud-storage/logs/** + + **/.generated/** **/src/main/java/org/apache/hugegraph/pd/grpc/** **/src/main/java/org/apache/hugegraph/store/grpc/** @@ -441,6 +448,12 @@ **/*.txt **/.flattened-pom.xml **/apache-hugegraph-*/**/* + + docker/cloud-storage/.artifacts/** + docker/cloud-storage/.generated/** + docker/cloud-storage/data/** + docker/cloud-storage/logs/** + **/.generated/**