From 680a0a7dd750de3d0854e793f8eba07236c87475 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 02:36:25 +0100 Subject: [PATCH 1/2] feat(dawarich): run managed Dawarich 1.19.0 on Redis 8 Move the managed bundle from Dawarich 1.15.3 to 1.19.0, which includes the 1.16 fixes for stale two-factor challenges and shared-photo authorization, and its dedicated Redis from 7.4 to 8.10, which reads the data 7.4 saved. The separate Sidekiq worker stays: 1.19 embeds jobs in the web process only with SIDEKIQ_EMBEDDED=true, which the managed app does not set. Re-audit the privacy subject projection against the live 1.19.0 schema in both the Rails collector and the backup exporter. Export geofence events, webhooks and their delivery state, legacy area-to-place mappings and the point/track revision counters. Omit webhook signing secrets and delivery response bodies and error messages, and reduce webhook URLs to their origin, since a path or query is often the receiver's credential. Add the 1.19 preference keys to the settings allowlist. Compare the complete table inventory as well as the FK graph before collecting: the revision counters carry user_id without a constraint, so the graph alone would have missed them. The new schema fingerprint comes from the live 1.19.0 database, and the runner reproduces the committed 1.15.3 fingerprint. Backups now record the dawarich-1.19.0 contract. Document the upgrade's background jobs and the self-hosted reach of geofencing webhooks. --- .../__tests__/dawarich-source-part.test.ts | 6 +- .../src/privacy/backup-source-part.test.ts | 2 +- .../controller-wide-export-postgres.test.ts | 4 +- .../routes/__tests__/admin-dawarich.test.ts | 12 +- .../__tests__/admin-services-sdk.test.ts | 6 +- .../routes/__tests__/admin-services.test.ts | 2 +- apps/api/src/routes/admin-dawarich.ts | 12 +- apps/api/src/server.ts | 2 +- .../__tests__/managed-resolver.test.ts | 2 +- .../dawarich/__tests__/provisioning.test.ts | 4 +- .../dawarich/provisioning-contract.ts | 2 +- apps/ops-agent/src/administrative-runtime.ts | 8 +- ...awarich-subject-export.integration.test.ts | 77 ++++++++ .../src/privacy-backup-runtime.test.ts | 8 +- apps/ops-agent/src/privacy-backup-runtime.ts | 2 +- docs/docs/features/personal-timeline.md | 2 +- docs/docs/guides/dawarich.md | 6 +- docs/self-hosting/dawarich.md | 40 +++- packages/cli/__tests__/cli-backup.test.ts | 2 +- packages/cli/src/commands/backup.ts | 16 +- .../src/__tests__/services-end-to-end.test.ts | 18 +- .../src/privacy/dawarich-contract.test.ts | 4 +- .../core/src/privacy/dawarich-contract.ts | 22 ++- release-notes/managed-dawarich-1-19.md | 23 +++ renovate.json | 6 +- .../scripts/openmapx-subject-export.rb | 66 ++++++- .../scripts/subject_export/schema.rb | 58 +++++- services/dawarich-app/service.json | 8 +- services/dawarich-redis/service.json | 8 +- services/dawarich-sidekiq/service.json | 8 +- .../openmapx-backup-subject-export.py | 172 ++++++++++++++---- .../ops-agent/privacy-backup/test-fixture.mjs | 82 +++++++-- services/service-file-history.json | 6 +- 33 files changed, 541 insertions(+), 155 deletions(-) create mode 100644 release-notes/managed-dawarich-1-19.md diff --git a/apps/api/src/privacy/__tests__/dawarich-source-part.test.ts b/apps/api/src/privacy/__tests__/dawarich-source-part.test.ts index 8f24874c2..9d26eb5d4 100644 --- a/apps/api/src/privacy/__tests__/dawarich-source-part.test.ts +++ b/apps/api/src/privacy/__tests__/dawarich-source-part.test.ts @@ -20,9 +20,9 @@ import { const tuple = { version: 1, - image: "freikin/dawarich:1.15.3", - imageDigest: "sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498", - upstreamCommit: "e32b70701b2467b862d4b5bc27ac428999997c1b", + image: "freikin/dawarich:1.19.0", + imageDigest: "sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304", + upstreamCommit: "752641768f37fdc07d5a23e0559fa7c5ff8fae0f", subjectUserIdDigest: "a".repeat(64), cutoff: "2026-01-01T00:00:00Z", snapshotAt: "2026-01-01T00:00:00Z", diff --git a/apps/api/src/privacy/backup-source-part.test.ts b/apps/api/src/privacy/backup-source-part.test.ts index 7ba51a72c..22f39e913 100644 --- a/apps/api/src/privacy/backup-source-part.test.ts +++ b/apps/api/src/privacy/backup-source-part.test.ts @@ -31,7 +31,7 @@ function fixture( const dawarichManifest = Buffer.from( JSON.stringify({ version: 1, - image: "freikin/dawarich:1.15.3", + image: "freikin/dawarich:1.19.0", imageDigest: DAWARICH_SUPPORTED_IMAGE_DIGEST, upstreamCommit: DAWARICH_SUPPORTED_COMMIT, subjectUserIdDigest: digest(Buffer.from("subject-1")), diff --git a/apps/api/src/privacy/controller-wide-export-postgres.test.ts b/apps/api/src/privacy/controller-wide-export-postgres.test.ts index 0de3c0260..50268c6d0 100644 --- a/apps/api/src/privacy/controller-wide-export-postgres.test.ts +++ b/apps/api/src/privacy/controller-wide-export-postgres.test.ts @@ -69,7 +69,7 @@ function dawarichTar(subjectId: string, cutoff: string): Buffer { const manifest = Buffer.from( JSON.stringify({ version: 1, - image: "freikin/dawarich:1.15.3", + image: "freikin/dawarich:1.19.0", imageDigest: DAWARICH_SUPPORTED_IMAGE_DIGEST, upstreamCommit: DAWARICH_SUPPORTED_COMMIT, subjectUserIdDigest: sha256(Buffer.from(subjectId)), @@ -103,7 +103,7 @@ function backupTar(subjectId: string, cutoff: string): Buffer { const dawarichManifest = Buffer.from( JSON.stringify({ version: 1, - image: "freikin/dawarich:1.15.3", + image: "freikin/dawarich:1.19.0", imageDigest: DAWARICH_SUPPORTED_IMAGE_DIGEST, upstreamCommit: DAWARICH_SUPPORTED_COMMIT, subjectUserIdDigest: sha256(Buffer.from(subjectId)), diff --git a/apps/api/src/routes/__tests__/admin-dawarich.test.ts b/apps/api/src/routes/__tests__/admin-dawarich.test.ts index 6a4c3ce75..b139c9fef 100644 --- a/apps/api/src/routes/__tests__/admin-dawarich.test.ts +++ b/apps/api/src/routes/__tests__/admin-dawarich.test.ts @@ -19,8 +19,8 @@ const bundle = new Map([ { manifest: { id: "dawarich-app", - version: "1.15.3", - container: { image: "freikin/dawarich", tag: "1.15.3" }, + version: "1.19.0", + container: { image: "freikin/dawarich", tag: "1.19.0" }, }, }, ], @@ -29,8 +29,8 @@ const bundle = new Map([ { manifest: { id: "dawarich-sidekiq", - version: "1.15.3", - container: { image: "freikin/dawarich", tag: "1.15.3" }, + version: "1.19.0", + container: { image: "freikin/dawarich", tag: "1.19.0" }, }, }, ], @@ -49,8 +49,8 @@ const bundle = new Map([ { manifest: { id: "dawarich-redis", - version: "7.4", - container: { image: "redis", tag: "7.4-alpine" }, + version: "8.10", + container: { image: "redis", tag: "8.10-alpine" }, }, }, ], diff --git a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts index 415980385..65eb3d19b 100644 --- a/apps/api/src/routes/__tests__/admin-services-sdk.test.ts +++ b/apps/api/src/routes/__tests__/admin-services-sdk.test.ts @@ -46,10 +46,10 @@ const service = { }, }; const bundle = new Map([ - ["dawarich-app", ["1.15.3", "freikin/dawarich", "1.15.3"]], - ["dawarich-sidekiq", ["1.15.3", "freikin/dawarich", "1.15.3"]], + ["dawarich-app", ["1.19.0", "freikin/dawarich", "1.19.0"]], + ["dawarich-sidekiq", ["1.19.0", "freikin/dawarich", "1.19.0"]], ["dawarich-postgis", ["18-3.6", "ghcr.io/baosystems/postgis", "18-3.6"]], - ["dawarich-redis", ["7.4", "redis", "7.4-alpine"]], + ["dawarich-redis", ["8.10", "redis", "8.10-alpine"]], ]); const getService = vi.fn((id: string) => { const item = bundle.get(id); diff --git a/apps/api/src/routes/__tests__/admin-services.test.ts b/apps/api/src/routes/__tests__/admin-services.test.ts index 33bc4be6e..c36a854e6 100644 --- a/apps/api/src/routes/__tests__/admin-services.test.ts +++ b/apps/api/src/routes/__tests__/admin-services.test.ts @@ -567,7 +567,7 @@ describe("service credentials", () => { manifest: { id: serviceId, name: serviceId, - version: "1.15.3", + version: "1.19.0", quality: "built-in", configSchema: { properties: { diff --git a/apps/api/src/routes/admin-dawarich.ts b/apps/api/src/routes/admin-dawarich.ts index 36c638afb..72e4bf0f5 100644 --- a/apps/api/src/routes/admin-dawarich.ts +++ b/apps/api/src/routes/admin-dawarich.ts @@ -22,15 +22,15 @@ const ROTATION_CONFIRMATION = "ROTATE DAWARICH OIDC SECRET"; const EXPECTED_BUNDLE = [ { id: DAWARICH_APP_SERVICE_ID, - version: "1.15.3", + version: "1.19.0", image: "freikin/dawarich", - tag: "1.15.3", + tag: "1.19.0", }, { id: DAWARICH_WORKER_SERVICE_ID, - version: "1.15.3", + version: "1.19.0", image: "freikin/dawarich", - tag: "1.15.3", + tag: "1.19.0", }, { id: DAWARICH_POSTGIS_SERVICE_ID, @@ -40,9 +40,9 @@ const EXPECTED_BUNDLE = [ }, { id: DAWARICH_REDIS_SERVICE_ID, - version: "7.4", + version: "8.10", image: "redis", - tag: "7.4-alpine", + tag: "8.10-alpine", }, ] as const; diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index c0506c93c..1f736999f 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -485,7 +485,7 @@ const managedDawarichState = () => { managed.manifest.subjectData?.strategy === "collector" && managed.manifest.subjectData.registrationIds?.includes("managed-dawarich") && managed.manifest.container.image === DAWARICH_SUPPORTED_IMAGE.split(":")[0] && - managed.manifest.container.tag === "1.15.3" && + managed.manifest.container.tag === "1.19.0" && managed.manifest.container.digest === DAWARICH_SUPPORTED_IMAGE_DIGEST && managed.manifest.container.environment?.OPENMAPX_DAWARICH_COMMIT === DAWARICH_SUPPORTED_COMMIT, diff --git a/apps/api/src/services/dawarich/__tests__/managed-resolver.test.ts b/apps/api/src/services/dawarich/__tests__/managed-resolver.test.ts index d8cf8164f..4332ffd0b 100644 --- a/apps/api/src/services/dawarich/__tests__/managed-resolver.test.ts +++ b/apps/api/src/services/dawarich/__tests__/managed-resolver.test.ts @@ -19,7 +19,7 @@ function readyClient(overrides: Partial = {}): ManagedOAuthC client_name: "OpenMapX Managed Dawarich", client_uri: "https://timeline.example.test", software_id: DAWARICH_SOFTWARE_ID, - software_version: "1.15.3", + software_version: "1.19.0", reference_id: MANAGED_REFERENCE_ID, redirect_uris: ["https://timeline.example.test/users/auth/openid_connect/callback"], token_endpoint_auth_method: "client_secret_basic", diff --git a/apps/api/src/services/dawarich/__tests__/provisioning.test.ts b/apps/api/src/services/dawarich/__tests__/provisioning.test.ts index 2a0abb852..8abe8fcf0 100644 --- a/apps/api/src/services/dawarich/__tests__/provisioning.test.ts +++ b/apps/api/src/services/dawarich/__tests__/provisioning.test.ts @@ -30,7 +30,7 @@ function baseClient(overrides: Partial = {}): ManagedOAuthCl client_name: "OpenMapX Managed Dawarich", client_uri: "https://timeline.example.test", software_id: DAWARICH_SOFTWARE_ID, - software_version: "1.15.3", + software_version: "1.19.0", reference_id: MANAGED_REFERENCE_ID, redirect_uris: ["https://timeline.example.test/users/auth/openid_connect/callback"], token_endpoint_auth_method: "client_secret_basic", @@ -222,7 +222,7 @@ describe("provisionManagedDawarich", () => { client_name: "OpenMapX Managed Dawarich", client_uri: "https://timeline.example.test", software_id: DAWARICH_SOFTWARE_ID, - software_version: "1.15.3", + software_version: "1.19.0", redirect_uris: ["https://timeline.example.test/users/auth/openid_connect/callback"], token_endpoint_auth_method: "client_secret_basic", grant_types: ["authorization_code"], diff --git a/apps/api/src/services/dawarich/provisioning-contract.ts b/apps/api/src/services/dawarich/provisioning-contract.ts index cd8861f4d..9421f4be5 100644 --- a/apps/api/src/services/dawarich/provisioning-contract.ts +++ b/apps/api/src/services/dawarich/provisioning-contract.ts @@ -7,7 +7,7 @@ export const DAWARICH_APP_SERVICE_ID = "dawarich-app"; export const DAWARICH_WORKER_SERVICE_ID = "dawarich-sidekiq"; export const DAWARICH_POSTGIS_SERVICE_ID = "dawarich-postgis"; export const DAWARICH_REDIS_SERVICE_ID = "dawarich-redis"; -export const DAWARICH_VERSION = "1.15.3"; +export const DAWARICH_VERSION = "1.19.0"; export const DAWARICH_PROVISIONING_GENERATION_KEY = "OPENMAPX_PROVISIONING_GENERATION"; export const DAWARICH_OIDC_RECOVERY_REQUIRED_KEY = "OPENMAPX_OIDC_RECOVERY_REQUIRED"; diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index c28cccfe8..4fb045fa6 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -440,12 +440,12 @@ function parseManifest(rootDir: string, backupId: string): ParsedBackupManifest const managed = provenance?.managedDawarich; if ( managed && - (managed.version !== "1.15.3" || + (managed.version !== "1.19.0" || managed.image !== "freikin/dawarich" || managed.imageDigest !== - "sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498" || - managed.upstreamCommit !== "e32b70701b2467b862d4b5bc27ac428999997c1b" || - managed.schemaContract !== "dawarich-1.15.3") + "sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304" || + managed.upstreamCommit !== "752641768f37fdc07d5a23e0559fa7c5ff8fae0f" || + managed.schemaContract !== "dawarich-1.19.0") ) throw new Error("Backup authority rejected"); return { diff --git a/apps/ops-agent/src/dawarich-subject-export.integration.test.ts b/apps/ops-agent/src/dawarich-subject-export.integration.test.ts index 70a8853ee..a7138d8ac 100644 --- a/apps/ops-agent/src/dawarich-subject-export.integration.test.ts +++ b/apps/ops-agent/src/dawarich-subject-export.integration.test.ts @@ -343,6 +343,21 @@ VALUES (1,1,1,'Subject unplanned place',TIMESTAMP '${at}',TIMESTAMP '${at}'), INSERT INTO point_sources (id,digest,tracker_id,topic,created_at,updated_at) VALUES (1,'subject-source-digest','subject-dimension-device','subject/topic',TIMESTAMP '${at}',TIMESTAMP '${at}'); UPDATE points SET source_id=1, tracker_id=NULL, topic=NULL WHERE id=1000; +INSERT INTO geofence_events (id,user_id,place_id,place_name,event_type,source,synthetic,occurred_at,received_at,lonlat,accuracy_m,device_id,metadata,created_at,updated_at) +VALUES (1,1,1,'Subject geofence place',0,0,false,TIMESTAMP '${at}',TIMESTAMP '${at}',ST_GeogFromText('POINT(13.4 52.5)'),12,'subject-geofence-device',${sql('{"note":"Subject geofence metadata","auth_token":"GEOFENCE_SECRET"}')}::jsonb,TIMESTAMP '${at}',TIMESTAMP '${at}'), + (2,2,2,'FOREIGN_SENTINEL',1,1,false,TIMESTAMP '${at}',TIMESTAMP '${at}',ST_GeogFromText('POINT(1 1)'),5,'FOREIGN_SENTINEL','{}'::jsonb,TIMESTAMP '${at}',TIMESTAMP '${at}'); +INSERT INTO webhooks (id,user_id,name,url,secret,event_types,place_ids,active,consecutive_failures,created_at,updated_at) +VALUES (1,1,'Subject webhook','https://hooks.subject.test/api/webhook/WEBHOOK_PATH_TOKEN?token=WEBHOOK_QUERY_TOKEN','WEBHOOK_SIGNING_SECRET','{0,1}','{1}',true,0,TIMESTAMP '${at}',TIMESTAMP '${at}'), + (2,2,'FOREIGN_SENTINEL','https://foreign.test/FOREIGN_SENTINEL','FOREIGN_SECRET','{0}','{}',true,0,TIMESTAMP '${at}',TIMESTAMP '${at}'); +INSERT INTO webhook_deliveries (id,webhook_id,geofence_event_id,status,attempt_count,response_status,response_body,error_message,delivered_at,created_at,updated_at) +VALUES (1,1,1,1,1,200,'DELIVERY_RESPONSE_SECRET','DELIVERY_ERROR_SECRET',TIMESTAMP '${at}',TIMESTAMP '${at}',TIMESTAMP '${at}'), + (2,2,2,2,3,500,'FOREIGN_SENTINEL','FOREIGN_SENTINEL',NULL,TIMESTAMP '${at}',TIMESTAMP '${at}'); +INSERT INTO legacy_area_place_mappings (id,area_id,place_id,created_at,updated_at) +VALUES (1,1,1,TIMESTAMP '${at}',TIMESTAMP '${at}'), (2,2,2,TIMESTAMP '${at}',TIMESTAMP '${at}'); +INSERT INTO point_revisions (id,user_id,month,revision) +VALUES (9001,1,DATE '2025-06-01','11111111-1111-4111-8111-111111111111'), (9002,2,DATE '2025-06-01','22222222-2222-4222-8222-222222222222'); +INSERT INTO track_revisions (id,user_id,month,revision) +VALUES (9001,1,DATE '2025-06-01','33333333-3333-4333-8333-333333333333'), (9002,2,DATE '2025-06-01','44444444-4444-4444-8444-444444444444'); COMMIT; `; } @@ -453,6 +468,12 @@ describe.skipIf(!runExactImage)("managed Dawarich exact-image export", () => { "route-videos", "service-settings", "trip-sources", + "geofence-events", + "webhooks", + "webhook-deliveries", + "legacy-area-place-mappings", + "point-revisions", + "track-revisions", "planned-days", "planned-day-notes", "planned-reservations", @@ -498,9 +519,65 @@ describe.skipIf(!runExactImage)("managed Dawarich exact-image export", () => { expect(plaintext).not.toContain("SERVICE_CREDENTIAL_SECRET"); expect(plaintext).not.toContain("TRIP_API_KEY_SECRET"); expect(plaintext).not.toContain("TRIP_SELECTION_SECRET"); + expect(plaintext).toContain("Subject geofence place"); + expect(plaintext).toContain("subject-geofence-device"); + expect(plaintext).toContain("11111111-1111-4111-8111-111111111111"); + expect(plaintext).toContain("33333333-3333-4333-8333-333333333333"); + for (const secret of [ + "GEOFENCE_SECRET", + "WEBHOOK_PATH_TOKEN", + "WEBHOOK_QUERY_TOKEN", + "WEBHOOK_SIGNING_SECRET", + "DELIVERY_RESPONSE_SECRET", + "DELIVERY_ERROR_SECRET", + "22222222-2222-4222-8222-222222222222", + "44444444-4444-4444-8444-444444444444", + ]) { + expect(plaintext, secret).not.toContain(secret); + } + const webhookRows = entries + .find((entry) => entry.id === "webhooks") + ?.content.toString("utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); + expect(webhookRows).toHaveLength(1); + expect(webhookRows?.[0]?.url).toBe("https://hooks.subject.test"); + const geofenceRows = entries + .find((entry) => entry.id === "geofence-events") + ?.content.toString("utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); + expect(geofenceRows).toHaveLength(1); + expect(geofenceRows?.[0]).toMatchObject({ + event_type: "enter", + source: "native_app", + lonlat: [13.4, 52.5], + }); expect(plaintext).not.toContain("aaaaaaaa-0000-4000-8000-000000000001"); expect(plaintext).not.toContain("bbbbbbbb-0000-4000-8000-000000000002"); expect(plaintext).not.toContain("other-family-member@example.test"); + + // An owner column without a constraint is invisible to the FK graph; the + // reviewed table inventory must still refuse the unreviewed relation. + await runPsql( + database, + "CREATE TABLE openmapx_unreviewed_owner (id bigserial PRIMARY KEY, user_id bigint);", + ); + const refused = await command( + "docker", + imageArgs(database, password, storageDirectory, [ + "ruby", + "/opt/openmapx/openmapx-subject-export.rb", + ]), + { input: request }, + ); + expect(refused.code).not.toBe(0); + expect(refused.stdout.length).toBe(0); + expect(refused.stderr.toString("utf8")).toContain( + "openmapx_subject_export:unsupported_schema", + ); } finally { await command("docker", ["rm", "--force", "--volumes", DATABASE_CONTAINER]); if (existsSync(storageDirectory)) rmSync(storageDirectory, { recursive: true, force: true }); diff --git a/apps/ops-agent/src/privacy-backup-runtime.test.ts b/apps/ops-agent/src/privacy-backup-runtime.test.ts index ac272239d..998831c27 100644 --- a/apps/ops-agent/src/privacy-backup-runtime.test.ts +++ b/apps/ops-agent/src/privacy-backup-runtime.test.ts @@ -126,11 +126,11 @@ describe("privacy backup extraction runtime", () => { openmapxVersion: "1.0.0", privacySourceProvenance: { managedDawarich: { - version: "1.15.3", + version: "1.19.0", image: "freikin/dawarich", imageDigest: `sha256:${"d".repeat(64)}`, upstreamCommit: "c".repeat(40), - schemaContract: "dawarich-1.15.3", + schemaContract: "dawarich-1.19.0", }, }, }, @@ -183,8 +183,8 @@ describe("privacy backup extraction runtime", () => { for await (const _chunk of output) void _chunk; expect(JSON.parse(body)).toMatchObject({ sources: [ - { family: "dawarich", schemaContract: "dawarich-1.15.3" }, - { family: "dawarich-storage", schemaContract: "dawarich-storage-1.15.3" }, + { family: "dawarich", schemaContract: "dawarich-1.19.0" }, + { family: "dawarich-storage", schemaContract: "dawarich-storage-1.19.0" }, ], }); }); diff --git a/apps/ops-agent/src/privacy-backup-runtime.ts b/apps/ops-agent/src/privacy-backup-runtime.ts index 1b3913120..ef4006adf 100644 --- a/apps/ops-agent/src/privacy-backup-runtime.ts +++ b/apps/ops-agent/src/privacy-backup-runtime.ts @@ -371,7 +371,7 @@ export function createPrivacyBackupExtractionRuntime(options: PrivacyBackupRunti input.serviceId === "postgis" ? "openmapx-v1" : input.serviceId === "dawarich-app" - ? "dawarich-storage-1.15.3" + ? "dawarich-storage-1.19.0" : verified.manifest.privacySourceProvenance?.managedDawarich?.schemaContract, })); if (sources.length < 1 || sources.some((source) => !source.schemaContract)) diff --git a/docs/docs/features/personal-timeline.md b/docs/docs/features/personal-timeline.md index 2ceafe04f..1ee7999f5 100644 --- a/docs/docs/features/personal-timeline.md +++ b/docs/docs/features/personal-timeline.md @@ -51,7 +51,7 @@ The two credentials have different jobs: email. Dawarich maintains a separate browser session; signing out of OpenMapX is not single logout. - The personal Dawarich API key authorizes OpenMapX's server-side, read-only API - calls. Dawarich 1.15.3 does not let the OIDC browser session or Better Auth + calls. Dawarich 1.19.0 does not let the OIDC browser session or Better Auth token replace this key. If a pre-existing Dawarich account has the same email, Dawarich requires its diff --git a/docs/docs/guides/dawarich.md b/docs/docs/guides/dawarich.md index 934b98085..6cf8bdbe6 100644 --- a/docs/docs/guides/dawarich.md +++ b/docs/docs/guides/dawarich.md @@ -45,10 +45,10 @@ the separate Dawarich browser session. | Service | Role | Default memory ceiling | | ------------------ | ----------------------------------- | ---------------------- | -| `dawarich-app` | Dawarich 1.15.3 web app | 4 GiB | +| `dawarich-app` | Dawarich 1.19.0 web app | 4 GiB | | `dawarich-sidekiq` | Background worker, concurrency 3 | 2 GiB | | `dawarich-postgis` | Dedicated PostgreSQL 18/PostGIS 3.6 | 2 GiB | -| `dawarich-redis` | Dedicated Redis 7.4 queue/cache | 512 MiB | +| `dawarich-redis` | Dedicated Redis 8.10 queue/cache | 512 MiB | The app and worker use the same digest-pinned Dawarich image. Their database, Redis, credentials, cookies, and volumes are separate from the OpenMapX platform @@ -253,7 +253,7 @@ available. Wait before retrying rather than repeatedly reconnecting the same key | `TIMELINE_PLAN_RESTRICTED` | Check whether the source account permits the required API access. | | `TIMELINE_RATE_LIMITED` | Wait for the retry interval. | | `TIMELINE_UPSTREAM_UNAVAILABLE` | Inspect container/network health and source availability. | -| `TIMELINE_RESPONSE_INVALID` | Check API compatibility with the managed 1.15.3 baseline and bounded response limits. | +| `TIMELINE_RESPONSE_INVALID` | Check API compatibility with the managed 1.19.0 baseline and bounded response limits. | ```bash pnpm openmapx services status dawarich-app diff --git a/docs/self-hosting/dawarich.md b/docs/self-hosting/dawarich.md index 466cf474b..d97b4a40f 100644 --- a/docs/self-hosting/dawarich.md +++ b/docs/self-hosting/dawarich.md @@ -6,8 +6,8 @@ user API-key connection, tracking clients, and the day viewer. This file records maintenance and release checks for the managed bundle; it is not evidence that a particular deployment passed them. -The current manifests select Dawarich 1.15.3 for app and worker, dedicated -PostgreSQL 18/PostGIS 3.6, and dedicated Redis 7.4. Their combined memory ceilings +The current manifests select Dawarich 1.19.0 for app and worker, dedicated +PostgreSQL 18/PostGIS 3.6, and dedicated Redis 8.10. Their combined memory ceilings are 8.5 GiB before OpenMapX and host overhead. App, worker, and data images are digest pinned. Database and Redis have no host-published ports. @@ -166,9 +166,9 @@ fresh runtime evidence. Before changing image pins, inspect the candidate indexes: ```bash -docker buildx imagetools inspect freikin/dawarich:1.15.3 +docker buildx imagetools inspect freikin/dawarich:1.19.0 docker buildx imagetools inspect ghcr.io/baosystems/postgis:18-3.6 -docker buildx imagetools inspect redis:7.4-alpine +docker buildx imagetools inspect redis:8.10-alpine ``` These inspect mutable tags, while the managed manifests run pinned digests. @@ -217,7 +217,37 @@ Do not change the bundle to `latest`. Read the target release notes and 5. Apply in a maintenance window and verify logs, health, sign-in, and a day read. The current compatibility baseline is -[Dawarich 1.15.3](https://github.com/Freika/dawarich/releases/tag/1.15.3). +[Dawarich 1.19.0](https://github.com/Freika/dawarich/releases/tag/1.19.0). + +### From 1.15.3 to 1.19.0 + +Dawarich 1.18 and 1.19 ask for a database backup before updating, because the +first start runs migrations that cannot be rolled back and then long background +jobs: Areas become Places, tracks from the last 12 months are repaired, flight +months and digests are recalculated, and visits are detected again across each +user's whole history. Keep the worker running until those jobs finish. On a small +server, `SKIP_VISITS_FLEET_REDETECT=1` defers the full-history visit detection, +which a user can start later from Visits → Re-run detection on full history. + +The bundle keeps its separate Sidekiq worker. Dawarich 1.17 can run jobs inside +the web process, but only when `SIDEKIQ_EMBEDDED=true`, which the managed app +does not set, so jobs never run twice. Redis 8.10 reads the data Redis 7.4 saved, +so queued jobs survive the update. + +The privacy collectors accept only the 1.19.0 schema. A backup taken on 1.15.3 +records the `dawarich-1.15.3` contract, and the backup subject export refuses it. + +## Geofencing webhooks and network reach + +Dawarich 1.18 lets each user send geofence arrival and departure events to a +webhook URL. The managed app runs with `SELF_HOSTED=true`, in which Dawarich +accepts private addresses and Docker service names, as it already does for the +Immich and PhotoPrism integration URLs; it blocks only cloud-metadata, multicast +and reserved ranges. Dawarich containers share the `openmapx` Docker network, so +a Dawarich user can make the worker send a signed POST with a fixed JSON body to +another OpenMapX service and read its status code and error message. Dawarich +stores, but does not show, the response body. Review this before onboarding users +you would not give that reach, and keep internal endpoints authenticated. ## Disable and purge diff --git a/packages/cli/__tests__/cli-backup.test.ts b/packages/cli/__tests__/cli-backup.test.ts index 2f7a85c9d..087eeed55 100644 --- a/packages/cli/__tests__/cli-backup.test.ts +++ b/packages/cli/__tests__/cli-backup.test.ts @@ -735,7 +735,7 @@ describe("discoverBackupableServices", () => { expect(targets).toEqual([ { id: "dawarich-app", - version: "1.15.3", + version: "1.19.0", volumes: [ { serviceId: "dawarich-app", diff --git a/packages/cli/src/commands/backup.ts b/packages/cli/src/commands/backup.ts index 0977fc587..f0281b8bd 100644 --- a/packages/cli/src/commands/backup.ts +++ b/packages/cli/src/commands/backup.ts @@ -84,11 +84,11 @@ export interface BackupManifest { services: BackupServiceEntry[]; privacySourceProvenance?: { managedDawarich?: { - version: "1.15.3"; + version: "1.19.0"; image: "freikin/dawarich"; imageDigest: string; upstreamCommit: string; - schemaContract: "dawarich-1.15.3"; + schemaContract: "dawarich-1.19.0"; }; }; } @@ -231,11 +231,11 @@ export function readBackupManifest(filePath: string): BackupManifest { const managed = raw.privacySourceProvenance?.managedDawarich; if ( managed && - (managed.version !== "1.15.3" || + (managed.version !== "1.19.0" || managed.image !== "freikin/dawarich" || !/^sha256:[a-f0-9]{64}$/.test(managed.imageDigest) || !/^[a-f0-9]{40}$/.test(managed.upstreamCommit) || - managed.schemaContract !== "dawarich-1.15.3") + managed.schemaContract !== "dawarich-1.19.0") ) throw new Error(`Malformed Dawarich privacy provenance in ${filePath}`); if (!isValidBackupName(raw.name)) { @@ -638,12 +638,12 @@ export async function createBackup(opts: CreateBackupOptions = {}): Promise { it("rejects compatibility drift", () => { const base = { version: 1, - image: "freikin/dawarich:1.15.3", + image: "freikin/dawarich:1.19.0", imageDigest: DAWARICH_SUPPORTED_IMAGE_DIGEST, upstreamCommit: DAWARICH_SUPPORTED_COMMIT, subjectUserIdDigest: "a".repeat(64), @@ -96,7 +96,7 @@ describe("managed Dawarich contract", () => { it("accepts only content-addressed dynamic members", () => { const base = { version: 1 as const, - image: "freikin/dawarich:1.15.3", + image: "freikin/dawarich:1.19.0", imageDigest: DAWARICH_SUPPORTED_IMAGE_DIGEST, upstreamCommit: DAWARICH_SUPPORTED_COMMIT, subjectUserIdDigest: "a".repeat(64), diff --git a/packages/core/src/privacy/dawarich-contract.ts b/packages/core/src/privacy/dawarich-contract.ts index 65f1e648e..1147b8570 100644 --- a/packages/core/src/privacy/dawarich-contract.ts +++ b/packages/core/src/privacy/dawarich-contract.ts @@ -1,17 +1,17 @@ import z from "zod/v4"; /** The only managed Dawarich runtime accepted by the controller. */ -export const DAWARICH_SUPPORTED_IMAGE = "freikin/dawarich:1.15.3" as const; +export const DAWARICH_SUPPORTED_IMAGE = "freikin/dawarich:1.19.0" as const; export const DAWARICH_SUPPORTED_IMAGE_DIGEST = - "sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498" as const; -export const DAWARICH_SUPPORTED_COMMIT = "e32b70701b2467b862d4b5bc27ac428999997c1b" as const; + "sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304" as const; +export const DAWARICH_SUPPORTED_COMMIT = "752641768f37fdc07d5a23e0559fa7c5ff8fae0f" as const; export const DAWARICH_PROTOCOL_VERSION = 1 as const; export const DAWARICH_TAR_MEDIA_TYPE = "application/vnd.openmapx.dawarich-subject-tar.v1" as const; -/** SHA-256 of the reviewed 1.15.3 ownership/schema projection. The Ruby +/** SHA-256 of the reviewed 1.19.0 ownership/schema projection. The Ruby * collector emits this value only after validating the live Rails schema; * accepting a different digest would make a changed source look complete. */ export const DAWARICH_EXPECTED_SCHEMA_FINGERPRINT = - "4ae830ea67dc4894d814d4e2e2d19cb92571c30f2df1df017417f671e26a1e09" as const; + "d728906287ec3c4f3f38a80c2c4fe6ce77cc40f03b56a5ba0e89b811dd6e631a" as const; const subjectId = z .string() @@ -84,6 +84,12 @@ export const DAWARICH_SOURCE_ENTRY_IDS = [ "route-videos", "service-settings", "trip-sources", + "geofence-events", + "webhooks", + "webhook-deliveries", + "legacy-area-place-mappings", + "point-revisions", + "track-revisions", "planned-days", "planned-day-notes", "planned-reservations", @@ -138,6 +144,12 @@ export const DAWARICH_SOURCE_PATHS: Readonly %w[id uid provider email first_name last_name created_at updated_at active_until plan status theme], "Area" => %w[id name latitude longitude radius created_at updated_at], - "Place" => %w[id name city country latitude longitude source note geodata created_at updated_at], + "Place" => %w[id name city country latitude longitude source note geodata visit_radius geofence archived_at created_at updated_at], "Tag" => %w[id name color icon privacy_radius_meters created_at updated_at], "Tagging" => %w[id tag_id taggable_id taggable_type created_at updated_at], "Import" => %w[id name source status demo points_count raw_points processed doubles created_at updated_at processing_started_at], @@ -62,7 +63,7 @@ def initialize(code) "Trip" => %w[id name started_at ended_at distance path visited_countries demo created_at updated_at last_recalculated_at], "Notification" => %w[id title content kind read_at created_at updated_at], "Point" => %w[id lon lat altitude altitude_decimal accuracy battery battery_status city connection country country_name course course_accuracy external_track_id geodata mode motion_data recorded_at timestamp tracker_id trigger velocity vertical_accuracy visit_id track_id import_id anomaly reverse_geocoded_at created_at updated_at], - "Visit" => %w[id name started_at ended_at duration confidence confidence_breakdown status area_id place_id demo created_at updated_at], + "Visit" => %w[id name location_label started_at ended_at duration confidence confidence_breakdown status area_id place_id demo created_at updated_at], "Stat" => %w[id year month distance daily_distance h3_hex_ids toponyms created_at updated_at], "Track" => %w[id start_at end_at distance duration avg_speed dominant_mode elevation_gain elevation_loss elevation_max elevation_min tracker_id demo original_path created_at updated_at], "TrackSegment" => %w[id track_id start_index end_index distance duration avg_speed max_speed confidence corrected_at source transportation_mode created_at updated_at], @@ -86,7 +87,17 @@ def initialize(code) "UserAchievement" => %w[id achievement_key earned_at metadata created_at updated_at], "RouteVideo" => %w[id name settings status expired_at created_at updated_at], "ServiceSetting" => %w[id service provider active config created_at updated_at], - "TripSource" => %w[id provider base_url status importing last_error last_synced_at created_at updated_at], + "TripSource" => %w[id provider base_url status importing skip_ssl_verification last_error last_synced_at created_at updated_at], + "GeofenceEvent" => %w[id event_type occurred_at received_at place_id place_name lonlat accuracy_m device_id source synthetic metadata webhook_skip_reason created_at updated_at], + # The signing secret is omitted, and the endpoint is reduced to its origin + # in serialized_row: a webhook path or query often carries a bearer token. + "Webhook" => %w[id name url event_types place_ids active consecutive_failures last_delivery_at last_success_at created_at updated_at], + # The response body and error message come from the receiving endpoint and + # may echo its own data or credentials; only delivery state is exported. + "WebhookDelivery" => %w[id webhook_id geofence_event_id status attempt_count response_status delivered_at created_at updated_at], + "LegacyAreaPlaceMapping" => %w[id area_id place_id created_at updated_at], + "PointRevision" => %w[id month revision], + "TrackRevision" => %w[id month revision], "PlannedDay" => %w[id trip_id date title notes position created_at updated_at], "PlannedDayNote" => %w[id planned_day_id body noted_at position created_at updated_at], "PlannedReservation" => %w[id trip_id planned_day_id reservation_type title location starts_at ends_at status notes created_at updated_at], @@ -107,6 +118,10 @@ def initialize(code) "achievement-unlock-events" => "AchievementUnlockEvent", "user-achievements" => "UserAchievement", "route-videos" => "RouteVideo", "service-settings" => "ServiceSetting", "trip-sources" => "TripSource", + "geofence-events" => "GeofenceEvent", "webhooks" => "Webhook", + "webhook-deliveries" => "WebhookDelivery", + "legacy-area-place-mappings" => "LegacyAreaPlaceMapping", + "point-revisions" => "PointRevision", "track-revisions" => "TrackRevision", "planned-days" => "PlannedDay", "planned-day-notes" => "PlannedDayNote", "planned-reservations" => "PlannedReservation", "planned-stops" => "PlannedStop", "planned-accommodations" => "PlannedAccommodation", @@ -134,6 +149,8 @@ def initialize(code) max_gap_minutes_in_city gps_filtering_enabled gps_accuracy_threshold timezone visit_radius_meters visit_min_points visit_min_duration_minutes visit_density_fill_enabled stay_max_gap_minutes point_dragging_enabled news_emails_enabled show_supporter_badge + achievements_enabled family_enabled date_format time_format first_day_of_week + gps_accuracy_filter_enabled ].freeze def fail_code(code) @@ -231,7 +248,9 @@ def model_for(model_name) "FamilyLocationRequest" => "Family::LocationRequest", "RichText" => "ActionText::RichText", "AchievementProgress" => "Achievements::Progress", - "AchievementUnlockEvent" => "Achievements::UnlockEvent" + "AchievementUnlockEvent" => "Achievements::UnlockEvent", + "PointRevision" => "Points::Revision", + "TrackRevision" => "Tracks::Revision" }.fetch(model_name, model_name) model = constant.split("::").reduce(Object) { |scope, part| scope.const_get(part, false) } return nil unless model.respond_to?(:where) && model.respond_to?(:column_names) @@ -267,6 +286,12 @@ def relation_scope(entry_id, model, user, cutoff) trip_ids = model_for("Trip")&.where(user_id: user.id)&.select(:id) day_ids = model_for("PlannedDay")&.where(trip_id: trip_ids)&.select(:id) model.where(planned_day_id: day_ids) + when "webhook-deliveries" + webhook_ids = model_for("Webhook")&.where(user_id: user.id)&.select(:id) + model.where(webhook_id: webhook_ids) + when "legacy-area-place-mappings" + area_ids = model_for("Area")&.where(user_id: user.id)&.select(:id) + model.where(area_id: area_ids) when "family-memberships" model.where(user_id: user.id) else @@ -319,6 +344,9 @@ def validate_schema!(user) end end.sort fail_code("unsupported_schema") unless observed_foreign_keys == EXPECTED_FOREIGN_KEYS.sort + # A table whose owner column has no constraint never appears in the FK + # graph, so the reviewed table inventory must match as well. + fail_code("unsupported_schema") unless connection.tables.map(&:to_s).sort == EXPECTED_TABLES.sort fingerprint_rows = REQUIRED_COLUMNS.map do |model_name, _required_columns| model = model_for(model_name) @@ -357,6 +385,22 @@ def each_relation_record(model, foreign_key, user, cutoff, entry_id: nil, &block fail_code("collector_failed") end + WEBHOOK_REDACTIONS = %w[dawarich-webhook-secret-omitted dawarich-webhook-endpoint-reduced].freeze + WEBHOOK_DELIVERY_REDACTIONS = %w[dawarich-webhook-response-omitted].freeze + + # Scheme, host and non-default port only: a webhook path or query string is + # commonly the receiver's bearer credential (for example a Home Assistant + # webhook id), so it never leaves the instance in an export. + def endpoint_origin(value) + uri = URI.parse(value.to_s) + return nil unless uri.is_a?(URI::HTTP) && !uri.host.to_s.empty? + + port = uri.port == uri.default_port ? "" : ":#{uri.port}" + "#{uri.scheme}://#{uri.host}#{port}" + rescue URI::Error + nil + end + def serialized_row(record, model_name) data = attributes(record, model_name) if model_name == "RichText" && record.respond_to?(:body) @@ -366,6 +410,12 @@ def serialized_row(record, model_name) data["bodyText"] = normalize_value(text) data["redactionCodes"] = ["dawarich-rich-text-markup-stripped"] end + if model_name == "Webhook" + data["url"] = endpoint_origin(data["url"]) + data["redactionCodes"] = WEBHOOK_REDACTIONS + elsif model_name == "WebhookDelivery" + data["redactionCodes"] = WEBHOOK_DELIVERY_REDACTIONS + end JSON.generate(data) + "\n" rescue CollectorError raise @@ -408,8 +458,12 @@ def relation_descriptor(entry_id, model_name, user, cutoff, scope: nil, portabil records: records, sha256: expected_digest, article15: true, - portability: portability.nil? ? (%w[places points raw-archives imports flights notes route-videos planned-days planned-day-notes planned-reservations planned-stops planned-accommodations planned-travellers planned-unplanned-places].include?(entry_id) || entry_id.start_with?("points-")) : portability, - redaction_codes: entry_id == "notes" ? ["dawarich-rights-of-others-review"] : [], + portability: portability.nil? ? (%w[places points raw-archives imports flights notes route-videos geofence-events planned-days planned-day-notes planned-reservations planned-stops planned-accommodations planned-travellers planned-unplanned-places].include?(entry_id) || entry_id.start_with?("points-")) : portability, + redaction_codes: { + "notes" => ["dawarich-rights-of-others-review"], + "webhooks" => WEBHOOK_REDACTIONS, + "webhook-deliveries" => WEBHOOK_DELIVERY_REDACTIONS + }.fetch(entry_id, []), write: lambda do |io| actual_digest = Digest::SHA256.new actual_bytes = 0 diff --git a/services/dawarich-app/scripts/subject_export/schema.rb b/services/dawarich-app/scripts/subject_export/schema.rb index 319d32f16..e230978ec 100644 --- a/services/dawarich-app/scripts/subject_export/schema.rb +++ b/services/dawarich-app/scripts/subject_export/schema.rb @@ -1,13 +1,13 @@ # Frozen compatibility tuple for the audited managed-Dawarich collector. module OpenMapxSubjectExport - IMAGE = "freikin/dawarich:1.15.3".freeze - IMAGE_DIGEST = "sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498".freeze - UPSTREAM_COMMIT = "e32b70701b2467b862d4b5bc27ac428999997c1b".freeze - EXPECTED_SCHEMA_FINGERPRINT = "4ae830ea67dc4894d814d4e2e2d19cb92571c30f2df1df017417f671e26a1e09".freeze + IMAGE = "freikin/dawarich:1.19.0".freeze + IMAGE_DIGEST = "sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304".freeze + UPSTREAM_COMMIT = "752641768f37fdc07d5a23e0559fa7c5ff8fae0f".freeze + EXPECTED_SCHEMA_FINGERPRINT = "d728906287ec3c4f3f38a80c2c4fe6ce77cc40f03b56a5ba0e89b811dd6e631a".freeze MAX_INPUT_BYTES = 16 * 1024 MAX_ENTRY_BYTES = 256 * 1024 * 1024 MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 - # This is the reviewed ownership surface for 1.15.3. It intentionally + # This is the reviewed ownership surface for 1.19.0. It intentionally # contains the columns needed by the collector rather than permitting a # reflection-based `select *`; a missing column or ownership FK makes the # source unavailable until the tuple is audited again. @@ -50,7 +50,13 @@ module OpenMapxSubjectExport "PlannedAccommodation" => %w[id trip_id name address latitude longitude starts_on ends_on check_in_at check_out_at notes created_at updated_at], "PlannedTraveller" => %w[id trip_id name owner created_at updated_at], "PlannedUnplannedPlace" => %w[id trip_id name address latitude longitude starts_at ends_at duration_minutes position category notes transport_mode created_at updated_at], - "PointSource" => %w[id digest tracker_id topic ssid bssid connection trigger battery_status inrids in_regions created_at updated_at] + "PointSource" => %w[id digest tracker_id topic ssid bssid connection trigger battery_status inrids in_regions created_at updated_at], + "GeofenceEvent" => %w[id user_id place_id event_type occurred_at lonlat device_id created_at updated_at], + "Webhook" => %w[id user_id name url secret event_types place_ids created_at updated_at], + "WebhookDelivery" => %w[id webhook_id geofence_event_id status response_body error_message created_at updated_at], + "LegacyAreaPlaceMapping" => %w[id area_id place_id created_at updated_at], + "PointRevision" => %w[id user_id month revision], + "TrackRevision" => %w[id user_id month revision] }.freeze REQUIRED_FOREIGN_KEYS = { @@ -69,10 +75,15 @@ module OpenMapxSubjectExport "PlannedDay" => %w[trip_id], "PlannedDayNote" => %w[planned_day_id], "PlannedReservation" => %w[planned_day_id trip_id], "PlannedStop" => %w[planned_day_id], "PlannedAccommodation" => %w[trip_id], "PlannedTraveller" => %w[trip_id], - "PlannedUnplannedPlace" => %w[trip_id] + "PlannedUnplannedPlace" => %w[trip_id], + "GeofenceEvent" => %w[user_id], "Webhook" => %w[user_id], + "WebhookDelivery" => %w[webhook_id], "LegacyAreaPlaceMapping" => %w[area_id], + # Revision counters carry user_id without a database constraint; the table + # inventory below is what makes such an unconstrained relation reviewable. + "PointRevision" => %w[user_id], "TrackRevision" => %w[user_id] }.freeze - # Generated from the reviewed 1.15.3 migration graph. This inventory is + # Generated from the reviewed 1.19.0 migration graph. This inventory is # intentionally wider than the direct user-owned relations above: a # polymorphic attachment or a visit/point edge can make a record reachable # from a subject-owned row even when the FK does not point directly at @@ -94,6 +105,10 @@ module OpenMapxSubjectExport family_memberships.family_id->families.id family_memberships.user_id->users.id flights.user_id->users.id + geofence_events.place_id->places.id + geofence_events.user_id->users.id + legacy_area_place_mappings.area_id->areas.id + legacy_area_place_mappings.place_id->places.id notes.user_id->users.id notifications.user_id->users.id pending_imports.claimed_by_user_id->users.id @@ -128,7 +143,28 @@ module OpenMapxSubjectExport visits.area_id->areas.id visits.place_id->places.id visits.user_id->users.id + webhook_deliveries.geofence_event_id->geofence_events.id + webhook_deliveries.webhook_id->webhooks.id + webhooks.user_id->users.id ].freeze + # Every table the reviewed release creates, including reference and + # framework tables. A foreign-key graph cannot reveal a new table whose + # owner column is unconstrained (as the revision counters are), so the + # complete inventory must match before any relation is opened. + EXPECTED_TABLES = %w[ + achievement_progresses achievement_unlock_events action_text_rich_texts + active_storage_attachments active_storage_blobs active_storage_variant_records + ar_internal_metadata areas countries data_migrations digests disputed_areas exports families + family_invitations family_location_requests family_memberships flights flipper_features + flipper_gates geofence_events imports instance_settings legacy_area_place_mappings notes + notifications pending_imports place_visits places planned_accommodations planned_day_notes + planned_days planned_reservations planned_stops planned_travellers planned_unplanned_places + point_revisions point_sources points points_raw_data_archives posters regions route_videos + schema_migrations service_settings shared_links spatial_ref_sys stats taggings tags + track_revisions track_segments tracks trip_sources trips user_achievements users visits + webhook_deliveries webhooks + ].freeze + ENTRY_PATHS = { "source-manifest" => "dawarich/source-manifest.json", "account" => "dawarich/account.json", @@ -158,6 +194,12 @@ module OpenMapxSubjectExport "route-videos" => "dawarich/route-videos.jsonl", "service-settings" => "dawarich/service-settings.jsonl", "trip-sources" => "dawarich/trip-sources.jsonl", + "geofence-events" => "dawarich/geofence-events.jsonl", + "webhooks" => "dawarich/webhooks.jsonl", + "webhook-deliveries" => "dawarich/webhook-deliveries.jsonl", + "legacy-area-place-mappings" => "dawarich/legacy-area-place-mappings.jsonl", + "point-revisions" => "dawarich/point-revisions.jsonl", + "track-revisions" => "dawarich/track-revisions.jsonl", "planned-days" => "dawarich/planned-days.jsonl", "planned-day-notes" => "dawarich/planned-day-notes.jsonl", "planned-reservations" => "dawarich/planned-reservations.jsonl", diff --git a/services/dawarich-app/service.json b/services/dawarich-app/service.json index 5a55be8d2..7a0151369 100644 --- a/services/dawarich-app/service.json +++ b/services/dawarich-app/service.json @@ -1,7 +1,7 @@ { "id": "dawarich-app", "name": "Dawarich Timeline", - "version": "1.15.3", + "version": "1.19.0", "description": "Managed multi-user Dawarich web application with OpenMapX OIDC sign-in.", "license": "AGPL-3.0", "quality": "built-in", @@ -15,8 +15,8 @@ }, "container": { "image": "freikin/dawarich", - "tag": "1.15.3", - "digest": "sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498", + "tag": "1.19.0", + "digest": "sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304", "expose": [3000], "entrypoint": ["/usr/local/bin/openmapx-entrypoint.sh"], "command": ["bin/rails", "server", "-p", "3000", "-b", "::"], @@ -44,7 +44,7 @@ "OIDC_PROVIDER_NAME": "OpenMapX", "OIDC_AUTO_REGISTER": "true", "OIDC_PKCE_ENABLED": "true", - "OPENMAPX_DAWARICH_COMMIT": "e32b70701b2467b862d4b5bc27ac428999997c1b", + "OPENMAPX_DAWARICH_COMMIT": "752641768f37fdc07d5a23e0559fa7c5ff8fae0f", "ALLOW_EMAIL_PASSWORD_REGISTRATION": "true", "ALLOW_EMAIL_PASSWORD_LOGIN": "true" }, diff --git a/services/dawarich-redis/service.json b/services/dawarich-redis/service.json index 2849efa7d..156060e3d 100644 --- a/services/dawarich-redis/service.json +++ b/services/dawarich-redis/service.json @@ -1,9 +1,9 @@ { "id": "dawarich-redis", "name": "Dawarich Redis", - "version": "7.4", + "version": "8.10", "description": "Dedicated transient job queue and cache for the managed Dawarich timeline.", - "license": "RSALv2 OR SSPL-1.0", + "license": "RSALv2 OR SSPL-1.0 OR AGPL-3.0", "quality": "built-in", "homepage": "https://redis.io/", "subjectData": { @@ -14,8 +14,8 @@ }, "container": { "image": "redis", - "tag": "7.4-alpine", - "digest": "sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f", + "tag": "8.10-alpine", + "digest": "sha256:21090af93f8324db2f0f0f4f38a7c60369d47d917e000c3423d89d2ae2883537", "expose": [6379], "command": ["redis-server", "--save", "900", "1", "--save", "300", "10", "--appendonly", "no"], "memory": "512m", diff --git a/services/dawarich-sidekiq/service.json b/services/dawarich-sidekiq/service.json index e2b709c53..9ec6b37f9 100644 --- a/services/dawarich-sidekiq/service.json +++ b/services/dawarich-sidekiq/service.json @@ -1,7 +1,7 @@ { "id": "dawarich-sidekiq", "name": "Dawarich Sidekiq", - "version": "1.15.3", + "version": "1.19.0", "description": "Background job worker for the managed multi-user Dawarich timeline.", "license": "AGPL-3.0", "quality": "built-in", @@ -15,8 +15,8 @@ }, "container": { "image": "freikin/dawarich", - "tag": "1.15.3", - "digest": "sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498", + "tag": "1.19.0", + "digest": "sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304", "entrypoint": ["/usr/local/bin/openmapx-entrypoint.sh"], "command": ["sidekiq"], "environment": { @@ -43,7 +43,7 @@ "OIDC_PROVIDER_NAME": "OpenMapX", "OIDC_AUTO_REGISTER": "true", "OIDC_PKCE_ENABLED": "true", - "OPENMAPX_DAWARICH_COMMIT": "e32b70701b2467b862d4b5bc27ac428999997c1b", + "OPENMAPX_DAWARICH_COMMIT": "752641768f37fdc07d5a23e0559fa7c5ff8fae0f", "ALLOW_EMAIL_PASSWORD_REGISTRATION": "true", "ALLOW_EMAIL_PASSWORD_LOGIN": "true", "BACKGROUND_PROCESSING_CONCURRENCY": "3" diff --git a/services/ops-agent/privacy-backup/openmapx-backup-subject-export.py b/services/ops-agent/privacy-backup/openmapx-backup-subject-export.py index 55d237cb2..c8253614d 100644 --- a/services/ops-agent/privacy-backup/openmapx-backup-subject-export.py +++ b/services/ops-agent/privacy-backup/openmapx-backup-subject-export.py @@ -18,10 +18,10 @@ MAX_MANIFEST = 256 * 1024 MAX_TOTAL = 2 * 1024 * 1024 * 1024 MAX_ENTRIES = 256 -DAWARICH_IMAGE = "freikin/dawarich:1.15.3" -DAWARICH_IMAGE_DIGEST = "sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498" -DAWARICH_COMMIT = "e32b70701b2467b862d4b5bc27ac428999997c1b" -DAWARICH_SCHEMA_FINGERPRINT = "4ae830ea67dc4894d814d4e2e2d19cb92571c30f2df1df017417f671e26a1e09" +DAWARICH_IMAGE = "freikin/dawarich:1.19.0" +DAWARICH_IMAGE_DIGEST = "sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304" +DAWARICH_COMMIT = "752641768f37fdc07d5a23e0559fa7c5ff8fae0f" +DAWARICH_SCHEMA_FINGERPRINT = "d728906287ec3c4f3f38a80c2c4fe6ce77cc40f03b56a5ba0e89b811dd6e631a" DAWARICH_RELATION_FINGERPRINT_VERSION = "dawarich-relations-v2" SAFE_EXTENSIONS = {"bin", "csv", "fit", "gpx", "json", "jsonl", "jpg", "jpeg", "kml", "mp4", "pdf", "png", "tcx", "txt"} @@ -68,31 +68,89 @@ ("PlannedTraveller", "planned_travellers", ["id", "trip_id", "name", "owner", "created_at", "updated_at"], ["trip_id"]), ("PlannedUnplannedPlace", "planned_unplanned_places", ["id", "trip_id", "name", "address", "latitude", "longitude", "starts_at", "ends_at", "duration_minutes", "position", "category", "notes", "transport_mode", "created_at", "updated_at"], ["trip_id"]), ("PointSource", "point_sources", ["id", "digest", "tracker_id", "topic", "ssid", "bssid", "connection", "trigger", "battery_status", "inrids", "in_regions", "created_at", "updated_at"], []), + ("GeofenceEvent", "geofence_events", ["id", "user_id", "place_id", "event_type", "occurred_at", "lonlat", "device_id", "created_at", "updated_at"], ["user_id"]), + ("Webhook", "webhooks", ["id", "user_id", "name", "url", "secret", "event_types", "place_ids", "created_at", "updated_at"], ["user_id"]), + ("WebhookDelivery", "webhook_deliveries", ["id", "webhook_id", "geofence_event_id", "status", "response_body", "error_message", "created_at", "updated_at"], ["webhook_id"]), + ("LegacyAreaPlaceMapping", "legacy_area_place_mappings", ["id", "area_id", "place_id", "created_at", "updated_at"], ["area_id"]), + ("PointRevision", "point_revisions", ["id", "user_id", "month", "revision"], ["user_id"]), + ("TrackRevision", "track_revisions", ["id", "user_id", "month", "revision"], ["user_id"]), ] DAWARICH_EXPECTED_FOREIGN_KEYS = sorted([ - "achievement_progresses.user_id->users.id", "achievement_unlock_events.user_id->users.id", + "achievement_progresses.user_id->users.id", + "achievement_unlock_events.user_id->users.id", "active_storage_attachments.blob_id->active_storage_blobs.id", "active_storage_variant_records.blob_id->active_storage_blobs.id", - "areas.user_id->users.id", "digests.user_id->users.id", "families.creator_id->users.id", - "family_invitations.family_id->families.id", "family_invitations.invited_by_id->users.id", - "family_location_requests.family_id->families.id", "family_location_requests.requester_id->users.id", - "family_location_requests.target_user_id->users.id", "family_memberships.family_id->families.id", - "family_memberships.user_id->users.id", "flights.user_id->users.id", "notes.user_id->users.id", - "notifications.user_id->users.id", "pending_imports.claimed_by_user_id->users.id", - "place_visits.place_id->places.id", "place_visits.visit_id->visits.id", - "planned_accommodations.trip_id->trips.id", "planned_day_notes.planned_day_id->planned_days.id", - "planned_days.trip_id->trips.id", "planned_reservations.planned_day_id->planned_days.id", - "planned_reservations.trip_id->trips.id", "planned_stops.planned_day_id->planned_days.id", - "planned_travellers.trip_id->trips.id", "planned_unplanned_places.trip_id->trips.id", - "points.raw_data_archive_id->points_raw_data_archives.id", "points.user_id->users.id", - "points.track_id->tracks.id", "points.visit_id->visits.id", "points_raw_data_archives.user_id->users.id", - "posters.user_id->users.id", "route_videos.user_id->users.id", "service_settings.user_id->users.id", - "shared_links.user_id->users.id", "stats.user_id->users.id", - "taggings.tag_id->tags.id", "tags.user_id->users.id", "track_segments.track_id->tracks.id", - "tracks.user_id->users.id", "trip_sources.user_id->users.id", "trips.trip_source_id->trip_sources.id", - "trips.user_id->users.id", "user_achievements.user_id->users.id", "visits.area_id->areas.id", - "visits.place_id->places.id", "visits.user_id->users.id", + "areas.user_id->users.id", + "digests.user_id->users.id", + "families.creator_id->users.id", + "family_invitations.family_id->families.id", + "family_invitations.invited_by_id->users.id", + "family_location_requests.family_id->families.id", + "family_location_requests.requester_id->users.id", + "family_location_requests.target_user_id->users.id", + "family_memberships.family_id->families.id", + "family_memberships.user_id->users.id", + "flights.user_id->users.id", + "geofence_events.place_id->places.id", + "geofence_events.user_id->users.id", + "legacy_area_place_mappings.area_id->areas.id", + "legacy_area_place_mappings.place_id->places.id", + "notes.user_id->users.id", + "notifications.user_id->users.id", + "pending_imports.claimed_by_user_id->users.id", + "place_visits.place_id->places.id", + "place_visits.visit_id->visits.id", + "planned_accommodations.trip_id->trips.id", + "planned_day_notes.planned_day_id->planned_days.id", + "planned_days.trip_id->trips.id", + "planned_reservations.planned_day_id->planned_days.id", + "planned_reservations.trip_id->trips.id", + "planned_stops.planned_day_id->planned_days.id", + "planned_travellers.trip_id->trips.id", + "planned_unplanned_places.trip_id->trips.id", + "points.raw_data_archive_id->points_raw_data_archives.id", + "points.track_id->tracks.id", + "points.user_id->users.id", + "points.visit_id->visits.id", + "points_raw_data_archives.user_id->users.id", + "posters.user_id->users.id", + "route_videos.user_id->users.id", + "service_settings.user_id->users.id", + "shared_links.user_id->users.id", + "stats.user_id->users.id", + "taggings.tag_id->tags.id", + "tags.user_id->users.id", + "track_segments.track_id->tracks.id", + "tracks.user_id->users.id", + "trip_sources.user_id->users.id", + "trips.trip_source_id->trip_sources.id", + "trips.user_id->users.id", + "user_achievements.user_id->users.id", + "visits.area_id->areas.id", + "visits.place_id->places.id", + "visits.user_id->users.id", + "webhook_deliveries.geofence_event_id->geofence_events.id", + "webhook_deliveries.webhook_id->webhooks.id", + "webhooks.user_id->users.id", +]) + +# Every table the reviewed release creates. An owner column without a +# constraint never appears in the FK graph, so the inventory must match too. +DAWARICH_EXPECTED_TABLES = sorted([ + "achievement_progresses", "achievement_unlock_events", "action_text_rich_texts", + "active_storage_attachments", "active_storage_blobs", "active_storage_variant_records", + "ar_internal_metadata", "areas", "countries", "data_migrations", "digests", + "disputed_areas", "exports", "families", "family_invitations", "family_location_requests", + "family_memberships", "flights", "flipper_features", "flipper_gates", "geofence_events", + "imports", "instance_settings", "legacy_area_place_mappings", "notes", "notifications", + "pending_imports", "place_visits", "places", "planned_accommodations", "planned_day_notes", + "planned_days", "planned_reservations", "planned_stops", "planned_travellers", + "planned_unplanned_places", "point_revisions", "point_sources", "points", + "points_raw_data_archives", "posters", "regions", "route_videos", "schema_migrations", + "service_settings", "shared_links", "spatial_ref_sys", "stats", "taggings", "tags", + "track_revisions", "track_segments", "tracks", "trip_sources", "trips", "user_achievements", + "users", "visits", "webhook_deliveries", "webhooks", ]) @@ -149,8 +207,8 @@ def verified_sources(request): declared = {(service.get("id"), volume.get("file")): volume for service in manifest.get("services", []) for volume in service.get("volumes", [])} contracts = { "openmapx-v1": ("openmapx", "postgis", "pg_dump"), - "dawarich-1.15.3": ("dawarich", "dawarich-postgis", "pg_dump"), - "dawarich-storage-1.15.3": ("dawarich-storage", "dawarich-app", "tar"), + "dawarich-1.19.0": ("dawarich", "dawarich-postgis", "pg_dump"), + "dawarich-storage-1.19.0": ("dawarich-storage", "dawarich-app", "tar"), } result = [] seen = set() @@ -162,7 +220,7 @@ def verified_sources(request): volume = declared.get((source.get("serviceId"), source.get("file"))) if contract in seen or source.get("family") != family or source.get("serviceId") != service_id or not volume or volume.get("mode") != mode: fail("unsupported_schema") - if contract == "dawarich-storage-1.15.3" and volume.get("name") != "openmapx-dawarich-storage": + if contract == "dawarich-storage-1.19.0" and volume.get("name") != "openmapx-dawarich-storage": fail("unsupported_schema") path = root / source["file"] if path.parent != root or path.is_symlink() or not path.is_file(): @@ -172,11 +230,11 @@ def verified_sources(request): fail("backup_digest_changed") seen.add(contract) result.append({**source, "path": path}) - if "dawarich-1.15.3" in seen: + if "dawarich-1.19.0" in seen: managed = manifest.get("privacySourceProvenance", {}).get("managedDawarich") - if managed != {"version": "1.15.3", "image": "freikin/dawarich", "imageDigest": DAWARICH_IMAGE_DIGEST, "upstreamCommit": DAWARICH_COMMIT, "schemaContract": "dawarich-1.15.3"}: + if managed != {"version": "1.19.0", "image": "freikin/dawarich", "imageDigest": DAWARICH_IMAGE_DIGEST, "upstreamCommit": DAWARICH_COMMIT, "schemaContract": "dawarich-1.19.0"}: fail("unsupported_schema") - if "dawarich-storage-1.15.3" in seen and "dawarich-1.15.3" not in seen: + if "dawarich-storage-1.19.0" in seen and "dawarich-1.19.0" not in seen: fail("unsupported_schema") return result, manifest @@ -236,6 +294,9 @@ def validate_dawarich_schema(socket): """) or b"[]") if observed != DAWARICH_EXPECTED_FOREIGN_KEYS: fail("unsupported_schema") + tables = json.loads(psql(socket, "SELECT COALESCE(json_agg(table_name::text ORDER BY table_name),'[]'::json) FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';") or b"[]") + if tables != DAWARICH_EXPECTED_TABLES: + fail("unsupported_schema") document = { "version": DAWARICH_RELATION_FINGERPRINT_VERSION, "user": sorted(columns(socket, "users")), @@ -475,13 +536,13 @@ def project_openmapx(socket): DAWARICH = { "areas": ("areas", "user_id", ["id", "name", "latitude", "longitude", "radius", "created_at", "updated_at"]), - "places": ("places", "user_id", ["id", "name", "city", "country", "latitude", "longitude", "source", "note", "geodata", "created_at", "updated_at"]), + "places": ("places", "user_id", ["id", "name", "city", "country", "latitude", "longitude", "source", "note", "geodata", "visit_radius", "geofence", "archived_at", "created_at", "updated_at"]), "tags": ("tags", "user_id", ["id", "name", "color", "icon", "privacy_radius_meters", "created_at", "updated_at"]), "imports": ("imports", "user_id", ["id", "name", "source", "status", "demo", "points_count", "raw_points", "processed", "doubles", "created_at", "updated_at", "processing_started_at"]), "export-records": ("exports", "user_id", ["id", "name", "file_type", "file_format", "status", "start_at", "end_at", "created_at", "updated_at", "processing_started_at"]), "trips": ("trips", "user_id", ["id", "name", "started_at", "ended_at", "distance", "path", "visited_countries", "demo", "created_at", "updated_at", "last_recalculated_at"]), "notifications": ("notifications", "user_id", ["id", "title", "content", "kind", "read_at", "created_at", "updated_at"]), - "visits": ("visits", "user_id", ["id", "name", "started_at", "ended_at", "duration", "confidence", "confidence_breakdown", "status", "area_id", "place_id", "demo", "created_at", "updated_at"]), + "visits": ("visits", "user_id", ["id", "name", "location_label", "started_at", "ended_at", "duration", "confidence", "confidence_breakdown", "status", "area_id", "place_id", "demo", "created_at", "updated_at"]), "stats": ("stats", "user_id", ["id", "year", "month", "distance", "daily_distance", "h3_hex_ids", "toponyms", "created_at", "updated_at"]), "tracks": ("tracks", "user_id", ["id", "start_at", "end_at", "distance", "duration", "avg_speed", "dominant_mode", "elevation_gain", "elevation_loss", "elevation_max", "elevation_min", "tracker_id", "demo", "original_path", "created_at", "updated_at"]), "digests": ("digests", "user_id", ["id", "year", "month", "period_type", "distance", "sent_at", "all_time_stats", "first_time_visits", "monthly_distances", "time_spent_by_location", "toponyms", "travel_patterns", "year_over_year", "created_at", "updated_at"]), @@ -495,9 +556,23 @@ def project_openmapx(socket): "user-achievements": ("user_achievements", "user_id", ["id", "achievement_key", "earned_at", "metadata", "created_at", "updated_at"]), "route-videos": ("route_videos", "user_id", ["id", "name", "settings", "status", "expired_at", "created_at", "updated_at"]), "service-settings": ("service_settings", "user_id", ["id", "service", "provider", "active", "config", "created_at", "updated_at"]), - "trip-sources": ("trip_sources", "user_id", ["id", "provider", "base_url", "status", "importing", "last_error", "last_synced_at", "created_at", "updated_at"]), + "trip-sources": ("trip_sources", "user_id", ["id", "provider", "base_url", "status", "importing", "skip_ssl_verification", "last_error", "last_synced_at", "created_at", "updated_at"]), + # lonlat, the enum labels and the reduced url are added as SQL extras. + "geofence-events": ("geofence_events", "user_id", ["id", "occurred_at", "received_at", "place_id", "place_name", "accuracy_m", "device_id", "synthetic", "metadata", "created_at", "updated_at"]), + "webhooks": ("webhooks", "user_id", ["id", "name", "event_types", "place_ids", "active", "consecutive_failures", "last_delivery_at", "last_success_at", "created_at", "updated_at"]), } +WEBHOOK_REDACTIONS = ["dawarich-webhook-secret-omitted", "dawarich-webhook-endpoint-reduced"] +WEBHOOK_DELIVERY_REDACTIONS = ["dawarich-webhook-response-omitted"] +# Scheme, host and non-default port only, as in the Rails collector: a webhook +# path or query string is commonly the receiver's bearer credential. +WEBHOOK_ORIGIN_SQL = ( + "(SELECT CASE WHEN o.scheme IN ('http','https') AND o.host<>'' THEN o.scheme || '://' || o.host" + " || CASE WHEN o.port IS NULL OR (o.scheme='http' AND o.port='80') OR (o.scheme='https' AND o.port='443') THEN '' ELSE ':' || o.port END END" + " FROM (SELECT lower(m[1]) AS scheme, m[2] AS host, m[3] AS port" + " FROM (SELECT regexp_match(t.url, '^([A-Za-z][A-Za-z0-9+.-]*)://(?:[^/?#@]*@)?(\\[[^]/?#]*\\]|[^:/?#]*)(?::([0-9]+))?(?:[/?#]|$)') AS m) parsed) o)" +) + DAWARICH_PLANNED = { "planned-days": ("planned_days", "trip_id", ["id", "trip_id", "date", "title", "notes", "position", "created_at", "updated_at"]), "planned-day-notes": ("planned_day_notes", "planned_day_id", ["id", "planned_day_id", "body", "noted_at", "position", "created_at", "updated_at"]), @@ -525,7 +600,7 @@ def project_dawarich(socket): [("redactionCodes", "jsonb_build_array('dawarich-authentication-secrets-redacted')")], ) entries.append(query_descriptor(socket, "dawarich/account.json", f"SELECT {account_projection} FROM users u WHERE u.id={user_id}", False, ["dawarich-authentication-secrets-redacted"])) - safe_settings = ["fog_of_war_meters", "fog_of_war_threshold", "fog_of_war_mode", "meters_between_routes", "preferred_map_layer", "speed_colored_routes", "points_rendering_mode", "minutes_between_routes", "time_threshold_minutes", "merge_threshold_minutes", "live_map_enabled", "route_opacity", "route_color", "track_color", "maps", "visits_suggestions_enabled", "enabled_map_layers", "maps_maplibre_style", "maps_maplibre_custom_theme", "globe_projection", "transportation_thresholds", "transportation_expert_thresholds", "transportation_expert_mode", "min_minutes_spent_in_city", "max_gap_minutes_in_city", "gps_filtering_enabled", "gps_accuracy_threshold", "timezone", "visit_radius_meters", "visit_min_points", "visit_min_duration_minutes", "visit_density_fill_enabled", "stay_max_gap_minutes", "point_dragging_enabled", "news_emails_enabled", "show_supporter_badge"] + safe_settings = ["fog_of_war_meters", "fog_of_war_threshold", "fog_of_war_mode", "meters_between_routes", "preferred_map_layer", "speed_colored_routes", "points_rendering_mode", "minutes_between_routes", "time_threshold_minutes", "merge_threshold_minutes", "live_map_enabled", "route_opacity", "route_color", "track_color", "maps", "visits_suggestions_enabled", "enabled_map_layers", "maps_maplibre_style", "maps_maplibre_custom_theme", "globe_projection", "transportation_thresholds", "transportation_expert_thresholds", "transportation_expert_mode", "min_minutes_spent_in_city", "max_gap_minutes_in_city", "gps_filtering_enabled", "gps_accuracy_threshold", "timezone", "visit_radius_meters", "visit_min_points", "visit_min_duration_minutes", "visit_density_fill_enabled", "stay_max_gap_minutes", "point_dragging_enabled", "news_emails_enabled", "show_supporter_badge", "achievements_enabled", "family_enabled", "date_format", "time_format", "first_day_of_week", "gps_accuracy_filter_enabled"] allow = "ARRAY[" + ",".join(literal(value) for value in safe_settings) + "]::text[]" settings = f"SELECT jsonb_build_object('values',COALESCE((SELECT jsonb_object_agg(key,value) FROM jsonb_each(u.settings) WHERE key=ANY({allow})),'{{}}'::jsonb),'redactedKeys',COALESCE((SELECT jsonb_agg(key ORDER BY key) FROM jsonb_each(u.settings) WHERE NOT key=ANY({allow})),'[]'::jsonb),'redactionCodes',jsonb_build_array('dawarich-settings-secrets-redacted')) FROM users u WHERE u.id={user_id}" entries.append(query_descriptor(socket, "dawarich/settings.json", settings, False, ["dawarich-settings-secrets-redacted"])) @@ -536,13 +611,23 @@ def project_dawarich(socket): extras.append(("path", "CASE WHEN t.path IS NULL THEN NULL ELSE ST_AsGeoJSON(t.path)::jsonb->'coordinates' END")) elif entry_id == "tracks": extras.append(("original_path", "ST_AsGeoJSON(t.original_path)::jsonb->'coordinates'")) + elif entry_id == "geofence-events": + extras.extend([ + ("lonlat", "jsonb_build_array(ST_X(t.lonlat::geometry), ST_Y(t.lonlat::geometry))"), + ("event_type", "CASE t.event_type WHEN 0 THEN 'enter' WHEN 1 THEN 'leave' END"), + ("source", "CASE t.source WHEN 0 THEN 'native_app' WHEN 1 THEN 'server_inferred' WHEN 2 THEN 'owntracks_native' END"), + ("webhook_skip_reason", "CASE t.webhook_skip_reason WHEN 0 THEN 'late' WHEN 1 THEN 'throttled' END"), + ]) + elif entry_id == "webhooks": + extras.extend([("url", WEBHOOK_ORIGIN_SQL), ("redactionCodes", "jsonb_build_array(" + ",".join(literal(code) for code in WEBHOOK_REDACTIONS) + ")")]) elif entry_id == "notes": extras.extend([ ("latitude", "CASE WHEN t.lonlat IS NULL THEN NULL ELSE ST_Y(t.lonlat::geometry) END"), ("longitude", "CASE WHEN t.lonlat IS NULL THEN NULL ELSE ST_X(t.lonlat::geometry) END"), ]) select = f"SELECT {projection('t', fields, extras)} FROM {table} t WHERE t.{owner}={user_id} AND t.created_at<={literal(REQUEST['cutoff'])}::timestamptz ORDER BY t.id" - entries.append(query_descriptor(socket, f"dawarich/{entry_id}.jsonl", select, entry_id in {"places", "imports", "raw-archives", "flights", "notes", "route-videos"}, ["dawarich-rights-of-others-review"] if entry_id == "notes" else [])) + redactions = {"notes": ["dawarich-rights-of-others-review"], "webhooks": WEBHOOK_REDACTIONS}.get(entry_id, []) + entries.append(query_descriptor(socket, f"dawarich/{entry_id}.jsonl", select, entry_id in {"places", "imports", "raw-archives", "flights", "notes", "route-videos", "geofence-events"}, redactions)) for entry_id, (table, owner, fields) in DAWARICH_PLANNED.items(): require(socket, table, ["id", owner, "created_at", "updated_at"]) if owner == "trip_id": @@ -555,6 +640,17 @@ def project_dawarich(socket): tagging_fields = ["id", "tag_id", "taggable_id", "taggable_type", "created_at", "updated_at"] taggings = f"SELECT {projection('t', tagging_fields)} FROM taggings t WHERE (t.tag_id IN (SELECT id FROM tags WHERE user_id={user_id}) OR (t.taggable_type='Place' AND t.taggable_id IN (SELECT id FROM places WHERE user_id={user_id}))) AND t.created_at<={literal(REQUEST['cutoff'])}::timestamptz ORDER BY t.id" entries.append(query_descriptor(socket, "dawarich/taggings.jsonl", taggings)) + delivery_fields = ["id", "webhook_id", "geofence_event_id", "attempt_count", "response_status", "delivered_at", "created_at", "updated_at"] + delivery_extras = [ + ("status", "CASE d.status WHEN 0 THEN 'pending' WHEN 1 THEN 'success' WHEN 2 THEN 'failure' WHEN 3 THEN 'retrying' END"), + ("redactionCodes", "jsonb_build_array(" + ",".join(literal(code) for code in WEBHOOK_DELIVERY_REDACTIONS) + ")"), + ] + entries.append(query_descriptor(socket, "dawarich/webhook-deliveries.jsonl", f"SELECT {projection('d', delivery_fields, delivery_extras)} FROM webhook_deliveries d WHERE d.webhook_id IN (SELECT id FROM webhooks WHERE user_id={user_id}) AND d.created_at<={literal(REQUEST['cutoff'])}::timestamptz ORDER BY d.id", False, WEBHOOK_DELIVERY_REDACTIONS)) + mapping_fields = ["id", "area_id", "place_id", "created_at", "updated_at"] + entries.append(query_descriptor(socket, "dawarich/legacy-area-place-mappings.jsonl", f"SELECT {projection('m', mapping_fields)} FROM legacy_area_place_mappings m WHERE m.area_id IN (SELECT id FROM areas WHERE user_id={user_id}) AND m.created_at<={literal(REQUEST['cutoff'])}::timestamptz ORDER BY m.id")) + # Revision counters have no timestamps, so the cutoff cannot apply to them. + for revision_entry, revision_table in (("point-revisions", "point_revisions"), ("track-revisions", "track_revisions")): + entries.append(query_descriptor(socket, f"dawarich/{revision_entry}.jsonl", f"SELECT {projection('r', ['id', 'month', 'revision'])} FROM {revision_table} r WHERE r.user_id={user_id} ORDER BY r.id")) segment_fields = ["id", "track_id", "start_index", "end_index", "distance", "duration", "avg_speed", "max_speed", "confidence", "corrected_at", "source", "transportation_mode", "created_at", "updated_at"] require(socket, "track_segments", ["id", "track_id", "created_at", "updated_at"]) entries.append(query_descriptor(socket, "dawarich/track-segments.jsonl", f"SELECT {projection('s', segment_fields)} FROM track_segments s WHERE s.track_id IN (SELECT id FROM tracks WHERE user_id={user_id}) AND s.created_at<={literal(REQUEST['cutoff'])}::timestamptz ORDER BY s.id")) @@ -717,7 +813,7 @@ def restore(source, data, socket, storage): def dawarich_id(path): - fixed = {name: name.removesuffix(".jsonl").removesuffix(".json") for name in ["account.json", "settings.json", "areas.jsonl", "places.jsonl", "tags.jsonl", "taggings.jsonl", "imports.jsonl", "trips.jsonl", "notifications.jsonl", "visits.jsonl", "stats.jsonl", "tracks.jsonl", "digests.jsonl", "flights.jsonl", "notes.jsonl", "posters.jsonl", "attachments.jsonl", "rich-text.jsonl", "family.jsonl", "achievement-progress.jsonl", "achievement-unlock-events.jsonl", "user-achievements.jsonl", "route-videos.jsonl", "service-settings.jsonl", "trip-sources.jsonl", "planned-days.jsonl", "planned-day-notes.jsonl", "planned-reservations.jsonl", "planned-stops.jsonl", "planned-accommodations.jsonl", "planned-travellers.jsonl", "planned-unplanned-places.jsonl"]} + fixed = {name: name.removesuffix(".jsonl").removesuffix(".json") for name in ["account.json", "settings.json", "areas.jsonl", "places.jsonl", "tags.jsonl", "taggings.jsonl", "imports.jsonl", "trips.jsonl", "notifications.jsonl", "visits.jsonl", "stats.jsonl", "tracks.jsonl", "digests.jsonl", "flights.jsonl", "notes.jsonl", "posters.jsonl", "attachments.jsonl", "rich-text.jsonl", "family.jsonl", "achievement-progress.jsonl", "achievement-unlock-events.jsonl", "user-achievements.jsonl", "route-videos.jsonl", "service-settings.jsonl", "trip-sources.jsonl", "planned-days.jsonl", "planned-day-notes.jsonl", "planned-reservations.jsonl", "planned-stops.jsonl", "planned-accommodations.jsonl", "planned-travellers.jsonl", "planned-unplanned-places.jsonl", "geofence-events.jsonl", "webhooks.jsonl", "webhook-deliveries.jsonl", "legacy-area-place-mappings.jsonl", "point-revisions.jsonl", "track-revisions.jsonl"]} fixed.update({"export-records.jsonl": "export-records", "track-segments.jsonl": "track-segments", "raw-archives.jsonl": "raw-archives", "shared-links.jsonl": "shared-links"}) relative = path.removeprefix("dawarich/") if relative in fixed: @@ -771,8 +867,8 @@ def emit(entries, warnings, schema, backup_manifest): try: OUTPUT, WARNINGS, DAWARICH_SCHEMA = [], [], None ACTIVE_DATABASES = [] - STORAGE = next((source for source in SOURCES if source["schemaContract"] == "dawarich-storage-1.15.3"), None) - databases = [source for source in SOURCES if source["schemaContract"] != "dawarich-storage-1.15.3"] + STORAGE = next((source for source in SOURCES if source["schemaContract"] == "dawarich-storage-1.19.0"), None) + databases = [source for source in SOURCES if source["schemaContract"] != "dawarich-storage-1.19.0"] for index, source in enumerate(databases): data = WORK / f"pg-{index}" socket = WORK / f"socket-{index}" diff --git a/services/ops-agent/privacy-backup/test-fixture.mjs b/services/ops-agent/privacy-backup/test-fixture.mjs index 44f545095..046a90d46 100644 --- a/services/ops-agent/privacy-backup/test-fixture.mjs +++ b/services/ops-agent/privacy-backup/test-fixture.mjs @@ -21,7 +21,7 @@ const databaseContainer = `openmapx-privacy-backup-${randomBytes(8).toString("he const databaseImage = "ghcr.io/baosystems/postgis:18-3.6@sha256:dd7515b7d8b8b8bac3516a9c3e12584df5890b0864c29295c1b934ecd1db81a7"; const dawarichImage = - "freikin/dawarich@sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498"; + "freikin/dawarich@sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304"; const databasePassword = "isolated-backup-fixture"; const outputFlag = process.argv.indexOf("--output"); const retainedOutput = outputFlag >= 0 ? process.argv[outputFlag + 1] : undefined; @@ -90,7 +90,7 @@ function dawarichDriftRequest(directory, dump, dumpName, backupId) { services: [ { id: "dawarich-postgis", - version: "1.15.3", + version: "1.19.0", volumes: [ { name: "database", @@ -104,11 +104,11 @@ function dawarichDriftRequest(directory, dump, dumpName, backupId) { ], privacySourceProvenance: { managedDawarich: { - version: "1.15.3", + version: "1.19.0", image: "freikin/dawarich", - imageDigest: "sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498", - upstreamCommit: "e32b70701b2467b862d4b5bc27ac428999997c1b", - schemaContract: "dawarich-1.15.3", + imageDigest: "sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304", + upstreamCommit: "752641768f37fdc07d5a23e0559fa7c5ff8fae0f", + schemaContract: "dawarich-1.19.0", }, }, }), @@ -128,7 +128,7 @@ function dawarichDriftRequest(directory, dump, dumpName, backupId) { family: "dawarich", serviceId: "dawarich-postgis", file: dumpName, - schemaContract: "dawarich-1.15.3", + schemaContract: "dawarich-1.19.0", }, ], })}\n`, @@ -291,6 +291,21 @@ INSERT INTO tracks (id,user_id,start_at,end_at,original_path,distance,duration,a VALUES (1,7,'2026-01-01','2026-01-01',ST_GeomFromText('LINESTRING(13.4 52.5,13.5 52.6)',4326),100,60,6,'2026-01-01','2026-01-01'); INSERT INTO notes (id,user_id,title,body,noted_at,lonlat,created_at,updated_at) VALUES (1,7,'Subject note','Subject note body','2026-01-01',ST_SetSRID(ST_MakePoint(13.45,52.55),4326)::geography,'2026-01-01','2026-01-01'); +INSERT INTO places (id,user_id,name,latitude,longitude,created_at,updated_at) +VALUES (1,7,'Subject place',52.5,13.4,'2026-01-01','2026-01-01'),(2,8,'FOREIGN_ROW_SENTINEL',1,2,'2026-01-01','2026-01-01'); +INSERT INTO geofence_events (id,user_id,place_id,place_name,event_type,source,synthetic,occurred_at,received_at,lonlat,device_id,metadata,created_at,updated_at) +VALUES (1,7,1,'Subject geofence place',1,2,false,'2026-01-01','2026-01-01',ST_GeogFromText('POINT(13.4 52.5)'),'subject-geofence-device','{"auth_token":"GEOFENCE_SECRET_SENTINEL"}','2026-01-01','2026-01-01'), + (2,8,2,'FOREIGN_ROW_SENTINEL',0,0,false,'2026-01-01','2026-01-01',ST_GeogFromText('POINT(1 1)'),'FOREIGN_ROW_SENTINEL','{}','2026-01-01','2026-01-01'); +INSERT INTO webhooks (id,user_id,name,url,secret,event_types,place_ids,active,consecutive_failures,created_at,updated_at) +VALUES (1,7,'Subject webhook','https://user:WEBHOOK_USERINFO_SENTINEL@hooks.subject.test:8443/api/webhook/WEBHOOK_PATH_SENTINEL?token=WEBHOOK_QUERY_SENTINEL','WEBHOOK_SECRET_SENTINEL','{0,1}','{1}',true,0,'2026-01-01','2026-01-01'), + (2,8,'FOREIGN_ROW_SENTINEL','https://foreign.test/FOREIGN_ROW_SENTINEL','WEBHOOK_SECRET_SENTINEL','{0}','{}',true,0,'2026-01-01','2026-01-01'); +INSERT INTO webhook_deliveries (id,webhook_id,geofence_event_id,status,attempt_count,response_status,response_body,error_message,created_at,updated_at) +VALUES (1,1,1,3,2,502,'DELIVERY_RESPONSE_SENTINEL','DELIVERY_ERROR_SENTINEL','2026-01-01','2026-01-01'), + (2,2,2,1,1,200,'FOREIGN_ROW_SENTINEL','FOREIGN_ROW_SENTINEL','2026-01-01','2026-01-01'); +INSERT INTO legacy_area_place_mappings (id,area_id,place_id,created_at,updated_at) +VALUES (1,1,1,'2026-01-01','2026-01-01'),(2,2,2,'2026-01-01','2026-01-01'); +INSERT INTO point_revisions (id,user_id,month,revision) +VALUES (9001,7,'2025-06-01','11111111-1111-4111-8111-111111111111'),(9002,8,'2025-06-01','22222222-2222-4222-8222-222222222222'); INSERT INTO active_storage_blobs (id,key,filename,content_type,byte_size,checksum,service_name,created_at) VALUES (1,'${attachmentKey}','subject.txt','text/plain',${attachmentBytes.length},'fixture','local','2026-01-01'), (2,'${routeVideoKey}','subject.mp4','video/mp4',${routeVideoBytes.length},'fixture','local','2026-01-01'); @@ -342,7 +357,7 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), }, { id: "dawarich-postgis", - version: "1.15.3", + version: "1.19.0", volumes: [ { name: "database", @@ -355,7 +370,7 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), }, { id: "dawarich-app", - version: "1.15.3", + version: "1.19.0", volumes: [ { name: "openmapx-dawarich-storage", @@ -369,11 +384,11 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), ], privacySourceProvenance: { managedDawarich: { - version: "1.15.3", + version: "1.19.0", image: "freikin/dawarich", - imageDigest: "sha256:589e3606b11b61c3fa2d9661832cb73645a9b93a9cc7d807a071e2bc6c584498", - upstreamCommit: "e32b70701b2467b862d4b5bc27ac428999997c1b", - schemaContract: "dawarich-1.15.3", + imageDigest: "sha256:d9385242202ae385a5d8064f41a59a0b58218399e60e7e499d2a6afb449bc304", + upstreamCommit: "752641768f37fdc07d5a23e0559fa7c5ff8fae0f", + schemaContract: "dawarich-1.19.0", }, }, }), @@ -394,13 +409,13 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), family: "dawarich", serviceId: "dawarich-postgis", file: dawarichFile, - schemaContract: "dawarich-1.15.3", + schemaContract: "dawarich-1.19.0", }, { family: "dawarich-storage", serviceId: "dawarich-app", file: storageFile, - schemaContract: "dawarich-storage-1.15.3", + schemaContract: "dawarich-storage-1.19.0", }, ], })}\n`, @@ -485,6 +500,31 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), ); if (plannedStop.name !== "Subject planned stop") throw new Error("collector omitted the Dawarich planned-itinerary graph"); + const jsonl = (name) => + spawnSync("tar", ["-xOf", tar, `dawarich/${name}.jsonl`]) + .stdout.toString() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + const [geofence] = jsonl("geofence-events"); + const [webhook] = jsonl("webhooks"); + const [delivery] = jsonl("webhook-deliveries"); + const revision = jsonl("point-revisions").find( + (row) => row.revision === "11111111-1111-4111-8111-111111111111", + ); + if ( + geofence?.event_type !== "leave" || + geofence.source !== "owntracks_native" || + JSON.stringify(geofence.lonlat) !== "[13.4,52.5]" || + webhook?.url !== "https://hooks.subject.test:8443" || + delivery?.status !== "retrying" || + "response_body" in delivery || + !revision || + jsonl("legacy-area-place-mappings").length !== 1 + ) + throw new Error( + `collector mis-projected the 1.19 geofence/webhook relations: ${JSON.stringify({ geofence, webhook, delivery, revision })}`, + ); if (!run.stdout.includes(routeVideoBytes)) throw new Error("collector omitted the Dawarich route-video attachment"); const dawarichManifest = JSON.parse( @@ -493,8 +533,8 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), const pointRelation = dawarichManifest.schemaRelations?.find((row) => row.entry === "Point"); if ( dawarichManifest.schemaFingerprint !== - "4ae830ea67dc4894d814d4e2e2d19cb92571c30f2df1df017417f671e26a1e09" || - dawarichManifest.schemaRelations?.length !== 39 || + "d728906287ec3c4f3f38a80c2c4fe6ce77cc40f03b56a5ba0e89b811dd6e631a" || + dawarichManifest.schemaRelations?.length !== 45 || !pointRelation?.columns.includes("lonlat") || pointRelation.columns.includes("lon") ) @@ -514,6 +554,14 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), "SERVICE_CREDENTIAL_SECRET", "TRIP_API_KEY_SECRET", "TRIP_SELECTION_SECRET", + "GEOFENCE_SECRET_SENTINEL", + "WEBHOOK_USERINFO_SENTINEL", + "WEBHOOK_PATH_SENTINEL", + "WEBHOOK_QUERY_SENTINEL", + "WEBHOOK_SECRET_SENTINEL", + "DELIVERY_RESPONSE_SENTINEL", + "DELIVERY_ERROR_SENTINEL", + "22222222-2222-4222-8222-222222222222", ]) { if (run.stdout.includes(Buffer.from(sentinel))) throw new Error(`collector leaked ${sentinel}`); } diff --git a/services/service-file-history.json b/services/service-file-history.json index 5cdd8e252..afe53ada2 100644 --- a/services/service-file-history.json +++ b/services/service-file-history.json @@ -13,12 +13,14 @@ ], "dawarich-app/scripts/openmapx-subject-export.rb": [ "2deeac1f6f0f8910aa2cdc428686e07622b26ad72c0c333a2b546786386b373b", - "662bd9a0015a26eb6b9fe7675bed21e61ba657579711b19742d365cc2e26f342" + "662bd9a0015a26eb6b9fe7675bed21e61ba657579711b19742d365cc2e26f342", + "d48fae1127f819d1e25baa95565898de6d8ec2f905967a04f681119592609c09" ], "dawarich-app/scripts/subject_export/schema.rb": [ "06729586a89a50288718a8d71fdeabfe87b7e108cbc02542248b30be48648f76", "1be08123f767dbc8a28a1a78c53eec33c52ea107a29d96a7a1154ff816efdb65", - "dd4aa7143242206642ebeeb1aa7c359e89c81489b9493b9a007a515c19853f81" + "dd4aa7143242206642ebeeb1aa7c359e89c81489b9493b9a007a515c19853f81", + "f7484d778ca2250da6ca1ed6c558c07bb983b778067445b6e0386e6188b5e810" ], "dawarich-sidekiq/scripts/openmapx-entrypoint.sh": [ "1d92f852851ba815ef4d147d381fddc6d429ac1e605654e953b5e1ced043a166" From 10aa100185ee822f7e48255e93544364c7c7ff48 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 30 Oct 2026 03:12:34 +0100 Subject: [PATCH 2/2] fix(privacy): keep URL credentials out of the backup webhook origin The backup exporter's host pattern accepted "@", so a URL with two "@" signs, such as https://user:pa@ss@host/x, exported part of its password as https://ss@host. Exclude "@" and whitespace from the host, accept only hexadecimal IPv6 literals, and assert the malformed URL exports as null in both the exact-image and retained-backup gates, as the Rails collector already does. --- .../src/dawarich-subject-export.integration.test.ts | 7 +++++-- .../privacy-backup/openmapx-backup-subject-export.py | 2 +- services/ops-agent/privacy-backup/test-fixture.mjs | 7 +++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/apps/ops-agent/src/dawarich-subject-export.integration.test.ts b/apps/ops-agent/src/dawarich-subject-export.integration.test.ts index a7138d8ac..a6eb5bf35 100644 --- a/apps/ops-agent/src/dawarich-subject-export.integration.test.ts +++ b/apps/ops-agent/src/dawarich-subject-export.integration.test.ts @@ -348,7 +348,8 @@ VALUES (1,1,1,'Subject geofence place',0,0,false,TIMESTAMP '${at}',TIMESTAMP '${ (2,2,2,'FOREIGN_SENTINEL',1,1,false,TIMESTAMP '${at}',TIMESTAMP '${at}',ST_GeogFromText('POINT(1 1)'),5,'FOREIGN_SENTINEL','{}'::jsonb,TIMESTAMP '${at}',TIMESTAMP '${at}'); INSERT INTO webhooks (id,user_id,name,url,secret,event_types,place_ids,active,consecutive_failures,created_at,updated_at) VALUES (1,1,'Subject webhook','https://hooks.subject.test/api/webhook/WEBHOOK_PATH_TOKEN?token=WEBHOOK_QUERY_TOKEN','WEBHOOK_SIGNING_SECRET','{0,1}','{1}',true,0,TIMESTAMP '${at}',TIMESTAMP '${at}'), - (2,2,'FOREIGN_SENTINEL','https://foreign.test/FOREIGN_SENTINEL','FOREIGN_SECRET','{0}','{}',true,0,TIMESTAMP '${at}',TIMESTAMP '${at}'); + (2,2,'FOREIGN_SENTINEL','https://foreign.test/FOREIGN_SENTINEL','FOREIGN_SECRET','{0}','{}',true,0,TIMESTAMP '${at}',TIMESTAMP '${at}'), + (3,1,'Malformed webhook','https://user:pa@WEBHOOK_SPLIT_TOKEN@hooks.subject.test/x','WEBHOOK_SIGNING_SECRET','{0}','{}',false,0,TIMESTAMP '${at}',TIMESTAMP '${at}'); INSERT INTO webhook_deliveries (id,webhook_id,geofence_event_id,status,attempt_count,response_status,response_body,error_message,delivered_at,created_at,updated_at) VALUES (1,1,1,1,1,200,'DELIVERY_RESPONSE_SECRET','DELIVERY_ERROR_SECRET',TIMESTAMP '${at}',TIMESTAMP '${at}',TIMESTAMP '${at}'), (2,2,2,2,3,500,'FOREIGN_SENTINEL','FOREIGN_SENTINEL',NULL,TIMESTAMP '${at}',TIMESTAMP '${at}'); @@ -527,6 +528,7 @@ describe.skipIf(!runExactImage)("managed Dawarich exact-image export", () => { "GEOFENCE_SECRET", "WEBHOOK_PATH_TOKEN", "WEBHOOK_QUERY_TOKEN", + "WEBHOOK_SPLIT_TOKEN", "WEBHOOK_SIGNING_SECRET", "DELIVERY_RESPONSE_SECRET", "DELIVERY_ERROR_SECRET", @@ -541,8 +543,9 @@ describe.skipIf(!runExactImage)("managed Dawarich exact-image export", () => { .split("\n") .filter(Boolean) .map((line) => JSON.parse(line) as Record); - expect(webhookRows).toHaveLength(1); + expect(webhookRows).toHaveLength(2); expect(webhookRows?.[0]?.url).toBe("https://hooks.subject.test"); + expect(webhookRows?.[1]?.url).toBeNull(); const geofenceRows = entries .find((entry) => entry.id === "geofence-events") ?.content.toString("utf8") diff --git a/services/ops-agent/privacy-backup/openmapx-backup-subject-export.py b/services/ops-agent/privacy-backup/openmapx-backup-subject-export.py index c8253614d..8e2952d2f 100644 --- a/services/ops-agent/privacy-backup/openmapx-backup-subject-export.py +++ b/services/ops-agent/privacy-backup/openmapx-backup-subject-export.py @@ -570,7 +570,7 @@ def project_openmapx(socket): "(SELECT CASE WHEN o.scheme IN ('http','https') AND o.host<>'' THEN o.scheme || '://' || o.host" " || CASE WHEN o.port IS NULL OR (o.scheme='http' AND o.port='80') OR (o.scheme='https' AND o.port='443') THEN '' ELSE ':' || o.port END END" " FROM (SELECT lower(m[1]) AS scheme, m[2] AS host, m[3] AS port" - " FROM (SELECT regexp_match(t.url, '^([A-Za-z][A-Za-z0-9+.-]*)://(?:[^/?#@]*@)?(\\[[^]/?#]*\\]|[^:/?#]*)(?::([0-9]+))?(?:[/?#]|$)') AS m) parsed) o)" + " FROM (SELECT regexp_match(t.url, '^([A-Za-z][A-Za-z0-9+.-]*)://(?:[^/?#@]*@)?(\\[[0-9A-Fa-f:.]+\\]|[^]:/?#@[[:space:]]+)(?::([0-9]+))?(?:[/?#]|$)') AS m) parsed) o)" ) DAWARICH_PLANNED = { diff --git a/services/ops-agent/privacy-backup/test-fixture.mjs b/services/ops-agent/privacy-backup/test-fixture.mjs index 046a90d46..3492ec68f 100644 --- a/services/ops-agent/privacy-backup/test-fixture.mjs +++ b/services/ops-agent/privacy-backup/test-fixture.mjs @@ -298,7 +298,8 @@ VALUES (1,7,1,'Subject geofence place',1,2,false,'2026-01-01','2026-01-01',ST_Ge (2,8,2,'FOREIGN_ROW_SENTINEL',0,0,false,'2026-01-01','2026-01-01',ST_GeogFromText('POINT(1 1)'),'FOREIGN_ROW_SENTINEL','{}','2026-01-01','2026-01-01'); INSERT INTO webhooks (id,user_id,name,url,secret,event_types,place_ids,active,consecutive_failures,created_at,updated_at) VALUES (1,7,'Subject webhook','https://user:WEBHOOK_USERINFO_SENTINEL@hooks.subject.test:8443/api/webhook/WEBHOOK_PATH_SENTINEL?token=WEBHOOK_QUERY_SENTINEL','WEBHOOK_SECRET_SENTINEL','{0,1}','{1}',true,0,'2026-01-01','2026-01-01'), - (2,8,'FOREIGN_ROW_SENTINEL','https://foreign.test/FOREIGN_ROW_SENTINEL','WEBHOOK_SECRET_SENTINEL','{0}','{}',true,0,'2026-01-01','2026-01-01'); + (2,8,'FOREIGN_ROW_SENTINEL','https://foreign.test/FOREIGN_ROW_SENTINEL','WEBHOOK_SECRET_SENTINEL','{0}','{}',true,0,'2026-01-01','2026-01-01'), + (3,7,'Malformed webhook','https://user:pa@WEBHOOK_SPLIT_SENTINEL@hooks.subject.test/x','WEBHOOK_SECRET_SENTINEL','{0}','{}',false,0,'2026-01-01','2026-01-01'); INSERT INTO webhook_deliveries (id,webhook_id,geofence_event_id,status,attempt_count,response_status,response_body,error_message,created_at,updated_at) VALUES (1,1,1,3,2,502,'DELIVERY_RESPONSE_SENTINEL','DELIVERY_ERROR_SENTINEL','2026-01-01','2026-01-01'), (2,2,2,1,1,200,'FOREIGN_ROW_SENTINEL','FOREIGN_ROW_SENTINEL','2026-01-01','2026-01-01'); @@ -507,7 +508,7 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), .filter(Boolean) .map((line) => JSON.parse(line)); const [geofence] = jsonl("geofence-events"); - const [webhook] = jsonl("webhooks"); + const [webhook, malformedWebhook] = jsonl("webhooks"); const [delivery] = jsonl("webhook-deliveries"); const revision = jsonl("point-revisions").find( (row) => row.revision === "11111111-1111-4111-8111-111111111111", @@ -517,6 +518,7 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), geofence.source !== "owntracks_native" || JSON.stringify(geofence.lonlat) !== "[13.4,52.5]" || webhook?.url !== "https://hooks.subject.test:8443" || + malformedWebhook?.url !== null || delivery?.status !== "retrying" || "response_body" in delivery || !revision || @@ -558,6 +560,7 @@ VALUES (1,'Import',1,'source',1,'2026-01-01'), "WEBHOOK_USERINFO_SENTINEL", "WEBHOOK_PATH_SENTINEL", "WEBHOOK_QUERY_SENTINEL", + "WEBHOOK_SPLIT_SENTINEL", "WEBHOOK_SECRET_SENTINEL", "DELIVERY_RESPONSE_SENTINEL", "DELIVERY_ERROR_SENTINEL",