From 45f1940f2600e0be5048409d3940caf8c22817b4 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 23 Oct 2026 09:54:03 +0000 Subject: [PATCH 1/3] Add a migrations directory, with schema.sql as its baseline schema.sql stays the schema as it is now. A change to an existing cluster goes in a numbered file in migrations/, applied once, in order, on one node. 000_baseline.sql is schema.sql as of this PR and is never edited; the README says how to run, document and record a migration. schema.sql's header points to it. --- .../migrations/000_baseline.sql | 610 ++++++++++++++++++ .../cluster-migration/migrations/README.md | 15 + scripts/cluster-migration/schema.sql | 2 +- 3 files changed, 626 insertions(+), 1 deletion(-) create mode 100644 scripts/cluster-migration/migrations/000_baseline.sql create mode 100644 scripts/cluster-migration/migrations/README.md diff --git a/scripts/cluster-migration/migrations/000_baseline.sql b/scripts/cluster-migration/migrations/000_baseline.sql new file mode 100644 index 00000000..48cdf331 --- /dev/null +++ b/scripts/cluster-migration/migrations/000_baseline.sql @@ -0,0 +1,610 @@ +-- The tables and views of the ooni database, as production has them (dumped from every replica on +-- 2026-10-20 after asnmeta was made replicated again on data1, OOPRIV-181). Create the database on every +-- node, then load this file into it: +-- clickhouse-client -q "CREATE DATABASE IF NOT EXISTS ooni ON CLUSTER oonidata_cluster" +-- clickhouse-client --database ooni --multiquery < schema.sql +-- Any database name works, e.g. for CI: ON CLUSTER creates each table in the client's database on every +-- node, and the Keeper paths include {database}, so a second database on the same cluster doesn't share +-- production's replicas. +-- Where the replicas differ, this is the definition of the replica that holds the data: +-- event_detector_cusums, labeling_frames: data1 (the only replica with rows, and with their newer columns) +-- fastpath: data1 and data3 (data2 alone has deduplicate_merge_projection_mode = 'rebuild') +-- event_detector_cusums, event_detector_v2_changepoints, labeling_frames and changepoint_label are not +-- replicated: ON CLUSTER creates an empty local table on each node, as production has them. +-- Not included: the updaters' *_tmp tables (recreated on every run), msmt_feedback_old (superseded by +-- msmt_feedback), session_expunge (unused since backend #972) and the leftover inner table of the dropped +-- counters_test_list view. + +CREATE TABLE IF NOT EXISTS analysis_web_measurement ON CLUSTER oonidata_cluster +( + `domain` String, + `input` String, + `test_name` String, + `probe_asn` UInt32, + `probe_as_org_name` String, + `probe_cc` String, + `resolver_asn` UInt32, + `resolver_as_cc` String, + `network_type` String, + `measurement_start_time` DateTime64(3, 'UTC'), + `measurement_uid` String, + `ooni_run_link_id` String, + `top_probe_analysis` Nullable(String), + `top_dns_failure` Nullable(String), + `top_tcp_failure` Nullable(String), + `top_tls_failure` Nullable(String), + `dns_blocked` Float32, + `dns_down` Float32, + `dns_ok` Float32, + `tcp_blocked` Float32, + `tcp_down` Float32, + `tcp_ok` Float32, + `tls_blocked` Float32, + `tls_down` Float32, + `tls_ok` Float32, + `top_dns_rule_id` LowCardinality(String), + `top_tcp_rule_id` LowCardinality(String), + `top_tls_rule_id` LowCardinality(String), + `probe_id` FixedString(64), + `analysis_rules_version` UInt16, + INDEX measurement_start_time_idx measurement_start_time TYPE minmax GRANULARITY 2, + INDEX probe_cc_idx probe_cc TYPE minmax GRANULARITY 1, + INDEX probe_asn_idx probe_asn TYPE minmax GRANULARITY 1 +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/analysis_web_measurement/{shard}', '{replica}') +PARTITION BY substring(measurement_uid, 1, 6) +PRIMARY KEY measurement_uid +ORDER BY (measurement_uid, measurement_start_time, probe_cc, probe_asn, domain) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS asnmeta ON CLUSTER oonidata_cluster +( + `asn` UInt32, + `org_name` String, + `cc` String, + `changed` Date, + `aut_name` String, + `source` String +) +ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/asnmeta', '{replica}') +ORDER BY (asn, changed) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS changepoint_label ON CLUSTER oonidata_cluster +( + -- Ground truth, set by a person. Represents the network state for + -- the (probe_cc, probe_asn, resolver_asn, domain) series of the + -- changepoint it labels + `id` UUID, + -- UUID in event_detector_v2_changepoints (foreign key) + `changepoint_id` UUID, + `created_at` DateTime64(3, 'UTC') DEFAULT now64(), + `author` String, + -- blocked, ok or undecided + `verdict` LowCardinality(String), + -- Additional context the author might share + `notes` String, + `last_ok_time` Nullable(DateTime64(3, 'UTC')), + `first_block_time` Nullable(DateTime64(3, 'UTC')), + `last_block_time` Nullable(DateTime64(3, 'UTC')), + `first_ok_time` Nullable(DateTime64(3, 'UTC')) +) +ENGINE = MergeTree +ORDER BY (changepoint_id, created_at, id) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS citizenlab ON CLUSTER oonidata_cluster +( + `domain` String, + `url` String, + `cc` FixedString(32), + `category_code` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/citizenlab', '{replica}') +ORDER BY (domain, url, cc, category_code) +SETTINGS index_granularity = 4; + +CREATE TABLE IF NOT EXISTS event_detector_changepoints ON CLUSTER oonidata_cluster +( + `probe_asn` UInt32, + `probe_cc` String, + `domain` String, + `ts` DateTime64(3, 'UTC'), + `count_isp_resolver` Nullable(UInt32), + `count_other_resolver` Nullable(UInt32), + `count` Nullable(UInt32), + `dns_isp_blocked` Nullable(Float32), + `dns_other_blocked` Nullable(Float32), + `tcp_blocked` Nullable(Float32), + `tls_blocked` Nullable(Float32), + `change_dir` Nullable(Int8), + `s_pos` Nullable(Float32), + `s_neg` Nullable(Float32), + `current_state` String, + `h` Nullable(Float32), + `block_type` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/event_detector_changepoints/{shard}', '{replica}') +ORDER BY (probe_asn, probe_cc, ts, domain) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS event_detector_cusums ON CLUSTER oonidata_cluster +( + `probe_asn` UInt32, + `probe_cc` String, + `domain` String, + `ts` DateTime64(3, 'UTC'), + `dns_isp_blocked_current_state` String DEFAULT 'ok', + `dns_isp_blocked_s_pos` Nullable(Float64), + `dns_isp_blocked_s_neg` Nullable(Float64), + `dns_other_blocked_current_state` String DEFAULT 'ok', + `dns_other_blocked_s_pos` Nullable(Float64), + `dns_other_blocked_s_neg` Nullable(Float64), + `tcp_blocked_current_state` String DEFAULT 'ok', + `tcp_blocked_s_pos` Nullable(Float64), + `tcp_blocked_s_neg` Nullable(Float64), + `tls_blocked_current_state` String DEFAULT 'ok', + `tls_blocked_s_pos` Nullable(Float64), + `tls_blocked_s_neg` Nullable(Float64), + `dns_isp_blocked_last_change` Int8 DEFAULT 0, + `dns_isp_blocked_last_ts` Nullable(DateTime64(3, 'UTC')), + `dns_other_blocked_last_change` Int8 DEFAULT 0, + `dns_other_blocked_last_ts` Nullable(DateTime64(3, 'UTC')), + `tcp_blocked_last_change` Int8 DEFAULT 0, + `tcp_blocked_last_ts` Nullable(DateTime64(3, 'UTC')), + `tls_blocked_last_change` Int8 DEFAULT 0, + `tls_blocked_last_ts` Nullable(DateTime64(3, 'UTC')) +) +ENGINE = ReplacingMergeTree(ts) +ORDER BY (probe_asn, probe_cc, domain) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS event_detector_v2_changepoints ON CLUSTER oonidata_cluster +( + `uuid` UUID, + `domain` String, + `probe_cc` String, + `probe_asn` UInt32, + `resolver_asn` UInt32, + `layer` LowCardinality(String), + `ts_hour` DateTime64(3, 'UTC'), + `s_neg` Float64, + `s_pos` Float64, + `h` Float64, + `state` LowCardinality(String), + -- JSON-encoded parameters the detector was run with + `run_parameters` String, + `created_at` DateTime64(3, 'UTC') DEFAULT now64() +) +ENGINE = MergeTree +ORDER BY (ts_hour, domain, layer, probe_cc, probe_asn, resolver_asn, state, uuid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS fastpath ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `report_id` String, + `input` String, + `probe_cc` LowCardinality(String), + `probe_asn` Int32, + `test_name` LowCardinality(String), + `test_start_time` DateTime, + `measurement_start_time` DateTime, + `filename` String, + `scores` String, + `platform` String, + `anomaly` LowCardinality(String), + `confirmed` LowCardinality(String), + `msm_failure` LowCardinality(String), + `domain` String, + `software_name` String, + `software_version` String, + `control_failure` String, + `blocking_general` Float32, + `is_ssl_expected` Int8, + `page_len` Int32, + `page_len_ratio` Float32, + `server_cc` String, + `server_asn` Int8, + `server_as_name` String, + `update_time` DateTime64(3) MATERIALIZED now64(), + `test_version` String, + `architecture` String, + `engine_name` LowCardinality(String), + `engine_version` String, + `test_runtime` Float32, + `blocking_type` String, + `test_helper_address` LowCardinality(String), + `test_helper_type` LowCardinality(String), + `ooni_run_link_id` Nullable(UInt64), + `is_verified` LowCardinality(String) DEFAULT 'u', + INDEX fastpath_rid_idx report_id TYPE minmax GRANULARITY 1, + INDEX measurement_uid_idx measurement_uid TYPE minmax GRANULARITY 8, + INDEX input_bloom_idx input TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/fastpath/{shard}', '{replica}', update_time) +ORDER BY (measurement_start_time, report_id, input, measurement_uid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS faulty_measurements ON CLUSTER oonidata_cluster +( + `ts` DateTime64(3, 'UTC') DEFAULT now64(), + `type` String, + `uid` UUID DEFAULT generateUUIDv4(), + -- geoip lookup result for the probe IP + `probe_cc` String, + `probe_asn` UInt32, + -- JSON-encoded details about the anomaly + `details` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/faulty_measurements/{shard}', '{replica}') +ORDER BY (ts, type, probe_cc, probe_asn, uid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS fingerprints_dns ON CLUSTER oonidata_cluster +( + `name` String, + `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6), + `other_names` String, + `location_found` String, + `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4), + `pattern` String, + `confidence_no_fp` UInt8, + `expected_countries` String, + `source` String, + `exp_url` String, + `notes` String +) +ENGINE = EmbeddedRocksDB +PRIMARY KEY name; + +CREATE TABLE IF NOT EXISTS fingerprints_http ON CLUSTER oonidata_cluster +( + `name` String, + `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6, 'injb' = 7, 'prov' = 8), + `other_names` String, + `location_found` String, + `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4), + `pattern` String, + `confidence_no_fp` UInt8, + `expected_countries` String, + `source` String, + `exp_url` String, + `notes` String +) +ENGINE = EmbeddedRocksDB +PRIMARY KEY name; + +CREATE TABLE IF NOT EXISTS jsonl ON CLUSTER oonidata_cluster +( + `report_id` String, + `input` String, + `s3path` String, + `linenum` Int32, + `measurement_uid` String, + `date` Date, + `source` String, + `update_time` DateTime64(3) MATERIALIZED now64() +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/jsonl/{shard}', '{replica}', update_time) +ORDER BY (report_id, input, measurement_uid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS labeling_frames ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `blocked_fastpath` Bool, + `blocked_analysis` Bool, + `stratum` LowCardinality(String), + `test_name` LowCardinality(String), + `day` DateTime('UTC'), + `fastpath_query` LowCardinality(String), + `analysis_query` LowCardinality(String), + `failed_fastpath` Bool +) +ENGINE = MergeTree +ORDER BY (stratum, measurement_uid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS msmt_feedback ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `account_id` String, + `status` String, + `update_time` DateTime64(3) DEFAULT now64(), + `comment` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/msmt_feedback_new/{shard}', '{replica}') +ORDER BY (measurement_uid, account_id, update_time) +SETTINGS index_granularity = 4; + +CREATE TABLE IF NOT EXISTS obs_http_middlebox ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `observation_idx` UInt16, + `input` Nullable(String), + `report_id` String, + `ooni_run_link_id` String DEFAULT '', + `measurement_start_time` DateTime64(3, 'UTC'), + `software_name` String, + `software_version` String, + `test_name` String, + `test_version` String, + `bucket_date` String, + `probe_asn` UInt32, + `probe_cc` String, + `probe_as_org_name` String, + `probe_as_cc` String, + `probe_as_name` String, + `network_type` String, + `platform` String, + `origin` String, + `engine_name` String, + `engine_version` String, + `architecture` String, + `resolver_ip` String, + `resolver_asn` UInt32, + `resolver_cc` String, + `resolver_as_org_name` String, + `resolver_as_cc` String, + `resolver_is_scrubbed` UInt8, + `resolver_asn_probe` UInt32, + `resolver_as_org_name_probe` String, + `created_at` Nullable(DateTime64(3, 'UTC')), + `hirl_sent_0` Nullable(String), + `hirl_sent_1` Nullable(String), + `hirl_sent_2` Nullable(String), + `hirl_sent_3` Nullable(String), + `hirl_sent_4` Nullable(String), + `hirl_received_0` Nullable(String), + `hirl_received_1` Nullable(String), + `hirl_received_2` Nullable(String), + `hirl_received_3` Nullable(String), + `hirl_received_4` Nullable(String), + `hirl_failure` Nullable(String), + `hirl_success` Nullable(UInt8), + `hfm_diff` Nullable(String), + `hfm_failure` Nullable(String), + `hfm_success` Nullable(UInt8), + `probe_id` FixedString(64) +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/obs_http_middlebox/{shard}', '{replica}') +PARTITION BY concat(substring(bucket_date, 1, 4), substring(bucket_date, 6, 2)) +PRIMARY KEY (measurement_uid, observation_idx) +ORDER BY (measurement_uid, observation_idx, measurement_start_time) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS obs_openvpn ON CLUSTER oonidata_cluster +( + `anomaly` Int8, + `bootstrap_time` Float32, + `confirmed` Int8, + `error` String, + `failure` String, + `input` String, + `last_handshake_transaction_id` Int32, + `measurement_start_time` DateTime, + `measurement_uid` String, + `minivpn_version` String, + `obfs4_version` String, + `obfuscation` String, + `platform` String, + `probe_asn` Int32, + `probe_cc` String, + `probe_network_name` String, + `provider` String, + `remote` String, + `report_id` String, + `resolver_asn` Int32, + `resolver_ip` String, + `resolver_network_name` String, + `software_name` String, + `software_version` String, + `success` Int8, + `success_handshake` Int8, + `success_icmp` Int8, + `success_urlgrab` Int8, + `tcp_connect_status_success` Int8, + `test_runtime` Float32, + `test_start_time` DateTime, + `transport` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/obs_openvpn/{shard}', '{replica}') +ORDER BY (measurement_start_time, report_id, input) +SETTINGS index_granularity = 8; + +CREATE TABLE IF NOT EXISTS obs_web ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `observation_idx` UInt16, + `input` Nullable(String), + `report_id` String, + `ooni_run_link_id` String DEFAULT '', + `measurement_start_time` DateTime64(3, 'UTC'), + `software_name` String, + `software_version` String, + `test_name` String, + `test_version` String, + `bucket_date` String, + `probe_asn` UInt32, + `probe_cc` String, + `probe_as_org_name` String, + `probe_as_cc` String, + `probe_as_name` String, + `network_type` String, + `platform` String, + `origin` String, + `engine_name` String, + `engine_version` String, + `architecture` String, + `resolver_ip` String, + `resolver_asn` UInt32, + `resolver_cc` String, + `resolver_as_org_name` String, + `resolver_as_cc` String, + `resolver_is_scrubbed` UInt8, + `resolver_asn_probe` UInt32, + `resolver_as_org_name_probe` String, + `created_at` Nullable(DateTime('UTC')), + `target_id` Nullable(String), + `hostname` Nullable(String), + `transaction_id` Nullable(UInt16), + `ip` Nullable(String), + `port` Nullable(UInt16), + `ip_asn` Nullable(UInt32), + `ip_as_org_name` Nullable(String), + `ip_as_cc` Nullable(String), + `ip_cc` Nullable(String), + `ip_is_bogon` Nullable(UInt8), + `dns_query_type` Nullable(String), + `dns_failure` Nullable(String), + `dns_engine` Nullable(String), + `dns_engine_resolver_address` Nullable(String), + `dns_answer_type` Nullable(String), + `dns_answer` Nullable(String), + `dns_answer_asn` Nullable(UInt32), + `dns_answer_as_org_name` Nullable(String), + `dns_t` Nullable(Float64), + `tcp_failure` Nullable(String), + `tcp_success` Nullable(UInt8), + `tcp_t` Nullable(Float64), + `tls_failure` Nullable(String), + `tls_server_name` Nullable(String), + `tls_outer_server_name` Nullable(String), + `tls_echconfig` Nullable(String), + `tls_version` Nullable(String), + `tls_cipher_suite` Nullable(String), + `tls_is_certificate_valid` Nullable(UInt8), + `tls_end_entity_certificate_fingerprint` Nullable(String), + `tls_end_entity_certificate_subject` Nullable(String), + `tls_end_entity_certificate_subject_common_name` Nullable(String), + `tls_end_entity_certificate_issuer` Nullable(String), + `tls_end_entity_certificate_issuer_common_name` Nullable(String), + `tls_end_entity_certificate_san_list` Array(String), + `tls_end_entity_certificate_not_valid_after` Nullable(DateTime64(3, 'UTC')), + `tls_end_entity_certificate_not_valid_before` Nullable(DateTime64(3, 'UTC')), + `tls_certificate_chain_length` Nullable(UInt16), + `tls_certificate_chain_fingerprints` Array(String), + `tls_handshake_read_count` Nullable(UInt16), + `tls_handshake_write_count` Nullable(UInt16), + `tls_handshake_read_bytes` Nullable(UInt32), + `tls_handshake_write_bytes` Nullable(UInt32), + `tls_handshake_last_operation` Nullable(String), + `tls_handshake_time` Nullable(Float64), + `tls_t` Nullable(Float64), + `http_request_url` Nullable(String), + `http_network` Nullable(String), + `http_alpn` Nullable(String), + `http_failure` Nullable(String), + `http_request_body_length` Nullable(UInt32), + `http_request_method` Nullable(String), + `http_runtime` Nullable(Float64), + `http_response_body_length` Nullable(Int32), + `http_response_body_is_truncated` Nullable(UInt8), + `http_response_body_sha1` Nullable(String), + `http_response_status_code` Nullable(UInt16), + `http_response_header_location` Nullable(String), + `http_response_header_server` Nullable(String), + `http_request_redirect_from` Nullable(String), + `http_request_body_is_truncated` Nullable(UInt8), + `http_t` Nullable(Float64), + `probe_analysis` Nullable(String), + `probe_id` FixedString(64), + INDEX measurement_start_time_idx measurement_start_time TYPE minmax GRANULARITY 2, + INDEX probe_cc_idx probe_cc TYPE minmax GRANULARITY 1, + INDEX probe_asn_idx probe_asn TYPE minmax GRANULARITY 1 +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/obs_web_repl/{shard}', '{replica}') +PARTITION BY concat(substring(bucket_date, 1, 4), substring(bucket_date, 6, 2)) +PRIMARY KEY (measurement_uid, observation_idx) +ORDER BY (measurement_uid, observation_idx, measurement_start_time, probe_cc, probe_asn) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS obs_web_ctrl ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `observation_idx` UInt16, + `input` Nullable(String), + `report_id` String, + `ooni_run_link_id` String DEFAULT '', + `measurement_start_time` DateTime64(3, 'UTC'), + `software_name` String, + `software_version` String, + `test_name` String, + `test_version` String, + `bucket_date` String, + `hostname` String, + `created_at` Nullable(DateTime64(3, 'UTC')), + `ip` String, + `port` Nullable(UInt16), + `ip_asn` Nullable(UInt32), + `ip_as_org_name` Nullable(String), + `ip_as_cc` Nullable(String), + `ip_cc` Nullable(String), + `ip_is_bogon` Nullable(UInt8), + `dns_failure` Nullable(String), + `dns_success` Nullable(UInt8), + `tcp_failure` Nullable(String), + `tcp_success` Nullable(UInt8), + `tls_failure` Nullable(String), + `tls_success` Nullable(UInt8), + `tls_server_name` Nullable(String), + `http_request_url` Nullable(String), + `http_failure` Nullable(String), + `http_success` Nullable(UInt8), + `http_response_body_length` Nullable(Int32), + INDEX measurement_start_time_idx measurement_start_time TYPE minmax GRANULARITY 2 +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/obs_web_ctrl/{shard}', '{replica}') +PARTITION BY concat(substring(bucket_date, 1, 4), substring(bucket_date, 6, 2)) +PRIMARY KEY (measurement_uid, observation_idx) +ORDER BY (measurement_uid, observation_idx, measurement_start_time, hostname) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS test_groups ON CLUSTER oonidata_cluster +( + `test_name` String, + `test_group` String +) +ENGINE = Join(ANY, LEFT, test_name); + +CREATE TABLE IF NOT EXISTS url_priorities ON CLUSTER oonidata_cluster +( + `sign` Int8, + `category_code` String, + `cc` String, + `domain` String, + `url` String, + `priority` Int32 +) +ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/{cluster}/tables/{database}/url_priorities/{shard}', '{replica}', sign) +ORDER BY (category_code, cc, domain, url, priority) +SETTINGS index_granularity = 1024; + +-- Its rows are in a hidden table, .inner_id.. The Keeper path keeps production's name +-- for it, so a replica added later joins production's table whatever UUID its view gets. +CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list ON CLUSTER oonidata_cluster +( + `week` DateTime, + `probe_cc` String, + `probe_asn` UInt64, + `input` String, + `msmt_cnt` UInt64 +) +ENGINE = ReplicatedSummingMergeTree('/clickhouse/{cluster}/tables/{database}/.inner_id.6a72ceec-7e0e-4b65-a3ab-da2809202114/{shard}', '{replica}') +ORDER BY (probe_cc, probe_asn, input) +SETTINGS index_granularity = 8192 +AS SELECT + toStartOfWeek(measurement_start_time) AS week, + probe_cc, + probe_asn, + input, + count() AS msmt_cnt +FROM fastpath +INNER JOIN citizenlab ON fastpath.input = citizenlab.url +WHERE (measurement_start_time < now()) AND (measurement_start_time > (now() - toIntervalDay(8))) AND (test_name = 'web_connectivity') +GROUP BY + week, + probe_cc, + probe_asn, + input; diff --git a/scripts/cluster-migration/migrations/README.md b/scripts/cluster-migration/migrations/README.md new file mode 100644 index 00000000..da3a671c --- /dev/null +++ b/scripts/cluster-migration/migrations/README.md @@ -0,0 +1,15 @@ +# ClickHouse schema migrations + +`../schema.sql` is the schema as it is now: read it to see a table, and load it to build a new +cluster. A change to an existing cluster's schema goes in a numbered file here, applied once, in +order, on one node (its DDL uses `ON CLUSTER`): + + clickhouse-client --database ooni --multiquery < migrations/NNN_description.sql + +- Every schema change updates `schema.sql` and adds a migration. `../test/check.sh`, which CI runs, + fails unless `schema.sql` gives the same tables as all the migrations in order. +- `000_baseline.sql` is `schema.sql` as of ooni/devops#521. Don't edit it, or a migration that has + been applied: fix forward with a new one. +- Start each file with what it does to production: whether it rewrites data (a mutation, e.g. + `MODIFY COLUMN` or `MATERIALIZE`), how long it took on a copy, and what readers see meanwhile. +- Say in the PR when it was applied to production. diff --git a/scripts/cluster-migration/schema.sql b/scripts/cluster-migration/schema.sql index 48cdf331..30ee6a85 100644 --- a/scripts/cluster-migration/schema.sql +++ b/scripts/cluster-migration/schema.sql @@ -5,7 +5,7 @@ -- clickhouse-client --database ooni --multiquery < schema.sql -- Any database name works, e.g. for CI: ON CLUSTER creates each table in the client's database on every -- node, and the Keeper paths include {database}, so a second database on the same cluster doesn't share --- production's replicas. +-- production's replicas. A change to an existing cluster also goes in migrations/: see its README. -- Where the replicas differ, this is the definition of the replica that holds the data: -- event_detector_cusums, labeling_frames: data1 (the only replica with rows, and with their newer columns) -- fastpath: data1 and data3 (data2 alone has deduplicate_merge_projection_mode = 'rebuild') From c9df565564c1bad95dcf64ebb799bcdf31923f93 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 23 Oct 2026 09:54:03 +0000 Subject: [PATCH 2/3] Check that schema.sql equals its migrations test/check.sh starts a one-node oonidata_cluster with Keeper at production's version (24.8.6.70), loads schema.sql into one database and every migration, in order, into another, and compares each table's definition and Keeper path with the database name and UUIDs blanked. It takes about 10 seconds. CI runs it when schema.sql, the migrations, the test or the workflow change. It fails when a migration is missing, and when schema.sql changes without one: both checked by hand before this commit. --- .github/workflows/check_cluster_schema.yml | 16 +++++++ scripts/cluster-migration/test/check.sh | 46 +++++++++++++++++++ scripts/cluster-migration/test/clickhouse.xml | 22 +++++++++ scripts/cluster-migration/test/compose.yml | 11 +++++ 4 files changed, 95 insertions(+) create mode 100644 .github/workflows/check_cluster_schema.yml create mode 100755 scripts/cluster-migration/test/check.sh create mode 100644 scripts/cluster-migration/test/clickhouse.xml create mode 100644 scripts/cluster-migration/test/compose.yml diff --git a/.github/workflows/check_cluster_schema.yml b/.github/workflows/check_cluster_schema.yml new file mode 100644 index 00000000..17c6ac96 --- /dev/null +++ b/.github/workflows/check_cluster_schema.yml @@ -0,0 +1,16 @@ +# Checks that scripts/cluster-migration/schema.sql and its migrations give the same tables +on: + pull_request: + paths: + - "scripts/cluster-migration/schema.sql" + - "scripts/cluster-migration/migrations/**" + - "scripts/cluster-migration/test/**" + - ".github/workflows/check_cluster_schema.yml" + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Compare schema.sql with the migrations + run: scripts/cluster-migration/test/check.sh diff --git a/scripts/cluster-migration/test/check.sh b/scripts/cluster-migration/test/check.sh new file mode 100755 index 00000000..2b0ab45b --- /dev/null +++ b/scripts/cluster-migration/test/check.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Checks that schema.sql and the migrations agree: loads schema.sql into one database and every +# migrations/NNN_*.sql, in order, into another, then compares each table's definition and Keeper path. +# Needs docker compose; set COMPOSE="podman compose" to use podman. +set -euo pipefail +cd "$(dirname "$0")" +compose=${COMPOSE:-docker compose} +q() { $compose exec -T clickhouse clickhouse-client -u admin --password admin "$@"; } +work=$(mktemp -d) +trap '$compose down -v >/dev/null 2>&1; rm -rf "$work"' EXIT + +$compose up -d +for _ in $(seq 60); do + q -q "SELECT 1 FROM system.zookeeper WHERE path = '/'" /dev/null 2>&1 && break + sleep 1 +done + +for db in from_schema from_migrations; do + q -q "CREATE DATABASE $db ON CLUSTER oonidata_cluster FORMAT Null" /dev/null +for f in ../migrations/[0-9][0-9][0-9]_*.sql; do + echo "applying $(basename "$f")" + q --database from_migrations --multiquery <"$f" >/dev/null +done + +# each table's definition and Keeper path, with the database name and UUIDs blanked +snapshot() { + q -q " + SELECT t.name, + replaceAll(replaceAll(replaceRegexpAll(t.create_table_query, '[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', ''), + '$1.', '.'), '/$1/', '//'), + replaceAll(r.zookeeper_path, '/$1/', '//') + FROM system.tables AS t + LEFT JOIN system.replicas AS r ON r.database = t.database AND r.table = t.name + WHERE t.database = '$1' + ORDER BY t.name + FORMAT TSV" "$work/schema.tsv" +snapshot from_migrations >"$work/migrations.tsv" +if ! diff -u "$work/schema.tsv" "$work/migrations.tsv"; then + echo "schema.sql and the migrations disagree: the diff above is schema.sql (-) against the migrations (+)" + exit 1 +fi +echo "schema.sql and the migrations agree on all $(wc -l <"$work/schema.tsv") tables" diff --git a/scripts/cluster-migration/test/clickhouse.xml b/scripts/cluster-migration/test/clickhouse.xml new file mode 100644 index 00000000..f393db4b --- /dev/null +++ b/scripts/cluster-migration/test/clickhouse.xml @@ -0,0 +1,22 @@ + + + + + + + + + clickhouse9000adminadmin + + + + oonidata_cluster01clickhouse + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + 1clickhouse9234 + + clickhouse9181 + diff --git a/scripts/cluster-migration/test/compose.yml b/scripts/cluster-migration/test/compose.yml new file mode 100644 index 00000000..5ebd8d0f --- /dev/null +++ b/scripts/cluster-migration/test/compose.yml @@ -0,0 +1,11 @@ +# A one-node oonidata_cluster with Keeper, at production's ClickHouse version, for check.sh +name: cluster-schema-check +services: + clickhouse: + image: docker.io/clickhouse/clickhouse-server:24.8.6.70 + hostname: clickhouse + environment: {CLICKHOUSE_USER: admin, CLICKHOUSE_PASSWORD: admin} + volumes: + - ./clickhouse.xml:/etc/clickhouse-server/config.d/cluster.xml:ro,z + ulimits: {nofile: 262144} + mem_limit: 2g From 01fa0846eaf45ab7ecec8622ba8d2eafe3b7ee36 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 23 Oct 2026 09:54:03 +0000 Subject: [PATCH 3/3] Give counters_asn_test_list a target table, with week in its key The view kept its rows in a hidden table, .inner_id., sorted by (probe_cc, probe_asn, input). It's a SummingMergeTree, so merges add up the weeks of each key and keep one arbitrary week. Its reader, ooniprobe's test-list/urls (common/prio.py), sums the current and previous week, so it misses every count whose merged row says an older week, which in production is most of them. Tested on 24.8: 2 rows from last week and 5 from this one merge into one row of 7 that says last week. The view now writes TO counters_asn_test_list_data, sorted by (probe_cc, probe_asn, input, week), with a plain {database} Keeper path. It keeps its name and column types, so the reader's query doesn't change. The table is backed up and restored by name, like any other table, by the clickhouse_backup role's "*" job; the view's backup holds only its definition. Restore the table before the view: a view restored without its table makes every insert into fastpath fail. Migration 001 does it in production. The old rows are wrong, so it doesn't copy them: it counts the two weeks the reader uses again from fastpath, from the rows inserted before the new view existed (fastpath's update_time), and the new view counts the rest. That also counts uploads that came more than 8 days late, which the view leaves out. It isn't a mutation. Queries on the view fail between the DROP and the RENAME, for about a second; from then on test-list/urls gets the corrected counts. Tested on a two-node 24.8.6.70 cluster set up like production (the view on one node, its rows already merged): the weeks come out right, a late upload is counted once, later inserts are counted once on both nodes, and test/check.sh passes. --- .../001_counters_asn_test_list_target.sql | 73 +++++++++++++++++++ .../cluster-migration/migrations/README.md | 18 +++++ scripts/cluster-migration/schema.sql | 19 +++-- 3 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 scripts/cluster-migration/migrations/001_counters_asn_test_list_target.sql diff --git a/scripts/cluster-migration/migrations/001_counters_asn_test_list_target.sql b/scripts/cluster-migration/migrations/001_counters_asn_test_list_target.sql new file mode 100644 index 00000000..57f8a487 --- /dev/null +++ b/scripts/cluster-migration/migrations/001_counters_asn_test_list_target.sql @@ -0,0 +1,73 @@ +-- Give counters_asn_test_list an explicit target table, counters_asn_test_list_data, instead of the +-- hidden .inner_id. table it made itself, and add week to its sorting key. The hidden table is +-- sorted by (probe_cc, probe_asn, input) only, so merges add up the weeks of each key and keep one +-- arbitrary week: the reader, ooniprobe's test-list/urls (common/prio.py), sums the current and previous +-- week and misses every count whose merged row says an older week. The new table keeps one row per +-- week, can be backed up and restored by name, and lets the view be dropped or changed without losing +-- its rows. +-- In production: not a mutation. The old rows are wrong, so they aren't copied: the two weeks the +-- reader uses are counted again from fastpath, from rows inserted before the new view existed (the +-- view counts the rest). Unlike the view, this also counts uploads that came more than 8 days after +-- their measurement. Queries on counters_asn_test_list fail between the DROP and the RENAME, for about +-- a second; test-list/urls returns the corrected counts from then on. + +CREATE TABLE IF NOT EXISTS counters_asn_test_list_data ON CLUSTER oonidata_cluster +( + `week` DateTime, + `probe_cc` String, + `probe_asn` UInt64, + `input` String, + `msmt_cnt` UInt64 +) +ENGINE = ReplicatedSummingMergeTree('/clickhouse/{cluster}/tables/{database}/counters_asn_test_list_data/{shard}', '{replica}') +ORDER BY (probe_cc, probe_asn, input, week) +SETTINGS index_granularity = 8192; + +-- Count fastpath's new rows from now on, alongside the old view +CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list_new ON CLUSTER oonidata_cluster TO counters_asn_test_list_data +( + `week` DateTime, + `probe_cc` String, + `probe_asn` UInt64, + `input` String, + `msmt_cnt` UInt64 +) +AS SELECT + toStartOfWeek(measurement_start_time) AS week, + probe_cc, + probe_asn, + input, + count() AS msmt_cnt +FROM fastpath +INNER JOIN citizenlab ON fastpath.input = citizenlab.url +WHERE (measurement_start_time < now()) AND (measurement_start_time > (now() - toIntervalDay(8))) AND (test_name = 'web_connectivity') +GROUP BY + week, + probe_cc, + probe_asn, + input; + +-- Count the two weeks the reader sums, from the rows the new view didn't see +INSERT INTO counters_asn_test_list_data SELECT + toStartOfWeek(measurement_start_time) AS week, + probe_cc, + probe_asn, + input, + count() AS msmt_cnt +FROM fastpath +INNER JOIN citizenlab ON fastpath.input = citizenlab.url +WHERE (measurement_start_time >= toStartOfWeek(now() - toIntervalWeek(1))) AND (test_name = 'web_connectivity') AND (update_time < ( + SELECT metadata_modification_time + FROM system.tables + WHERE (database = currentDatabase()) AND (name = 'counters_asn_test_list_new') +)) +GROUP BY + week, + probe_cc, + probe_asn, + input; + +-- The old view and its hidden table. They exist only on data1 and data3, hence IF EXISTS. +DROP TABLE IF EXISTS counters_asn_test_list ON CLUSTER oonidata_cluster SYNC; + +RENAME TABLE counters_asn_test_list_new TO counters_asn_test_list ON CLUSTER oonidata_cluster; diff --git a/scripts/cluster-migration/migrations/README.md b/scripts/cluster-migration/migrations/README.md index da3a671c..c0093add 100644 --- a/scripts/cluster-migration/migrations/README.md +++ b/scripts/cluster-migration/migrations/README.md @@ -13,3 +13,21 @@ order, on one node (its DDL uses `ON CLUSTER`): - Start each file with what it does to production: whether it rewrites data (a mutation, e.g. `MODIFY COLUMN` or `MATERIALIZE`), how long it took on a copy, and what readers see meanwhile. - Say in the PR when it was applied to production. + +## Rebuilding counters_asn_test_list + +Nothing rebuilds the view's counts by itself: it only counts new fastpath inserts. Restore +`counters_asn_test_list_data` from a backup before the view (a view without its table makes every +fastpath insert fail). If the table is lost with no backup, recreate it from `schema.sql` and count the +two weeks that test-list/urls reads again from fastpath, from the rows inserted before the table +existed (the view counts the rest): + +```sql +INSERT INTO ooni.counters_asn_test_list_data +SELECT toStartOfWeek(measurement_start_time) AS week, probe_cc, probe_asn, input, count() AS msmt_cnt +FROM ooni.fastpath INNER JOIN ooni.citizenlab ON fastpath.input = citizenlab.url +WHERE measurement_start_time >= toStartOfWeek(now() - INTERVAL 1 WEEK) AND test_name = 'web_connectivity' + AND update_time < (SELECT metadata_modification_time FROM system.tables + WHERE database = 'ooni' AND name = 'counters_asn_test_list_data') +GROUP BY week, probe_cc, probe_asn, input +``` diff --git a/scripts/cluster-migration/schema.sql b/scripts/cluster-migration/schema.sql index 30ee6a85..402a0dbc 100644 --- a/scripts/cluster-migration/schema.sql +++ b/scripts/cluster-migration/schema.sql @@ -581,9 +581,19 @@ ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/{cluster}/tables/{database}/ ORDER BY (category_code, cc, domain, url, priority) SETTINGS index_granularity = 1024; --- Its rows are in a hidden table, .inner_id.. The Keeper path keeps production's name --- for it, so a replica added later joins production's table whatever UUID its view gets. -CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list ON CLUSTER oonidata_cluster +CREATE TABLE IF NOT EXISTS counters_asn_test_list_data ON CLUSTER oonidata_cluster +( + `week` DateTime, + `probe_cc` String, + `probe_asn` UInt64, + `input` String, + `msmt_cnt` UInt64 +) +ENGINE = ReplicatedSummingMergeTree('/clickhouse/{cluster}/tables/{database}/counters_asn_test_list_data/{shard}', '{replica}') +ORDER BY (probe_cc, probe_asn, input, week) +SETTINGS index_granularity = 8192; + +CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list ON CLUSTER oonidata_cluster TO counters_asn_test_list_data ( `week` DateTime, `probe_cc` String, @@ -591,9 +601,6 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list ON CLUSTER oonidat `input` String, `msmt_cnt` UInt64 ) -ENGINE = ReplicatedSummingMergeTree('/clickhouse/{cluster}/tables/{database}/.inner_id.6a72ceec-7e0e-4b65-a3ab-da2809202114/{shard}', '{replica}') -ORDER BY (probe_cc, probe_asn, input) -SETTINGS index_granularity = 8192 AS SELECT toStartOfWeek(measurement_start_time) AS week, probe_cc,