|
10 | 10 | //! |
11 | 11 | //! A migration that rebuilds the metric table is only safe if it is invisible from |
12 | 12 | //! the outside. That is not a claim to assert, it is a thing to demonstrate: these |
13 | | -//! tests seed a database in the shape the migration finds, capture the raw bytes of |
14 | | -//! every response that reads a metric, run the migration, and capture them again. |
15 | | -//! The two captures must be equal byte for byte. |
| 13 | +//! tests seed a database in the shape the migration finds, capture what the outside |
| 14 | +//! could see, run the migration, and capture it again. The two captures must be |
| 15 | +//! equal byte for byte. |
16 | 16 | //! |
17 | | -//! The same binary serves both captures because every reader captured here goes |
18 | | -//! through the `metric_boundary` view, whose column list the migration holds |
19 | | -//! unchanged. |
| 17 | +//! What both captures can read is the `metric_boundary` view, which the migration |
| 18 | +//! holds unchanged and which is defined on either shape of the table. No endpoint |
| 19 | +//! reads it any more: every reader now drives on the named `metric` rows, and those |
| 20 | +//! rows do not exist before the migration, so no response has a pre-migration form |
| 21 | +//! to compare. The view is therefore captured directly, and the responses that read |
| 22 | +//! the named rows are anchored on the first migrated capture instead. |
20 | 23 |
|
21 | 24 | use bencher_api_tests::{ |
22 | 25 | TestServer, |
@@ -51,25 +54,25 @@ struct Fixture { |
51 | 54 | metric_uuids: Vec<MetricUuid>, |
52 | 55 | } |
53 | 56 |
|
54 | | -/// The raw bytes of every response that reads a metric through the |
55 | | -/// `metric_boundary` view, which is the view this migration holds unchanged. |
| 57 | +/// Every row of the `metric_boundary` view, rendered, which is the artifact this |
| 58 | +/// migration holds unchanged. |
56 | 59 | /// |
57 | | -/// The report, perf, and metric responses are deliberately absent. All three are |
58 | | -/// built from the named `metric` rows rather than from the view, and those rows do |
59 | | -/// not exist before the migration, so none of them has a pre-migration form to |
60 | | -/// compare. None is stable across a down and up round trip either, because the |
61 | | -/// migration mints a fresh uuid for every bound row it recreates and all three now |
62 | | -/// echo those uuids. That is not a regression: a server runs its migrations before |
63 | | -/// it serves, and the `value` row, which every reader of the old shape saw, keeps |
64 | | -/// its identity. What the perf response owes older clients is pinned where that |
65 | | -/// response lives, by the byte compatibility test in `perf.rs`; the metric endpoint |
66 | | -/// is pinned the same way in `metrics.rs`, and what it owes this migration is that |
67 | | -/// a down and up round trip leaves it unmoved, which |
| 60 | +/// The report, perf, alert, and metric responses are deliberately absent. All four |
| 61 | +/// are built from the named `metric` rows, and those rows do not exist before the |
| 62 | +/// migration, so none of them has a pre-migration form to compare. None is stable |
| 63 | +/// across a down and up round trip either, because the migration mints a fresh uuid |
| 64 | +/// for every bound row it recreates and all four now echo or read those rows. That |
| 65 | +/// is not a regression: a server runs its migrations before it serves, and the |
| 66 | +/// `value` row, which every reader of the old shape saw, keeps its identity. What |
| 67 | +/// the perf response owes older clients is pinned where that response lives, by the |
| 68 | +/// byte compatibility test in `perf.rs`; the metric endpoint is pinned the same way |
| 69 | +/// in `metrics.rs` and the alert responses in `parameters.rs`. What they owe this |
| 70 | +/// migration is that a down and up round trip leaves them unmoved, which |
68 | 71 | /// [`migration_down_and_up_round_trips_the_metric_triple`] asserts against |
69 | | -/// [`capture_metrics`]. |
| 72 | +/// [`capture_metrics`] and [`capture_alerts`]. |
70 | 73 | #[derive(Debug, PartialEq, Eq)] |
71 | 74 | struct Captured { |
72 | | - alerts: String, |
| 75 | + view: String, |
73 | 76 | } |
74 | 77 |
|
75 | 78 | /// The fixture rows, chosen to reach every branch of the explosion and the pivot. |
@@ -111,24 +114,23 @@ const FIXTURE_METRICS: [LegacyMetric; 4] = [ |
111 | 114 | /// table hands back the same numbering either way. |
112 | 115 | const FIXTURE_METRIC_IDS: [i32; 4] = [10, 20, 30, 40]; |
113 | 116 |
|
114 | | -// Every response that reads a metric through the view is byte identical across the |
115 | | -// migration. |
| 117 | +// The view every reader used to go through is byte identical across the migration. |
116 | 118 | #[tokio::test] |
117 | 119 | async fn responses_are_byte_identical_across_the_migration() { |
118 | 120 | let server = TestServer::new().await; |
119 | 121 | let fixture = seed_legacy_project(&server, "equiv").await; |
120 | 122 |
|
121 | | - let before = capture(&server, &fixture).await; |
| 123 | + let before = capture(&server); |
122 | 124 | assert!( |
123 | | - before.alerts.contains("40.5"), |
124 | | - "the fixture reaches the alert response before anything is compared" |
| 125 | + before.view.contains("40.5"), |
| 126 | + "the fixture reaches the view before anything is compared" |
125 | 127 | ); |
126 | 128 | apply_migration(&server); |
127 | | - let after = capture(&server, &fixture).await; |
| 129 | + let after = capture(&server); |
128 | 130 |
|
129 | 131 | assert_eq!( |
130 | | - before.alerts, after.alerts, |
131 | | - "the alerts response changed across the migration" |
| 132 | + before.view, after.view, |
| 133 | + "the metric boundary view changed across the migration" |
132 | 134 | ); |
133 | 135 |
|
134 | 136 | // The metric endpoint reads the named rows, so the migration is what gives it a |
@@ -348,41 +350,51 @@ async fn migration_down_and_up_round_trips_the_metric_triple() { |
348 | 350 | let server = TestServer::new().await; |
349 | 351 | let fixture = seed_legacy_project(&server, "roundtrip").await; |
350 | 352 |
|
351 | | - let before = capture(&server, &fixture).await; |
| 353 | + let before = capture(&server); |
352 | 354 | apply_migration(&server); |
353 | | - let after_first = capture(&server, &fixture).await; |
| 355 | + let after_first = capture(&server); |
354 | 356 | let metrics_first = capture_metrics(&server, &fixture).await; |
| 357 | + let alerts_first = capture_alerts(&server, &fixture).await; |
355 | 358 | assert_eq!( |
356 | 359 | before, after_first, |
357 | | - "applying the migration leaves every response unchanged" |
| 360 | + "applying the migration leaves the view unchanged" |
358 | 361 | ); |
359 | 362 |
|
360 | 363 | let mut conn = server.db_conn(); |
361 | 364 | revert_migration(&mut conn); |
362 | 365 | drop(conn); |
363 | 366 |
|
364 | | - let after_revert = capture(&server, &fixture).await; |
| 367 | + let after_revert = capture(&server); |
365 | 368 | assert_eq!( |
366 | 369 | before, after_revert, |
367 | | - "reverting the migration leaves every response unchanged" |
| 370 | + "reverting the migration leaves the view unchanged" |
368 | 371 | ); |
369 | 372 |
|
370 | 373 | apply_migration(&server); |
371 | | - let after_round_trip = capture(&server, &fixture).await; |
| 374 | + let after_round_trip = capture(&server); |
372 | 375 | assert_eq!( |
373 | 376 | before, after_round_trip, |
374 | | - "a down and up round trip leaves every response unchanged" |
| 377 | + "a down and up round trip leaves the view unchanged" |
375 | 378 | ); |
376 | 379 |
|
377 | | - // The metric endpoint has no response to capture in the legacy shape, so its |
378 | | - // anchor is the first migrated capture rather than the pre-migration one. The |
379 | | - // bound rows the round trip recreates carry fresh uuids, and the metric endpoint |
380 | | - // never echoes a bound row's uuid, so the response is stable across it. |
| 380 | + // The metric endpoint and the alert responses have nothing to capture in the |
| 381 | + // legacy shape, so their anchor is the first migrated capture rather than the |
| 382 | + // pre-migration one. The bound rows the round trip recreates carry fresh uuids, |
| 383 | + // and neither reader echoes a bound row's uuid, so both are stable across it. |
381 | 384 | assert_eq!( |
382 | 385 | metrics_first, |
383 | 386 | capture_metrics(&server, &fixture).await, |
384 | 387 | "a down and up round trip leaves every metric response unchanged" |
385 | 388 | ); |
| 389 | + assert!( |
| 390 | + alerts_first.contains("40.5"), |
| 391 | + "the fixture reaches the alert response before anything is compared" |
| 392 | + ); |
| 393 | + assert_eq!( |
| 394 | + alerts_first, |
| 395 | + capture_alerts(&server, &fixture).await, |
| 396 | + "a down and up round trip leaves the alerts response unchanged" |
| 397 | + ); |
386 | 398 |
|
387 | 399 | // The responses can only be trusted if the schema they were served from is the |
388 | 400 | // schema the code compiles against, so the round trip also has to land back on it. |
@@ -1178,21 +1190,98 @@ fn seed_threshold_and_alert( |
1178 | 1190 | /// |
1179 | 1191 | /// The bytes are compared, not parsed values: parsing and re-serializing would |
1180 | 1192 | /// hide exactly the float formatting drift this migration has to rule out. |
1181 | | -async fn capture(server: &TestServer, fixture: &Fixture) -> Captured { |
| 1193 | +fn capture(server: &TestServer) -> Captured { |
| 1194 | + let mut conn = server.db_conn(); |
| 1195 | + let view = diesel::sql_query(VIEW_ROWS_SQL) |
| 1196 | + .load::<ViewRow>(&mut conn) |
| 1197 | + .expect("Failed to read the metric boundary view") |
| 1198 | + .into_iter() |
| 1199 | + .map( |
| 1200 | + |ViewRow { |
| 1201 | + metric_id, |
| 1202 | + metric_uuid, |
| 1203 | + report_benchmark_id, |
| 1204 | + measure_id, |
| 1205 | + value, |
| 1206 | + lower_value, |
| 1207 | + upper_value, |
| 1208 | + boundary_id, |
| 1209 | + boundary_uuid, |
| 1210 | + threshold_id, |
| 1211 | + model_id, |
| 1212 | + baseline, |
| 1213 | + lower_limit, |
| 1214 | + upper_limit, |
| 1215 | + }| { |
| 1216 | + format!( |
| 1217 | + "{metric_id}|{metric_uuid}|{report_benchmark_id}|{measure_id}|{value:?}|{lower_value:?}|{upper_value:?}|{boundary_id:?}|{boundary_uuid:?}|{threshold_id:?}|{model_id:?}|{baseline:?}|{lower_limit:?}|{upper_limit:?}" |
| 1218 | + ) |
| 1219 | + }, |
| 1220 | + ) |
| 1221 | + .collect::<Vec<_>>() |
| 1222 | + .join("\n"); |
| 1223 | + Captured { view } |
| 1224 | +} |
| 1225 | + |
| 1226 | +/// Every row of the `metric_boundary` view, in a stable order. |
| 1227 | +const VIEW_ROWS_SQL: &str = "SELECT * FROM metric_boundary ORDER BY metric_id"; |
| 1228 | + |
| 1229 | +/// One row of the `metric_boundary` view. |
| 1230 | +/// |
| 1231 | +/// The floats are read as floats and rendered with Rust's shortest round trip |
| 1232 | +/// formatting, which is what makes formatting drift visible: it is exact, and it |
| 1233 | +/// prints the sign of a negative zero that `==` cannot see. The render destructures |
| 1234 | +/// the row, so a column added to the view is a build error here rather than a |
| 1235 | +/// silent omission. |
| 1236 | +#[derive(diesel::QueryableByName)] |
| 1237 | +struct ViewRow { |
| 1238 | + #[diesel(sql_type = Integer)] |
| 1239 | + metric_id: i32, |
| 1240 | + #[diesel(sql_type = Text)] |
| 1241 | + metric_uuid: String, |
| 1242 | + #[diesel(sql_type = Integer)] |
| 1243 | + report_benchmark_id: i32, |
| 1244 | + #[diesel(sql_type = Integer)] |
| 1245 | + measure_id: i32, |
| 1246 | + #[diesel(sql_type = Double)] |
| 1247 | + value: f64, |
| 1248 | + #[diesel(sql_type = Nullable<Double>)] |
| 1249 | + lower_value: Option<f64>, |
| 1250 | + #[diesel(sql_type = Nullable<Double>)] |
| 1251 | + upper_value: Option<f64>, |
| 1252 | + #[diesel(sql_type = Nullable<Integer>)] |
| 1253 | + boundary_id: Option<i32>, |
| 1254 | + #[diesel(sql_type = Nullable<Text>)] |
| 1255 | + boundary_uuid: Option<String>, |
| 1256 | + #[diesel(sql_type = Nullable<Integer>)] |
| 1257 | + threshold_id: Option<i32>, |
| 1258 | + #[diesel(sql_type = Nullable<Integer>)] |
| 1259 | + model_id: Option<i32>, |
| 1260 | + #[diesel(sql_type = Nullable<Double>)] |
| 1261 | + baseline: Option<f64>, |
| 1262 | + #[diesel(sql_type = Nullable<Double>)] |
| 1263 | + lower_limit: Option<f64>, |
| 1264 | + #[diesel(sql_type = Nullable<Double>)] |
| 1265 | + upper_limit: Option<f64>, |
| 1266 | +} |
| 1267 | + |
| 1268 | +/// Capture the raw bytes of the alerts endpoint. |
| 1269 | +/// |
| 1270 | +/// Separate from [`capture`] because the alert responses read the named `metric` |
| 1271 | +/// rows, so they only have a response to capture once the migration has made them. |
| 1272 | +async fn capture_alerts(server: &TestServer, fixture: &Fixture) -> String { |
1182 | 1273 | let Fixture { |
1183 | 1274 | project_slug, |
1184 | 1275 | token, |
1185 | 1276 | metric_uuids: _, |
1186 | 1277 | } = fixture; |
1187 | 1278 |
|
1188 | | - let alerts = get( |
| 1279 | + get( |
1189 | 1280 | server, |
1190 | 1281 | token, |
1191 | 1282 | &format!("/v0/projects/{project_slug}/alerts"), |
1192 | 1283 | ) |
1193 | | - .await; |
1194 | | - |
1195 | | - Captured { alerts } |
| 1284 | + .await |
1196 | 1285 | } |
1197 | 1286 |
|
1198 | 1287 | /// Capture the raw bytes of the metric endpoint, one response per seeded metric. |
|
0 commit comments