diff --git a/examples/data/dolfyn/sigvm_bt_09.AD2CP b/examples/data/dolfyn/sigvm_bt_09.AD2CP new file mode 100644 index 000000000..f65318130 Binary files /dev/null and b/examples/data/dolfyn/sigvm_bt_09.AD2CP differ diff --git a/examples/data/dolfyn/test_data/BenchFile01.nc b/examples/data/dolfyn/test_data/BenchFile01.nc index 3b2af8fc4..c295c1492 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01.nc and b/examples/data/dolfyn/test_data/BenchFile01.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01.repr.txt b/examples/data/dolfyn/test_data/BenchFile01.repr.txt index 5e5d563c7..47e5983f7 100644 --- a/examples/data/dolfyn/test_data/BenchFile01.repr.txt +++ b/examples/data/dolfyn/test_data/BenchFile01.repr.txt @@ -17,10 +17,9 @@ - pressure ('time',) - pressure_alt ('time',) - pressure_altraw ('time_altraw',) + - pressure_b5 ('time_b5',) - amp ('beam', 'range', 'time') - amp_b5 ('range_b5', 'time_b5') - corr ('beam', 'range', 'time') - corr_b5 ('range_b5', 'time_b5') - - accel ('dirIMU', 'time') - - mag ('dirIMU', 'time') ... and others (see `.variables`) diff --git a/examples/data/dolfyn/test_data/BenchFile01_avg.nc b/examples/data/dolfyn/test_data/BenchFile01_avg.nc index 86e8abb00..8f03cd5a6 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_avg.nc and b/examples/data/dolfyn/test_data/BenchFile01_avg.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_crop.nc b/examples/data/dolfyn/test_data/BenchFile01_crop.nc index 278a872bf..eeef5ec31 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_crop.nc and b/examples/data/dolfyn/test_data/BenchFile01_crop.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc b/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc index 2004de5f4..ecdd55e3e 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc and b/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc b/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc index a71cbbbdd..71fc9b995 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc and b/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc b/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc index bbfeaf37e..ca45fc2f9 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc and b/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc b/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc index de929e2e4..f4a309035 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc and b/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU.nc b/examples/data/dolfyn/test_data/Sig1000_IMU.nc index 397475f51..ee53b2ad1 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc index b46762be3..962c5bbf3 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc index 6c3337157..0f41b34f4 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc index f7364c5f0..3524f27a4 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc index 05663f604..e22599a0c 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc b/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc index 28cef003d..c099abcd3 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc and b/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc b/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc index d2fb9f4e2..51cabb1a7 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc and b/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_tidal.nc b/examples/data/dolfyn/test_data/Sig1000_tidal.nc index 759fe4775..e2f2b2c1f 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_tidal.nc and b/examples/data/dolfyn/test_data/Sig1000_tidal.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_tidal_bin.nc b/examples/data/dolfyn/test_data/Sig1000_tidal_bin.nc index 6bc404fdf..cf2816917 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_tidal_bin.nc and b/examples/data/dolfyn/test_data/Sig1000_tidal_bin.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc b/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc index b78e028c2..17a74763d 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc and b/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo.dolfyn.log b/examples/data/dolfyn/test_data/Sig500_Echo.dolfyn.log index 2bfe04ac8..ef60a78f1 100644 --- a/examples/data/dolfyn/test_data/Sig500_Echo.dolfyn.log +++ b/examples/data/dolfyn/test_data/Sig500_Echo.dolfyn.log @@ -1,30 +1,101 @@ root - INFO - 4464: A5, 10, 18, 188, 0, 1, 1 - root - INFO - 4662: A5, 10, 15, 524, 0, 1, 1 - root - INFO - 5196: A5, 10, 1C, 214, 0, 1, 1 - -root - INFO - 5420: A5, 10, 18, 188, 1, 2, 2 - -root - INFO - 5618: A5, 10, 15, 524, 1, 2, 2 - -root - INFO - 6152: A5, 10, 1C, 214, 1, 2, 2 - -root - INFO - 6376: A5, 10, 18, 188, 2, 3, 3 - -root - INFO - 6574: A5, 10, 15, 524, 2, 3, 3 - -root - INFO - 7108: A5, 10, 1C, 214, 2, 3, 3 - -root - INFO - 7332: A5, 10, 18, 188, 3, 4, 4 - -root - INFO - 7530: A5, 10, 15, 524, 3, 4, 4 - -root - INFO - 8064: A5, 10, 1C, 214, 3, 4, 4 - -root - INFO - 8288: A5, 10, 18, 188, 4, 5, 5 - -root - INFO - 8486: A5, 10, 15, 524, 4, 5, 5 - -root - INFO - 9020: A5, 10, 1C, 214, 4, 5, 5 - +root - INFO - 5420: A5, 10, 18, 188, 1, 1, 1 +root - INFO - 5618: A5, 10, 15, 524, 1, 1, 1 +root - INFO - 6152: A5, 10, 1C, 214, 1, 1, 1 +root - INFO - 6376: A5, 10, 18, 188, 2, 1, 1 +root - INFO - 6574: A5, 10, 15, 524, 2, 1, 1 +root - INFO - 7108: A5, 10, 1C, 214, 2, 1, 1 +root - INFO - 7332: A5, 10, 18, 188, 3, 1, 1 +root - INFO - 7530: A5, 10, 15, 524, 3, 1, 1 +root - INFO - 8064: A5, 10, 1C, 214, 3, 1, 1 +root - INFO - 8288: A5, 10, 18, 188, 4, 1, 1 +root - INFO - 8486: A5, 10, 15, 524, 4, 1, 1 +root - INFO - 9020: A5, 10, 1C, 214, 4, 1, 1 +root - INFO - 9244: A5, 10, 18, 188, 5, 1, 1 +root - INFO - 9442: A5, 10, 15, 524, 5, 1, 1 +root - INFO - 9976: A5, 10, 1C, 214, 5, 1, 1 +root - INFO - 10200: A5, 10, 18, 188, 6, 1, 1 +root - INFO - 10398: A5, 10, 15, 524, 6, 1, 1 +root - INFO - 10932: A5, 10, 1C, 214, 6, 1, 1 +root - INFO - 11156: A5, 10, 18, 188, 7, 1, 1 +root - INFO - 11354: A5, 10, 15, 524, 7, 1, 1 +root - INFO - 11888: A5, 10, 1C, 214, 7, 1, 1 +root - INFO - 12112: A5, 10, 18, 188, 8, 1, 1 +root - INFO - 12310: A5, 10, 15, 524, 8, 1, 1 +root - INFO - 12844: A5, 10, 1C, 214, 8, 1, 1 +root - INFO - 13068: A5, 10, 18, 188, 9, 1, 1 +root - INFO - 13266: A5, 10, 15, 524, 9, 1, 1 +root - INFO - 13800: A5, 10, 1C, 214, 9, 1, 1 +root - INFO - 14024: A5, 10, 18, 188, 10, 1, 1 +root - INFO - 14222: A5, 10, 15, 524, 10, 1, 1 +root - INFO - 14756: A5, 10, 1C, 214, 10, 1, 1 +root - INFO - 14980: A5, 10, 18, 188, 11, 1, 1 +root - INFO - 15178: A5, 10, 15, 524, 11, 1, 1 +root - INFO - 15712: A5, 10, 1C, 214, 11, 1, 1 +root - INFO - 15936: A5, 10, 18, 188, 12, 1, 1 +root - INFO - 16134: A5, 10, 15, 524, 12, 1, 1 +root - INFO - 16668: A5, 10, 1C, 214, 12, 1, 1 +root - INFO - 16892: A5, 10, 18, 188, 13, 1, 1 +root - INFO - 17090: A5, 10, 15, 524, 13, 1, 1 +root - INFO - 17624: A5, 10, 1C, 214, 13, 1, 1 +root - INFO - 17848: A5, 10, 18, 188, 14, 1, 1 +root - INFO - 18046: A5, 10, 15, 524, 14, 1, 1 +root - INFO - 18580: A5, 10, 1C, 214, 14, 1, 1 +root - INFO - 18804: A5, 10, 18, 188, 15, 1, 1 +root - INFO - 19002: A5, 10, 15, 524, 15, 1, 1 +root - INFO - 19536: A5, 10, 1C, 214, 15, 1, 1 +root - INFO - 19760: A5, 10, 18, 188, 16, 1, 1 +root - INFO - 19958: A5, 10, 15, 524, 16, 1, 1 +root - INFO - 20492: A5, 10, 1C, 214, 16, 1, 1 +root - INFO - 20716: A5, 10, 18, 188, 17, 1, 1 +root - INFO - 20914: A5, 10, 15, 524, 17, 1, 1 +root - INFO - 21448: A5, 10, 1C, 214, 17, 1, 1 +root - INFO - 21672: A5, 10, 18, 188, 18, 1, 1 +root - INFO - 21870: A5, 10, 15, 524, 18, 1, 1 +root - INFO - 22404: A5, 10, 1C, 214, 18, 1, 1 +root - INFO - 22628: A5, 10, 18, 188, 19, 1, 1 +root - INFO - 22826: A5, 10, 15, 524, 19, 1, 1 +root - INFO - 23360: A5, 10, 1C, 214, 19, 1, 1 +root - INFO - 23584: A5, 10, 18, 188, 20, 1, 1 +root - INFO - 23782: A5, 10, 15, 524, 20, 1, 1 +root - INFO - 24316: A5, 10, 1C, 214, 20, 1, 1 +root - INFO - 24540: A5, 10, 18, 188, 21, 1, 1 +root - INFO - 24738: A5, 10, 15, 524, 21, 1, 1 +root - INFO - 25272: A5, 10, 1C, 214, 21, 1, 1 +root - INFO - 25496: A5, 10, 18, 188, 22, 1, 1 +root - INFO - 25694: A5, 10, 15, 524, 22, 1, 1 +root - INFO - 26228: A5, 10, 1C, 214, 22, 1, 1 +root - INFO - 26452: A5, 10, 18, 188, 23, 1, 1 +root - INFO - 26650: A5, 10, 15, 524, 23, 1, 1 +root - INFO - 27184: A5, 10, 1C, 214, 23, 1, 1 +root - INFO - 27408: A5, 10, 18, 188, 24, 1, 1 +root - INFO - 27606: A5, 10, 15, 524, 24, 1, 1 +root - INFO - 28140: A5, 10, 1C, 214, 24, 1, 1 +root - INFO - 28364: A5, 10, 18, 188, 25, 1, 1 +root - INFO - 28562: A5, 10, 15, 524, 25, 1, 1 +root - INFO - 29096: A5, 10, 1C, 214, 25, 1, 1 +root - INFO - 29320: A5, 10, 18, 188, 26, 1, 1 +root - INFO - 29518: A5, 10, 15, 524, 26, 1, 1 +root - INFO - 30052: A5, 10, 1C, 214, 26, 1, 1 +root - INFO - 30276: A5, 10, 18, 188, 27, 1, 1 +root - INFO - 30474: A5, 10, 15, 524, 27, 1, 1 +root - INFO - 31008: A5, 10, 1C, 214, 27, 1, 1 +root - INFO - 31232: A5, 10, 18, 188, 28, 1, 1 +root - INFO - 31430: A5, 10, 15, 524, 28, 1, 1 +root - INFO - 31964: A5, 10, 1C, 214, 28, 1, 1 +root - INFO - 32188: A5, 10, 18, 188, 29, 1, 1 +root - INFO - 32386: A5, 10, 15, 524, 29, 1, 1 +root - INFO - 32920: A5, 10, 1C, 214, 29, 1, 1 +root - INFO - 33144: A5, 10, 18, 188, 30, 1, 1 +root - INFO - 33342: A5, 10, 15, 524, 30, 1, 1 +root - INFO - 33876: A5, 10, 1C, 214, 30, 1, 1 +root - INFO - 34100: A5, 10, 18, 188, 31, 1, 1 +root - INFO - 34298: A5, 10, 15, 524, 31, 1, 1 +root - INFO - 34832: A5, 10, 1C, 214, 31, 1, 1 +root - INFO - 35056: A5, 10, 18, 188, 32, 1, 1 +root - INFO - 35254: A5, 10, 15, 524, 32, 1, 1 +root - INFO - 35788: A5, 10, 1C, 214, 32, 1, 1 +root - INFO - 36012: A5, 10, 18, 188, 33, 1, 1 +root - INFO - 36210: A5, 10, 15, 524, 33, 1, 1 diff --git a/examples/data/dolfyn/test_data/Sig500_Echo.nc b/examples/data/dolfyn/test_data/Sig500_Echo.nc index 78dbdd06f..9b4da5d4e 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo.nc and b/examples/data/dolfyn/test_data/Sig500_Echo.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc b/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc index d6f187116..112a320ff 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc b/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc index 80d4bca81..7a5230a7b 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc b/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc deleted file mode 100644 index b44b1d8ca..000000000 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc and /dev/null differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc b/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc deleted file mode 100644 index 8e12197df..000000000 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc and /dev/null differ diff --git a/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc b/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc index 2ac223c12..ac6210544 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc and b/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_dp_ice2.nc b/examples/data/dolfyn/test_data/Sig500_dp_ice2.nc index 7c02a066a..9816520b3 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_dp_ice2.nc and b/examples/data/dolfyn/test_data/Sig500_dp_ice2.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_dp_ice2_rotate_earth2inst.nc b/examples/data/dolfyn/test_data/Sig500_dp_ice2_rotate_earth2inst.nc new file mode 100644 index 000000000..c44b6d2e7 Binary files /dev/null and b/examples/data/dolfyn/test_data/Sig500_dp_ice2_rotate_earth2inst.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc b/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc index f52d8df45..b94122540 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc and b/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc differ diff --git a/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc b/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc index 90b41d3c7..15a248267 100644 Binary files a/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc and b/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc differ diff --git a/examples/data/dolfyn/test_data/VelEchoBT01.nc b/examples/data/dolfyn/test_data/VelEchoBT01.nc index f2fb6183e..0f5e34e30 100644 Binary files a/examples/data/dolfyn/test_data/VelEchoBT01.nc and b/examples/data/dolfyn/test_data/VelEchoBT01.nc differ diff --git a/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc b/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc index 6cefd020c..0d1bc6a47 100644 Binary files a/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc and b/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc differ diff --git a/examples/data/dolfyn/test_data/VelEchoBT01_rotate_inst2earth.nc b/examples/data/dolfyn/test_data/VelEchoBT01_rotate_inst2earth.nc new file mode 100644 index 000000000..06ac2e636 Binary files /dev/null and b/examples/data/dolfyn/test_data/VelEchoBT01_rotate_inst2earth.nc differ diff --git a/examples/data/dolfyn/test_data/sigvm_bt_09.nc b/examples/data/dolfyn/test_data/sigvm_bt_09.nc new file mode 100644 index 000000000..97d18d5ab Binary files /dev/null and b/examples/data/dolfyn/test_data/sigvm_bt_09.nc differ diff --git a/examples/data/dolfyn/test_data/vector_data01_avg.nc b/examples/data/dolfyn/test_data/vector_data01_avg.nc index 91ba2b135..9a235b607 100644 Binary files a/examples/data/dolfyn/test_data/vector_data01_avg.nc and b/examples/data/dolfyn/test_data/vector_data01_avg.nc differ diff --git a/mhkit/dolfyn/io/base.py b/mhkit/dolfyn/io/base.py index c52bbd4ee..90f3179bc 100644 --- a/mhkit/dolfyn/io/base.py +++ b/mhkit/dolfyn/io/base.py @@ -329,11 +329,8 @@ def _create_dataset(data): else: # amp, corr, prcnt_gd, status dim0 = "beam" - if not any(val in key for val in tag) or ("_avg" in key): - if "_avg" in key: - tg = "_avg" - else: - tg = "" + if not any(val in key for val in tag) or (tg in ["_avg", "_bt"]): + tg = "" if tg not in ["_avg", "_bt"] else tg ds_dict[key] = { "dims": (dim0, "range" + tg, "time" + tg), "data": data["data_vars"][key], diff --git a/mhkit/dolfyn/io/nortek2.py b/mhkit/dolfyn/io/nortek2.py index a536bca51..9b15f6b2d 100644 --- a/mhkit/dolfyn/io/nortek2.py +++ b/mhkit/dolfyn/io/nortek2.py @@ -81,10 +81,8 @@ def read_signature( ) d = rdr.readfile(nens[0], nens[1]) rdr.sci_data(d) - if rdr._dp: - _clean_dp_skips(d) - out = _reorg(d) - _reduce(out) + _clean_skips(d) + out = _reduce(_reorg(d)) # Convert time to dt64 and fill gaps coords = out["coords"] @@ -113,14 +111,13 @@ def read_signature( ds = _create_dataset(out) ds = _set_coords(ds, ref_frame=ds.coord_sys) - if ("orientmat" not in ds) and ("heading" in ds): - ds["orientmat"] = _euler2orient( - ds["time"], ds["heading"], ds["pitch"], ds["roll"] - ) - elif ("orientmat_avg" not in ds) and ("heading_avg" in ds): - ds["orientmat_avg"] = _euler2orient( - ds["time_avg"], ds["heading_avg"], ds["pitch_avg"], ds["roll_avg"] - ) + # Add orientation matrices + tags = ["", "_avg", "_b5", "_echo", "_bt"] + for tg in tags: + if ("orientmat" + tg not in ds) and ("heading" + tg in ds): + ds["orientmat" + tg] = _euler2orient( + ds["time" + tg], ds["heading" + tg], ds["pitch" + tg], ds["roll" + tg] + ) if declin is not None: set_declination(ds, declin, inplace=True) @@ -164,7 +161,6 @@ def __init__(self, fname, rebuild_index, debug, dual_profile): self._ens_pos = self._index["pos"][ lib._boolarray_firstensemble_ping(self._index) ] - self._lastblock_iswhole = self._calc_lastblock_iswhole() self._config = lib._calc_config(self._index) self._init_burst_readers() self.unknown_ID_count = {} @@ -228,12 +224,12 @@ def _read_filehead_config_string( out2[ky] = out[ky] return out2 - def _calc_lastblock_iswhole(self): + def _lastblock_iswhole(self): blocksize, blocksize_count = np.unique( np.diff(self._ens_pos), return_counts=True ) - standard_blocksize = blocksize[blocksize_count.argmax()] - return (self._eof - self._ens_pos[-1]) == standard_blocksize + last_block_size = self._eof - self._ens_pos[-1] + return last_block_size in blocksize def _init_burst_readers( self, @@ -246,7 +242,7 @@ def _init_burst_readers( ) elif rdr_id == 23: self._burst_readers[rdr_id] = defs._calc_bt_struct( - cfg["_config"], cfg["n_beams"] + cfg["_config"], cfg["n_beams"], cfg["n_cells"], cfg["_d_ver"] ) else: self._burst_readers[rdr_id] = defs._calc_burst_struct( @@ -257,7 +253,8 @@ def init_data(self, ens_start, ens_stop): outdat = {} nens = int(ens_stop - ens_start) - # ID 26 and 31 recorded infrequently + # If there are multiple pings in an ensemble, nens will + # be the total number of pings read in def n_id(id): return ( (self._index["ID"] == id) @@ -265,6 +262,7 @@ def n_id(id): & (self._index["ens"] < ens_stop) ).sum() + # ID 26 and 31 recorded infrequently n_altraw = {26: n_id(26), 31: n_id(31)} if not n_altraw[26] and 26 in self._burst_readers: self._burst_readers.pop(26) @@ -301,14 +299,16 @@ def _read_str(self, size): string = string[1:-1] return id, string - def _read_burst(self, id, dat, c, echo=False): + def _read_burst(self, id, dat, c, sz, echo=False): rdr = self._burst_readers[id] - rdr.read_into(self.f, dat, c) + to_skip = rdr.read_into(self.f, dat, c, sz) + if to_skip: + self.f.seek(to_skip, 1) def readfile(self, ens_start=0, ens_stop=None): # If the lastblock is not whole, we don't read it. # If it is, we do (don't subtract 1) - nens_total = len(self._ens_pos) - int(not self._lastblock_iswhole) + nens_total = len(self._ens_pos) - int(not self._lastblock_iswhole()) if ens_stop is None or ens_stop > nens_total: ens_stop = nens_total ens_start = int(ens_start) @@ -326,10 +326,13 @@ def readfile(self, ens_start=0, ens_stop=None): except IOError: return outdat id = hdr["id"] - if id in [21, 22, 23, 24, 28]: # "burst data record" (vel + ast), + if id not in outdat: + # If the ID was skipped in the indexer, it won't be in outdat + self.f.seek(hdr["sz"], 1) + elif id in [21, 22, 23, 24, 28]: # "burst data record" (vel + ast), # "avg data record" (vel_avg + ast_avg), "bottom track data record" (bt), # "interleaved burst data record" (vel_b5), "echosounder record" (echo) - self._read_burst(id, outdat[id], c) + self._read_burst(id, outdat[id], c, hdr["sz"]) elif id in [26, 31]: # "burst altimeter raw record" (_altraw), "avg altimeter raw record" (_altraw_avg) rdr = self._burst_readers[id] @@ -367,7 +370,7 @@ def readfile(self, ens_start=0, ens_stop=None): "The number of samples in this 'Altimeter Raw' " "burst is different from prior bursts." ) - self._read_burst(id, outdat[id], c_altraw[id]) + self._read_burst(id, outdat[id], c_altraw[id], hdr["sz"]) outdat[id]["ensemble"][c_altraw[id]] = c c_altraw[id] += 1 @@ -506,10 +509,6 @@ def _reorg(dat): (28, "_echo"), (31, "raw_avg"), ]: - if id in [24, 26]: - collapse_exclude = [0] - else: - collapse_exclude = [] if id not in dat: continue dnow = dat[id] @@ -520,7 +519,8 @@ def _reorg(dat): dnow["standard_name"].pop(ky) outdat["standard_name"].update(dnow["standard_name"]) cfg["burst_config" + tag] = lib._headconfig_int2dict( - lib._collapse(dnow["config"], exclude=collapse_exclude, name="config") + lib._collapse(dnow["config"], exclude=[0], name="config"), + mode="bt" if "bt" in tag else "burst", ) outdat["coords"]["time" + tag] = lib._calc_time( dnow["year"] + np.uint16(1900), @@ -531,34 +531,30 @@ def _reorg(dat): dnow["second"], dnow["usec100"].astype("uint32") * 100, ) - tmp = lib._beams_cy_int2dict( - lib._collapse( - dnow["beam_config"], exclude=collapse_exclude, name="beam_config" - ), - 21, # always 21 here - ) + + d_ver = lib._collapse(dnow["ver"], exclude=[0], name="ver") + beams_cy = lib._collapse(dnow["beam_config"], exclude=[0], name="beam_config") + # Deal with data version 9 for bottom track + if (id == 23) and (d_ver >= 9): + n_cells = lib._collapse(dnow["n_cells"], exclude=[0], name="n_cells") + beams_cy = lib._repack_bt_beams_cy(beams_cy, n_cells) + + # Unpack cells, coordinate axes and beams + tmp = lib._beams_cy_int2dict(beams_cy, id, d_ver) cfg["n_cells" + tag] = int(tmp["n_cells"]) cfg["coord_sys_axes" + tag] = tmp["cy"] cfg["n_beams" + tag] = int(tmp["n_beams"]) - cfg["ambig_vel" + tag] = round( - float(lib._collapse(dnow["ambig_vel"], name="ambig_vel")), 3 - ) - for ky in [ - "SerialNum", - "nominal_corr", - ]: - cfg[ky + tag] = int( - lib._collapse(dnow[ky], exclude=collapse_exclude, name=ky) + cfg["SerialNum" + tag] = int( + lib._collapse(dnow["SerialNum"], exclude=[0], name="SerialNum") + ) + for ky in ["cell_size", "power_level_dB", "ambig_vel"]: + cfg[ky + tag] = round( + float(lib._collapse(dnow[ky], exclude=[0], name=ky)), 3 ) - - for ky in [ - "cell_size", - "blank_dist", - "power_level_dB", - ]: - cfg[ky + tag] = float( - lib._collapse(dnow[ky], exclude=collapse_exclude, name=ky) + if id != 23: + cfg["blank_dist" + tag] = float( + lib._collapse(dnow["blank_dist"], exclude=[0], name="blank_dist") ) for ky in [ @@ -674,9 +670,7 @@ def _reorg(dat): cfg["rotate_vars"] = [] cy = cfg["coord_sys_axes_avg"] else: - cfg["rotate_vars"] = [ - "vel", - ] + cfg["rotate_vars"] = ["vel"] cy = cfg["coord_sys_axes"] outdat["attrs"]["coord_sys"] = {"XYZ": "inst", "ENU": "earth", "beam": "beam"}[cy] @@ -693,10 +687,9 @@ def _reorg(dat): return outdat -def _clean_dp_skips(data): +def _clean_skips(data): """ - Removes zeros from interwoven measurements taken in a dual profile - configuration. + Removes zeros from interwoven measurements """ for id in data: @@ -724,14 +717,6 @@ def _reduce(data): dc = data["coords"] da = data["attrs"] - # Average these fields - for ky in ["c_sound", "temp", "pressure", "temp_press", "temp_clock", "batt"]: - lib._reduce_by_average(dv, ky, ky + "_b5") - - # Angle-averaging is treated separately - for ky in ["heading", "pitch", "roll"]: - lib._reduce_by_average_angle(dv, ky, ky + "_b5") - if "vel" in dv: dc["range"] = (np.arange(dv["vel"].shape[1]) + 1) * da["cell_size"] + da[ "blank_dist" @@ -784,6 +769,8 @@ def _reduce(data): tm[irow, icol] = tmat["M" + str(irow + 1) + str(icol + 1)] dv["beam2inst_orientmat"] = tm + return data + def split_dp_datasets(ds): """ diff --git a/mhkit/dolfyn/io/nortek2_defs.py b/mhkit/dolfyn/io/nortek2_defs.py index 56cc0e052..f08f635ee 100644 --- a/mhkit/dolfyn/io/nortek2_defs.py +++ b/mhkit/dolfyn/io/nortek2_defs.py @@ -71,13 +71,14 @@ def init_data(self, npings): out[nm] = _nans(shp + [npings], dtype=np.dtype(fmt[0])) return out - def read_into(self, fobj, data, ens, cs=None): - dat_tuple = self.read(fobj, cs=cs) + def read_into(self, fobj, data, ens, sz, cs=None): + dat_tuple, to_skip = self.read(fobj, sz=sz, cs=cs) for nm, shp, d in zip(self._names, self._shape, dat_tuple): try: data[nm][..., ens] = d except ValueError: data[nm][..., ens] = np.asarray(d).reshape(shp) + return to_skip @property def format( @@ -85,8 +86,16 @@ def format( ): return _format(self._format, self._N) - def read(self, fobj, cs=None): + def read(self, fobj, sz=None, cs=None): bytes = fobj.read(self.nbyte) + # For newer versions of id data packets + if sz is not None: + extra_bytes = sz - self.nbyte + if extra_bytes < 0: + raise ValueError( + "Data definition contains different number of bytes than " + f"data packet: {extra_bytes}" + ) if len(bytes) != self.nbyte: raise IOError("End of file.") data = self._struct.unpack(bytes) @@ -110,7 +119,10 @@ def read(self, fobj, cs=None): else: out.append(data[c : (c + n)]) c += n - return out + if sz is not None: + return out, extra_bytes + else: + return out def read2dict(self, fobj, cs=False): return {self._names[idx]: dat for idx, dat in enumerate(self.read(fobj, cs=cs))} @@ -291,98 +303,129 @@ def __call__(self, array): ("_ensemble", "I", [], None), ] -_bt_hdr = [ - ("ver", "B", [], None), - ("DatOffset", "B", [], None), - ("config", "H", [], None), - ("SerialNum", "I", [], None), - ("year", "B", [], None), - ("month", "B", [], None), - ("day", "B", [], None), - ("hour", "B", [], None), - ("minute", "B", [], None), - ("second", "B", [], None), - ("usec100", "H", [], None), - ( - "c_sound", - "H", - [], - _LinFunc(0.1, dtype=dt32), - "m s-1", - "Speed of Sound", - "speed_of_sound_in_sea_water", - ), - ( - "temp", - "h", - [], - _LinFunc(0.01, dtype=dt32), - "degree_C", - "Temperature", - "sea_water_temperature", - ), - ( - "pressure", - "I", - [], - _LinFunc(0.001, dtype=dt32), - "dbar", - "Pressure", - "sea_water_pressure", - ), - ( - "heading", - "H", - [], - _LinFunc(0.01, dtype=dt32), - "degree", - "Heading", - "platform_orientation", - ), - ("pitch", "h", [], _LinFunc(0.01, dtype=dt32), "degree", "Pitch", "platform_pitch"), - ("roll", "h", [], _LinFunc(0.01, dtype=dt32), "degree", "Roll", "platform_roll"), - ("beam_config", "H", [], None), - ("cell_size", "H", [], _LinFunc(0.001), "m"), - ("blank_dist", "H", [], _LinFunc(0.01), "m"), - ("nominal_corr", "B", [], None, "%"), - ("unused", "B", [], None), - ( - "batt", - "H", - [], - _LinFunc(0.1, dtype=dt32), - "V", - "Battery Voltage", - "battery_voltage", - ), - ("mag", "h", [3], None, "uT", "Compass"), - ( - "accel", - "h", - [3], - _LinFunc(1.0 / 16384 * grav, dtype=dt32), - "m s-2", - "Acceleration", - "", - ), - ("ambig_vel", "I", [], _LinFunc(0.001, dtype=dt32), "m s-1"), - ("data_desc", "H", [], None), - ("xmit_energy", "H", [], None, "dB", "Sound Pressure Level of Acoustic Signal"), - ("vel_scale", "b", [], None), - ("power_level_dB", "b", [], _LinFunc(dtype=dt32), "dB"), - ("temp_mag", "h", [], None), # uncalibrated - ( - "temp_clock", - "h", - [], - _LinFunc(0.01, dtype=dt32), - "degree_C", - "Internal Clock Temperature", - ), - ("error", "I", [], None, "1", "Error Code"), - ("status", "I", [], None, "1", "Status Code"), - ("_ensemble", "I", [], None), -] + +def _bt_hdr(ver): + if ver == 9: # version 9 + options = [ + ("beam_config", "B", [], None), + ("pad1", "B", [], None), + ("n_cells", "H", [], None), + ("cell_size", "H", [], _LinFunc(0.001), "m"), + ("unused", "B", [], None), + ("temp_pres", "B", [], None), + ] + else: # version 1 + options = [ + ("beam_config", "H", [], None), + ("cell_size", "H", [], _LinFunc(0.001), "m"), + ("blank_dist", "H", [], None), + ("vel_range", "H", [], None), + ] + out = [ + ("ver", "B", [], None), + ("DatOffset", "B", [], None), + ("config", "H", [], None), + ("SerialNum", "L", [], None), + ("year", "B", [], None), + ("month", "B", [], None), + ("day", "B", [], None), + ("hour", "B", [], None), + ("minute", "B", [], None), + ("second", "B", [], None), + ("usec100", "H", [], None), + ( + "c_sound", + "H", + [], + _LinFunc(0.1, dtype=dt32), + "m s-1", + "Speed of Sound", + "speed_of_sound_in_sea_water", + ), + ( + "temp", + "h", + [], + _LinFunc(0.01, dtype=dt32), + "degree_C", + "Temperature", + "sea_water_temperature", + ), + ( + "pressure", + "L", + [], + _LinFunc(0.001, dtype=dt32), + "dbar", + "Pressure", + "sea_water_pressure", + ), + ( + "heading", + "H", + [], + _LinFunc(0.01, dtype=dt32), + "degree", + "Heading", + "platform_orientation", + ), + ( + "pitch", + "h", + [], + _LinFunc(0.01, dtype=dt32), + "degree", + "Pitch", + "platform_pitch", + ), + ( + "roll", + "h", + [], + _LinFunc(0.01, dtype=dt32), + "degree", + "Roll", + "platform_roll", + ), + *options, + ( + "batt", + "H", + [], + _LinFunc(0.1, dtype=dt32), + "V", + "Battery Voltage", + "battery_voltage", + ), + ("mag", "h", [3], _LinFunc(0.1, dtype=dt32), "uT", "Compass"), + ( + "accel", + "h", + [3], + _LinFunc(1.0 / 16384 * grav, dtype=dt32), + "m s-2", + "Acceleration", + "", + ), + ("ambig_vel", "I", [], _LinFunc(0.001, dtype=dt32), "m s-1"), + ("data_desc", "H", [], None), + ("xmit_energy", "H", [], None, "dB", "Sound Pressure Level of Acoustic Signal"), + ("vel_scale", "b", [], None), + ("power_level_dB", "b", [], _LinFunc(dtype=dt32), "dB"), + ("temp_mag", "h", [], None), # uncalibrated + ( + "temp_clock", + "h", + [], + _LinFunc(0.01, dtype=dt32), + "degree_C", + "Internal Clock Temperature", + ), + ("error", "I", [], None, "1", "Error Code"), + ("status", "I", [], None, "1", "Status Code"), + ("_ensemble", "I", [], None), + ] + return out _ahrs_def = [ @@ -399,29 +442,50 @@ def __call__(self, array): ] -def _calc_bt_struct(config, nb): +def _calc_bt_struct(config, nb, nc, d_ver=1): flags = lib._headconfig_int2dict(config, mode="bt") - dd = copy(_bt_hdr) - if flags["vel"]: - # units handled in Ad2cpReader.sci_data - dd.append( - ("vel", "i", [nb], None, "m s-1", "Platform Velocity from Bottom Track") - ) - if flags["dist"]: - dd.append( - ( - "dist", - "i", - [nb], - _LinFunc(0.001, dtype=dt32), - "m", - "Bottom Track Measured Depth", + dd = _bt_hdr(d_ver) + + if d_ver == 1: + if flags["vel"]: + # units handled in Ad2cpReader.sci_data + dd.append( + ("vel", "i", [nb], None, "m s-1", "Platform Velocity from Bottom Track") ) - ) - if flags["fom"]: - dd.append(("fom", "H", [nb], None, "1", "Figure of Merit")) - if flags["ahrs"]: - dd += _ahrs_def + if flags["dist"]: + dd.append( + ( + "dist", + "i", + [nb], + _LinFunc(0.001, dtype=dt32), + "m", + "Bottom Track Measured Depth", + ) + ) + if flags["fom"]: + dd.append(("fom", "H", [nb], None, "1", "Figure of Merit")) + # Does not appear to contain AHRS data even if flag says so + else: # Data record version 9 + if flags["fom"]: + dd.append(("fom", "H", [nb], None, "1", "Figure of Merit")) + if flags["vel"]: + # units handled in Ad2cpReader.sci_data + dd.append( + ("vel", "i", [nb], None, "m s-1", "Platform Velocity from Bottom Track") + ) + if flags["dist"]: + dd.append( + ( + "dist", + "i", + [nb], + _LinFunc(0.001, dtype=dt32), + "m", + "Bottom Track Measured Depth", + ) + ) + # Does not appear to contain AHRS data even if flag says so return _DataDef(dd) diff --git a/mhkit/dolfyn/io/nortek2_lib.py b/mhkit/dolfyn/io/nortek2_lib.py index 7c1f17eef..5d928ce0e 100644 --- a/mhkit/dolfyn/io/nortek2_lib.py +++ b/mhkit/dolfyn/io/nortek2_lib.py @@ -7,37 +7,6 @@ from .. import time from .base import _abspath - -def _reduce_by_average(data, ky0, ky1): - # Average two arrays together, if they both exist. - if ky1 in data: - tmp = data.pop(ky1) - if ky0 in data: - data[ky0] += tmp - data[ky0] = data[ky0] / 2 - else: - data[ky0] = tmp - - -def _reduce_by_average_angle(data, ky0, ky1, degrees=True): - # Average two arrays of angles together, if they both exist. - if degrees: - rad_fact = np.pi / 180 - else: - rad_fact = 1 - if ky1 in data: - if ky0 in data: - data[ky0] = ( - np.angle( - np.exp(1j * data.pop(ky0) * rad_fact) - + np.exp(1j * data.pop(ky1) * rad_fact) - ) - / rad_fact - ) - else: - data[ky0] = data.pop(ky1) - - # This is the data-type of the index file. # This must match what is written-out by the create_index function. _index_version = 1 @@ -152,23 +121,6 @@ def _create_index(infile, outfile, init_pos, eof, debug): N = dict.fromkeys(ids, 0) config = 0 last_ens = dict.fromkeys(ids, -1) - seek_2ens = { - 21: 40, # 0x15 burst - 22: 40, # 0x16 average - 23: 42, # 0x17 bottom track, starts from "42" - 24: 40, # 0x18 interleaved burst (beam 5) - 26: 40, # 0x1A burst altimeter - 27: 40, # 0x1B DVL bottom track - 28: 40, # 0x1C echo sounder - 29: 40, # 0x1D DVL water track - 30: 40, # 0x1E altimeter - 31: 40, # 0x1F avg altimeter - 35: 40, # 0x23 raw echo sounder - 36: 40, # 0x24 raw tx echo sounder - 48: 40, # 0x30 processed wave - # 160: 40, # 0xA0 string (header info, GPS NMEA data) - # 192: 40, # 0xC0 Nortek Data format 8 record - } pos = 0 header_check_flag = 0 # leave room for header plus other data (12 + 76) @@ -176,7 +128,7 @@ def _create_index(infile, outfile, init_pos, eof, debug): if init_pos and not pos: fin.seek(init_pos, 1) try: - dat = struct.unpack(" 0: - if (ens[idk] == 1) or (ens[idk] < last_ens[idk]): - # Covers all id keys saved in "burst mode" - # Covers ID keys not saved in sequential order - ens[idk] = last_ens[idk] + 1 - - if last_ens[idk] > 0 and last_ens[idk] != ens[idk]: N[idk] += 1 + # Update our last ensemble before writing to index file + last_ens[idk] = ens[idk] + # Skip to next ID number + fin.seek(dat[4] - d_off, 1) + # Write to index file fout.write( struct.pack( " N_id)[0] + ibad = np.nonzero(diff > N_id)[0] # Check if spacing is equal for dual profiling ADCPs if dp: @@ -326,11 +309,12 @@ def _check_index(idx, infile, fix_hw_ens=False, dp=False): for ib in ibad: # The ping number reported here may not be quite right if # the ensemble count is wrong. - logging.warning( - "Skipped ping (ID: {}) in file {} at ensemble {}.".format( - id, infile, idx["ens"][inds[ib + 1] - 1] - ) + msg = ( + f"Skipped ping (ID: {hex(id)}) in file {infile} at ensemble " + f"{idx['ens'][inds[ib + 1] - 1]}." ) + warnings.warn(msg) + logging.warning(msg) hwe[inds[(ib + 1) :]] += 1 ens[inds[(ib + 1) :]] += 1 @@ -488,7 +472,7 @@ def _headconfig_int2dict(val, mode="burst"): temp_valid=_getbit(val, 1), compass_valid=_getbit(val, 2), tilt_valid=_getbit(val, 3), - # bit 4 is unused + # ext_sensor=_getbit(val, 4), vel=_getbit(val, 5), amp=_getbit(val, 6), corr=_getbit(val, 7), @@ -499,7 +483,7 @@ def _headconfig_int2dict(val, mode="burst"): ahrs=_getbit(val, 12), p_gd=_getbit(val, 13), std=_getbit(val, 14), - # bit 15 is unused + # spec=_getbit(val, 15), ) elif mode == "bt": return dict( @@ -509,11 +493,13 @@ def _headconfig_int2dict(val, mode="burst"): tilt_valid=_getbit(val, 3), # bit 4 is unused vel=_getbit(val, 5), - # bits 6-7 unused + amp=_getbit(val, 6), + corr=_getbit(val, 7), dist=_getbit(val, 8), fom=_getbit(val, 9), ahrs=_getbit(val, 10), - # bits 10-15 unused + aux=_getbit(val, 11), + # bits 12-15 unused ) @@ -565,12 +551,28 @@ def _alt_status2data(val): return out -def _beams_cy_int2dict(val, id): +def _repack_bt_beams_cy(beams_cy, n_cells): + # "beams_cy": < bit 7-5: Number of beams, bit 4-3: coordinate system + n_beams = (beams_cy >> 5) & 0b111 + cy_idx = (beams_cy >> 3) & 0b11 + return (n_cells << 5) | (n_beams << 2) | cy_idx + + +def _beams_cy_int2dict(val, id, d_ver=3): """Convert the beams/coordinate-system bytes to a dict of values.""" if id == 28: # 0x1C (echosounder) - return dict(n_cells=val) - elif id in [26, 31]: + return dict(n_cells=val, cy="beam", n_beams=1) + elif id in [26, 31]: # Altimeter return dict(n_cells=val & (2**10 - 1), cy="beam", n_beams=1) + elif (id == 23) and (d_ver == 9): + # Unpack custom beams_cy for bottom track data version 9 + return dict( + n_cells=val >> 5, + cy=["ENU", "XYZ", "beam", None][(val & 31) & 3], + n_beams=((val & 31) >> 2) & 7, # usually = 4 + ) + # Data definition version 1 and 3 + # "ncells from bits 9-0, nbeams from bits 15-12, and coordinate system from bits 11-10". return dict( n_cells=val & (2**10 - 1), cy=["ENU", "XYZ", "beam", None][val >> 10 & 3], @@ -640,6 +642,7 @@ def _calc_config(index): inds = index["ID"] == id _config = index["config"][inds] _beams_cy = index["beams_cy"][inds] + _data_ver = index["d_ver"][inds] # Check that these variables are consistent if not _isuniform(_config): @@ -649,7 +652,14 @@ def _calc_config(index): if id == 23: # change in "n_cells" doesn't matter lob = np.unique(_beams_cy) - beams = list(map(_beams_cy_int2dict, lob, 23 * np.ones(lob.size))) + beams = list( + map( + _beams_cy_int2dict, + lob, + 23 * np.ones(lob.size), + _data_ver[0] * np.ones(lob.size), + ) + ) if all([d["cy"] for d in beams]) and all([d["n_beams"] for d in beams]): err = False if err: @@ -657,9 +667,10 @@ def _calc_config(index): # Now that we've confirmed they are the same: config[id] = _headconfig_int2dict(_config[0], mode=type) - config[id].update(_beams_cy_int2dict(_beams_cy[0], id)) + config[id].update(_beams_cy_int2dict(_beams_cy[0], id, _data_ver[0])) config[id]["_config"] = _config[0] config[id]["_beams_cy"] = _beams_cy[0] + config[id]["_d_ver"] = _data_ver[0] config[id]["type"] = type config[id].pop("cy", None) diff --git a/mhkit/dolfyn/rotate/signature.py b/mhkit/dolfyn/rotate/signature.py index 3619be60f..662d58111 100644 --- a/mhkit/dolfyn/rotate/signature.py +++ b/mhkit/dolfyn/rotate/signature.py @@ -6,6 +6,44 @@ from numpy.linalg import inv +def _update_orientation_matrix(omat, reverse: bool): + # Take the transpose of the orientation to get the inst->earth rotation + # matrix. + rmat = np.rollaxis(omat.data, 1) + + _dcheck = rotb._check_rotmat_det(rmat) + if not _dcheck.all(): + warnings.warn( + "Invalid orientation matrix (determinant != 1) at indices: {}. " + "If rotated, data at these indices will be erroneous.".format( + np.nonzero(~_dcheck)[0] + ), + UserWarning, + ) + + # The dictionary of rotation matrices for different sized arrays. + rmd = { + 3: rmat, + } + + # The 4-row rotation matrix assume that rows 0,1 are u,v, + # and 2,3 are independent estimates of w. + tmp = rmd[4] = np.zeros((4, 4, rmat.shape[-1]), dtype=np.float64) + tmp[:3, :3] = rmat + # Copy row 2 to 3 + tmp[3, :2] = rmat[2, :2] + tmp[3, 3] = rmat[2, 2] + # Extend rows 0,1 + tmp[0, 2:] = rmat[0, 2] / 2 + tmp[1, 2:] = rmat[1, 2] / 2 + + if reverse: + # 3-element inverse handled by sumstr definition (transpose) + rmd[4] = np.moveaxis(inv(np.moveaxis(rmd[4], -1, 0)), 0, -1) + + return rmd + + def _inst2earth(adcpo, reverse=False, rotate_vars=None, force=False): """ Rotate data in an ADCP object to the earth from the instrument @@ -55,60 +93,40 @@ def _inst2earth(adcpo, reverse=False, rotate_vars=None, force=False): "Data must be in the '%s' frame when using this function" % cs_now ) - if "orientmat" in adcpo: - omat = adcpo["orientmat"] - elif "orientmat_avg" in adcpo: - omat = adcpo["orientmat_avg"] - elif "time" in adcpo: - omat = _euler2orient( - adcpo["time"], - adcpo["heading"].values, - adcpo["pitch"].values, - adcpo["roll"].values, - ) - elif "time_avg" in adcpo: - omat = _euler2orient( - adcpo["time_avg"], - adcpo["heading_avg"].values, - adcpo["pitch_avg"].values, - adcpo["roll_avg"].values, - ) - - # Take the transpose of the orientation to get the inst->earth rotation - # matrix. - rmat = np.rollaxis(omat.data, 1) - - _dcheck = rotb._check_rotmat_det(rmat) - if not _dcheck.all(): - warnings.warn( - "Invalid orientation matrix (determinant != 1) at indices: {}. " - "If rotated, data at these indices will be erroneous.".format( - np.nonzero(~_dcheck)[0] - ), - UserWarning, - ) - - # The dictionary of rotation matrices for different sized arrays. - rmd = { - 3: rmat, - } - - # The 4-row rotation matrix assume that rows 0,1 are u,v, - # and 2,3 are independent estimates of w. - tmp = rmd[4] = np.zeros((4, 4, rmat.shape[-1]), dtype=np.float64) - tmp[:3, :3] = rmat - # Copy row 2 to 3 - tmp[3, :2] = rmat[2, :2] - tmp[3, 3] = rmat[2, 2] - # Extend rows 0,1 - tmp[0, 2:] = rmat[0, 2] / 2 - tmp[1, 2:] = rmat[1, 2] / 2 - - if reverse: - # 3-element inverse handled by sumstr definition (transpose) - rmd[4] = np.moveaxis(inv(np.moveaxis(rmd[4], -1, 0)), 0, -1) + # Make sure there is an orientation matrix in the dataset + tags = ["", "_avg", "_echo", "_b5", "_bt"] + omat_list = ["orientmat" + tg for tg in tags] + orientmat = None + # Fetch the stored orientation matrix if there is one + # Always want the default to be "orientmat" or "orientmat_avg" + for tg, nm in zip(tags, omat_list): + if nm in adcpo: + orientmat = adcpo[nm] + break + # Recalculate the orientation matrix if there is not + # Always want the default to be "orientmat" or "orientmat_avg" + # Primarily for backwards-compatibility + if orientmat is None: + for tg in tags: + if "time" + tg in adcpo: + orientmat = _euler2orient( + adcpo["time" + tg], + adcpo["heading" + tg].values, + adcpo["pitch" + tg].values, + adcpo["roll" + tg].values, + ) + break for nm in rotate_vars: + # Fetch the correct orientation matrix + if "_" in nm: + tag = "_" + nm.split("_")[-1] + else: + tag = "" + # Pick the tagged orientation matrix, otherwise use the default + omat = getattr(adcpo, "orientmat" + tag, orientmat) + rmd = _update_orientation_matrix(omat, reverse=reverse) + dat = adcpo[nm].values n = dat.shape[0] # Nortek documents sign change for upside-down instruments diff --git a/mhkit/dolfyn/velocity.py b/mhkit/dolfyn/velocity.py index ce8372816..c8574ef7e 100644 --- a/mhkit/dolfyn/velocity.py +++ b/mhkit/dolfyn/velocity.py @@ -634,11 +634,13 @@ def _bin(self, function, raw_ds, out_ds=None, names=None): # Non-time variables if not any([ar for ar in dims_list if "time" in ar]): out_ds[ky] = raw_ds[ky] - # Variables with time coordinate + # Don't take the variance of orientation matrices + if (function == self.variance) and ("orientmat" in ky): + continue + # Variables with time coordinate (use different n_bin for ADV system vars) n_bin = self.n_bin if "time_sys" not in dims_list else n_bin_sys out_ds[ky + suffix] = xr.DataArray( function(raw_ds[ky].values, n_bin=n_bin), - # coords=coords_dict, dims=raw_ds[ky].dims, attrs=raw_ds[ky].attrs, ).astype("float32") diff --git a/mhkit/tests/dolfyn/test_api.py b/mhkit/tests/dolfyn/test_api.py index 1769d8135..fc44dcd6f 100644 --- a/mhkit/tests/dolfyn/test_api.py +++ b/mhkit/tests/dolfyn/test_api.py @@ -8,7 +8,7 @@ class api_testcase(unittest.TestCase): - def test_repr(self): + def test_repr(self, make_data=False): _str = [] for dat, fnm in [ (vec, rfnm("vector_data01.repr.txt")), @@ -16,10 +16,12 @@ def test_repr(self): (rdi, rfnm("RDI_test01.repr.txt")), ]: _str = dat.velds.__repr__() + if make_data: with open(fnm, "w") as fl: fl.write(_str) - else: - with open(fnm, "r") as fl: - test_str = fl.read() - assert test_str == _str + return + + with open(fnm, "r") as fl: + test_str = fl.read() + assert test_str == _str diff --git a/mhkit/tests/dolfyn/test_motion.py b/mhkit/tests/dolfyn/test_motion.py index 9cb682524..2a9cc7834 100644 --- a/mhkit/tests/dolfyn/test_motion.py +++ b/mhkit/tests/dolfyn/test_motion.py @@ -91,7 +91,12 @@ def test_duty_cycle(self): for i in range(n_ensembles): cd0 = tdc.isel(time=slice(n_burst * i, n_burst * i + n_burst)) cd0 = correct_motion(cd0, accel_filtfreq=0.03, to_earth=False) - cd = xr.merge((cd, cd0), combine_attrs="no_conflicts") + cd = xr.merge( + (cd, cd0), + combine_attrs="no_conflicts", + join="outer", + compat="no_conflicts", + ) cd.attrs["duty_cycle_n_burst"] = n_burst cd_ENU = cd.velds.rotate2("earth", inplace=False) diff --git a/mhkit/tests/dolfyn/test_read_adp.py b/mhkit/tests/dolfyn/test_read_adp.py index 1ac29179f..e04c7be17 100644 --- a/mhkit/tests/dolfyn/test_read_adp.py +++ b/mhkit/tests/dolfyn/test_read_adp.py @@ -47,6 +47,7 @@ dat_sig_dp1_ice = load("Sig500_dp_ice2.nc") dat_sig_dp2_echo = load("Sig1000_dp_echo1.nc") dat_sig_dp2_avg = load("Sig1000_dp_echo2.nc") +dat_sigvm_bt_09 = load("sigvm_bt_09.nc") class io_adp_testcase(unittest.TestCase): @@ -172,6 +173,7 @@ def test_io_nortek2(self): td_sig_dp2_echo, td_sig_dp2_avg = read( "Sig1000_dp_echo.ad2cp", rebuild_index=True ) + td_sigvm_bt_09 = read("sigvm_bt_09.AD2CP", rebuild_index=True) with pytest.warns(UserWarning): # This issues a warning... @@ -197,6 +199,7 @@ def test_io_nortek2(self): os.remove(tb.rfnm("Sig1000_BadTime01.ad2cp.index")) os.remove(tb.exdt("Sig500_dp_ice.ad2cp.index")) os.remove(tb.exdt("Sig1000_dp_echo.ad2cp.index")) + os.remove(tb.exdt("sigvm_bt_09.AD2CP.index")) if make_data: save(td_sig, "BenchFile01.nc") @@ -215,6 +218,7 @@ def test_io_nortek2(self): save(td_sig_dp1_ice, "Sig500_dp_ice2.nc") save(td_sig_dp2_echo, "Sig1000_dp_echo1.nc") save(td_sig_dp2_avg, "Sig1000_dp_echo2.nc") + save(td_sigvm_bt_09, "sigvm_bt_09.nc") return assert_allclose(td_sig, dat_sig, atol=1e-6) @@ -226,13 +230,14 @@ def test_io_nortek2(self): assert_allclose(td_sig_raw_avg, dat_sig_raw_avg, atol=1e-6) assert_allclose(td_sig_avg, dat_sig_avg, atol=1e-6) assert_allclose(td_sig_rt, dat_sig_rt, atol=1e-6) - assert_allclose(td_sig5_leiw, dat_sig5_leiw, atol=1e-6) assert_allclose(td_sig_skip, dat_sig_skip, atol=1e-6) assert_allclose(td_sig_badt, dat_sig_badt, atol=1e-6) + assert_allclose(td_sig5_leiw, dat_sig5_leiw, atol=1e-6) assert_allclose(td_sig_dp1_all, dat_sig_dp1_all, atol=1e-6) assert_allclose(td_sig_dp1_ice, dat_sig_dp1_ice, atol=1e-6) assert_allclose(td_sig_dp2_echo, dat_sig_dp2_echo, atol=1e-6) assert_allclose(td_sig_dp2_avg, dat_sig_dp2_avg, atol=1e-6) + assert_allclose(td_sigvm_bt_09, dat_sigvm_bt_09, atol=1e-6) def test_nortek2_crop(self): # Test file cropping function diff --git a/mhkit/tests/dolfyn/test_rotate_adp.py b/mhkit/tests/dolfyn/test_rotate_adp.py index 80dec8ba6..6e426353f 100644 --- a/mhkit/tests/dolfyn/test_rotate_adp.py +++ b/mhkit/tests/dolfyn/test_rotate_adp.py @@ -41,19 +41,15 @@ def test_rotate_inst2beam(self): rotate2(td_sig, "beam", inplace=True) td_sig_i = load("Sig1000_IMU_rotate_beam2inst.nc") rotate2(td_sig_i, "beam", inplace=True) - td_sig_ie = load("Sig500_Echo_earth2inst.nc") - rotate2(td_sig_ie, "beam", inplace=True) if make_data: save(td_awac, "AWAC_test01_inst2beam.nc") - save(td_sig_ie, "Sig500_Echo_inst2beam.nc") return cd_td = tr.dat_rdi.copy(deep=True) cd_awac = load("AWAC_test01_inst2beam.nc") cd_sig = tr.dat_sig.copy(deep=True) cd_sig_i = tr.dat_sig_i.copy(deep=True) - cd_sig_ie = load("Sig500_Echo_inst2beam.nc") # # The reverse RDI rotation doesn't work b/c of NaN's in one beam # # that propagate to others, so we impose that here. @@ -63,86 +59,81 @@ def test_rotate_inst2beam(self): assert_allclose(td_awac, cd_awac, atol=1e-5) assert_allclose(td_sig, cd_sig, atol=1e-5) assert_allclose(td_sig_i, cd_sig_i, atol=1e-5) - assert_allclose(td_sig_ie, cd_sig_ie, atol=1e-5) def test_rotate_inst2earth(self): td = rotate2(tr.dat_rdi.copy(deep=True), "earth", inplace=False) - tdwr2 = rotate2(tr.dat_wr2.copy(deep=True), "earth", inplace=False) + td_wr2 = rotate2(tr.dat_wr2.copy(deep=True), "earth", inplace=False) td_sig = load("BenchFile01_rotate_beam2inst.nc") rotate2(td_sig, "earth", inplace=True) td_sig_i = load("Sig1000_IMU_rotate_beam2inst.nc") rotate2(td_sig_i, "earth", inplace=True) + td_sig_ieb = load("VelEchoBT01_rotate_beam2inst.nc") + rotate2(td_sig_ieb, "earth", inplace=True) + # Test that loss of orientmat doesn't error out + td_sig_o = tr.dat_sig_ie.copy(deep=True) + rotate2(td_sig_o, "inst", inplace=True) + td_sig_o = rotate2(td_sig_o.drop_vars("orientmat"), "earth", inplace=False) if make_data: - # AWAC & Sig500 are loaded in earth, so save them in inst - td_awac = tr.dat_awac.copy(deep=True) - rotate2(td_awac, "inst", inplace=True) - td_sig_ie = tr.dat_sig_ie.copy(deep=True) - rotate2(td_sig_ie, "inst", inplace=True) - - save(td_awac, "AWAC_test01_earth2inst.nc") save(td, "RDI_test01_rotate_inst2earth.nc") - save(tdwr2, "winriver02_rotate_ship2earth.nc") + save(td_wr2, "winriver02_rotate_ship2earth.nc") save(td_sig, "BenchFile01_rotate_inst2earth.nc") save(td_sig_i, "Sig1000_IMU_rotate_inst2earth.nc") - save(td_sig_ie, "Sig500_Echo_earth2inst.nc") + save(td_sig_ieb, "VelEchoBT01_rotate_inst2earth.nc") return - td_awac = rotate2(load("AWAC_test01_earth2inst.nc"), "earth", inplace=False) - cd_awac = tr.dat_awac.copy(deep=True) - td_sig_ie = rotate2(load("Sig500_Echo_earth2inst.nc"), "earth", inplace=False) - cd_sig_ie = tr.dat_sig_ie.copy(deep=True) - td_sig_o = rotate2(cd_sig_ie.drop_vars("orientmat"), "earth", inplace=False) - cd = load("RDI_test01_rotate_inst2earth.nc") cdwr2 = load("winriver02_rotate_ship2earth.nc") cd_sig = load("BenchFile01_rotate_inst2earth.nc") cd_sig_i = load("Sig1000_IMU_rotate_inst2earth.nc") + cd_sig_ieb = load("VelEchoBT01_rotate_inst2earth.nc") assert_allclose(td, cd, atol=1e-5) - assert_allclose(tdwr2, cdwr2, atol=1e-5) - assert_allclose(td_awac, cd_awac, atol=1e-5) + assert_allclose(td_wr2, cdwr2, atol=1e-5) assert_allclose(td_sig, cd_sig, atol=1e-5) assert_allclose(td_sig_i, cd_sig_i, atol=1e-5) - assert_allclose(td_sig_ie, cd_sig_ie, atol=1e-5) - npt.assert_allclose(td_sig_o.vel, cd_sig_ie.vel, atol=1e-5) + assert_allclose(td_sig_ieb, cd_sig_ieb, atol=1e-5) + npt.assert_allclose(td_sig_o.vel, tr.dat_sig_ie.vel, atol=1e-5) def test_rotate_earth2inst(self): td_rdi = load("RDI_test01_rotate_inst2earth.nc") rotate2(td_rdi, "inst", inplace=True) - tdwr2 = load("winriver02_rotate_ship2earth.nc") - rotate2(tdwr2, "inst", inplace=True) - + td_wr2 = load("winriver02_rotate_ship2earth.nc") + rotate2(td_wr2, "inst", inplace=True) td_awac = tr.dat_awac.copy(deep=True) - rotate2(td_awac, "inst", inplace=True) # AWAC is in earth coords + rotate2(td_awac, "inst", inplace=True) td_sig = load("BenchFile01_rotate_inst2earth.nc") rotate2(td_sig, "inst", inplace=True) td_sig_i = load("Sig1000_IMU_rotate_inst2earth.nc") rotate2(td_sig_i, "inst", inplace=True) + td_sig_dp1_ice = tr.dat_sig_dp1_ice.copy(deep=True) + rotate2(td_sig_dp1_ice, "inst", inplace=True) - # ship and inst are considered equivalent in dolfyn cd_rdi = load("RDI_test01_rotate_beam2inst.nc") cd_wr2 = tr.dat_wr2.copy(deep=True) cd_wr2.attrs["coord_sys"] = "inst" cd_awac = load("AWAC_test01_earth2inst.nc") cd_sig = load("BenchFile01_rotate_beam2inst.nc") cd_sig_i = load("Sig1000_IMU_rotate_beam2inst.nc") + cd_sig_dp1_ice = load("Sig500_dp_ice2_rotate_earth2inst.nc") # Just check that these run without error - td_sig_avg = load("Sig100_avg.nc") - rotate2(td_sig_avg, "inst", inplace=True) - td_sig_dp1_ice = load("Sig500_dp_ice2.nc") - rotate2(td_sig_dp1_ice, "inst", inplace=True) - td_aqd = load("AQD_test01.nc") - rotate2(td_aqd, "inst", inplace=True) + td_sig_avg = rotate2(tr.dat_sig_avg.copy(deep=True), "inst", inplace=False) + td_aqd = rotate2(tr.dat_aqd.copy(deep=True), "inst", inplace=False) + + if make_data: + save(td_awac, "AWAC_test01_earth2inst.nc") + save(td_sig_dp1_ice, "Sig500_dp_ice2_rotate_earth2inst.nc") + return assert_allclose(td_rdi, cd_rdi, atol=1e-5) - assert_allclose(tdwr2, cd_wr2, atol=1e-5) + assert_allclose(td_wr2, cd_wr2, atol=1e-5) assert_allclose(td_awac, cd_awac, atol=1e-5) assert_allclose(td_sig, cd_sig, atol=1e-5) # known failure due to orientmat, see test_vs_nortek # assert_allclose(td_sig_i, cd_sig_i, atol=1e-3) npt.assert_allclose(td_sig_i.accel.values, cd_sig_i.accel.values, atol=1e-3) + assert_allclose(td_sig_dp1_ice, cd_sig_dp1_ice, atol=1e-5) def test_rotate_earth2principal(self): td_rdi = load("RDI_test01_rotate_inst2earth.nc") diff --git a/mhkit/tests/dolfyn/test_vs_nortek.py b/mhkit/tests/dolfyn/test_vs_nortek.py index d0ca8ff89..4c996cdbb 100644 --- a/mhkit/tests/dolfyn/test_vs_nortek.py +++ b/mhkit/tests/dolfyn/test_vs_nortek.py @@ -102,7 +102,7 @@ def rotate(axis): ) # bottom-track - assert_allclose(td_sig_ieb.vel_bt.values, vel_bt[axis][..., :nens], atol=5e-3) + assert_allclose(td_sig_ieb.vel_bt.values, vel_bt[axis][..., :nens], atol=1e-5) class nortek_testcase(unittest.TestCase):