From 382921b9ff40665ff0c7cf0a50ce11312d8e123e Mon Sep 17 00:00:00 2001 From: Omar Omar Date: Wed, 11 Nov 2026 18:20:45 -0600 Subject: [PATCH 1/4] python(feat): support custom time formats on CSV and Parquet imports --- python/changelog.d/884.feat.md | 1 + .../_tests/resources/test_data_imports.py | 58 +++++++++++++++++++ .../lib/sift_client/sift_types/data_import.py | 23 ++++++++ 3 files changed, 82 insertions(+) create mode 100644 python/changelog.d/884.feat.md diff --git a/python/changelog.d/884.feat.md b/python/changelog.d/884.feat.md new file mode 100644 index 000000000..fe6383ddf --- /dev/null +++ b/python/changelog.d/884.feat.md @@ -0,0 +1 @@ +Added `TimeFormat.CUSTOM` and `custom_format` on `CsvTimeColumn` and `ParquetTimeColumn` for importing time columns in a custom Unix date format, such as `%m-%d-%Y %H:%M:%S`. Importing with `TimeFormat.CUSTOM` requires a non-empty `custom_format`; TDMS and HDF5 imports do not support it. diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 75aa9e465..d46c0d048 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -220,6 +220,11 @@ def test_run_id_takes_precedence(self): proto = config._to_proto() assert proto.run_id == "run_123" + def test_custom_time_format_rejected(self): + config = TdmsImportConfig(asset_name="a", time_format=TimeFormat.CUSTOM) + with pytest.raises(ValueError, match="only supported for CSV and Parquet"): + config._to_proto() + class TestHdf5Config: def test_to_proto(self): @@ -709,6 +714,27 @@ def test_from_proto_round_trip(self, csv_config): assert restored.time_column.column == csv_config.time_column.column assert len(restored.data_columns) == len(csv_config.data_columns) + def test_custom_format_round_trip(self, csv_config): + csv_config.time_column = CsvTimeColumn( + column=4, format=TimeFormat.CUSTOM, custom_format="%m-%d-%Y %H:%M:%S" + ) + proto = csv_config._to_proto() + assert proto.time_column.format == TimeFormat.CUSTOM.value + assert proto.time_column.custom_format == "%m-%d-%Y %H:%M:%S" + + restored = CsvImportConfig._from_proto(proto) + assert restored.time_column.format == TimeFormat.CUSTOM + assert restored.time_column.custom_format == "%m-%d-%Y %H:%M:%S" + + def test_custom_format_requires_custom_format(self): + col = CsvTimeColumn(column=1, format=TimeFormat.CUSTOM) + with pytest.raises(ValueError, match="custom_format"): + col._to_proto() + + def test_custom_format_defaults_to_empty(self, csv_config): + assert csv_config.time_column.custom_format == "" + assert csv_config._to_proto().time_column.custom_format == "" + class TestParquetToProto: def test_flat_dataset_to_proto(self, parquet_config): @@ -763,6 +789,33 @@ def test_missing_format_raises(self): with pytest.raises(ValueError, match="format must be set"): col._to_proto() + def test_custom_format_round_trip(self): + col = ParquetTimeColumn( + path="timestamp", format=TimeFormat.CUSTOM, custom_format="%m-%d-%Y %H:%M:%S" + ) + proto = col._to_proto() + assert proto.custom_format == "%m-%d-%Y %H:%M:%S" + + restored = ParquetTimeColumn._from_proto(proto) + assert restored.format == TimeFormat.CUSTOM + assert restored.custom_format == "%m-%d-%Y %H:%M:%S" + + def test_custom_format_requires_custom_format(self): + col = ParquetTimeColumn(path="timestamp", format=TimeFormat.CUSTOM) + with pytest.raises(ValueError, match="custom_format"): + col._to_proto() + + def test_custom_format_without_custom_format_reads_back(self): + proto = ParquetTimeColumnProto(path="timestamp", format=TimeFormat.CUSTOM.value) + col = ParquetTimeColumn._from_proto(proto) + assert col.format == TimeFormat.CUSTOM + assert col.custom_format == "" + + def test_custom_format_defaults_to_empty(self): + col = ParquetTimeColumn(path="timestamp", format=TimeFormat.ABSOLUTE_RFC3339) + assert col.custom_format == "" + assert col._to_proto().custom_format == "" + class TestResolveDataTypeKey: def test_parquet_requires_data_type(self): @@ -996,6 +1049,11 @@ def test_hdf5_to_proto_rejects_missing_format(self): with pytest.raises(ValueError, match="time_format is required"): config._to_proto() + def test_hdf5_to_proto_rejects_custom_format(self): + config = Hdf5ImportConfig(asset_name="a", data=[], time_format=TimeFormat.CUSTOM) + with pytest.raises(ValueError, match="only supported for CSV and Parquet"): + config._to_proto() + class TestDetectConfigRelativeStartTime: START = pd.Timestamp("2026-01-01T00:00:00.000000001Z") diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index b75ba5134..715f0a42e 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -92,6 +92,7 @@ class TimeFormat(Enum): ABSOLUTE_UNIX_MILLISECONDS = TimeFormatProto.TIME_FORMAT_ABSOLUTE_UNIX_MILLISECONDS ABSOLUTE_UNIX_MICROSECONDS = TimeFormatProto.TIME_FORMAT_ABSOLUTE_UNIX_MICROSECONDS ABSOLUTE_UNIX_NANOSECONDS = TimeFormatProto.TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS + CUSTOM = TimeFormatProto.TIME_FORMAT_CUSTOM @property def is_relative(self) -> bool: @@ -143,6 +144,8 @@ class TimeColumnBase(BaseModel, ABC): (caller-passed ``time_format`` > server-detected format > ``ABSOLUTE_UNIX_NANOSECONDS`` fallback). Importing without a format set raises ``ValueError``. + custom_format: Unix date format, e.g. ``%m-%d-%Y %H:%M:%S``. Required at + import when ``format`` is ``TimeFormat.CUSTOM``; ignored otherwise. relative_start_time: Required when using a relative time format. Pass a ``pd.Timestamp`` for nanosecond precision. """ @@ -151,8 +154,18 @@ class TimeColumnBase(BaseModel, ABC): model_config = ConfigDict(arbitrary_types_allowed=True) format: TimeFormat | None = None + custom_format: str = "" relative_start_time: pd.Timestamp | datetime | None = None + def _check_custom_format(self) -> None: + """Raise if the format is custom with no format string. + + Checked at import rather than construction, so a config can be read + back or edited before ``custom_format`` is set. + """ + if self.format == TimeFormat.CUSTOM and not self.custom_format.strip(): + raise ValueError("'custom_format' is required when using TimeFormat.CUSTOM.") + @model_validator(mode="after") def _check_relative_start_time(self) -> TimeColumnBase: if self.format is not None and self.format.is_relative and self.relative_start_time is None: @@ -361,9 +374,11 @@ def _to_proto(self) -> CsvTimeColumnProto: "Pass time_format to detect_config/import_from_path, or set " "config.time_column.format explicitly." ) + self._check_custom_format() proto = CsvTimeColumnProto( column_number=self.column, format=self.format.value, + custom_format=self.custom_format, ) if self.relative_start_time is not None: proto.relative_start_time.CopyFrom(to_timestamp_pb(self.relative_start_time)) @@ -442,6 +457,7 @@ def _from_proto(cls, proto: CsvConfigProto) -> CsvImportConfig: time_column = CsvTimeColumn( column=proto.time_column.column_number, format=time_format, + custom_format=proto.time_column.custom_format, relative_start_time=relative_start_time, ) data_columns = [ @@ -493,9 +509,11 @@ def _to_proto(self) -> ParquetTimeColumnProto: "Pass time_format to detect_config/import_from_path, or set " "config.time_column.format explicitly." ) + self._check_custom_format() proto = ParquetTimeColumnProto( path=self.path, format=self.format.value, + custom_format=self.custom_format, ) if self.relative_start_time is not None: proto.relative_start_time.CopyFrom(to_timestamp_pb(self.relative_start_time)) @@ -511,6 +529,7 @@ def _from_proto(cls, proto: ParquetTimeColumnProto) -> ParquetTimeColumn: return cls( path=proto.path or "", format=fmt, + custom_format=proto.custom_format, relative_start_time=relative_start_time, ) @@ -887,6 +906,8 @@ def _to_proto(self) -> TDMSConfigProto: proto.start_time_override.CopyFrom(to_timestamp_pb(self.start_time_override)) proto.fallback_method = self.fallback_method.value if self.time_format is not None: + if self.time_format == TimeFormat.CUSTOM: + raise ValueError("TimeFormat.CUSTOM is only supported for CSV and Parquet imports.") proto.time_format = self.time_format.value if self.relative_start_time is not None: proto.relative_start_time.CopyFrom(to_timestamp_pb(self.relative_start_time)) @@ -1045,6 +1066,8 @@ def _to_proto(self) -> Hdf5ConfigProto: "config.time_format explicitly (e.g. TimeFormat.ABSOLUTE_UNIX_NANOSECONDS " "or TimeFormat.ABSOLUTE_DATETIME) before importing." ) + if self.time_format == TimeFormat.CUSTOM: + raise ValueError("TimeFormat.CUSTOM is only supported for CSV and Parquet imports.") proto = Hdf5ConfigProto( asset_name=self.asset_name, run_name=self.run_name or "", From b9df5b5c339c4dcf9d2f749cc35629ac4cf5fa8b Mon Sep 17 00:00:00 2001 From: Omar Omar Date: Wed, 11 Nov 2026 18:21:01 -0600 Subject: [PATCH 2/4] rust(feat): support custom time formats in sift-cli CSV and Parquet imports --- rust/crates/sift_cli/CHANGELOG.md | 3 ++ .../assets/docs/src/data/importing.md | 4 ++ rust/crates/sift_cli/src/cli/mod.rs | 15 ++++++ rust/crates/sift_cli/src/cli/time.rs | 3 ++ rust/crates/sift_cli/src/cmd/import/csv.rs | 54 ++++++++++++++++++- .../sift_cli/src/cmd/import/hdf5/import.rs | 2 +- .../sift_cli/src/cmd/import/hdf5/tests.rs | 12 +++++ .../import/parquet/detect_parquet_schema.rs | 20 ++++--- .../sift_cli/src/cmd/import/parquet/tests.rs | 45 ++++++++++++++++ .../src/cmd/import/tdms/detect_tdms_config.rs | 1 + .../sift_cli/src/cmd/import/tdms/tests.rs | 12 +++++ rust/crates/sift_cli/src/cmd/import/utils.rs | 33 ++++++++++++ 12 files changed, 195 insertions(+), 9 deletions(-) diff --git a/rust/crates/sift_cli/CHANGELOG.md b/rust/crates/sift_cli/CHANGELOG.md index 471b90fac..cd5ed4a09 100644 --- a/rust/crates/sift_cli/CHANGELOG.md +++ b/rust/crates/sift_cli/CHANGELOG.md @@ -30,6 +30,9 @@ This project adheres to [Semantic Versioning](http://semver.org/). - `create_rule` now documents the annotation action, and both rule tools refuse a rule without exactly one condition instead of passing on Sift's internal error. +- `sift-cli import csv` and `sift-cli import parquet` accept `--time-format custom` + with `--time-format-str` for timestamps in a custom Unix date format, such as + `%m-%d-%Y %H:%M:%S`. ## [v0.9.0] - November 16, 2026 diff --git a/rust/crates/sift_cli/assets/docs/src/data/importing.md b/rust/crates/sift_cli/assets/docs/src/data/importing.md index 498487428..947ce64d7 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/importing.md +++ b/rust/crates/sift_cli/assets/docs/src/data/importing.md @@ -104,11 +104,15 @@ timestamps are encoded. Supported values: | `relative-seconds` | Offset from a start time, in seconds | | `relative-minutes` | Offset from a start time, in minutes | | `relative-hours` | Offset from a start time, in hours | +| `custom` (CSV and Parquet) | Custom Unix date format set with `--time-format-str`, e.g. `%m-%d-%Y %H:%M:%S` | When the time format is **relative**, supply a start time with `--relative-start-time` (`-s`) in RFC 3339 so Sift can anchor the offsets to absolute time. +When the time format is **custom**, supply the format with `--time-format-str`. +Only `import csv` and `import parquet` accept `custom`. + ### Channel data types Where a command lets you declare channel types, the accepted values are: diff --git a/rust/crates/sift_cli/src/cli/mod.rs b/rust/crates/sift_cli/src/cli/mod.rs index f2ce2c3cc..87b99c1ff 100644 --- a/rust/crates/sift_cli/src/cli/mod.rs +++ b/rust/crates/sift_cli/src/cli/mod.rs @@ -482,6 +482,10 @@ pub struct ImportCsvArgs { #[arg(short = 'f', long, default_value_t = TimeFormat::default(), hide_possible_values = true, help_heading = "Time")] pub time_format: TimeFormat, + /// Unix date format, e.g. "%m-%d-%Y %H:%M:%S", used with `--time-format custom` + #[arg(long, default_value_t = String::default(), hide_possible_values = true, help_heading = "Time")] + pub time_format_str: String, + /// Start time (RFC3339) to use if time format is relative #[arg(short = 's', long, help_heading = "Time")] pub relative_start_time: Option, @@ -629,6 +633,10 @@ pub struct FlatDatasetArgs { #[arg(short = 'f', long, hide_possible_values = true, help_heading = "Time")] pub time_format: Option, + /// Unix date format, e.g. "%m-%d-%Y %H:%M:%S", used with `--time-format custom` + #[arg(long, default_value_t = String::default(), help_heading = "Time")] + pub time_format_str: String, + /// Start time (RFC3339) to use if time format is relative #[arg(short = 's', long, help_heading = "Time")] pub relative_start_time: Option, @@ -670,6 +678,10 @@ pub struct ChannelPerRowCommonArgs { #[arg(short = 'f', long, hide_possible_values = true, help_heading = "Time")] pub time_format: Option, + /// Unix date format, e.g. "%m-%d-%Y %H:%M:%S", used with `--time-format custom` + #[arg(long, default_value_t = String::default(), help_heading = "Time")] + pub time_format_str: String, + /// Start time (RFC3339) to use if time format is relative #[arg(short = 's', long, help_heading = "Time")] pub relative_start_time: Option, @@ -720,6 +732,7 @@ pub struct ChannelPerRowArgs { pub mode: ChannelMode, pub time_path: Option, pub time_format: Option, + pub time_format_str: String, pub relative_start_time: Option, pub data_path: String, pub channel_name: Option, @@ -737,6 +750,7 @@ impl From for ChannelPerRowArgs { mode: ChannelMode::Single, time_path: args.common.time_path, time_format: args.common.time_format, + time_format_str: args.common.time_format_str, relative_start_time: args.common.relative_start_time, data_path: args.common.data_path, channel_name: Some(args.channel_name), @@ -756,6 +770,7 @@ impl From for ChannelPerRowArgs { mode: ChannelMode::Multi, time_path: args.common.time_path, time_format: args.common.time_format, + time_format_str: args.common.time_format_str, relative_start_time: args.common.relative_start_time, data_path: args.common.data_path, channel_name: None, diff --git a/rust/crates/sift_cli/src/cli/time.rs b/rust/crates/sift_cli/src/cli/time.rs index a70c1dac0..9809e9298 100644 --- a/rust/crates/sift_cli/src/cli/time.rs +++ b/rust/crates/sift_cli/src/cli/time.rs @@ -18,6 +18,7 @@ pub enum TimeFormat { RelativeSeconds, RelativeMinutes, RelativeHours, + Custom, } impl From for ProtoTimeFormat { @@ -35,6 +36,7 @@ impl From for ProtoTimeFormat { TimeFormat::AbsoluteUnixMilliseconds => Self::AbsoluteUnixMilliseconds, TimeFormat::AbsoluteUnixMicroseconds => Self::AbsoluteUnixMicroseconds, TimeFormat::AbsoluteUnixNanoseconds => Self::AbsoluteUnixNanoseconds, + TimeFormat::Custom => Self::Custom, } } } @@ -54,6 +56,7 @@ impl Display for TimeFormat { Self::RelativeSeconds => write!(f, "relative-seconds"), Self::RelativeMinutes => write!(f, "relative-minutes"), Self::RelativeHours => write!(f, "relative-hours"), + Self::Custom => write!(f, "custom"), } } } diff --git a/rust/crates/sift_cli/src/cmd/import/csv.rs b/rust/crates/sift_cli/src/cmd/import/csv.rs index 522c9e951..6cee21c5e 100644 --- a/rust/crates/sift_cli/src/cmd/import/csv.rs +++ b/rust/crates/sift_cli/src/cmd/import/csv.rs @@ -106,7 +106,11 @@ fn create_data_import_request( .context("keep in mind that --units and --descriptions can be empty strings"); } - validate_time_format(args.time_format, &args.relative_start_time)?; + validate_time_format( + args.time_format, + &args.relative_start_time, + Some(&args.time_format_str), + )?; let relative_start_time = match &args.relative_start_time { Some(start) => { @@ -343,7 +347,7 @@ fn create_data_import_request( relative_start_time, column_number: args.time_column as u32, format: PbTimeFormat::from(args.time_format).into(), - custom_format: String::default(), + custom_format: args.time_format_str.clone(), }), ..Default::default() }), @@ -391,6 +395,7 @@ mod test_create_data_import_request { bit_field_config: Vec::default(), time_column: 1, time_format: TimeFormat::default(), + time_format_str: String::default(), relative_start_time: None, wait: false, preview: false, @@ -444,6 +449,7 @@ mod test_create_data_import_request { bit_field_config: Vec::default(), time_column: 1, time_format: TimeFormat::default(), + time_format_str: String::default(), relative_start_time: None, wait: false, preview: false, @@ -457,6 +463,47 @@ mod test_create_data_import_request { assert!(csv_config.run_name.is_empty()); } + #[test] + fn custom_time_format() { + let test_csv = indoc! {" + time,channel + 10-04-2025 21:58:13,1.0 + "}; + let csv_reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(test_csv.as_bytes()); + + let req = create_data_import_request( + csv_reader, + &ImportCsvArgs { + path: PathBuf::default(), + asset: "test_asset".into(), + run: None, + run_id: None, + header_row: 1, + first_data_row: 2, + channel_column: Vec::default(), + data_type: Vec::default(), + unit: Vec::default(), + description: Vec::default(), + enum_config: Vec::default(), + bit_field_config: Vec::default(), + time_column: 1, + time_format: TimeFormat::Custom, + time_format_str: "%m-%d-%Y %H:%M:%S".into(), + relative_start_time: None, + wait: false, + preview: false, + leave_run_open: false, + }, + ) + .expect("expected Result::Ok"); + + let time_config = req.csv_config.unwrap().time_column.unwrap(); + assert_eq!(PbTimeFormat::Custom as i32, time_config.format); + assert_eq!("%m-%d-%Y %H:%M:%S", time_config.custom_format); + } + #[test] fn simple_type_override() { let test_csv = indoc! {" @@ -484,6 +531,7 @@ mod test_create_data_import_request { bit_field_config: Vec::default(), time_column: 1, time_format: TimeFormat::default(), + time_format_str: String::default(), relative_start_time: None, wait: false, preview: false, @@ -530,6 +578,7 @@ mod test_create_data_import_request { bit_field_config: Vec::default(), time_column: 1, time_format: TimeFormat::default(), + time_format_str: String::default(), relative_start_time: None, wait: false, preview: false, @@ -592,6 +641,7 @@ mod test_create_data_import_request { bit_field_config: Vec::default(), time_column: 1, time_format: TimeFormat::default(), + time_format_str: String::default(), relative_start_time: None, wait: false, preview: false, diff --git a/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs b/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs index a98fadaf5..6845ae1fb 100644 --- a/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs +++ b/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs @@ -132,7 +132,7 @@ pub fn build_hdf5_config(args: &ImportHdf5Args) -> Result { let time_format = args .time_format .ok_or_else(|| anyhow!("--time-format is required for HDF5 imports"))?; - validate_time_format(time_format, &args.relative_start_time) + validate_time_format(time_format, &args.relative_start_time, None) .context("validating time format for hdf5")?; let relative_start_time = match &args.relative_start_time { diff --git a/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs b/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs index 7215fb73f..ea7f57b3c 100644 --- a/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs @@ -122,6 +122,18 @@ fn build_hdf5_config_relative_time_format_requires_start() { ); } +#[test] +fn build_hdf5_config_custom_time_format_errors() { + let mut args = make_args(); + args.time_format = Some(TimeFormat::Custom); + let err = build_hdf5_config(&args).unwrap_err(); + assert!( + err.chain() + .any(|e| e.to_string().contains("only supported by `import csv`")), + "expected validation error, got: {err:#}" + ); +} + #[test] fn is_time_dataset_name_recognizes_known_names() { assert!(is_time_dataset_name("time")); diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/detect_parquet_schema.rs b/rust/crates/sift_cli/src/cmd/import/parquet/detect_parquet_schema.rs index 3c9cdce12..7577ec981 100644 --- a/rust/crates/sift_cli/src/cmd/import/parquet/detect_parquet_schema.rs +++ b/rust/crates/sift_cli/src/cmd/import/parquet/detect_parquet_schema.rs @@ -57,8 +57,12 @@ pub fn detect_flat_dataset_config( let (resolved_format, format_source) = resolve_time_format(args.time_format, time_field); - validate_time_format(resolved_format, &args.relative_start_time) - .context("validating time format")?; + validate_time_format( + resolved_format, + &args.relative_start_time, + Some(&args.time_format_str), + ) + .context("validating time format")?; let relative_start_time_input = match &args.relative_start_time { Some(start) => { @@ -74,7 +78,7 @@ pub fn detect_flat_dataset_config( relative_start_time: relative_start_time_input, path: time_path.clone(), format: TimeFormat::from(resolved_format).into(), - custom_format: String::default(), + custom_format: args.time_format_str.clone(), }); let mut data_columns = Vec::new(); @@ -176,8 +180,12 @@ pub fn detect_channel_per_row_config( let (resolved_format, format_source) = resolve_time_format(args.time_format, time_field); - validate_time_format(resolved_format, &args.relative_start_time) - .context("validating time format")?; + validate_time_format( + resolved_format, + &args.relative_start_time, + Some(&args.time_format_str), + ) + .context("validating time format")?; let relative_start_time = match &args.relative_start_time { Some(start) => { @@ -192,7 +200,7 @@ pub fn detect_channel_per_row_config( relative_start_time, path: time_path, format: TimeFormat::from(resolved_format).into(), - custom_format: String::default(), + custom_format: args.time_format_str.clone(), }); let data_field = arrow_schema diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs b/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs index fa434c9a6..f13f77e39 100644 --- a/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs @@ -38,6 +38,7 @@ fn make_test_args(time_path: Option<&str>, time_format: Option) -> F bit_field_config: vec![], time_path: time_path.map(|s| s.to_string()), time_format, + time_format_str: String::default(), relative_start_time: None, complex_types_mode: ComplexTypesMode::default(), } @@ -313,6 +314,49 @@ fn test_relative_time_format_without_start_time_returns_error() { ); } +#[test] +fn test_custom_time_format_without_time_format_str_returns_error() { + let batch = create_test_batch(); + let bytes = write_to_parquet_bytes(&batch); + let args = make_test_args(Some("time"), Some(TimeFormat::Custom)); + + let err = detect_parquet_schema::detect_flat_dataset_config(&bytes, &args).unwrap_err(); + assert!(format!("{err:#}").contains("--time-format-str is required")); +} + +#[test] +fn test_custom_time_format_sets_custom_format() { + let batch = create_test_batch(); + let bytes = write_to_parquet_bytes(&batch); + let mut args = make_test_args(Some("time"), Some(TimeFormat::Custom)); + args.time_format_str = "%m-%d-%Y %H:%M:%S".into(); + + let (config, _) = detect_parquet_schema::detect_flat_dataset_config(&bytes, &args) + .expect("custom format with format string should succeed"); + let time_column = config.time_column.expect("time column"); + assert_eq!(ProtoTimeFormat::Custom as i32, time_column.format); + assert_eq!("%m-%d-%Y %H:%M:%S", time_column.custom_format); +} + +#[test] +fn test_channel_per_row_custom_time_format_sets_custom_format() { + let batch = create_channel_per_row_single_batch(); + let bytes = write_to_parquet_bytes(&batch); + let mut args = make_channel_per_row_args( + ChannelMode::Single, + Some("timestamp"), + Some(TimeFormat::Custom), + ); + args.channel_name = Some("temperature".into()); + args.time_format_str = "%m-%d-%Y %H:%M:%S".into(); + + let (cfg, _) = detect_parquet_schema::detect_channel_per_row_config(&bytes, &args) + .expect("custom format with format string should succeed"); + let time_column = cfg.time_column.expect("time column"); + assert_eq!(ProtoTimeFormat::Custom as i32, time_column.format); + assert_eq!("%m-%d-%Y %H:%M:%S", time_column.custom_format); +} + #[test] fn test_invalid_rfc3339_relative_start_time_returns_error() { let batch = create_test_batch(); @@ -557,6 +601,7 @@ fn make_channel_per_row_args( mode, time_path: time_path.map(|s| s.to_string()), time_format, + time_format_str: String::default(), relative_start_time: None, data_path: "value".into(), channel_name: None, diff --git a/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs b/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs index dc3eebd5a..3dddc510e 100644 --- a/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs +++ b/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs @@ -97,6 +97,7 @@ pub fn build_tdms_config(args: &ImportTdmsArgs) -> Result { validate_time_format( args.time_format.unwrap_or_default(), &args.relative_start_time, + None, ) .context("validating time format for tdms")?; diff --git a/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs b/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs index fb287d21c..8a72e3f3f 100644 --- a/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs @@ -181,6 +181,18 @@ fn build_tdms_config_invalid_start_time_override_errors() { ); } +#[test] +fn build_tdms_config_custom_time_format_errors() { + let mut args = make_args(); + args.time_format = Some(TimeFormat::Custom); + let err = build_tdms_config(&args).unwrap_err(); + assert!( + err.chain() + .any(|e| e.to_string().contains("only supported by `import csv`")), + "expected validation error, got: {err:#}" + ); +} + #[test] fn test_tdms_to_sift_data_type() { assert_eq!( diff --git a/rust/crates/sift_cli/src/cmd/import/utils.rs b/rust/crates/sift_cli/src/cmd/import/utils.rs index f8b829b21..7aa13bb41 100644 --- a/rust/crates/sift_cli/src/cmd/import/utils.rs +++ b/rust/crates/sift_cli/src/cmd/import/utils.rs @@ -84,11 +84,22 @@ pub fn gzip_file(file: File) -> Result> { Ok(compressed_data) } +/// `time_format_str` is `None` for imports that don't accept `--time-format-str`. pub fn validate_time_format( time_format: TimeFormat, relative_start_time: &Option, + time_format_str: Option<&str>, ) -> Result<()> { match time_format { + TimeFormat::Custom => match time_format_str { + None => Err(anyhow!( + "--time-format custom is only supported by `import csv` and `import parquet`" + )), + Some(s) if s.trim().is_empty() => Err(anyhow!( + "--time-format-str is required if time format is custom" + )), + Some(_) => Ok(()), + }, TimeFormat::RelativeNanoseconds | TimeFormat::RelativeMicroseconds | TimeFormat::RelativeMilliseconds @@ -170,3 +181,25 @@ pub fn try_parse_bit_field_config(val: &str) -> Result Date: Tue, 17 Nov 2026 10:34:32 -0600 Subject: [PATCH 3/4] sift_client: resolve pr feedback to check custom format at construction --- .../_tests/resources/test_data_imports.py | 20 +++++++++++-------- .../lib/sift_client/sift_types/data_import.py | 14 ++++++++++--- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index d46c0d048..f1bd0a782 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -727,7 +727,12 @@ def test_custom_format_round_trip(self, csv_config): assert restored.time_column.custom_format == "%m-%d-%Y %H:%M:%S" def test_custom_format_requires_custom_format(self): - col = CsvTimeColumn(column=1, format=TimeFormat.CUSTOM) + with pytest.raises(ValueError, match="Set config.time_column.custom_format"): + CsvTimeColumn(column=1, format=TimeFormat.CUSTOM) + + def test_custom_format_set_after_construction_rejected_at_import(self): + col = CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339) + col.format = TimeFormat.CUSTOM with pytest.raises(ValueError, match="custom_format"): col._to_proto() @@ -801,15 +806,14 @@ def test_custom_format_round_trip(self): assert restored.custom_format == "%m-%d-%Y %H:%M:%S" def test_custom_format_requires_custom_format(self): - col = ParquetTimeColumn(path="timestamp", format=TimeFormat.CUSTOM) with pytest.raises(ValueError, match="custom_format"): - col._to_proto() + ParquetTimeColumn(path="timestamp", format=TimeFormat.CUSTOM) - def test_custom_format_without_custom_format_reads_back(self): - proto = ParquetTimeColumnProto(path="timestamp", format=TimeFormat.CUSTOM.value) - col = ParquetTimeColumn._from_proto(proto) - assert col.format == TimeFormat.CUSTOM - assert col.custom_format == "" + def test_custom_format_set_after_construction_rejected_at_import(self): + col = ParquetTimeColumn(path="timestamp", format=TimeFormat.ABSOLUTE_RFC3339) + col.format = TimeFormat.CUSTOM + with pytest.raises(ValueError, match="custom_format"): + col._to_proto() def test_custom_format_defaults_to_empty(self): col = ParquetTimeColumn(path="timestamp", format=TimeFormat.ABSOLUTE_RFC3339) diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index 715f0a42e..54a1cabff 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -160,11 +160,19 @@ class TimeColumnBase(BaseModel, ABC): def _check_custom_format(self) -> None: """Raise if the format is custom with no format string. - Checked at import rather than construction, so a config can be read - back or edited before ``custom_format`` is set. + Runs at construction and again at import, because assigning ``format`` + or ``custom_format`` after construction is not validated. """ if self.format == TimeFormat.CUSTOM and not self.custom_format.strip(): - raise ValueError("'custom_format' is required when using TimeFormat.CUSTOM.") + raise ValueError( + "'custom_format' is required when using TimeFormat.CUSTOM. " + 'Set config.time_column.custom_format, e.g. "%m-%d-%Y %H:%M:%S".' + ) + + @model_validator(mode="after") + def _validate_custom_format(self) -> TimeColumnBase: + self._check_custom_format() + return self @model_validator(mode="after") def _check_relative_start_time(self) -> TimeColumnBase: From 89aaf3e9f4bf47688b828039b8260bcba4ad3729 Mon Sep 17 00:00:00 2001 From: Omar Omar Date: Tue, 17 Nov 2026 10:43:09 -0600 Subject: [PATCH 4/4] rust: error when time-format-str is set outside time-format=custom --- .../sift_cli/src/cmd/import/parquet/tests.rs | 11 ++++++++++ rust/crates/sift_cli/src/cmd/import/utils.rs | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs b/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs index f13f77e39..1a3012b2f 100644 --- a/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs @@ -324,6 +324,17 @@ fn test_custom_time_format_without_time_format_str_returns_error() { assert!(format!("{err:#}").contains("--time-format-str is required")); } +#[test] +fn test_time_format_str_without_custom_returns_error() { + let batch = create_test_batch(); + let bytes = write_to_parquet_bytes(&batch); + let mut args = make_test_args(Some("time"), None); + args.time_format_str = "%m-%d-%Y %H:%M:%S".into(); + + let err = detect_parquet_schema::detect_flat_dataset_config(&bytes, &args).unwrap_err(); + assert!(format!("{err:#}").contains("--time-format-str requires --time-format custom")); +} + #[test] fn test_custom_time_format_sets_custom_format() { let batch = create_test_batch(); diff --git a/rust/crates/sift_cli/src/cmd/import/utils.rs b/rust/crates/sift_cli/src/cmd/import/utils.rs index 7aa13bb41..d92e877cc 100644 --- a/rust/crates/sift_cli/src/cmd/import/utils.rs +++ b/rust/crates/sift_cli/src/cmd/import/utils.rs @@ -90,6 +90,13 @@ pub fn validate_time_format( relative_start_time: &Option, time_format_str: Option<&str>, ) -> Result<()> { + if !matches!(time_format, TimeFormat::Custom) + && time_format_str.is_some_and(|s| !s.trim().is_empty()) + { + return Err(anyhow!( + "--time-format-str requires --time-format custom, but the time format is {time_format}" + )); + } match time_format { TimeFormat::Custom => match time_format_str { None => Err(anyhow!( @@ -197,6 +204,21 @@ mod test_validate_time_format { validate_time_format(TimeFormat::Custom, &None, Some("%m-%d-%Y %H:%M:%S")).unwrap(); } + #[test] + fn time_format_str_rejected_without_custom() { + let err = + validate_time_format(TimeFormat::AbsoluteRfc3339, &None, Some("%m-%d-%Y")).unwrap_err(); + assert!( + err.to_string() + .contains("--time-format-str requires --time-format custom") + ); + } + + #[test] + fn empty_time_format_str_is_ok_without_custom() { + validate_time_format(TimeFormat::AbsoluteRfc3339, &None, Some("")).unwrap(); + } + #[test] fn custom_rejected_without_time_format_str_flag() { let err = validate_time_format(TimeFormat::Custom, &None, None).unwrap_err();