Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions python/changelog.d/884.feat.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added `TimeFormat.CUSTOM` and `custom_format` on `CsvTimeColumn` and `ParquetTimeColumn` for importing time columns in a custom Unix date format, such as `%m-%d-%Y %H:%M:%S`. Importing with `TimeFormat.CUSTOM` requires a non-empty `custom_format`; TDMS and HDF5 imports do not support it.
62 changes: 62 additions & 0 deletions python/lib/sift_client/_tests/resources/test_data_imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,11 @@ def test_run_id_takes_precedence(self):
proto = config._to_proto()
assert proto.run_id == "run_123"

def test_custom_time_format_rejected(self):
config = TdmsImportConfig(asset_name="a", time_format=TimeFormat.CUSTOM)
with pytest.raises(ValueError, match="only supported for CSV and Parquet"):
config._to_proto()


class TestHdf5Config:
def test_to_proto(self):
Expand Down Expand Up @@ -709,6 +714,32 @@ def test_from_proto_round_trip(self, csv_config):
assert restored.time_column.column == csv_config.time_column.column
assert len(restored.data_columns) == len(csv_config.data_columns)

def test_custom_format_round_trip(self, csv_config):
csv_config.time_column = CsvTimeColumn(
column=4, format=TimeFormat.CUSTOM, custom_format="%m-%d-%Y %H:%M:%S"
)
proto = csv_config._to_proto()
assert proto.time_column.format == TimeFormat.CUSTOM.value
assert proto.time_column.custom_format == "%m-%d-%Y %H:%M:%S"

restored = CsvImportConfig._from_proto(proto)
assert restored.time_column.format == TimeFormat.CUSTOM
assert restored.time_column.custom_format == "%m-%d-%Y %H:%M:%S"

def test_custom_format_requires_custom_format(self):
with pytest.raises(ValueError, match="Set config.time_column.custom_format"):
CsvTimeColumn(column=1, format=TimeFormat.CUSTOM)

def test_custom_format_set_after_construction_rejected_at_import(self):
col = CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339)
col.format = TimeFormat.CUSTOM
with pytest.raises(ValueError, match="custom_format"):
col._to_proto()

def test_custom_format_defaults_to_empty(self, csv_config):
assert csv_config.time_column.custom_format == ""
assert csv_config._to_proto().time_column.custom_format == ""


class TestParquetToProto:
def test_flat_dataset_to_proto(self, parquet_config):
Expand Down Expand Up @@ -763,6 +794,32 @@ def test_missing_format_raises(self):
with pytest.raises(ValueError, match="format must be set"):
col._to_proto()

def test_custom_format_round_trip(self):
col = ParquetTimeColumn(
path="timestamp", format=TimeFormat.CUSTOM, custom_format="%m-%d-%Y %H:%M:%S"
)
proto = col._to_proto()
assert proto.custom_format == "%m-%d-%Y %H:%M:%S"

restored = ParquetTimeColumn._from_proto(proto)
assert restored.format == TimeFormat.CUSTOM
assert restored.custom_format == "%m-%d-%Y %H:%M:%S"

def test_custom_format_requires_custom_format(self):
with pytest.raises(ValueError, match="custom_format"):
ParquetTimeColumn(path="timestamp", format=TimeFormat.CUSTOM)

def test_custom_format_set_after_construction_rejected_at_import(self):
col = ParquetTimeColumn(path="timestamp", format=TimeFormat.ABSOLUTE_RFC3339)
col.format = TimeFormat.CUSTOM
with pytest.raises(ValueError, match="custom_format"):
col._to_proto()

def test_custom_format_defaults_to_empty(self):
col = ParquetTimeColumn(path="timestamp", format=TimeFormat.ABSOLUTE_RFC3339)
assert col.custom_format == ""
assert col._to_proto().custom_format == ""


class TestResolveDataTypeKey:
def test_parquet_requires_data_type(self):
Expand Down Expand Up @@ -996,6 +1053,11 @@ def test_hdf5_to_proto_rejects_missing_format(self):
with pytest.raises(ValueError, match="time_format is required"):
config._to_proto()

def test_hdf5_to_proto_rejects_custom_format(self):
config = Hdf5ImportConfig(asset_name="a", data=[], time_format=TimeFormat.CUSTOM)
with pytest.raises(ValueError, match="only supported for CSV and Parquet"):
config._to_proto()


class TestDetectConfigRelativeStartTime:
START = pd.Timestamp("2026-01-01T00:00:00.000000001Z")
Expand Down
31 changes: 31 additions & 0 deletions python/lib/sift_client/sift_types/data_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ class TimeFormat(Enum):
ABSOLUTE_UNIX_MILLISECONDS = TimeFormatProto.TIME_FORMAT_ABSOLUTE_UNIX_MILLISECONDS
ABSOLUTE_UNIX_MICROSECONDS = TimeFormatProto.TIME_FORMAT_ABSOLUTE_UNIX_MICROSECONDS
ABSOLUTE_UNIX_NANOSECONDS = TimeFormatProto.TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS
CUSTOM = TimeFormatProto.TIME_FORMAT_CUSTOM

@property
def is_relative(self) -> bool:
Expand Down Expand Up @@ -143,6 +144,8 @@ class TimeColumnBase(BaseModel, ABC):
(caller-passed ``time_format`` > server-detected format >
``ABSOLUTE_UNIX_NANOSECONDS`` fallback). Importing without a
format set raises ``ValueError``.
custom_format: Unix date format, e.g. ``%m-%d-%Y %H:%M:%S``. Required at
import when ``format`` is ``TimeFormat.CUSTOM``; ignored otherwise.
relative_start_time: Required when using a relative time format.
Pass a ``pd.Timestamp`` for nanosecond precision.
"""
Expand All @@ -151,8 +154,26 @@ class TimeColumnBase(BaseModel, ABC):
model_config = ConfigDict(arbitrary_types_allowed=True)

format: TimeFormat | None = None
custom_format: str = ""
relative_start_time: pd.Timestamp | datetime | None = None

def _check_custom_format(self) -> None:
Comment thread
omar-sift marked this conversation as resolved.
Resolved
"""Raise if the format is custom with no format string.

Runs at construction and again at import, because assigning ``format``
or ``custom_format`` after construction is not validated.
"""
if self.format == TimeFormat.CUSTOM and not self.custom_format.strip():
raise ValueError(
"'custom_format' is required when using TimeFormat.CUSTOM. "
'Set config.time_column.custom_format, e.g. "%m-%d-%Y %H:%M:%S".'
)

@model_validator(mode="after")
def _validate_custom_format(self) -> TimeColumnBase:
self._check_custom_format()
return self

@model_validator(mode="after")
def _check_relative_start_time(self) -> TimeColumnBase:
if self.format is not None and self.format.is_relative and self.relative_start_time is None:
Expand Down Expand Up @@ -361,9 +382,11 @@ def _to_proto(self) -> CsvTimeColumnProto:
"Pass time_format to detect_config/import_from_path, or set "
"config.time_column.format explicitly."
)
self._check_custom_format()
proto = CsvTimeColumnProto(
column_number=self.column,
format=self.format.value,
custom_format=self.custom_format,
)
if self.relative_start_time is not None:
proto.relative_start_time.CopyFrom(to_timestamp_pb(self.relative_start_time))
Expand Down Expand Up @@ -442,6 +465,7 @@ def _from_proto(cls, proto: CsvConfigProto) -> CsvImportConfig:
time_column = CsvTimeColumn(
column=proto.time_column.column_number,
format=time_format,
custom_format=proto.time_column.custom_format,
relative_start_time=relative_start_time,
)
data_columns = [
Expand Down Expand Up @@ -493,9 +517,11 @@ def _to_proto(self) -> ParquetTimeColumnProto:
"Pass time_format to detect_config/import_from_path, or set "
"config.time_column.format explicitly."
)
self._check_custom_format()
proto = ParquetTimeColumnProto(
path=self.path,
format=self.format.value,
custom_format=self.custom_format,
)
if self.relative_start_time is not None:
proto.relative_start_time.CopyFrom(to_timestamp_pb(self.relative_start_time))
Expand All @@ -511,6 +537,7 @@ def _from_proto(cls, proto: ParquetTimeColumnProto) -> ParquetTimeColumn:
return cls(
path=proto.path or "",
format=fmt,
custom_format=proto.custom_format,
relative_start_time=relative_start_time,
)

Expand Down Expand Up @@ -887,6 +914,8 @@ def _to_proto(self) -> TDMSConfigProto:
proto.start_time_override.CopyFrom(to_timestamp_pb(self.start_time_override))
proto.fallback_method = self.fallback_method.value
if self.time_format is not None:
if self.time_format == TimeFormat.CUSTOM:
raise ValueError("TimeFormat.CUSTOM is only supported for CSV and Parquet imports.")
proto.time_format = self.time_format.value
if self.relative_start_time is not None:
proto.relative_start_time.CopyFrom(to_timestamp_pb(self.relative_start_time))
Expand Down Expand Up @@ -1045,6 +1074,8 @@ def _to_proto(self) -> Hdf5ConfigProto:
"config.time_format explicitly (e.g. TimeFormat.ABSOLUTE_UNIX_NANOSECONDS "
"or TimeFormat.ABSOLUTE_DATETIME) before importing."
)
if self.time_format == TimeFormat.CUSTOM:
raise ValueError("TimeFormat.CUSTOM is only supported for CSV and Parquet imports.")
proto = Hdf5ConfigProto(
asset_name=self.asset_name,
run_name=self.run_name or "",
Expand Down
3 changes: 3 additions & 0 deletions rust/crates/sift_cli/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
- `create_rule` now documents the annotation action, and both rule tools
refuse a rule without exactly one condition instead of passing on Sift's
internal error.
- `sift-cli import csv` and `sift-cli import parquet` accept `--time-format custom`
with `--time-format-str` for timestamps in a custom Unix date format, such as
`%m-%d-%Y %H:%M:%S`.

## [v0.9.0] - November 16, 2026

Expand Down
4 changes: 4 additions & 0 deletions rust/crates/sift_cli/assets/docs/src/data/importing.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,15 @@ timestamps are encoded. Supported values:
| `relative-seconds` | Offset from a start time, in seconds |
| `relative-minutes` | Offset from a start time, in minutes |
| `relative-hours` | Offset from a start time, in hours |
| `custom` (CSV and Parquet) | Custom Unix date format set with `--time-format-str`, e.g. `%m-%d-%Y %H:%M:%S` |

When the time format is **relative**, supply a start time with
`--relative-start-time` (`-s`) in RFC 3339 so Sift can anchor the offsets to
absolute time.

When the time format is **custom**, supply the format with `--time-format-str`.
Only `import csv` and `import parquet` accept `custom`.

### Channel data types

Where a command lets you declare channel types, the accepted values are:
Expand Down
15 changes: 15 additions & 0 deletions rust/crates/sift_cli/src/cli/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,10 @@ pub struct ImportCsvArgs {
#[arg(short = 'f', long, default_value_t = TimeFormat::default(), hide_possible_values = true, help_heading = "Time")]
pub time_format: TimeFormat,

/// Unix date format, e.g. "%m-%d-%Y %H:%M:%S", used with `--time-format custom`
#[arg(long, default_value_t = String::default(), hide_possible_values = true, help_heading = "Time")]
pub time_format_str: String,

/// Start time (RFC3339) to use if time format is relative
#[arg(short = 's', long, help_heading = "Time")]
pub relative_start_time: Option<String>,
Expand Down Expand Up @@ -629,6 +633,10 @@ pub struct FlatDatasetArgs {
#[arg(short = 'f', long, hide_possible_values = true, help_heading = "Time")]
pub time_format: Option<TimeFormat>,

/// Unix date format, e.g. "%m-%d-%Y %H:%M:%S", used with `--time-format custom`
#[arg(long, default_value_t = String::default(), help_heading = "Time")]
pub time_format_str: String,

/// Start time (RFC3339) to use if time format is relative
#[arg(short = 's', long, help_heading = "Time")]
pub relative_start_time: Option<String>,
Expand Down Expand Up @@ -670,6 +678,10 @@ pub struct ChannelPerRowCommonArgs {
#[arg(short = 'f', long, hide_possible_values = true, help_heading = "Time")]
pub time_format: Option<TimeFormat>,

/// Unix date format, e.g. "%m-%d-%Y %H:%M:%S", used with `--time-format custom`
#[arg(long, default_value_t = String::default(), help_heading = "Time")]
pub time_format_str: String,

/// Start time (RFC3339) to use if time format is relative
#[arg(short = 's', long, help_heading = "Time")]
pub relative_start_time: Option<String>,
Expand Down Expand Up @@ -720,6 +732,7 @@ pub struct ChannelPerRowArgs {
pub mode: ChannelMode,
pub time_path: Option<String>,
pub time_format: Option<TimeFormat>,
pub time_format_str: String,
pub relative_start_time: Option<String>,
pub data_path: String,
pub channel_name: Option<String>,
Expand All @@ -737,6 +750,7 @@ impl From<ChannelPerRowSingleArgs> for ChannelPerRowArgs {
mode: ChannelMode::Single,
time_path: args.common.time_path,
time_format: args.common.time_format,
time_format_str: args.common.time_format_str,
relative_start_time: args.common.relative_start_time,
data_path: args.common.data_path,
channel_name: Some(args.channel_name),
Expand All @@ -756,6 +770,7 @@ impl From<ChannelPerRowMultiArgs> for ChannelPerRowArgs {
mode: ChannelMode::Multi,
time_path: args.common.time_path,
time_format: args.common.time_format,
time_format_str: args.common.time_format_str,
relative_start_time: args.common.relative_start_time,
data_path: args.common.data_path,
channel_name: None,
Expand Down
3 changes: 3 additions & 0 deletions rust/crates/sift_cli/src/cli/time.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ pub enum TimeFormat {
RelativeSeconds,
RelativeMinutes,
RelativeHours,
Custom,
}

impl From<TimeFormat> for ProtoTimeFormat {
Expand All @@ -35,6 +36,7 @@ impl From<TimeFormat> for ProtoTimeFormat {
TimeFormat::AbsoluteUnixMilliseconds => Self::AbsoluteUnixMilliseconds,
TimeFormat::AbsoluteUnixMicroseconds => Self::AbsoluteUnixMicroseconds,
TimeFormat::AbsoluteUnixNanoseconds => Self::AbsoluteUnixNanoseconds,
TimeFormat::Custom => Self::Custom,
}
}
}
Expand All @@ -54,6 +56,7 @@ impl Display for TimeFormat {
Self::RelativeSeconds => write!(f, "relative-seconds"),
Self::RelativeMinutes => write!(f, "relative-minutes"),
Self::RelativeHours => write!(f, "relative-hours"),
Self::Custom => write!(f, "custom"),
}
}
}
Loading
Loading