Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 25 additions & 33 deletions energy_fault_detector/fault_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,45 +124,15 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
"floats during imputation. Non-declared categorical features will be dropped.")
clear_session()

# --- Resolve conditional features against available data ---
self._resolve_conditional_features(sensor_data)

# Preprocess data and fit autoencoder
model_path = None
x_prepped, x, y = self.preprocess_train_data(
sensor_data=sensor_data, normal_index=normal_index, fit_preprocessor=fit_preprocessor
)

# Capture conditions available in the raw data before the preprocessor may expand
# categorical conditions into one-hot encoded column names (e.g. 'state' -> 'state_A').
declared_conditions = list(self.autoencoder.conditional_features or [])

# Resolve declared conditions against preprocessed data (may expand categorical names)
self._resolve_conditional_features(x_prepped)

# Check conditionals: available conditions in original data and surviving conditions during preprocessing.
if self.autoencoder.is_conditional:
# Uses nested for loop in case categorical cols have been encoded already and contain the original
# conditional feature name as a substring.
available = [declared_condition for declared_condition in declared_conditions
if any(declared_condition in col for col in sensor_data.columns)]
missing = [declared_condition for declared_condition in declared_conditions
if declared_condition not in available]
if missing:
logger.warning(f"Declared conditions not found in sensor_data will be ignored: "
f"{sorted(missing)}. Using: {available or 'none'}")

if not self.config.protect_conditional_features:
# Uses nested for loop in case categorical cols have been encoded already and contain the original
# conditional feature name as a substring.
surviving = [declared_condition for declared_condition in available
if any(declared_condition in col for col in x_prepped.columns)]
dropped_by_pipeline = set(available or []) - set(surviving)

if dropped_by_pipeline:
logger.warning(f"Declared conditions dropped by preprocessing pipeline: "
f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}")

train_recon_error, val_recon_error = None, None
x_train, x_val = self.train_val_split(x_prepped)
logger.info('Train autoencoder.')
Expand Down Expand Up @@ -478,15 +448,15 @@ def _fit_threshold(self, x: pd.DataFrame, y: pd.Series, x_val: pd.DataFrame, fit
else:
self.threshold_selector.fit(x=scores, y=y.loc[scores.index])

def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]:
def _resolve_conditional_features(self, prepped_sensor_data: pd.DataFrame) -> List[str]:
"""Resolve which declared conditions are actually available in the data.

If all conditions are missing and the autoencoder is a ConditionalAE,
falls back to MultilayerAutoencoder. For sequence models, simply clears the
conditional_features list (they handle None gracefully).

Args:
sensor_data: Input DataFrame to check column availability.
prepped_sensor_data: Input DataFrame to check column availability.

Returns:
List of available conditional feature names (may be empty).
Expand All @@ -497,14 +467,36 @@ def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]:

# TODO: this can produce undesired results if any column in the original data contains a substring of a declared
# conditional feature name.
available = [col for col in sensor_data.columns if any(declared_condition in col for declared_condition in configured)]
available = [col for col in prepped_sensor_data.columns
if any(declared_condition in col for declared_condition in configured)]

if not available:
self._fallback_if_no_conditionals()
else:
self.autoencoder.conditional_features = available
self.autoencoder.is_conditional = True

# Check conditionals: available conditions in original data and surviving conditions during preprocessing.
if self.autoencoder.is_conditional:
# Uses nested for loop in case categorical cols have been encoded already and contain the original
# conditional feature name as a substring.
missing = [declared_condition for declared_condition in self.autoencoder.conditional_features
if declared_condition not in available]
if missing:
logger.warning(f"Declared conditions not found in sensor_data will be ignored: "
f"{sorted(missing)}. Using: {available or 'none'}")

if not self.config.protect_conditional_features:
# Uses nested for loop in case categorical cols have been encoded already and contain the original
# conditional feature name as a substring.
surviving = [declared_condition for declared_condition in available
if any(declared_condition in col for col in prepped_sensor_data.columns)]
dropped_by_pipeline = set(available or []) - set(surviving)

if dropped_by_pipeline:
logger.warning(f"Declared conditions dropped by preprocessing pipeline: "
f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}")

return available

def _fallback_if_no_conditionals(self) -> None:
Expand Down
6 changes: 5 additions & 1 deletion tests/test_data/test_conditional_ae_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ train:
params:
pca: true
pca_min_var: 0.85
data_preprocessor: # no further spec, so defaults are applied
data_preprocessor:
steps:
- name: timestamp_transform
params:
features: ['hour_of_day']
autoencoder:
name: 'ConditionalAE'
verbose: 0
Expand Down
26 changes: 20 additions & 6 deletions tests/test_fault_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -555,13 +555,14 @@ def setUp(self) -> None:

np.random.seed(42)
length = 100
self.timestamp_idx = pd.date_range(start='2022-01-01', periods=length, freq='15min')
self.sensor_data = pd.DataFrame({
'feature_a': np.random.random(size=length),
'feature_b': np.random.random(size=length),
'feature_c': np.random.random(size=length),
'feature_d': np.random.random(size=length),
})
self.normal_index = pd.Series([True] * 80 + [False] * 20)
}, index=self.timestamp_idx)
self.normal_index = pd.Series([True] * 80 + [False] * 20, index=self.timestamp_idx)

def tearDown(self) -> None:
shutil.rmtree(self.test_dir)
Expand All @@ -581,6 +582,18 @@ def test_resolve_partial_missing_conditionals(self):
self.assertEqual(fd.autoencoder.conditional_features, ['feature_a'])
self.assertTrue(fd.autoencoder.is_conditional)

def test_resolve_partial_missing_conditionals_after_fit(self):
"""When some conditional features are missing, only available ones are kept after fit."""
fd = FaultDetector(config=self.conf, model_directory=self.test_dir)
fd.autoencoder.conditional_features = ['feature_a', 'feature_b', 'hour_of_day_sine', 'hour_of_day_cosine']

# Drop feature_b from sensor data
sensor_data_partial = self.sensor_data.drop(columns=['feature_b'])

fd.fit(sensor_data=sensor_data_partial, normal_index=self.normal_index)
self.assertEqual(fd.autoencoder.conditional_features,
['feature_a', 'hour_of_day_sine', 'hour_of_day_cosine'])

def test_resolve_all_missing_conditionals_fallback_to_multilayer(self):
"""When ALL conditional features are missing, ConditionalAE falls back to MultilayerAutoencoder."""
from energy_fault_detector.autoencoders import MultilayerAutoencoder
Expand Down Expand Up @@ -687,24 +700,25 @@ def setUp(self) -> None:
np.random.seed(42)
length = 100
# feature_a is constant → will be dropped by LowUniqueValueFilter when unprotected
self.timestamp_idx = pd.date_range(start='2022-01-01', periods=length, freq='15min')
self.sensor_data = pd.DataFrame({
'feature_a': [1.0] * length,
'feature_b': np.random.random(size=length),
'feature_c': np.random.random(size=length),
'feature_d': np.random.random(size=length),
})
self.normal_index = pd.Series([True] * 80 + [False] * 20)
}, index=self.timestamp_idx)
self.normal_index = pd.Series([True] * 80 + [False] * 20, index=self.timestamp_idx)

def tearDown(self) -> None:
shutil.rmtree(self.test_dir)

def test_conditional_features_can_be_dropped_when_unprotected(self):
"""When protect=False, pipeline can drop constant conditional features."""
from energy_fault_detector.autoencoders import MultilayerAutoencoder

self.conf.config_dict['train']['protect_conditional_features'] = False
self.conf.config_dict['train']['data_preprocessor']['steps'].append({'name': 'low_unique_value_filter', 'params': {'min_unique_value_count': 2}})
fd = FaultDetector(config=self.conf, model_directory=self.test_dir)
self.assertFalse(fd.config.protect_conditional_features)

result = fd.fit(sensor_data=self.sensor_data, normal_index=self.normal_index, save_models=False)

# feature_a was constant, should have been dropped, triggering fallback
Expand Down
Loading