diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py index a9ea5c6..ca7f953 100644 --- a/energy_fault_detector/fault_detector.py +++ b/energy_fault_detector/fault_detector.py @@ -124,45 +124,15 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo "floats during imputation. Non-declared categorical features will be dropped.") clear_session() - # --- Resolve conditional features against available data --- - self._resolve_conditional_features(sensor_data) - # Preprocess data and fit autoencoder model_path = None x_prepped, x, y = self.preprocess_train_data( sensor_data=sensor_data, normal_index=normal_index, fit_preprocessor=fit_preprocessor ) - # Capture conditions available in the raw data before the preprocessor may expand - # categorical conditions into one-hot encoded column names (e.g. 'state' -> 'state_A'). - declared_conditions = list(self.autoencoder.conditional_features or []) - # Resolve declared conditions against preprocessed data (may expand categorical names) self._resolve_conditional_features(x_prepped) - # Check conditionals: available conditions in original data and surviving conditions during preprocessing. - if self.autoencoder.is_conditional: - # Uses nested for loop in case categorical cols have been encoded already and contain the original - # conditional feature name as a substring. - available = [declared_condition for declared_condition in declared_conditions - if any(declared_condition in col for col in sensor_data.columns)] - missing = [declared_condition for declared_condition in declared_conditions - if declared_condition not in available] - if missing: - logger.warning(f"Declared conditions not found in sensor_data will be ignored: " - f"{sorted(missing)}. Using: {available or 'none'}") - - if not self.config.protect_conditional_features: - # Uses nested for loop in case categorical cols have been encoded already and contain the original - # conditional feature name as a substring. - surviving = [declared_condition for declared_condition in available - if any(declared_condition in col for col in x_prepped.columns)] - dropped_by_pipeline = set(available or []) - set(surviving) - - if dropped_by_pipeline: - logger.warning(f"Declared conditions dropped by preprocessing pipeline: " - f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}") - train_recon_error, val_recon_error = None, None x_train, x_val = self.train_val_split(x_prepped) logger.info('Train autoencoder.') @@ -478,7 +448,7 @@ def _fit_threshold(self, x: pd.DataFrame, y: pd.Series, x_val: pd.DataFrame, fit else: self.threshold_selector.fit(x=scores, y=y.loc[scores.index]) - def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]: + def _resolve_conditional_features(self, prepped_sensor_data: pd.DataFrame) -> List[str]: """Resolve which declared conditions are actually available in the data. If all conditions are missing and the autoencoder is a ConditionalAE, @@ -486,7 +456,7 @@ def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]: conditional_features list (they handle None gracefully). Args: - sensor_data: Input DataFrame to check column availability. + prepped_sensor_data: Input DataFrame to check column availability. Returns: List of available conditional feature names (may be empty). @@ -497,7 +467,8 @@ def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]: # TODO: this can produce undesired results if any column in the original data contains a substring of a declared # conditional feature name. - available = [col for col in sensor_data.columns if any(declared_condition in col for declared_condition in configured)] + available = [col for col in prepped_sensor_data.columns + if any(declared_condition in col for declared_condition in configured)] if not available: self._fallback_if_no_conditionals() @@ -505,6 +476,27 @@ def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]: self.autoencoder.conditional_features = available self.autoencoder.is_conditional = True + # Check conditionals: available conditions in original data and surviving conditions during preprocessing. + if self.autoencoder.is_conditional: + # Uses nested for loop in case categorical cols have been encoded already and contain the original + # conditional feature name as a substring. + missing = [declared_condition for declared_condition in self.autoencoder.conditional_features + if declared_condition not in available] + if missing: + logger.warning(f"Declared conditions not found in sensor_data will be ignored: " + f"{sorted(missing)}. Using: {available or 'none'}") + + if not self.config.protect_conditional_features: + # Uses nested for loop in case categorical cols have been encoded already and contain the original + # conditional feature name as a substring. + surviving = [declared_condition for declared_condition in available + if any(declared_condition in col for col in prepped_sensor_data.columns)] + dropped_by_pipeline = set(available or []) - set(surviving) + + if dropped_by_pipeline: + logger.warning(f"Declared conditions dropped by preprocessing pipeline: " + f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}") + return available def _fallback_if_no_conditionals(self) -> None: diff --git a/tests/test_data/test_conditional_ae_config.yaml b/tests/test_data/test_conditional_ae_config.yaml index 66983fe..d9d6482 100644 --- a/tests/test_data/test_conditional_ae_config.yaml +++ b/tests/test_data/test_conditional_ae_config.yaml @@ -4,7 +4,11 @@ train: params: pca: true pca_min_var: 0.85 - data_preprocessor: # no further spec, so defaults are applied + data_preprocessor: + steps: + - name: timestamp_transform + params: + features: ['hour_of_day'] autoencoder: name: 'ConditionalAE' verbose: 0 diff --git a/tests/test_fault_detector.py b/tests/test_fault_detector.py index 10e4c8d..8a25932 100644 --- a/tests/test_fault_detector.py +++ b/tests/test_fault_detector.py @@ -555,13 +555,14 @@ def setUp(self) -> None: np.random.seed(42) length = 100 + self.timestamp_idx = pd.date_range(start='2022-01-01', periods=length, freq='15min') self.sensor_data = pd.DataFrame({ 'feature_a': np.random.random(size=length), 'feature_b': np.random.random(size=length), 'feature_c': np.random.random(size=length), 'feature_d': np.random.random(size=length), - }) - self.normal_index = pd.Series([True] * 80 + [False] * 20) + }, index=self.timestamp_idx) + self.normal_index = pd.Series([True] * 80 + [False] * 20, index=self.timestamp_idx) def tearDown(self) -> None: shutil.rmtree(self.test_dir) @@ -581,6 +582,18 @@ def test_resolve_partial_missing_conditionals(self): self.assertEqual(fd.autoencoder.conditional_features, ['feature_a']) self.assertTrue(fd.autoencoder.is_conditional) + def test_resolve_partial_missing_conditionals_after_fit(self): + """When some conditional features are missing, only available ones are kept after fit.""" + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + fd.autoencoder.conditional_features = ['feature_a', 'feature_b', 'hour_of_day_sine', 'hour_of_day_cosine'] + + # Drop feature_b from sensor data + sensor_data_partial = self.sensor_data.drop(columns=['feature_b']) + + fd.fit(sensor_data=sensor_data_partial, normal_index=self.normal_index) + self.assertEqual(fd.autoencoder.conditional_features, + ['feature_a', 'hour_of_day_sine', 'hour_of_day_cosine']) + def test_resolve_all_missing_conditionals_fallback_to_multilayer(self): """When ALL conditional features are missing, ConditionalAE falls back to MultilayerAutoencoder.""" from energy_fault_detector.autoencoders import MultilayerAutoencoder @@ -687,13 +700,14 @@ def setUp(self) -> None: np.random.seed(42) length = 100 # feature_a is constant → will be dropped by LowUniqueValueFilter when unprotected + self.timestamp_idx = pd.date_range(start='2022-01-01', periods=length, freq='15min') self.sensor_data = pd.DataFrame({ 'feature_a': [1.0] * length, 'feature_b': np.random.random(size=length), 'feature_c': np.random.random(size=length), 'feature_d': np.random.random(size=length), - }) - self.normal_index = pd.Series([True] * 80 + [False] * 20) + }, index=self.timestamp_idx) + self.normal_index = pd.Series([True] * 80 + [False] * 20, index=self.timestamp_idx) def tearDown(self) -> None: shutil.rmtree(self.test_dir) @@ -701,10 +715,10 @@ def tearDown(self) -> None: def test_conditional_features_can_be_dropped_when_unprotected(self): """When protect=False, pipeline can drop constant conditional features.""" from energy_fault_detector.autoencoders import MultilayerAutoencoder - + self.conf.config_dict['train']['protect_conditional_features'] = False + self.conf.config_dict['train']['data_preprocessor']['steps'].append({'name': 'low_unique_value_filter', 'params': {'min_unique_value_count': 2}}) fd = FaultDetector(config=self.conf, model_directory=self.test_dir) self.assertFalse(fd.config.protect_conditional_features) - result = fd.fit(sensor_data=self.sensor_data, normal_index=self.normal_index, save_models=False) # feature_a was constant, should have been dropped, triggering fallback