From 2733d3f98a4b541f1a9e9d85018c4c88e11f9f00 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Mon, 15 Jun 2026 16:18:05 +0200
Subject: [PATCH 01/35] Added todos to refactor training data preprocessing to
 handle feature types.

Todos to update the preprocessing method to distinguish between categorical, numerical, and boolean features based on configuration. Added TODOs for implementing separate pipelines for categorical and numerical processing. Included plans to protect relevant features based on configuration.
---
 energy_fault_detector/fault_detector.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index df4b2fc0..d44dc20d 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -39,7 +39,14 @@ def __init__(self, config: Optional[Config] = None, model_directory: str | Path
 
     def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Series, fit_preprocessor: bool = True
                               ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.Series]:
-        """Preprocesses the training data using the configured data_preprocessor."""
+        """Preprocesses the training data using the configured data_preprocessor.
+        If categorical features are declared in config file, data is split into categorical and numerical features.
+        Boolean features are treated the same as numerical features. Scaling is not applied to encoded categorical features.
+        """
+
+        # TODO: Check if categorical features are declared. If true, split data into categorical and numerical data.
+        # TODO: Fit categorical data -> categorical pipeline
+        # TODO: Fit numerical and boolean data. -> numerical pipeline
 
         x = sensor_data.sort_index()
         if normal_index is not None:
@@ -51,6 +58,7 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
             raise ValueError('There are duplicated indices in the input dataframe `sensor_data` and/or in the '
                              '`normal_index`, please check your input data.')
 
+        # TODO: add list of relevant features to config and protect relevant features (if present) as well.
         # Determine which features to protect based on config flag
         protect = self.config.protect_conditional_features if self.config else True
         protected_features = (
@@ -70,6 +78,7 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
 
         x_normal = x[y.values]
         if fit_preprocessor:
+            # TODO: Here we have to run two separate pipelines, one for categorical and one for numerical + boolean.
             logger.info('Fit preprocessor pipeline.')
             fit_params = {}
             for step_name in self.data_preprocessor.named_steps.keys():

From 8c5a2703162da77a8a63ef2ffd91a43e25767513 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Tue, 16 Jun 2026 17:24:28 +0200
Subject: [PATCH 02/35] Add CategoricalEncoder, Scaler, and Imputer to
 preprocessing pipeline. Scaler is a wrap-up estimator for any scaler used
 that can passthrough encoded features. Imputer is a wrap-up estimator
 covering simple imputer and categorical imputer. Categorical econder
 one-hot-encodes categorical features.

---
 .../data_preprocessing/categorical_encoder.py |  36 +++++
 .../data_preprocessing/data_preprocessor.py   |   9 +-
 .../data_preprocessing/imputer.py             |  33 +++++
 .../data_preprocessing/scaler.py              | 130 ++++++++++++++++++
 4 files changed, 207 insertions(+), 1 deletion(-)
 create mode 100644 energy_fault_detector/data_preprocessing/categorical_encoder.py
 create mode 100644 energy_fault_detector/data_preprocessing/imputer.py
 create mode 100644 energy_fault_detector/data_preprocessing/scaler.py

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
new file mode 100644
index 00000000..ebd0525a
--- /dev/null
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -0,0 +1,36 @@
+from typing import Optional, List, Union, Callable
+import pandas as pd
+from sklearn.preprocessing import OneHotEncoder
+from sklearn.utils.validation import check_is_fitted
+from energy_fault_detector.core.data_transformer import DataTransformer
+
+
+class CategoricalEncoder(DataTransformer):
+    " Class containing the one-hot-code step of categorical features to be used in the pre-pocessing pipeline"
+    def __init__(self, categorical_features):
+        super().__init__()
+        self.categorical_features = categorical_features
+        self.one_hot_encoder = OneHotEncoder(sparse_output=False)
+
+    def fit(self, x: pd.DataFrame, y=None):
+        self.feature_names_in_ = x.columns.tolist()
+        self.n_features_in_ = len(self.feature_names_in_)
+        self.input_index_ = x.index
+        # Do the one-hot-encode
+        self.one_hot_encoder.fit(x)
+        return self
+    
+    def transform(self, x: pd.DataFrame):
+        check_is_fitted(self)
+        x_ = self.one_hot_encoder.transform(x)
+        self.feature_names_out_ = self.get_feature_names_out()
+        return pd.DataFrame(x_, columns=self.feature_names_out_, index=self.input_index_)
+    
+    def inverse_transform(self, x: pd.DataFrame):
+        check_is_fitted(self)
+        x_ = self.one_hot_encoder.inverse_transform(x)
+        return pd.DataFrame(x_, columns=self.feature_names_in_, index=self.input_index_)
+    
+    def get_feature_names_out(self, input_features=None):
+        check_is_fitted(self)
+        return self.one_hot_encoder.get_feature_names_out(self.feature_names_in_)
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index 2387ae75..03f536f8 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -15,6 +15,8 @@
 from .duplicate_value_to_nan import DuplicateValuesToNan
 from .counter_diff_transformer import CounterDiffTransformer
 from .timestamp_transformer import TimestampTransformer
+from .categorical_encoder import CategoricalEncoder
+from .imputer import Imputer
 
 
 class DataPreprocessor(Pipeline, SaveLoadMixin):
@@ -28,6 +30,8 @@ class DataPreprocessor(Pipeline, SaveLoadMixin):
         'standard_scaler': StandardScaler,
         'minmax_scaler': MinMaxScaler,
         'timestamp_transformer': TimestampTransformer,
+        'categorical_encoder': CategoricalEncoder,
+        'imputer': Imputer,
     }
 
     NAME_ALIASES: Dict[str, str] = {
@@ -43,6 +47,7 @@ class DataPreprocessor(Pipeline, SaveLoadMixin):
         "duplicate_values_to_nan": "duplicate_to_nan",
         "timestamp_features": "timestamp_transformer",
         "timestamp_transform": "timestamp_transformer",
+        "categorical_encoder": "categorical_encoder",
     }
 
     def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None:
@@ -301,6 +306,7 @@ def _build_from_steps_spec(self) -> List:
             cls = self.STEP_REGISTRY.get(name)
             if cls is None:
                 raise ValueError(f"Unknown step name '{name}'. Register it in STEP_REGISTRY.")
+            # TODO: review initialization of estimator classes
             estimator = cls(**params)
             step_name = spec.get("step_name", name)
             steps.append((step_name, estimator))
@@ -337,6 +343,7 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
         imputer = [s for s in steps_spec if s.get("name") == "simple_imputer"]
         scaler_names = {"standard_scaler", "minmax_scaler"}
         scalers = [s for s in steps_spec if s.get("name") in scaler_names]
+        encoders = [s for s in steps_spec if s.get("name") == "categorical_encoder"]
         if len(scalers) > 1:
             raise ValueError(f"Only one scaler can be used, two found in the steps specification: {scalers}")
         timestamp_step = [s for s in steps_spec if s.get("name") == "timestamp_transformer"]
@@ -345,7 +352,7 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
             s for s in steps_spec
             if s.get("name") not in {
                 "column_selector", "duplicate_to_nan", "counter_diff_transformer", "simple_imputer",
-                "low_unique_value_filter", "timestamp_transformer",
+                "categorical_encoder", "low_unique_value_filter", "timestamp_transformer",
             } | scaler_names
         ]
 
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
new file mode 100644
index 00000000..562e8eed
--- /dev/null
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -0,0 +1,33 @@
+from typing import Optional, List, Union, Callable
+import pandas as pd
+from sklearn.utils.validation import check_is_fitted
+from energy_fault_detector.core.data_transformer import DataTransformer
+
+class Imputer(DataTransformer):
+    " Class containing the imputation step of categorical features to be used in the pre-pocessing pipeline"
+    def __init__(self, strategy: str = 'most_frequent'):
+        super().__init__()
+        self.strategy = strategy
+
+    def fit(self, x: pd.DataFrame, y=None):
+        self.feature_names_in_ = x.columns.tolist()
+        self.n_features_in_ = len(self.feature_names_in_)
+        self.input_index_ = x.index
+        # Impute the most frequent value for each column
+        self.fill_values_ = x.mode().iloc[0]
+        return self
+    
+    def transform(self, x: pd.DataFrame):
+        check_is_fitted(self)
+        x_ = x.fillna(self.fill_values_)
+        self.feature_names_out_ = self.get_feature_names_out()
+        return pd.DataFrame(x_, columns=self.feature_names_out_, index=self.input_index_)
+    
+    def inverse_transform(self, x: pd.DataFrame):
+        check_is_fitted(self)
+        x_ = x
+        return pd.DataFrame(x_, columns=self.feature_names_in_, index=self.input_index_)
+    
+    def get_feature_names_out(self, input_features=None):
+        check_is_fitted(self)
+        return self.feature_names_in_
\ No newline at end of file
diff --git a/energy_fault_detector/data_preprocessing/scaler.py b/energy_fault_detector/data_preprocessing/scaler.py
new file mode 100644
index 00000000..eb8fce1b
--- /dev/null
+++ b/energy_fault_detector/data_preprocessing/scaler.py
@@ -0,0 +1,130 @@
+import pandas as pd
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+from sklearn.utils.validation import check_is_fitted
+
+from energy_fault_detector.core.data_transformer import DataTransformer
+import logging
+
+logger = logging.getLogger('energy_fault_detector')
+
+
+class Scaler(DataTransformer):
+    """Scaler pre-processor step for scaling datasets."""
+
+    SCALER_REGISTRY = {
+        'standard': StandardScaler,
+        'minmax': MinMaxScaler
+    }
+
+    def __init__(self, scaler_type: str = 'standard', scale_encoded_features: bool = True,
+                 encoded_feature_names: list = None, **params):
+        """
+        Initialize the Scaler object.
+
+        Args:
+            scaler_type: Type of scaler ('standard', 'minmax'). Defaults to 'standard'.
+            scale_encoded_features: Flag to scale encoded features. Defaults to True.
+            encoded_feature_names: List of names of encoded features. Defaults to None.
+        """
+        super().__init__()
+        self.scaler_type = scaler_type
+        self.scale_encoded_features = scale_encoded_features
+        self.encoded_feature_names = encoded_feature_names if encoded_feature_names else []
+        self.scaler = self.SCALER_REGISTRY.get(scaler_type)
+
+        # Attributes to be defined during fitting
+        self.n_features_in_ = None
+        self.feature_names_in_ = None
+        self.feature_names_out_ = None
+        self.columns_dropped_ = []
+
+        # Parameters of nested estimators
+        self.params = params
+        # Initialize nested estimators
+        self.scaler = self.scaler(**self.params)
+
+    def fit(self, x: pd.DataFrame, y: pd.Series = None) -> 'Scaler':
+        """
+        Fit the scaler to the dataset.
+
+        Args:
+            x: pandas DataFrame with input data.
+            y: (optional) labels. Defaults to None.
+
+        Returns:
+            Self.
+        """
+        logger.debug("Fitting Scaler transformer...")
+        self.feature_names_in_ = list(x.columns)
+        self.n_features_in_ = len(self.feature_names_in_)
+
+        # Determine features to scale
+        if self.scale_encoded_features:
+            subset_to_fit = x
+        else:
+            non_encoded_features = [col for col in x.columns if col not in self.encoded_feature_names]
+            subset_to_fit = x[non_encoded_features]
+
+        # Fit the scaler
+        self.scaler.fit(subset_to_fit)
+
+        # Define output feature names
+        self.feature_names_out_ = list(x.columns)
+        return self
+
+    def transform(self, x: pd.DataFrame) -> pd.DataFrame:
+        """
+        Apply the scaling transformation to the data.
+
+        Args:
+            x: pandas DataFrame of input data.
+
+        Returns:
+            Transformed DataFrame.
+        """
+        logger.debug("Transforming data with Scaler transformer...")
+        check_is_fitted(self)
+        x_transformed = x.copy()
+
+        # Transform the appropriate features
+        if self.scale_encoded_features:
+            x_transformed.iloc[:, :] = self.scaler.transform(x_transformed)
+        else:
+            non_encoded_features = [col for col in x.columns if col not in self.encoded_feature_names]
+            x_transformed[non_encoded_features] = self.scaler.transform(x[non_encoded_features])
+
+        return x_transformed
+
+    def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
+        """
+        Apply the inverse scaling transformation to the data.
+
+        Args:
+            x: pandas DataFrame of scaled input data.
+
+        Returns:
+            Inversely transformed DataFrame.
+        """
+        logger.debug("Applying inverse transformation with Scaler transformer...")
+        x_inverse_transformed = x.copy()
+
+        # Apply inverse transform to the appropriate features
+        if self.scale_encoded_features:
+            x_inverse_transformed.iloc[:, :] = self.scaler.inverse_transform(x_inverse_transformed)
+        else:
+            non_encoded_features = [col for col in x.columns if col not in self.encoded_feature_names]
+            x_inverse_transformed[non_encoded_features] = self.scaler.inverse_transform(x[non_encoded_features])
+
+        return x_inverse_transformed
+
+    def get_feature_names_out(self, input_features=None) -> list:
+        """
+        Get output feature names for the transformed data.
+
+        Args:
+            input_features: Optional list of input features.
+
+        Returns:
+            List of output feature names.
+        """
+        return self.feature_names_out_

From c613ad193731fce95faca35e1da87898ab36a8e1 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Mon, 22 Jun 2026 15:52:38 +0200
Subject: [PATCH 03/35] Refactor handling of categorical features in scaler and
 imputer

Updated the scaler to use `scale_categorical_features` instead of `scale_encoded_features` for clarity. The imputer was expanded to separately handle numerical and categorical features, introducing dedicated imputers for each and enhancing column-specific processing. Imputer is not tested yet and logging is missing.
---
 .../data_preprocessing/imputer.py             | 87 +++++++++++++++++--
 .../data_preprocessing/scaler.py              | 26 +++---
 2 files changed, 92 insertions(+), 21 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index 562e8eed..fe1a0281 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -1,32 +1,103 @@
 from typing import Optional, List, Union, Callable
 import pandas as pd
+from dask.dataframe.dispatch import categorical_dtype
 from sklearn.utils.validation import check_is_fitted
+from sklearn.impute import SimpleImputer
+from statsmodels.tools import categorical
+
 from energy_fault_detector.core.data_transformer import DataTransformer
+import logging
+
+logger = logging.getLogger('energy_fault_detector')
+
 
 class Imputer(DataTransformer):
-    " Class containing the imputation step of categorical features to be used in the pre-pocessing pipeline"
-    def __init__(self, strategy: str = 'most_frequent'):
+    """
+    Class containing the imputation step. It is a wrap around methods for
+    imputation of numerical and categorical features.
+    """
+
+    def __init__(self, strategy: str = 'mean', categorical_features: list = None, **params):
         super().__init__()
         self.strategy = strategy
+        self.categorical_features = categorical_features if categorical_features else []
+        # Initialize nested estimators
+        self.params = params
+        # Separate imputers for numerical and categorical data
+        self.numerical_imputer = SimpleImputer(strategy=self.strategy, **self.params)
+        self.categorical_imputer = SimpleImputer(strategy='most_frequent')  # Categorical default to 'most_frequent'
+
+        # Attributes to be defined during fitting
+        self.n_features_in_ = None
+        self.feature_names_in_ = None
+        self.feature_names_out_ = None
+        self.input_index_ = None
+        self.numerical_columns = None
+        self.categorical_columns = None
 
     def fit(self, x: pd.DataFrame, y=None):
+        """
+        Fits the imputer on the provided DataFrame by separately handling numerical
+        and categorical columns.
+        """
         self.feature_names_in_ = x.columns.tolist()
         self.n_features_in_ = len(self.feature_names_in_)
         self.input_index_ = x.index
-        # Impute the most frequent value for each column
-        self.fill_values_ = x.mode().iloc[0]
+
+        # Split data into numerical (and boolean) and categorical features
+        self.numerical_columns = [col for col in self.feature_names_in_ if not any(feature in col for feature in self.categorical_features)]
+        self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
+        numerical_data = x[self.numerical_columns]
+        categorical_data = x[self.categorical_columns]
+
+        # Fit the imputers
+        self.numerical_imputer.fit(numerical_data)
+        if not categorical_data.empty:
+            self.categorical_imputer.fit(categorical_data)
+
         return self
     
     def transform(self, x: pd.DataFrame):
+        """
+        Transforms the DataFrame by imputing missing values for numerical and
+        categorical columns separately. Rejoins the transformed dataframes afterward.
+        """
         check_is_fitted(self)
-        x_ = x.fillna(self.fill_values_)
+
+        # Separate data into numerical and categorical features
+        # TODO: what happens if x doesnt have all columns as during fit?
+        numerical_data = x[self.numerical_columns]
+        categorical_data = x[self.categorical_columns]
+
+        # Transform the data
+        numerical_transformed = pd.DataFrame(
+            self.numerical_imputer.transform(numerical_data),
+            columns=self.numerical_columns,
+            index=x.index
+        )
+        if not categorical_data.empty:
+            categorical_transformed = pd.DataFrame(
+                self.categorical_imputer.transform(categorical_data),
+                columns=self.categorical_columns,
+                index=x.index
+            )
+        else:
+            categorical_transformed = pd.DataFrame(index=x.index)  # Empty DataFrame for consistency
+
+        # Concatenate numerical and categorical data back together
         self.feature_names_out_ = self.get_feature_names_out()
-        return pd.DataFrame(x_, columns=self.feature_names_out_, index=self.input_index_)
+        transformed_data = pd.concat([numerical_transformed, categorical_transformed], axis=1)
+        transformed_data = transformed_data[self.feature_names_out_]  # Ensure original column order
+
+        return transformed_data
     
     def inverse_transform(self, x: pd.DataFrame):
+        """
+        For compatibility, this method returns the DataFrame with the original column
+        order but assumes no special inversions are necessary after imputation.
+        """
         check_is_fitted(self)
-        x_ = x
-        return pd.DataFrame(x_, columns=self.feature_names_in_, index=self.input_index_)
+        return pd.DataFrame(x, columns=self.feature_names_in_, index=self.input_index_)
     
     def get_feature_names_out(self, input_features=None):
         check_is_fitted(self)
diff --git a/energy_fault_detector/data_preprocessing/scaler.py b/energy_fault_detector/data_preprocessing/scaler.py
index eb8fce1b..bc5b0309 100644
--- a/energy_fault_detector/data_preprocessing/scaler.py
+++ b/energy_fault_detector/data_preprocessing/scaler.py
@@ -16,20 +16,20 @@ class Scaler(DataTransformer):
         'minmax': MinMaxScaler
     }
 
-    def __init__(self, scaler_type: str = 'standard', scale_encoded_features: bool = True,
-                 encoded_feature_names: list = None, **params):
+    def __init__(self, scaler_type: str = 'standard', scale_categorical_features: bool = True,
+                 categorical_features: list = None, **params):
         """
         Initialize the Scaler object.
 
         Args:
             scaler_type: Type of scaler ('standard', 'minmax'). Defaults to 'standard'.
-            scale_encoded_features: Flag to scale encoded features. Defaults to True.
-            encoded_feature_names: List of names of encoded features. Defaults to None.
+            scale_categorical_features: Flag to scale categorical features after encoding. Defaults to True.
+            categorical_features: List of names of categorical features. Defaults to None.
         """
         super().__init__()
         self.scaler_type = scaler_type
-        self.scale_encoded_features = scale_encoded_features
-        self.encoded_feature_names = encoded_feature_names if encoded_feature_names else []
+        self.scale_categorical_features = scale_categorical_features
+        self.categorical_features = categorical_features if categorical_features else []
         self.scaler = self.SCALER_REGISTRY.get(scaler_type)
 
         # Attributes to be defined during fitting
@@ -59,10 +59,10 @@ def fit(self, x: pd.DataFrame, y: pd.Series = None) -> 'Scaler':
         self.n_features_in_ = len(self.feature_names_in_)
 
         # Determine features to scale
-        if self.scale_encoded_features:
+        if self.scale_categorical_features:
             subset_to_fit = x
         else:
-            non_encoded_features = [col for col in x.columns if col not in self.encoded_feature_names]
+            non_encoded_features = [col for col in x.columns if not any(feature in col for feature in self.categorical_features)]
             subset_to_fit = x[non_encoded_features]
 
         # Fit the scaler
@@ -85,12 +85,12 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         logger.debug("Transforming data with Scaler transformer...")
         check_is_fitted(self)
         x_transformed = x.copy()
-
+        # TODO: not sure if this work when x has different columns as the ones during fit...
         # Transform the appropriate features
-        if self.scale_encoded_features:
+        if self.scale_categorical_features:
             x_transformed.iloc[:, :] = self.scaler.transform(x_transformed)
         else:
-            non_encoded_features = [col for col in x.columns if col not in self.encoded_feature_names]
+            non_encoded_features = [col for col in x.columns if not any(feature in col for feature in self.categorical_features)]
             x_transformed[non_encoded_features] = self.scaler.transform(x[non_encoded_features])
 
         return x_transformed
@@ -109,10 +109,10 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
         x_inverse_transformed = x.copy()
 
         # Apply inverse transform to the appropriate features
-        if self.scale_encoded_features:
+        if self.scale_categorical_features:
             x_inverse_transformed.iloc[:, :] = self.scaler.inverse_transform(x_inverse_transformed)
         else:
-            non_encoded_features = [col for col in x.columns if col not in self.encoded_feature_names]
+            non_encoded_features = [col for col in x.columns if not any(feature in col for feature in self.categorical_features)]
             x_inverse_transformed[non_encoded_features] = self.scaler.inverse_transform(x[non_encoded_features])
 
         return x_inverse_transformed

From a2f0ca4efcb59303d1553af00f7cd7d70ddaa328 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 3 Jul 2026 10:49:26 +0200
Subject: [PATCH 04/35] Refactor preprocessing steps to use custom Scaler and
 Imputer.

Replaced sklearn's StandardScaler and SimpleImputer with custom Scaler and Imputer classes. Improved CategoricalEncoder to handle missing columns gracefully and separate numerical and categorical processing. Added validation with `check_is_fitted` in data_preprocessor.py ensure transformers are correctly applied.
---
 .../data_preprocessing/categorical_encoder.py | 54 ++++++++++++++-----
 .../data_preprocessing/data_preprocessor.py   | 46 +++++++++-------
 .../data_preprocessing/imputer.py             |  2 -
 3 files changed, 69 insertions(+), 33 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index ebd0525a..77d997c5 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -7,29 +7,59 @@
 
 class CategoricalEncoder(DataTransformer):
     " Class containing the one-hot-code step of categorical features to be used in the pre-pocessing pipeline"
-    def __init__(self, categorical_features):
+    def __init__(self, categorical_features: list = None):
         super().__init__()
-        self.categorical_features = categorical_features
+        self.categorical_features = categorical_features if categorical_features else []
         self.one_hot_encoder = OneHotEncoder(sparse_output=False)
+        self.categorical_columns = None
 
     def fit(self, x: pd.DataFrame, y=None):
-        self.feature_names_in_ = x.columns.tolist()
+        # Only categorical features are fitted and therefore features_names_in_ contain only categorical features
+        self.categorical_columns = [col for col in x.columns if any(feature in col for feature in self.categorical_features)]
+        self.feature_names_in_ = self.categorical_columns
         self.n_features_in_ = len(self.feature_names_in_)
-        self.input_index_ = x.index
-        # Do the one-hot-encode
-        self.one_hot_encoder.fit(x)
+        categorical_data = x[self.categorical_columns]
+
+        # Do the one-hot-encode on categorical data
+        self.one_hot_encoder.fit(categorical_data)
         return self
     
     def transform(self, x: pd.DataFrame):
         check_is_fitted(self)
-        x_ = self.one_hot_encoder.transform(x)
-        self.feature_names_out_ = self.get_feature_names_out()
-        return pd.DataFrame(x_, columns=self.feature_names_out_, index=self.input_index_)
-    
+
+        # Separate data into numerical and categorical features. Only categorical features are transformed
+        # TODO: what happens if x doesnt have all columns as during fit?
+        numerical_data = x[[col for col in x.columns if col not in self.categorical_columns]]
+        try:
+            categorical_data = x[self.categorical_columns]
+        except KeyError:
+            raise KeyError(f"The categorical features {self.categorical_columns} are not present in the input data.")
+
+        if not categorical_data.empty:
+            x_categorical_ = self.one_hot_encoder.transform(categorical_data)
+            self.feature_names_out_ = self.get_feature_names_out(self.feature_names_in_)
+            categorical_transformed = pd.DataFrame(x_categorical_, columns=self.feature_names_out_, index=categorical_data.index)
+            transformed_data = pd.concat([numerical_data, categorical_transformed], axis=1)
+            return transformed_data
+        else:
+            return numerical_data  # Returns input df if no categorical features are specified in config file
+
     def inverse_transform(self, x: pd.DataFrame):
         check_is_fitted(self)
-        x_ = self.one_hot_encoder.inverse_transform(x)
-        return pd.DataFrame(x_, columns=self.feature_names_in_, index=self.input_index_)
+
+        # Separate numerical columns from one-hot-encoded categorical columns
+        categorical_encoded_columns = [col for col in x.columns if col in self.feature_names_out_]
+        numerical_columns = [col for col in x.columns if col not in self.feature_names_out_]
+
+        numerical_data = x[numerical_columns]
+
+        if categorical_encoded_columns:
+            categorical_data = x[categorical_encoded_columns]
+            x_categorical_ = self.one_hot_encoder.inverse_transform(categorical_data)
+            categorical_original = pd.DataFrame(x_categorical_, columns=self.categorical_columns, index=x.index)
+            return pd.concat([numerical_data, categorical_original], axis=1)
+        else:
+            return numerical_data # Returns input df if no categorical features are specified in config file
     
     def get_feature_names_out(self, input_features=None):
         check_is_fitted(self)
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index 03f536f8..b1c21514 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -5,8 +5,7 @@
 
 import pandas as pd
 from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import StandardScaler, MinMaxScaler
-from sklearn.impute import SimpleImputer
+from sklearn.utils.validation import check_is_fitted
 
 from ..core.save_load_mixin import SaveLoadMixin
 from .column_selector import ColumnSelector
@@ -16,6 +15,7 @@
 from .counter_diff_transformer import CounterDiffTransformer
 from .timestamp_transformer import TimestampTransformer
 from .categorical_encoder import CategoricalEncoder
+from .scaler import Scaler
 from .imputer import Imputer
 
 
@@ -26,9 +26,8 @@ class DataPreprocessor(Pipeline, SaveLoadMixin):
         'low_unique_value_filter': LowUniqueValueFilter,
         'angle_transformer': AngleTransformer,
         'counter_diff_transformer': CounterDiffTransformer,
-        'simple_imputer': SimpleImputer,
-        'standard_scaler': StandardScaler,
-        'minmax_scaler': MinMaxScaler,
+        'simple_imputer': Imputer,
+        'scaler': Scaler,
         'timestamp_transformer': TimestampTransformer,
         'categorical_encoder': CategoricalEncoder,
         'imputer': Imputer,
@@ -71,7 +70,7 @@ def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None:
               1) NaN introducing steps first (DuplicateValuesToNan, CounterDiffTransformer),
               2) ColumnSelector (if present),
               3) Other steps
-              4) SimpleImputer placed before scaler (always present; mean strategy by default),
+              4) Imputer placed before scaler (always present; mean strategy by default),
               5) Scaler always last (StandardScaler by default).
               6) TimestampTransformer (if present).
 
@@ -124,6 +123,7 @@ def inverse_transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
         Returns:
             DataFrame with inverse scaling and angle back-transformation.
         """
+        check_is_fitted(self)
         x_ = x.copy()  # avoid modifying the original DataFrame
 
         # Drop time features
@@ -157,6 +157,7 @@ def transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
         Returns:
             DataFrame with the same index as input.
         """
+        check_is_fitted(self)
         x_ = super().transform(X=x.copy())
         return pd.DataFrame(data=x_, columns=self.get_feature_names_out(), index=x.index)
 
@@ -254,7 +255,7 @@ def _build_default_pipeline(self) -> List:
         Steps:
             - Column selection: A ColumnSelector object filters out columns/features with too many NaN values.
             - Low unique value filter: Remove columns/features with <= 2 unique values.
-            - Imputation with sklearn's SimpleImputer
+            - Simple imputation
             - Scaling: Apply either sklearn's StandardScaler or MinMaxScaler.
 
         Returns:
@@ -264,8 +265,8 @@ def _build_default_pipeline(self) -> List:
         steps = [
             ("column_selector", ColumnSelector(max_nan_frac_per_col=0.05)),
             ("low_unique_value_filter", LowUniqueValueFilter(min_unique_value_count=2, max_col_zero_frac=1.0)),
-            ("simple_imputer", SimpleImputer(strategy="mean").set_output(transform="pandas")),
-            ("standard_scaler", StandardScaler(with_mean=True, with_std=True)),
+            ("simple_imputer", Imputer(strategy="mean").set_output(transform="pandas")),
+            ("standard_scaler", Scaler(with_mean=True, with_std=True)),
         ]
 
         return steps
@@ -320,7 +321,7 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
           - NaN introducing steps first (DuplicateValuesToNan and CounterDiffTransformer)
           - ColumnSelector (if present).
           - Other steps
-          - Any imputer placed at the end, before scaler. If no imputer was defined, the SimpleImputer with imputation
+          - Any imputer placed at the end, before scaler. If no imputer was defined, the a simple imputer with imputation
             strategy 'mean' is added.
           - Scaler last (if present). If no scaler is added, the StandardScaler with default values is added.
           - TimestampTransformer (if present).
@@ -341,8 +342,7 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
         duplicates = [s for s in steps_spec if s.get("name") == "duplicate_to_nan"]
         counter = [s for s in steps_spec if s.get("name") == "counter_diff_transformer"]
         imputer = [s for s in steps_spec if s.get("name") == "simple_imputer"]
-        scaler_names = {"standard_scaler", "minmax_scaler"}
-        scalers = [s for s in steps_spec if s.get("name") in scaler_names]
+        scalers = [s for s in steps_spec if s.get("name") == "scaler"]
         encoders = [s for s in steps_spec if s.get("name") == "categorical_encoder"]
         if len(scalers) > 1:
             raise ValueError(f"Only one scaler can be used, two found in the steps specification: {scalers}")
@@ -352,15 +352,20 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
             s for s in steps_spec
             if s.get("name") not in {
                 "column_selector", "duplicate_to_nan", "counter_diff_transformer", "simple_imputer",
-                "categorical_encoder", "low_unique_value_filter", "timestamp_transformer",
-            } | scaler_names
+                "categorical_encoder", "low_unique_value_filter", "timestamp_transformer", "scaler"
+            }
         ]
 
         # Add default scaler if empty
+        # TODO: Review default scaler and imputer. What happens if the data contains categorical features?
         if not scalers:
-            scalers = [{'name': 'standard_scaler',
+            scalers = [{'name': 'scaler',
                         'step_name': 'scaler',
-                        'params': {'with_mean': True, 'with_std': True}}]
+                        'params': {
+                            'scaler_typ': 'standard',
+                            'scale_categorical_features': True,
+                            'with_mean': True,
+                            'with_std': True}}]
         # Add default imputer if empty
         if not imputer:
             imputer = [{'name': 'simple_imputer',
@@ -377,11 +382,14 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
         ordered.extend(low_unique_value_filter)
         # other transformations
         ordered.extend(others)
-        # Imputation and scaling
+        # Imputation
         ordered.extend(imputer)
-        ordered.extend(scalers)
+        # Encoding categorical features before scaling
+        ordered.extend(encoders)
+        # Scaling
+        # ordered.extend(scalers)
         # No scaling needed for the time features
-        ordered.extend(timestamp_step)
+        # ordered.extend(timestamp_step)
         return ordered
 
     @staticmethod
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index fe1a0281..3fc0e39d 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -1,9 +1,7 @@
 from typing import Optional, List, Union, Callable
 import pandas as pd
-from dask.dataframe.dispatch import categorical_dtype
 from sklearn.utils.validation import check_is_fitted
 from sklearn.impute import SimpleImputer
-from statsmodels.tools import categorical
 
 from energy_fault_detector.core.data_transformer import DataTransformer
 import logging

From 99e3c554d04b327052915f20fc83311f2057ca63 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 3 Jul 2026 12:15:46 +0200
Subject: [PATCH 05/35] Refactor data preprocessing and enhance
 CategoricalEncoder. Enhanced `CategoricalEncoder` with detailed docstrings,
 improved handling of numerical and categorical features, and fixed feature
 name consistency issues. Data preprocessing pipeline successfully tested.

---
 .../data_preprocessing/categorical_encoder.py | 122 ++++++++++++++++--
 .../data_preprocessing/data_preprocessor.py   |   6 +-
 2 files changed, 115 insertions(+), 13 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index 77d997c5..98b8d4ce 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -1,4 +1,6 @@
 from typing import Optional, List, Union, Callable
+
+import numpy as np
 import pandas as pd
 from sklearn.preprocessing import OneHotEncoder
 from sklearn.utils.validation import check_is_fitted
@@ -6,17 +8,112 @@
 
 
 class CategoricalEncoder(DataTransformer):
-    " Class containing the one-hot-code step of categorical features to be used in the pre-pocessing pipeline"
+    """
+    CategoricalEncoder is a specialized transformer for encoding categorical features in a dataset.
+
+    This class is designed to handle preprocessing of datasets by encoding specified categorical
+    features using one-hot encoding while maintaining numerical features unchanged. It provides
+    methods to fit to the dataset, transform it into an encoded format, inversely transform
+    encoded data back to the original format, and retrieve the transformed feature names.
+    It assumes the input data will be in the form of a DataFrame.
+
+    It can be used in preprocessing pipelines for machine learning models.
+
+    Attributes:
+        categorical_features (list): A list of strings representing the categorical feature names to be one-hot encoded.
+                                     If not provided, the encoder will not encode any categorical features.
+        feature_names_out_ (list): The list of output feature names after transformation.
+        numerical_columns (list): The list of numerical feature names identified from the input data.
+        n_features_in_ (int): The total number of input features in the dataset.
+        feature_names_in_ (list): The list of input feature names identified during the fit process.
+        one_hot_encoder (OneHotEncoder): An instance of `OneHotEncoder` used for transforming categorical features.
+        categorical_columns (list): The list of categorical columns identified from the input data.
+
+    Methods:
+        fit(x: pd.DataFrame, y=None):
+            Fits the OneHotEncoder to the categorical features in the provided DataFrame.
+
+            Args:
+                x (pd.DataFrame): The input data containing all features.
+                                  Only the specified categorical features will be fitted.
+                y: Ignored. Retained for compatibility with scikit-learn API.
+
+            Returns:
+                self: The fitted CategoricalEncoder instance.
+
+        transform(x: pd.DataFrame) -> pd.DataFrame:
+            Transforms the input DataFrame by applying one-hot encoding to categorical features
+            and combining them with numerical features.
+
+            Args:
+                x (pd.DataFrame): The input data to be transformed.
+
+            Returns:
+                pd.DataFrame: The transformed DataFrame with one-hot encoded categorical features
+                              and numerical features.
+
+            Raises:
+                KeyError: If the input DataFrame is missing any of the features fitted during the `fit` step.
+
+        inverse_transform(x: pd.DataFrame) -> pd.DataFrame:
+            Reverts the transformed data back to its original form by mapping one-hot encoded
+            categorical features back to the original categorical values.
+
+            Args:
+                x (pd.DataFrame): The transformed input data.
+
+            Returns:
+                pd.DataFrame: The original data with categorical features restored to their original values.
+
+        get_feature_names_out(input_features=None) -> list:
+            Returns the names of features after the transformation.
+
+            Args:
+                input_features (list, optional): Unused. Retained for compatibility with scikit-learn API.
+
+            Returns:
+                list: List of output feature names including both numerical and one-hot encoded features.
+
+    Example:
+        ```python
+        import pandas as pd
+        from categorical_encoder import CategoricalEncoder
+
+        # Example dataset
+        data = pd.DataFrame({
+            'Category1': ['A', 'B', 'A'],
+            'Category2': ['X', 'Y', 'Z'],
+            'Numerical': [1, 2, 3]
+        })
+
+        # Initialize and fit the encoder
+        encoder = CategoricalEncoder(categorical_features=['Category1', 'Category2'])
+        encoder.fit(data)
+
+        # Transform the data
+        transformed_data = encoder.transform(data)
+        print(transformed_data)
+
+        # Inverse transform the data
+        original_data = encoder.inverse_transform(transformed_data)
+        print(original_data)
+
+    """
     def __init__(self, categorical_features: list = None):
         super().__init__()
+        self.feature_names_out_ = None
+        self.numerical_columns = None
+        self.n_features_in_ = None
+        self.feature_names_in_ = None
         self.categorical_features = categorical_features if categorical_features else []
         self.one_hot_encoder = OneHotEncoder(sparse_output=False)
         self.categorical_columns = None
 
     def fit(self, x: pd.DataFrame, y=None):
         # Only categorical features are fitted and therefore features_names_in_ contain only categorical features
-        self.categorical_columns = [col for col in x.columns if any(feature in col for feature in self.categorical_features)]
-        self.feature_names_in_ = self.categorical_columns
+        self.feature_names_in_ = x.columns.tolist()
+        self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
+        self.numerical_columns = [col for col in self.feature_names_in_ if col not in self.categorical_columns]
         self.n_features_in_ = len(self.feature_names_in_)
         categorical_data = x[self.categorical_columns]
 
@@ -29,17 +126,18 @@ def transform(self, x: pd.DataFrame):
 
         # Separate data into numerical and categorical features. Only categorical features are transformed
         # TODO: what happens if x doesnt have all columns as during fit?
-        numerical_data = x[[col for col in x.columns if col not in self.categorical_columns]]
         try:
+            numerical_data = x[self.numerical_columns]
             categorical_data = x[self.categorical_columns]
         except KeyError:
-            raise KeyError(f"The categorical features {self.categorical_columns} are not present in the input data.")
+            raise KeyError(f"The features {self.feature_names_in_} are not present in the input data.")
 
         if not categorical_data.empty:
             x_categorical_ = self.one_hot_encoder.transform(categorical_data)
+            x_numerical_ = numerical_data.values
+            x_ = np.concatenate((x_numerical_, x_categorical_), axis=1)
             self.feature_names_out_ = self.get_feature_names_out(self.feature_names_in_)
-            categorical_transformed = pd.DataFrame(x_categorical_, columns=self.feature_names_out_, index=categorical_data.index)
-            transformed_data = pd.concat([numerical_data, categorical_transformed], axis=1)
+            transformed_data = pd.DataFrame(x_, columns=self.feature_names_out_, index=x.index)
             return transformed_data
         else:
             return numerical_data  # Returns input df if no categorical features are specified in config file
@@ -47,9 +145,12 @@ def transform(self, x: pd.DataFrame):
     def inverse_transform(self, x: pd.DataFrame):
         check_is_fitted(self)
 
+        # Get the one-hot encoded column names from the encoder
+        encoded_categorical_columns = list(self.one_hot_encoder.get_feature_names_out(self.categorical_features))
+
         # Separate numerical columns from one-hot-encoded categorical columns
-        categorical_encoded_columns = [col for col in x.columns if col in self.feature_names_out_]
-        numerical_columns = [col for col in x.columns if col not in self.feature_names_out_]
+        categorical_encoded_columns = [col for col in x.columns if col in encoded_categorical_columns]
+        numerical_columns = [col for col in x.columns if col not in encoded_categorical_columns]
 
         numerical_data = x[numerical_columns]
 
@@ -63,4 +164,5 @@ def inverse_transform(self, x: pd.DataFrame):
     
     def get_feature_names_out(self, input_features=None):
         check_is_fitted(self)
-        return self.one_hot_encoder.get_feature_names_out(self.feature_names_in_)
+        self.feature_names_out_ = self.numerical_columns + list(self.one_hot_encoder.get_feature_names_out(self.categorical_columns))
+        return self.feature_names_out_
\ No newline at end of file
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index b1c21514..202cc0ab 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -132,7 +132,7 @@ def inverse_transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
             x_ = self.named_steps[timestamp_key].inverse_transform(x_.copy())
 
         # Find scaler by type and reverse scaling
-        scaler_key, _ = self._find_step_by_type((StandardScaler, MinMaxScaler))
+        scaler_key, _ = self._find_step_by_type((Scaler,))
         x_ = self.named_steps[scaler_key].inverse_transform(x_)
         x_ = pd.DataFrame(data=x_, columns=self.named_steps[scaler_key].get_feature_names_out())
 
@@ -387,9 +387,9 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
         # Encoding categorical features before scaling
         ordered.extend(encoders)
         # Scaling
-        # ordered.extend(scalers)
+        ordered.extend(scalers)
         # No scaling needed for the time features
-        # ordered.extend(timestamp_step)
+        ordered.extend(timestamp_step)
         return ordered
 
     @staticmethod

From 9efe8029719e1b6d6f885d3ee5564116ca6c1ab4 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 17 Jul 2026 09:35:34 +0200
Subject: [PATCH 06/35] Enhance data preprocessing components: - Update Imputer
 to drop non-declared categorical features and log details.. - Improve
 DataPreprocessor docstring and fix scaler_type parameter name. - Validate
 scaler_type in Scaler initialization. - Update FaultDetector to ensure
 preprocessor is fitted before use and log warnings for non-numeric data.

---
 .../data_preprocessing/categorical_encoder.py     |  2 +-
 .../data_preprocessing/data_preprocessor.py       |  6 +++++-
 .../data_preprocessing/imputer.py                 | 12 +++++++++++-
 .../data_preprocessing/scaler.py                  |  5 +++++
 energy_fault_detector/fault_detector.py           | 15 ++++++++++-----
 5 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index 98b8d4ce..dacc26e8 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -110,7 +110,7 @@ def __init__(self, categorical_features: list = None):
         self.categorical_columns = None
 
     def fit(self, x: pd.DataFrame, y=None):
-        # Only categorical features are fitted and therefore features_names_in_ contain only categorical features
+        # TODO: non-numerical features that are not specified in categorical_features should bbe dropped and warning should be raised
         self.feature_names_in_ = x.columns.tolist()
         self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
         self.numerical_columns = [col for col in self.feature_names_in_ if col not in self.categorical_columns]
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index 202cc0ab..b68928fc 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -20,6 +20,10 @@
 
 
 class DataPreprocessor(Pipeline, SaveLoadMixin):
+    """
+    A configurable data preprocessing pipeline for tabular data.
+    """
+
     STEP_REGISTRY = {
         'duplicate_to_nan': DuplicateValuesToNan,
         'column_selector': ColumnSelector,
@@ -362,7 +366,7 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
             scalers = [{'name': 'scaler',
                         'step_name': 'scaler',
                         'params': {
-                            'scaler_typ': 'standard',
+                            'scaler_type': 'standard',
                             'scale_categorical_features': True,
                             'with_mean': True,
                             'with_std': True}}]
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index 3fc0e39d..bafbb95e 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -32,12 +32,14 @@ def __init__(self, strategy: str = 'mean', categorical_features: list = None, **
         self.input_index_ = None
         self.numerical_columns = None
         self.categorical_columns = None
+        self.non_declared_categorical_features = None
 
     def fit(self, x: pd.DataFrame, y=None):
         """
         Fits the imputer on the provided DataFrame by separately handling numerical
         and categorical columns.
         """
+
         self.feature_names_in_ = x.columns.tolist()
         self.n_features_in_ = len(self.feature_names_in_)
         self.input_index_ = x.index
@@ -48,6 +50,14 @@ def fit(self, x: pd.DataFrame, y=None):
         numerical_data = x[self.numerical_columns]
         categorical_data = x[self.categorical_columns]
 
+        # Clean numerical columns from non_declared categorical features
+        self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist()
+        self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
+        numerical_data = numerical_data[self.numerical_columns]
+
+        logger.debug(f"Numerical columns: {self.numerical_columns}")
+        logger.debug(f"Categorical columns: {self.categorical_columns}")
+
         # Fit the imputers
         self.numerical_imputer.fit(numerical_data)
         if not categorical_data.empty:
@@ -99,4 +109,4 @@ def inverse_transform(self, x: pd.DataFrame):
     
     def get_feature_names_out(self, input_features=None):
         check_is_fitted(self)
-        return self.feature_names_in_
\ No newline at end of file
+        return self.numerical_columns + self.categorical_columns
\ No newline at end of file
diff --git a/energy_fault_detector/data_preprocessing/scaler.py b/energy_fault_detector/data_preprocessing/scaler.py
index bc5b0309..938ab715 100644
--- a/energy_fault_detector/data_preprocessing/scaler.py
+++ b/energy_fault_detector/data_preprocessing/scaler.py
@@ -27,6 +27,11 @@ def __init__(self, scaler_type: str = 'standard', scale_categorical_features: bo
             categorical_features: List of names of categorical features. Defaults to None.
         """
         super().__init__()
+
+        # Validate scaler_type
+        if scaler_type not in self.SCALER_REGISTRY:
+            raise ValueError(f"Unsupported scaler_type '{scaler_type}'. Valid types are: {list(self.SCALER_REGISTRY.keys())}")
+
         self.scaler_type = scaler_type
         self.scale_categorical_features = scale_categorical_features
         self.categorical_features = categorical_features if categorical_features else []
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index d44dc20d..7d271332 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -7,6 +7,7 @@
 
 import pandas as pd
 import numpy as np
+from sklearn.utils.validation import check_is_fitted
 
 from energy_fault_detector.core.fault_detection_model import FaultDetectionModel
 from energy_fault_detector.core.fault_detection_result import FaultDetectionResult, ModelMetadata
@@ -41,7 +42,8 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
                               ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.Series]:
         """Preprocesses the training data using the configured data_preprocessor.
         If categorical features are declared in config file, data is split into categorical and numerical features.
-        Boolean features are treated the same as numerical features. Scaling is not applied to encoded categorical features.
+        Boolean features are treated the same as numerical features. Encoded categorical features are scaled or not
+        depending on the config flag.
         """
 
         # TODO: Check if categorical features are declared. If true, split data into categorical and numerical data.
@@ -65,6 +67,7 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
             self.autoencoder.conditional_features or []
         ) if protect else []
 
+        # Data clipping (outlier clipping)
         if self.config.data_clipping:
             logger.debug('Clip data before scaling.')
             clipper_params = self.config.data_clipping_params.copy()
@@ -77,8 +80,8 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
             x = data_clipper.transform(x)
 
         x_normal = x[y.values]
+        # Data preprocessing pipeline
         if fit_preprocessor:
-            # TODO: Here we have to run two separate pipelines, one for categorical and one for numerical + boolean.
             logger.info('Fit preprocessor pipeline.')
             fit_params = {}
             for step_name in self.data_preprocessor.named_steps.keys():
@@ -97,6 +100,8 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
             overwrite_models: bool = False, fit_autoencoder_only: bool = False, fit_preprocessor: bool = True,
             **kwargs) -> ModelMetadata:
         """Fit models on the given sensor_data and save them locally and return the metadata."""
+        if not check_is_fitted(self.data_preprocessor) and not fit_preprocessor:
+            raise ValueError("Data preprocessor is not fitted. Consider setting `fit_preprocessor=True`.")
 
         try:
             from keras.backend import clear_session
@@ -110,9 +115,9 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
                              "QuantileThresholdSelector or AdaptiveThresholdSelector.")
 
         non_numeric = sensor_data.select_dtypes(exclude='number').columns.tolist()
-        if non_numeric:
-            raise ValueError(f"`sensor_data` must be numeric. Non-numeric columns: {non_numeric}")
-
+        if non_numeric and fit_preprocessor:
+                logger.warning("Fitting preprocessor on non-numeric columns. Boolean columns will be treated as "
+                               "floats during imputation. Non-declared categorical features will be dropped.")
         clear_session()
 
         model_path = None

From c82a3b41bfc32ea60b0e52fe44ab05db60800bba Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 17 Jul 2026 16:05:23 +0200
Subject: [PATCH 07/35] Added class ForwardFillImputer. The class is an
 alternative imputer based on ffill stratey up to a set limit of samples. Data
 is previously resmpled to 1 minute by default. Remaining nans ar dropped.
 Added 'median' strategy to simple imputer. Added the condition that only one
 type of imputer can be present in the steps config. Refatoring of other
 preprocessor steps. Transform method of DataPreprocessor now returns the
 transformed object as it is returned by the sklear pipeline transform method,
 i.e. without explicitly converting to dataframe.

---
 .../data_preprocessing/categorical_encoder.py |  8 +-
 .../data_preprocessing/data_preprocessor.py   | 16 ++--
 .../data_preprocessing/ffill_imputer.py       | 89 +++++++++++++++++++
 .../data_preprocessing/imputer.py             | 41 +++++----
 4 files changed, 128 insertions(+), 26 deletions(-)
 create mode 100644 energy_fault_detector/data_preprocessing/ffill_imputer.py

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index dacc26e8..d4a3096a 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -109,7 +109,7 @@ def __init__(self, categorical_features: list = None):
         self.one_hot_encoder = OneHotEncoder(sparse_output=False)
         self.categorical_columns = None
 
-    def fit(self, x: pd.DataFrame, y=None):
+    def fit(self, x: pd.DataFrame, y=None) -> "CategoricalEncoder":
         # TODO: non-numerical features that are not specified in categorical_features should bbe dropped and warning should be raised
         self.feature_names_in_ = x.columns.tolist()
         self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
@@ -121,7 +121,7 @@ def fit(self, x: pd.DataFrame, y=None):
         self.one_hot_encoder.fit(categorical_data)
         return self
     
-    def transform(self, x: pd.DataFrame):
+    def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         check_is_fitted(self)
 
         # Separate data into numerical and categorical features. Only categorical features are transformed
@@ -142,7 +142,7 @@ def transform(self, x: pd.DataFrame):
         else:
             return numerical_data  # Returns input df if no categorical features are specified in config file
 
-    def inverse_transform(self, x: pd.DataFrame):
+    def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
         check_is_fitted(self)
 
         # Get the one-hot encoded column names from the encoder
@@ -162,7 +162,7 @@ def inverse_transform(self, x: pd.DataFrame):
         else:
             return numerical_data # Returns input df if no categorical features are specified in config file
     
-    def get_feature_names_out(self, input_features=None):
+    def get_feature_names_out(self, input_features=None) -> List[str]:
         check_is_fitted(self)
         self.feature_names_out_ = self.numerical_columns + list(self.one_hot_encoder.get_feature_names_out(self.categorical_columns))
         return self.feature_names_out_
\ No newline at end of file
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index b68928fc..466263f0 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -17,6 +17,7 @@
 from .categorical_encoder import CategoricalEncoder
 from .scaler import Scaler
 from .imputer import Imputer
+from .ffill_imputer import ForwardFillImputer
 
 
 class DataPreprocessor(Pipeline, SaveLoadMixin):
@@ -34,7 +35,7 @@ class DataPreprocessor(Pipeline, SaveLoadMixin):
         'scaler': Scaler,
         'timestamp_transformer': TimestampTransformer,
         'categorical_encoder': CategoricalEncoder,
-        'imputer': Imputer,
+        'ffill_imputer': ForwardFillImputer,
     }
 
     NAME_ALIASES: Dict[str, str] = {
@@ -51,6 +52,7 @@ class DataPreprocessor(Pipeline, SaveLoadMixin):
         "timestamp_features": "timestamp_transformer",
         "timestamp_transform": "timestamp_transformer",
         "categorical_encoder": "categorical_encoder",
+        "ffill_imputer": "ffill_imputer",
     }
 
     def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None:
@@ -163,7 +165,7 @@ def transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
         """
         check_is_fitted(self)
         x_ = super().transform(X=x.copy())
-        return pd.DataFrame(data=x_, columns=self.get_feature_names_out(), index=x.index)
+        return x_ # x_ is already a dataframe. Converting to a dataframe here and adding the input index is not working in case rows are dropped during preprocessing (e.g. in ForwardFillImputer). The index of the returned dataframe should be the index of the transformed dataframe, not the input dataframe.
 
     # pylint: disable=arguments-renamed
     def fit(self, X: pd.DataFrame, y=None, **fit_params):
@@ -239,6 +241,7 @@ def _validate_singletons(steps_spec: List[Dict[str, Any]]) -> None:
             "column_selector",
             "low_unique_value_filter",
             "simple_imputer",
+            "ffill_imputer",
             "timestamp_transformer",
             # scaler handled separately (standard_scaler/minmax_scaler) in your code
         }
@@ -311,7 +314,6 @@ def _build_from_steps_spec(self) -> List:
             cls = self.STEP_REGISTRY.get(name)
             if cls is None:
                 raise ValueError(f"Unknown step name '{name}'. Register it in STEP_REGISTRY.")
-            # TODO: review initialization of estimator classes
             estimator = cls(**params)
             step_name = spec.get("step_name", name)
             steps.append((step_name, estimator))
@@ -345,23 +347,25 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
         low_unique_value_filter = [s for s in steps_spec if s.get("name") == "low_unique_value_filter"]
         duplicates = [s for s in steps_spec if s.get("name") == "duplicate_to_nan"]
         counter = [s for s in steps_spec if s.get("name") == "counter_diff_transformer"]
-        imputer = [s for s in steps_spec if s.get("name") == "simple_imputer"]
+        imputer = [s for s in steps_spec if s.get("name") in {"simple_imputer", "ffill_imputer"}]
         scalers = [s for s in steps_spec if s.get("name") == "scaler"]
         encoders = [s for s in steps_spec if s.get("name") == "categorical_encoder"]
         if len(scalers) > 1:
             raise ValueError(f"Only one scaler can be used, two found in the steps specification: {scalers}")
+        if len(imputer) > 1:
+            raise ValueError(f"Only one imputer can be used, two found in the steps specification: {imputer}")
         timestamp_step = [s for s in steps_spec if s.get("name") == "timestamp_transformer"]
 
         others = [
             s for s in steps_spec
             if s.get("name") not in {
                 "column_selector", "duplicate_to_nan", "counter_diff_transformer", "simple_imputer",
-                "categorical_encoder", "low_unique_value_filter", "timestamp_transformer", "scaler"
+                "categorical_encoder", "low_unique_value_filter", "timestamp_transformer", "scaler",
+                "ffill_imputer"
             }
         ]
 
         # Add default scaler if empty
-        # TODO: Review default scaler and imputer. What happens if the data contains categorical features?
         if not scalers:
             scalers = [{'name': 'scaler',
                         'step_name': 'scaler',
diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py
new file mode 100644
index 00000000..e1b1aa50
--- /dev/null
+++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py
@@ -0,0 +1,89 @@
+from typing import List, Optional
+
+import logging
+import pandas as pd
+from sklearn.utils.validation import check_is_fitted
+
+from energy_fault_detector.core.data_transformer import DataTransformer
+
+logger = logging.getLogger('energy_fault_detector')
+
+class ForwardFillImputer(DataTransformer):
+
+    """Impute missing values using forward fill after resampling frequency."""
+
+    def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_features: Optional[List[str]] = None):
+        super().__init__()
+        self.freq = freq
+        self.ffill_limit = ffill_limit
+
+        # Attributes set during fit.
+        self.n_features_in_ = None
+        self.feature_names_in_: List[str] = []
+        self.feature_names_out_: List[str] = []
+        self.categorical_features = categorical_features if categorical_features else []
+        self.numerical_columns: List[str] = []
+        self.categorical_columns: List[str] = []
+        self.non_declared_categorical_features: List[str] = []
+
+    def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImputer":
+        """Store input feature metadata required by the DataTransformer API."""
+        if not isinstance(x, pd.DataFrame):
+            raise TypeError("x must be a pandas DataFrame.")
+
+        self.feature_names_in_ = x.columns.tolist()
+        self.n_features_in_ = len(self.feature_names_in_)
+
+        self.numerical_columns = [col for col in self.feature_names_in_ if not any(feature in col for feature in self.categorical_features)]
+        self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
+        # Clean numerical columns from non_declared categorical features
+        numerical_data = x.loc[:, self.numerical_columns]
+        self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist()
+        self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
+
+        if self.non_declared_categorical_features:
+            logger.info(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
+                        f"They will be dropped. Consider adding them to the categorical_features list if they should be treated as categorical.")
+
+        return self
+
+    def transform(self, x: pd.DataFrame) -> pd.DataFrame:
+        """Apply impute_by_row logic: asfreq -> ffill(limit) -> drop_duplicates -> dropna."""
+        check_is_fitted(self, attributes=["feature_names_in_", "n_features_in_"])
+        feature_names_in = self.feature_names_in_
+        if feature_names_in is None:
+            raise ValueError("ForwardFillImputer is not fitted.")
+
+        if not isinstance(x, pd.DataFrame):
+            raise TypeError("x must be a pandas DataFrame.")
+
+        missing_columns = [col for col in feature_names_in if col not in x.columns]
+        if missing_columns:
+            raise ValueError(f"Input is missing columns seen during fit: {missing_columns}")
+
+        if not isinstance(x.index, (pd.DatetimeIndex, pd.TimedeltaIndex, pd.PeriodIndex)):
+            raise TypeError(
+                "x index must be a DatetimeIndex, TimedeltaIndex, or PeriodIndex to use asfreq()."
+            )
+
+        x_selected = x[self.numerical_columns + self.categorical_columns]
+        df_resampled = x_selected.asfreq(self.freq)
+        df_filled = df_resampled.ffill(limit=self.ffill_limit)
+        df_cleaned = df_filled.drop_duplicates(keep="first")
+        df_final = df_cleaned.dropna(how="any")
+
+        self.feature_names_out_ = self.get_feature_names_out()
+        return df_final[self.feature_names_out_]
+
+    def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
+        """
+        For compatibility, this method returns the DataFrame with the original column
+        order but assumes no special inversions are necessary after imputation.
+        """
+        check_is_fitted(self)
+        return pd.DataFrame(x, columns=self.feature_names_in_)
+
+    def get_feature_names_out(self, input_features=None) -> List[str]:
+        """Return output feature names for downstream transformers."""
+        check_is_fitted(self)
+        return self.numerical_columns + self.categorical_columns
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index bafbb95e..d21a278c 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -8,7 +8,6 @@
 
 logger = logging.getLogger('energy_fault_detector')
 
-
 class Imputer(DataTransformer):
     """
     Class containing the imputation step. It is a wrap around methods for
@@ -19,22 +18,28 @@ def __init__(self, strategy: str = 'mean', categorical_features: list = None, **
         super().__init__()
         self.strategy = strategy
         self.categorical_features = categorical_features if categorical_features else []
-        # Initialize nested estimators
         self.params = params
+        # Initialize nested estimators
         # Separate imputers for numerical and categorical data
-        self.numerical_imputer = SimpleImputer(strategy=self.strategy, **self.params)
-        self.categorical_imputer = SimpleImputer(strategy='most_frequent')  # Categorical default to 'most_frequent'
+        if self.strategy == 'mean':
+            self.numerical_imputer = SimpleImputer(strategy=self.strategy, **self.params)
+            self.categorical_imputer = SimpleImputer(strategy='most_frequent')  # Categorical default to 'most_frequent'
+        elif self.strategy == 'median':
+            self.numerical_imputer = SimpleImputer(strategy=self.strategy, **self.params)
+            self.categorical_imputer = SimpleImputer(strategy='most_frequent')  # Categorical default to 'most_frequent'
+        else:
+            raise ValueError(f"Unsupported strategy: {self.strategy}. Supported strategies are 'mean' and 'median'.")
 
         # Attributes to be defined during fitting
         self.n_features_in_ = None
         self.feature_names_in_ = None
         self.feature_names_out_ = None
         self.input_index_ = None
-        self.numerical_columns = None
-        self.categorical_columns = None
-        self.non_declared_categorical_features = None
+        self.numerical_columns: List[str] = []
+        self.categorical_columns: List[str] = []
+        self.non_declared_categorical_features: List[str] = []
 
-    def fit(self, x: pd.DataFrame, y=None):
+    def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
         """
         Fits the imputer on the provided DataFrame by separately handling numerical
         and categorical columns.
@@ -47,13 +52,16 @@ def fit(self, x: pd.DataFrame, y=None):
         # Split data into numerical (and boolean) and categorical features
         self.numerical_columns = [col for col in self.feature_names_in_ if not any(feature in col for feature in self.categorical_features)]
         self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
-        numerical_data = x[self.numerical_columns]
-        categorical_data = x[self.categorical_columns]
+        numerical_data = x.loc[:, self.numerical_columns]
+        categorical_data = x.loc[:, self.categorical_columns]
 
         # Clean numerical columns from non_declared categorical features
         self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist()
         self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
-        numerical_data = numerical_data[self.numerical_columns]
+        numerical_data = numerical_data.loc[:, self.numerical_columns]
+        if self.non_declared_categorical_features:
+            logger.info(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
+                        f"They will be dropped. Consider adding them to the categorical_features list if they should be treated as categorical.")
 
         logger.debug(f"Numerical columns: {self.numerical_columns}")
         logger.debug(f"Categorical columns: {self.categorical_columns}")
@@ -65,7 +73,7 @@ def fit(self, x: pd.DataFrame, y=None):
 
         return self
     
-    def transform(self, x: pd.DataFrame):
+    def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         """
         Transforms the DataFrame by imputing missing values for numerical and
         categorical columns separately. Rejoins the transformed dataframes afterward.
@@ -74,8 +82,8 @@ def transform(self, x: pd.DataFrame):
 
         # Separate data into numerical and categorical features
         # TODO: what happens if x doesnt have all columns as during fit?
-        numerical_data = x[self.numerical_columns]
-        categorical_data = x[self.categorical_columns]
+        numerical_data = x.loc[:, self.numerical_columns]
+        categorical_data = x.loc[:, self.categorical_columns]
 
         # Transform the data
         numerical_transformed = pd.DataFrame(
@@ -99,14 +107,15 @@ def transform(self, x: pd.DataFrame):
 
         return transformed_data
     
-    def inverse_transform(self, x: pd.DataFrame):
+    def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
         """
         For compatibility, this method returns the DataFrame with the original column
         order but assumes no special inversions are necessary after imputation.
         """
         check_is_fitted(self)
+        
         return pd.DataFrame(x, columns=self.feature_names_in_, index=self.input_index_)
     
-    def get_feature_names_out(self, input_features=None):
+    def get_feature_names_out(self, input_features=None) -> List[str]:
         check_is_fitted(self)
         return self.numerical_columns + self.categorical_columns
\ No newline at end of file

From 37e04dad342eee445d1c09160045d6e15d8abb19 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 24 Jul 2026 11:44:08 +0200
Subject: [PATCH 08/35] Enhance CategoricalEncoder and Imputer: Added logging
 for non-declared categorical features and harmonized data types in numerical
 columns converting numerical columns, which might inlcude booleans,  into
 float. Updated FaultDetector docstrings for clarity.

---
 .../data_preprocessing/categorical_encoder.py | 17 ++++++++++---
 .../data_preprocessing/ffill_imputer.py       |  8 +++++-
 .../data_preprocessing/imputer.py             | 14 +++++++++--
 energy_fault_detector/fault_detector.py       | 25 +++++++++++++------
 4 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index d4a3096a..45fa57ce 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -1,11 +1,12 @@
 from typing import Optional, List, Union, Callable
-
+import logging
 import numpy as np
 import pandas as pd
 from sklearn.preprocessing import OneHotEncoder
 from sklearn.utils.validation import check_is_fitted
 from energy_fault_detector.core.data_transformer import DataTransformer
 
+logger = logging.getLogger('energy_fault_detector')
 
 class CategoricalEncoder(DataTransformer):
     """
@@ -108,12 +109,23 @@ def __init__(self, categorical_features: list = None):
         self.categorical_features = categorical_features if categorical_features else []
         self.one_hot_encoder = OneHotEncoder(sparse_output=False)
         self.categorical_columns = None
+        self.non_declared_categorical_features = None
 
     def fit(self, x: pd.DataFrame, y=None) -> "CategoricalEncoder":
-        # TODO: non-numerical features that are not specified in categorical_features should bbe dropped and warning should be raised
         self.feature_names_in_ = x.columns.tolist()
         self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
         self.numerical_columns = [col for col in self.feature_names_in_ if col not in self.categorical_columns]
+
+        # Clean numerical columns from non_declared categorical features
+        numerical_data = x.loc[:, self.numerical_columns]
+        self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist()
+        self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
+
+        if self.non_declared_categorical_features:
+            logger.info(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
+                        f"They will be dropped. Consider adding them to the categorical_features list if they should be treated as categorical.")
+        
+
         self.n_features_in_ = len(self.feature_names_in_)
         categorical_data = x[self.categorical_columns]
 
@@ -125,7 +137,6 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         check_is_fitted(self)
 
         # Separate data into numerical and categorical features. Only categorical features are transformed
-        # TODO: what happens if x doesnt have all columns as during fit?
         try:
             numerical_data = x[self.numerical_columns]
             categorical_data = x[self.categorical_columns]
diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py
index e1b1aa50..93c94aba 100644
--- a/energy_fault_detector/data_preprocessing/ffill_imputer.py
+++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py
@@ -65,7 +65,13 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
             raise TypeError(
                 "x index must be a DatetimeIndex, TimedeltaIndex, or PeriodIndex to use asfreq()."
             )
-
+        # Harmonize data types in numerical columns to avoid issues during concatenation after one-hot encoding
+        for col in self.numerical_columns:
+            if col in x.columns:
+                try:
+                    x[col] = x[col].astype(float, errors='raise')
+                except ValueError as e:
+                    raise ValueError(f"Column '{col}' cannot be converted to float.") from e
         x_selected = x[self.numerical_columns + self.categorical_columns]
         df_resampled = x_selected.asfreq(self.freq)
         df_filled = df_resampled.ffill(limit=self.ffill_limit)
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index d21a278c..1b5b6322 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -32,7 +32,7 @@ def __init__(self, strategy: str = 'mean', categorical_features: list = None, **
 
         # Attributes to be defined during fitting
         self.n_features_in_ = None
-        self.feature_names_in_ = None
+        self.feature_names_in_: List[str] = []
         self.feature_names_out_ = None
         self.input_index_ = None
         self.numerical_columns: List[str] = []
@@ -79,9 +79,19 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         categorical columns separately. Rejoins the transformed dataframes afterward.
         """
         check_is_fitted(self)
+        missing_columns = [col for col in self.feature_names_in_ if col not in x.columns]
+        if missing_columns:
+            raise ValueError(f"Input is missing columns seen during fit: {missing_columns}")
+
+        # Harmonize data types in numerical columns to avoid issues during concatenation after one-hot encoding
+        for col in self.numerical_columns:
+            if col in x.columns:
+                try:
+                    x[col] = x[col].astype(float, errors='raise')
+                except ValueError as e:
+                    raise ValueError(f"Column '{col}' cannot be converted to float.") from e
 
         # Separate data into numerical and categorical features
-        # TODO: what happens if x doesnt have all columns as during fit?
         numerical_data = x.loc[:, self.numerical_columns]
         categorical_data = x.loc[:, self.categorical_columns]
 
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index 7d271332..ed0e7466 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -44,11 +44,11 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
         If categorical features are declared in config file, data is split into categorical and numerical features.
         Boolean features are treated the same as numerical features. Encoded categorical features are scaled or not
         depending on the config flag.
-        """
 
-        # TODO: Check if categorical features are declared. If true, split data into categorical and numerical data.
-        # TODO: Fit categorical data -> categorical pipeline
-        # TODO: Fit numerical and boolean data. -> numerical pipeline
+        Returns:
+            Tuple[pd.DataFrame, pd.DataFrame, pd.Series]: Preprocessed training data after filtering only normal index,
+            original sensor data (clipped if data clipping is enabled), and normal index.
+        """
 
         x = sensor_data.sort_index()
         if normal_index is not None:
@@ -100,8 +100,6 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
             overwrite_models: bool = False, fit_autoencoder_only: bool = False, fit_preprocessor: bool = True,
             **kwargs) -> ModelMetadata:
         """Fit models on the given sensor_data and save them locally and return the metadata."""
-        if not check_is_fitted(self.data_preprocessor) and not fit_preprocessor:
-            raise ValueError("Data preprocessor is not fitted. Consider setting `fit_preprocessor=True`.")
 
         try:
             from keras.backend import clear_session
@@ -120,6 +118,10 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
                                "floats during imputation. Non-declared categorical features will be dropped.")
         clear_session()
 
+        # --- Resolve conditional features against available data ---
+        self._resolve_conditional_features(sensor_data)
+
+        # Preprocess data and fit autoencoder
         model_path = None
         x_prepped, x, y = self.preprocess_train_data(
             sensor_data=sensor_data, normal_index=normal_index, fit_preprocessor=fit_preprocessor
@@ -422,12 +424,19 @@ def run_root_cause_analysis(self, sensor_data: pd.DataFrame, track_losses: bool
 
     def _fit_threshold(self, x: pd.DataFrame, y: pd.Series, x_val: pd.DataFrame, fit_on_validation: bool = False
                        ) -> None:
-        """Fit AnomalyScore and ThresholdSelector objects."""
+        """Fit AnomalyScore and ThresholdSelector objects.
+
+        Args:
+            x: pandas DataFrame with the sensor data.
+            y: pandas Series with the labels indicating whether each sample is normal (True) or anomalous (False).
+            x_val: pandas DataFrame with the validation sensor data.
+            fit_on_validation: bool indicating whether to fit the threshold on the validation data.
+        """
 
-        # Fit score object only on normal data (all training + validation data)
         x_prepped_all = self.data_preprocessor.transform(x)
         deviations = self.autoencoder.get_reconstruction_error(x_prepped_all)
         y_ = y.loc[deviations.index]
+        # Fit score object only on normal data (all training + validation data)
         self.anomaly_score.fit(deviations[y_.values])  # use series values for compatibility with a multi-index
 
         scores = self.anomaly_score.transform(deviations)

From d178a1d82512c2e4a56d88e6b8192a3a7f16b2be Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Tue, 4 Aug 2026 17:25:20 +0200
Subject: [PATCH 09/35] Added new test cases for CategoricalEncoder and
 ForwardFillImputer, updated .gitignore to include generated documentation
 files, and added TODO comments in FaultDetector for handling of encoded
 categorical features as conditions.

---
 .gitignore                                    |   5 +
 docs/conf.py                                  | 144 ++++-----
 energy_fault_detector/fault_detector.py       |   2 +
 .../test_categorical_encoder.py               | 277 ++++++++++++++++
 .../data_preprocessing/test_ffill_imputer.py  | 303 ++++++++++++++++++
 5 files changed, 659 insertions(+), 72 deletions(-)
 create mode 100644 tests/data_preprocessing/test_categorical_encoder.py
 create mode 100644 tests/data_preprocessing/test_ffill_imputer.py

diff --git a/.gitignore b/.gitignore
index 2283addc..6c605a1e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,8 @@ coverage.xml
 /docs/modules.rst
 /docs/anomaly_detection.rst
 /docs/anomaly_detection.*.rst
+/docs/energy_fault_detector.*.rst
+/docs/energy_fault_detector.rst
 public/
 
 # builds
@@ -45,3 +47,6 @@ notebooks/PreDist/predist_data/
 
 # logging
 *.log
+
+# coding agents
+.continue/
diff --git a/docs/conf.py b/docs/conf.py
index ed747664..5ced0440 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -209,78 +209,78 @@
 ]
 
 
-def run_apidoc(app):
-    """Generate API .rst files with sphinx-apidoc, like in CI:
-
-       sphinx-apidoc -o docs energy_fault_detector/ --module-first --force --separate --templatedir docs/_templates
-    """
-
-    # Only run this step if we're building the HTML docs
-    if app.builder.name != "html":
-        return
-
-    from sphinx.ext.apidoc import main as apidoc_main
-    from pathlib import Path
-
-    here = Path(__file__).parent
-    pkg_dir = here.parent / IMPORT_NAME
-    out_dir = here
-    templates = here / "_templates"
-
-    apidoc_main([
-        "-o", str(out_dir),
-        str(pkg_dir),
-        "--module-first",
-        "--force",
-        "--separate",
-        "--templatedir", str(templates),
-        # Following files are skipped
-        # Quick fault detection internals
-        str(pkg_dir / "main.py"),
-        str(pkg_dir / "quick_fault_detection" / "configuration.py"),
-        str(pkg_dir / "quick_fault_detection" / "data_loading.py"),
-        str(pkg_dir / "quick_fault_detection" / "optimization.py"),
-        str(pkg_dir / "quick_fault_detection" / "output.py"),
-        str(pkg_dir / "quick_fault_detection" / "pipeline.py"),
-        str(pkg_dir / "quick_fault_detection" / "quick_fault_detector.py"),
-        # core internals
-        str(pkg_dir / "core" / "anomaly_score.py"),
-        str(pkg_dir / "core" / "data_transformer.py"),
-        str(pkg_dir / "core" / "threshold_selector.py"),
-        str(pkg_dir / "core" / "fault_detection_result.py"),
-        str(pkg_dir / "core" / "fault_detection_model.py"),
-        str(pkg_dir / "core" / "model_factory.py"),
-        str(pkg_dir / "core" / "save_load_mixin.py"),
-        # internal utilities
-        str(pkg_dir / "utils" / "index_utils.py"),
-        # config internals
-        str(pkg_dir / "config" / "base_config.py"),
-        str(pkg_dir / "config" / "config.py"),
-        str(pkg_dir / "config" / "quickstart_config.py"),
-        # Class specific files (documented directly under the top module)
-        # autoencoder
-        str(pkg_dir / "autoencoders" / "multilayer_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "conditional_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "lstm_seq2one_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "cnn_seq2one_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "bidirectional_lstm_seq2one_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "cnn_seq_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "lstm_seq2seq_autoencoder.py"),
-        # anomaly score
-        str(pkg_dir / "anomaly_scores" / "rmse_score.py"),
-        str(pkg_dir / "anomaly_scores" / "mahalanobis_score.py"),
-        # threshold selector
-        str(pkg_dir / "threshold_selectors" / "adaptive_threshold.py"),
-        str(pkg_dir / "threshold_selectors" / "fdr_threshold.py"),
-        str(pkg_dir / "threshold_selectors" / "fbeta_threshold.py"),
-        str(pkg_dir / "threshold_selectors" / "quantile_threshold.py"),
-        # main class (documented directly under the top module)
-        str(pkg_dir / "fault_detector.py"),
-    ])
-
-
-def setup(app):
-    app.connect("builder-inited", run_apidoc)
+# def run_apidoc(app):
+#     """Generate API .rst files with sphinx-apidoc, like in CI:
+
+#        sphinx-apidoc -o docs energy_fault_detector/ --module-first --force --separate --templatedir docs/_templates
+#     """
+
+#     # Only run this step if we're building the HTML docs
+#     if app.builder.name != "html":
+#         return
+
+#     from sphinx.ext.apidoc import main as apidoc_main
+#     from pathlib import Path
+
+#     here = Path(__file__).parent
+#     pkg_dir = here.parent / IMPORT_NAME
+#     out_dir = here
+#     templates = here / "_templates"
+
+#     apidoc_main([
+#         "-o", str(out_dir),
+#         str(pkg_dir),
+#         "--module-first",
+#         "--force",
+#         "--separate",
+#         "--templatedir", str(templates),
+#         # Following files are skipped
+#         # Quick fault detection internals
+#         str(pkg_dir / "main.py"),
+#         str(pkg_dir / "quick_fault_detection" / "configuration.py"),
+#         str(pkg_dir / "quick_fault_detection" / "data_loading.py"),
+#         str(pkg_dir / "quick_fault_detection" / "optimization.py"),
+#         str(pkg_dir / "quick_fault_detection" / "output.py"),
+#         str(pkg_dir / "quick_fault_detection" / "pipeline.py"),
+#         str(pkg_dir / "quick_fault_detection" / "quick_fault_detector.py"),
+#         # core internals
+#         str(pkg_dir / "core" / "anomaly_score.py"),
+#         str(pkg_dir / "core" / "data_transformer.py"),
+#         str(pkg_dir / "core" / "threshold_selector.py"),
+#         str(pkg_dir / "core" / "fault_detection_result.py"),
+#         str(pkg_dir / "core" / "fault_detection_model.py"),
+#         str(pkg_dir / "core" / "model_factory.py"),
+#         str(pkg_dir / "core" / "save_load_mixin.py"),
+#         # internal utilities
+#         str(pkg_dir / "utils" / "index_utils.py"),
+#         # config internals
+#         str(pkg_dir / "config" / "base_config.py"),
+#         str(pkg_dir / "config" / "config.py"),
+#         str(pkg_dir / "config" / "quickstart_config.py"),
+#         # Class specific files (documented directly under the top module)
+#         # autoencoder
+#         str(pkg_dir / "autoencoders" / "multilayer_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "conditional_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "lstm_seq2one_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "cnn_seq2one_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "bidirectional_lstm_seq2one_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "cnn_seq_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "lstm_seq2seq_autoencoder.py"),
+#         # anomaly score
+#         str(pkg_dir / "anomaly_scores" / "rmse_score.py"),
+#         str(pkg_dir / "anomaly_scores" / "mahalanobis_score.py"),
+#         # threshold selector
+#         str(pkg_dir / "threshold_selectors" / "adaptive_threshold.py"),
+#         str(pkg_dir / "threshold_selectors" / "fdr_threshold.py"),
+#         str(pkg_dir / "threshold_selectors" / "fbeta_threshold.py"),
+#         str(pkg_dir / "threshold_selectors" / "quantile_threshold.py"),
+#         # main class (documented directly under the top module)
+#         str(pkg_dir / "fault_detector.py"),
+#     ])
+
+
+# def setup(app):
+#     app.connect("builder-inited", run_apidoc)
 
 
 def linkcode_resolve(domain, info):
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index ed0e7466..d0dc9539 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -68,6 +68,7 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
         ) if protect else []
 
         # Data clipping (outlier clipping)
+        # TODO: what happens in DataClipper if data contains non-numerical features?
         if self.config.data_clipping:
             logger.debug('Clip data before scaling.')
             clipper_params = self.config.data_clipping_params.copy()
@@ -131,6 +132,7 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
         self._resolve_conditional_features(x_prepped)
 
         # Post-preprocessing check: conditionals may have been dropped
+        # TODO: consider adding the encoded categorical features to conditional_features at this point. Declared categorical features can be used as conditions after encoding.
         if not self.config.protect_conditional_features and self.autoencoder.is_conditional:
             # Check if conditionals survived preprocessing
             surviving = [
diff --git a/tests/data_preprocessing/test_categorical_encoder.py b/tests/data_preprocessing/test_categorical_encoder.py
new file mode 100644
index 00000000..5be17a66
--- /dev/null
+++ b/tests/data_preprocessing/test_categorical_encoder.py
@@ -0,0 +1,277 @@
+import pytest
+import pandas as pd
+import numpy as np
+from unittest.mock import MagicMock, patch
+from energy_fault_detector.data_preprocessing.categorical_encoder import CategoricalEncoder
+
+class TestCategoricalEncoder:
+    @pytest.fixture
+    def sample_data(self):
+        """Create a sample DataFrame with mixed categorical and numerical features."""
+        return pd.DataFrame({
+            'Category1': ['A', 'B', 'A', 'C', 'B'],
+            'Category2': ['X', 'Y', 'Z', 'X', 'Y'],
+            'Numerical1': [1.0, 2.0, 3.0, 4.0, 5.0],
+            'Numerical2': [10, 20, 30, 40, 50],
+        })
+
+    @pytest.fixture
+    def encoder_with_config(self):
+        """Create an encoder with declared categorical features."""
+        return CategoricalEncoder(categorical_features=['Category1', 'Category2'])
+
+    @pytest.fixture
+    def empty_encoder(self):
+        """Create an encoder without declared categorical features."""
+        return CategoricalEncoder(categorical_features=[])
+
+    def test_initialization(self, encoder_with_config, empty_encoder):
+        """Test the initialization of CategoricalEncoder."""
+        assert encoder_with_config.categorical_features == ['Category1', 'Category2']
+        assert empty_encoder.categorical_features == []
+        assert isinstance(encoder_with_config.one_hot_encoder, type(MagicMock().__class__))
+
+    def test_fit_with_valid_data(self, encoder_with_config, sample_data):
+        """Test that fitting works correctly on valid data."""
+        encoder_with_config.fit(sample_data)
+        
+        # Check fitted attributes
+        assert encoder_with_config.feature_names_in_ == sample_data.columns.tolist()
+        assert encoder_with_config.categorical_columns == ['Category1', 'Category2']
+        assert encoder_with_config.numerical_columns == ['Numerical1', 'Numerical2']
+        assert encoder_with_config.n_features_in_ == 4
+        assert len(encoder_with_config.categorical_columns) == 2
+        assert len(encoder_with_config.numerical_columns) == 2
+
+    def test_fit_with_non_declared_categorical_in_numerical_columns(self, encoder_with_config):
+        """Test handling of non-declared categorical features in numerical columns."""
+        data_with_mixed = pd.DataFrame({
+            'Category1': ['A', 'B', 'A'],
+            'Numerical': ['1', '2', '3'],  # String instead of numeric
+            'RealNumerical': [1.0, 2.0, 3.0]
+        })
+        
+        encoder_with_config.categorical_features = ['Category1']
+        with patch('energy_fault_detector.data_preprocessing.categorical_encoder.logger') as mock_logger:
+            encoder_with_config.fit(data_with_mixed)
+            
+            # Verify non-declared categorical features were identified and dropped
+            assert 'Numerical' in encoder_with_config.non_declared_categorical_features
+            assert 'Numerical' not in encoder_with_config.numerical_columns
+            assert 'Numerical' not in encoder_with_config.categorical_columns
+            mock_logger.info.assert_called_once()
+
+    def test_fit_with_no_categorical_features(self, empty_encoder, sample_data):
+        """Test fit behavior when no categorical features are declared."""
+        empty_encoder.fit(sample_data)
+        
+        assert empty_encoder.categorical_columns == []
+        assert empty_encoder.numerical_columns == sample_data.columns.tolist()
+        assert empty_encoder.n_features_in_ == 4
+
+    def test_transform_with_valid_data(self, encoder_with_config, sample_data):
+        """Test transformation with valid data."""
+        encoder_with_config.fit(sample_data)
+        transformed = encoder_with_config.transform(sample_data)
+        
+        # Verify output shape and column names
+        expected_columns = [
+            'Numerical1', 'Numerical2', 
+            'Category1_A', 'Category1_B', 'Category1_C', 
+            'Category2_X', 'Category2_Y', 'Category2_Z'
+        ]
+        assert list(transformed.columns) == expected_columns
+        assert transformed.shape == (5, len(expected_columns))
+        
+        # Check values are binary for one-hot columns
+        onehot_columns = [col for col in transformed.columns if col.startswith('Category')]
+        assert all(transformed[col].isin([0, 1]).all() for col in onehot_columns)
+
+    def test_transform_missing_features_raises_key_error(self, encoder_with_config, sample_data):
+        """Test that transform raises KeyError when input is missing required features."""
+        encoder_with_config.fit(sample_data)
+        
+        incomplete_data = sample_data.drop(columns=['Category1'])
+        with pytest.raises(KeyError, match="Category1"):
+            encoder_with_config.transform(incomplete_data)
+
+    def test_transform_with_no_categorical_features(self, empty_encoder, sample_data):
+        """Test transform behavior when no categorical features are declared."""
+        empty_encoder.fit(sample_data)
+        transformed = empty_encoder.transform(sample_data)
+        
+        # Should return only numerical data without transformation
+        pd.testing.assert_frame_equal(transformed, sample_data)
+
+    def test_inverse_transform_roundtrip(self, encoder_with_config, sample_data):
+        """Test inverse_transform correctly reverses the transformation."""
+        encoder_with_config.fit(sample_data)
+        transformed = encoder_with_config.transform(sample_data)
+        reconstructed = encoder_with_config.inverse_transform(transformed)
+        
+        # Sort columns before comparison to handle potential column order differences
+        pd.testing.assert_frame_equal(
+            sample_data.sort_index(axis=1), 
+            reconstructed.sort_index(axis=1)
+        )
+
+    def test_inverse_transform_with_missing_categorical_columns(self, encoder_with_config, sample_data):
+        """Test inverse_transform handles missing categorical columns gracefully."""
+        encoder_with_config.fit(sample_data)
+        transformed = encoder_with_config.transform(sample_data)
+        
+        # Create a DataFrame missing some one-hot encoded columns
+        incomplete_transformed = transformed.drop(columns=['Category1_C', 'Category2_Z'])
+        reconstructed = encoder_with_config.inverse_transform(incomplete_transformed)
+        
+        # Original data should still be reconstructed correctly
+        pd.testing.assert_frame_equal(
+            sample_data.sort_index(axis=1), 
+            reconstructed.sort_index(axis=1)
+        )
+
+    def test_inverse_transform_with_no_categorical_features(self, empty_encoder, sample_data):
+        """Test inverse_transform behavior when no categorical features are declared."""
+        empty_encoder.fit(sample_data)
+        transformed = empty_encoder.transform(sample_data)
+        reconstructed = empty_encoder.inverse_transform(transformed)
+        
+        # Should return only numerical data without transformation
+        pd.testing.assert_frame_equal(reconstructed, sample_data)
+
+    def test_get_feature_names_out(self, encoder_with_config, sample_data):
+        """Test that get_feature_names_out returns correct feature names."""
+        encoder_with_config.fit(sample_data)
+        feature_names = encoder_with_config.get_feature_names_out()
+        
+        expected_names = [
+            'Numerical1', 'Numerical2',
+            'Category1_A', 'Category1_B', 'Category1_C',
+            'Category2_X', 'Category2_Y', 'Category2_Z'
+        ]
+        assert feature_names == expected_names
+
+    def test_get_feature_names_out_with_empty_categorical_features(self, empty_encoder, sample_data):
+        """Test get_feature_names_out when no categorical features are declared."""
+        empty_encoder.fit(sample_data)
+        feature_names = empty_encoder.get_feature_names_out()
+        
+        assert feature_names == ['Category1', 'Category2', 'Numerical1', 'Numerical2']
+
+    def test_unfitted_transform_raises_error(self, sample_data):
+        """Test that transform raises error when called before fitting."""
+        encoder = CategoricalEncoder()
+        with pytest.raises(NotImplementedError):
+            encoder.transform(sample_data)
+
+    def test_unfitted_inverse_transform_raises_error(self, sample_data):
+        """Test that inverse_transform raises error when called before fitting."""
+        encoder = CategoricalEncoder()
+        with pytest.raises(NotImplementedError):
+            encoder.inverse_transform(sample_data)
+
+    def test_unfitted_get_feature_names_out_raises_error(self, sample_data):
+        """Test that get_feature_names_out raises error when called before fitting."""
+        encoder = CategoricalEncoder()
+        with pytest.raises(NotImplementedError):
+            encoder.get_feature_names_out()
+
+    def test_handle_empty_categorical_data(self):
+        """Test behavior with empty categorical dataframes."""
+        data = pd.DataFrame({
+            'Numerical1': [1.0, 2.0, 3.0],
+            'Numerical2': [10, 20, 30]
+        })
+        encoder = CategoricalEncoder(categorical_features=[])
+        
+        encoder.fit(data)
+        transformed = encoder.transform(data)
+        
+        pd.testing.assert_frame_equal(transformed, data)
+
+    def test_handle_unknown_categories_in_transform(self, encoder_with_config):
+        """Test how transform handles categories not seen during fit."""
+        train_data = pd.DataFrame({
+            'Category1': ['A', 'B', 'A'],
+            'Category2': ['X', 'Y', 'Z'],
+            'Numerical1': [1.0, 2.0, 3.0]
+        })
+        test_data = pd.DataFrame({
+            'Category1': ['A', 'C', 'D'],  # 'D' is unknown category
+            'Category2': ['X', 'Y', 'Z'],
+            'Numerical1': [4.0, 5.0, 6.0]
+        })
+        
+        encoder_with_config.fit(train_data)
+        
+        # Note: With handle_unknown='ignore', unknown categories are encoded as all zeros
+        transformed = encoder_with_config.transform(test_data)
+        
+        # Verify unknown categories are encoded as zeros
+        assert transformed.loc[2, 'Category1_D'] == 0  # Unknown category encoded as 0
+
+    def test_pandas_dtype_preservation(self, encoder_with_config):
+        """Test that numerical dtypes are preserved in transformed data."""
+        data = pd.DataFrame({
+            'Category1': ['A', 'B', 'C'],
+            'Numerical1': [1, 2, 3],
+            'Numerical2': [1.1, 2.2, 3.3]
+        })
+        
+        encoder_with_config.fit(data)
+        transformed = encoder_with_config.transform(data)
+        
+        # Check dtypes for numerical columns in transformed data
+        assert transformed['Numerical1'].dtype == np.float64
+        assert transformed['Numerical2'].dtype == np.float64
+
+    def test_index_preservation_in_transform(self, encoder_with_config):
+        """Test that index is preserved during transform."""
+        data = pd.DataFrame({
+            'Category1': ['A', 'B', 'C'],
+            'Numerical1': [1.0, 2.0, 3.0]
+        }, index=['row1', 'row2', 'row3'])
+        
+        encoder_with_config.fit(data)
+        transformed = encoder_with_config.transform(data)
+        
+        assert list(transformed.index) == ['row1', 'row2', 'row3']
+
+    def test_index_preservation_in_inverse_transform(self, encoder_with_config):
+        """Test that index is preserved during inverse_transform."""
+        data = pd.DataFrame({
+            'Category1': ['A', 'B', 'C'],
+            'Numerical1': [1.0, 2.0, 3.0]
+        }, index=['row1', 'row2', 'row3'])
+        
+        encoder_with_config.fit(data)
+        transformed = encoder_with_config.transform(data)
+        reconstructed = encoder_with_config.inverse_transform(transformed)
+        
+        assert list(reconstructed.index) == ['row1', 'row2', 'row3']
+
+    def test_repeated_fit_and_transform(self, encoder_with_config, sample_data):
+        """Test that repeated fitting and transforming works correctly."""
+        encoder_with_config.fit(sample_data)
+        transformed1 = encoder_with_config.transform(sample_data)
+        
+        # Fit again with different subset
+        subset_data = sample_data.iloc[:3]
+        encoder_with_config.fit(subset_data)
+        transformed2 = encoder_with_config.transform(subset_data)
+        
+        # Check that results differ due to different fit
+        assert not transformed1.equals(transformed2)
+        assert transformed2.shape == (3, transformed1.shape[1])
+
+    def test_get_feature_names_out_with_custom_input_features(self, encoder_with_config, sample_data):
+        """Test that input_features argument is ignored in get_feature_names_out."""
+        encoder_with_config.fit(sample_data)
+        
+        # Provide arbitrary input_features - should be ignored
+        custom_features = ['custom1', 'custom2']
+        feature_names = encoder_with_config.get_feature_names_out(custom_features)
+        
+        # Result should not be affected by input_features
+        assert 'Category1' not in feature_names
+        assert feature_names == encoder_with_config.get_feature_names_out()
\ No newline at end of file
diff --git a/tests/data_preprocessing/test_ffill_imputer.py b/tests/data_preprocessing/test_ffill_imputer.py
new file mode 100644
index 00000000..3ee85694
--- /dev/null
+++ b/tests/data_preprocessing/test_ffill_imputer.py
@@ -0,0 +1,303 @@
+import unittest
+import pytest
+import pandas as pd
+import numpy as np
+from datetime import datetime, timedelta
+
+from energy_fault_detector.data_preprocessing.ffill_imputer import ForwardFillImputer
+
+
+class TestForwardFillImputer(unittest.TestCase):
+
+    def setUp(self):
+        """Set up common test fixtures."""
+        # Create datetime index with 1-minute frequency
+        base_time = datetime(2024, 1, 1, 0, 0, 0)
+        self.timestamps = pd.date_range(start=base_time, periods=10, freq='1Min')
+        
+        # Create numeric data with some NaNs
+        self.data_numeric = pd.DataFrame({
+            'temp': [20.0, np.nan, 22.0, np.nan, np.nan, 25.0, np.nan, np.nan, np.nan, 30.0],
+            'humidity': [60.0, 61.0, np.nan, np.nan, 64.0, 65.0, np.nan, 67.0, 68.0, 69.0],
+            'pressure': [1013.0, 1013.5, 1014.0, np.nan, np.nan, np.nan, 1015.5, 1016.0, np.nan, 1016.5]
+        }, index=self.timestamps)
+        
+        # Create categorical data
+        self.data_categorical = pd.DataFrame({
+            'status': ['A', 'B', 'A', np.nan, 'A', 'B', 'A', 'A', 'B', 'A'],
+            'region': ['North', 'South', 'East', 'West', 'North', 'South', 'East', 'West', 'North', 'South']
+        }, index=self.timestamps)
+        
+        # Combined DataFrame
+        self.data_full = pd.concat([self.data_numeric, self.data_categorical], axis=1)
+        
+        # Create non-declared categorical features in numeric columns (strings)
+        self.data_mixed_categorical = self.data_numeric.copy()
+        self.data_mixed_categorical['category_col'] = ['cat1', 'cat2', 'cat1', 'cat3', 'cat2', 'cat1', 'cat3', 'cat1', 'cat2', 'cat1']
+
+    def tearDown(self):
+        """Clean up after each test."""
+        del self.timestamps, self.data_numeric, self.data_categorical, self.data_full, self.data_mixed_categorical
+
+    def test_init_defaults(self):
+        """Test default initialization."""
+        imputer = ForwardFillImputer()
+        self.assertEqual(imputer.freq, "1Min")
+        self.assertEqual(imputer.ffill_limit, 15)
+        self.assertEqual(imputer.categorical_features, [])
+
+    def test_init_custom_params(self):
+        """Test initialization with custom parameters."""
+        imputer = ForwardFillImputer(freq="5Min", ffill_limit=10, categorical_features=["status", "region"])
+        self.assertEqual(imputer.freq, "5Min")
+        self.assertEqual(imputer.ffill_limit, 10)
+        self.assertEqual(imputer.categorical_features, ["status", "region"])
+
+    def test_fit_numerical_only(self):
+        """Test fitting on numerical-only data."""
+        imputer = ForwardFillImputer()
+        imputer.fit(self.data_numeric)
+        
+        self.assertEqual(imputer.n_features_in_, 3)
+        self.assertListEqual(imputer.feature_names_in_, ['temp', 'humidity', 'pressure'])
+        self.assertListEqual(imputer.numerical_columns, ['temp', 'humidity', 'pressure'])
+        self.assertEqual(len(imputer.categorical_columns), 0)
+        self.assertEqual(len(imputer.non_declared_categorical_features), 0)
+
+    def test_fit_with_categorical(self):
+        """Test fitting with categorical features declared."""
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_full)
+        
+        self.assertEqual(imputer.n_features_in_, 5)
+        self.assertListEqual(imputer.numerical_columns, ['temp', 'humidity', 'pressure'])
+        self.assertListEqual(imputer.categorical_columns, ['status', 'region'])
+        self.assertEqual(len(imputer.non_declared_categorical_features), 0)
+
+    def test_fit_non_declared_categorical_in_numerical_columns(self):
+        """Test detection of non-declared categorical features."""
+        imputer = ForwardFillImputer()
+        imputer.fit(self.data_mixed_categorical)
+        
+        # Should detect 'category_col' as non-declared categorical
+        self.assertIn('category_col', imputer.non_declared_categorical_features)
+        self.assertNotIn('category_col', imputer.numerical_columns)
+        self.assertEqual(imputer.numerical_columns, ['temp', 'humidity', 'pressure'])
+
+    def test_transform_basic(self):
+        """Test basic forward fill transformation."""
+        imputer = ForwardFillImputer(ffill_limit=10)
+        imputer.fit(self.data_numeric)
+        result = imputer.transform(self.data_numeric)
+        
+        # Should not have any NaNs
+        self.assertFalse(result.isna().any().any())
+        # Should maintain same columns
+        self.assertEqual(result.shape[1], 3)
+        # Should have same column names
+        self.assertListEqual(list(result.columns), ['temp', 'humidity', 'pressure'])
+
+    def test_transform_with_categorical(self):
+        """Test transformation with categorical features."""
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit=10)
+        imputer.fit(self.data_full)
+        result = imputer.transform(self.data_full)
+        
+        # Should not have any NaNs
+        self.assertFalse(result.isna().any().any())
+        # Should maintain same columns
+        self.assertEqual(result.shape[1], 5)
+        # Should preserve categorical data
+        self.assertListEqual(list(result.columns), ['temp', 'humidity', 'pressure', 'status', 'region'])
+        # Check that status column is preserved (with some imputed)
+        self.assertEqual(result['status'].iloc[3], 'A')  # Should be forward-filled
+
+    def test_transform_limit_exceeded(self):
+        """Test that values beyond ffill_limit are not filled."""
+        # Create data with long gaps (> ffill_limit)
+        long_gaps_data = pd.DataFrame({
+            'value': [1.0, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, 2.0]
+        }, index=pd.date_range('2024-01-01', periods=17, freq='1Min'))
+        
+        imputer = ForwardFillImputer(ffill_limit=5)
+        imputer.fit(long_gaps_data)
+        result = imputer.transform(long_gaps_data)
+        
+        # The last value (2.0) should still be NaN because 10 consecutive NaNs > ffill_limit=5
+        self.assertTrue(result.iloc[-1].isna()['value'])
+
+    def test_transform_drops_duplicates(self):
+        """Test that duplicate rows are removed."""
+        # Create data with duplicate rows
+        dup_data = pd.DataFrame({
+            'value': [1.0, 2.0, 2.0, 3.0]
+        }, index=pd.date_range('2024-01-01', periods=4, freq='1Min'))
+        
+        imputer = ForwardFillImputer()
+        imputer.fit(dup_data)
+        result = imputer.transform(dup_data)
+        
+        # Should have fewer rows after deduplication
+        self.assertLess(result.shape[0], dup_data.shape[0])
+
+    def test_transform_drops_rows_with_any_na(self):
+        """Test that rows with any NaN are dropped after imputation."""
+        # Create data where after ffill, some rows still have NaN
+        some_na_data = pd.DataFrame({
+            'value1': [1.0, np.nan, 3.0, np.nan],
+            'value2': [np.nan, 2.0, 3.0, 4.0]
+        }, index=pd.date_range('2024-01-01', periods=4, freq='1Min'))
+        
+        imputer = ForwardFillImputer(ffill_limit=1)
+        imputer.fit(some_na_data)
+        result = imputer.transform(some_na_data)
+        
+        # Should have dropped rows with any remaining NaNs
+        self.assertFalse(result.isna().any().any())
+        self.assertEqual(result.shape[0], 2)  # Should keep at least one row
+
+    def test_transform_preserves_index_type(self):
+        """Test that index type is preserved after transformation."""
+        imputer = ForwardFillImputer()
+        imputer.fit(self.data_numeric)
+        result = imputer.transform(self.data_numeric)
+        
+        self.assertIsInstance(result.index, pd.DatetimeIndex)
+
+    def test_transform_non_datetime_index_raises(self):
+        """Test that non-datetime index raises TypeError."""
+        imputer = ForwardFillImputer()
+        imputer.fit(self.data_numeric)
+        
+        # Create DataFrame with non-datetime index
+        bad_index_data = self.data_numeric.copy()
+        bad_index_data.index = range(len(bad_index_data))
+        
+        with self.assertRaises(TypeError) as context:
+            imputer.transform(bad_index_data)
+        self.assertIn("DatetimeIndex", str(context.exception))
+
+    def test_transform_missing_columns_raises(self):
+        """Test that missing columns raise ValueError."""
+        imputer = ForwardFillImputer()
+        imputer.fit(self.data_numeric)
+        
+        # Drop one column
+        partial_data = self.data_numeric.drop(columns=['temp'])
+        
+        with self.assertRaises(ValueError) as context:
+            imputer.transform(partial_data)
+        self.assertIn("missing columns", str(context.exception).lower())
+
+    def test_transform_invalid_type_raises(self):
+        """Test that non-DataFrame input raises TypeError."""
+        imputer = ForwardFillImputer()
+        imputer.fit(self.data_numeric)
+        
+        with self.assertRaises(TypeError) as context:
+            imputer.transform([1, 2, 3])
+        self.assertIn("pandas DataFrame", str(context.exception))
+
+    def test_transform_unfitted_raises(self):
+        """Test that transform on unfitted model raises error."""
+        imputer = ForwardFillImputer()
+        
+        with self.assertRaises(ValueError) as context:
+            imputer.transform(self.data_numeric)
+        self.assertIn("not fitted", str(context.exception).lower())
+
+    def test_inverse_transform(self):
+        """Test inverse transform maintains column order."""
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit=10)
+        imputer.fit(self.data_full)
+        transformed = imputer.transform(self.data_full)
+        inverse = imputer.inverse_transform(transformed)
+        
+        # Should match original column names
+        self.assertListEqual(list(inverse.columns), self.data_full.columns.tolist())
+        self.assertEqual(inverse.shape[1], 5)
+
+    def test_get_feature_names_out(self):
+        """Test feature names out method."""
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_full)
+        
+        feature_names = imputer.get_feature_names_out()
+        self.assertEqual(len(feature_names), 5)
+        self.assertIn('temp', feature_names)
+        self.assertIn('humidity', feature_names)
+        self.assertIn('pressure', feature_names)
+        self.assertIn('status', feature_names)
+        self.assertIn('region', feature_names)
+
+    def test_fit_transform_consistency(self):
+        """Test that fit_transform produces consistent results."""
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit=10)
+        result1 = imputer.fit_transform(self.data_full)
+        result2 = imputer.transform(self.data_full)
+        
+        pd.testing.assert_frame_equal(result1, result2)
+
+    def test_empty_dataframe_handling(self):
+        """Test behavior with empty DataFrame."""
+        empty_df = pd.DataFrame(columns=self.data_numeric.columns, 
+                               index=pd.DatetimeIndex([]))
+        
+        imputer = ForwardFillImputer()
+        imputer.fit(self.data_numeric)
+        
+        result = imputer.transform(empty_df)
+        
+        self.assertEqual(result.shape[0], 0)
+
+    def test_all_nan_column_handling(self):
+        """Test handling of columns that are all NaN."""
+        all_nan_data = pd.DataFrame({
+            'value1': [1.0, 2.0, 3.0],
+            'value2': [np.nan, np.nan, np.nan]
+        }, index=pd.date_range('2024-01-01', periods=3, freq='1Min'))
+        
+        imputer = ForwardFillImputer()
+        imputer.fit(all_nan_data)
+        result = imputer.transform(all_nan_data)
+        
+        # All-NaN column should be dropped during fit or transform
+        self.assertNotIn('value2', result.columns)
+        self.assertEqual(result.shape[1], 1)
+
+    def test_numerical_conversion_error(self):
+        """Test error handling for non-convertible numerical columns."""
+        # Add non-numeric column to numeric section
+        bad_numeric = self.data_numeric.copy()
+        bad_numeric['status'] = ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
+        
+        imputer = ForwardFillImputer(categorical_features=['status'])
+        imputer.fit(bad_numeric)
+        
+        # Try transform - should succeed because status is in categorical_features
+        result = imputer.transform(bad_numeric)
+        self.assertEqual(result.shape[1], 4)
+        
+        # Now test without declaring as categorical
+        bad_numeric_no_cat = bad_numeric.drop('status', axis=1)
+        bad_numeric_no_cat['bad_col'] = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+        
+        imputer2 = ForwardFillImputer()
+        imputer2.fit(bad_numeric_no_cat)
+        
+        # Try transform - should raise error because 'bad_col' cannot convert to float
+        with self.assertRaises(ValueError) as context:
+            imputer2.transform(bad_numeric_no_cat)
+        self.assertIn("cannot be converted to float", str(context.exception))
+
+    def test_feature_names_out_after_fit(self):
+        """Test feature names out are available after fit."""
+        imputer = ForwardFillImputer()
+        imputer.fit(self.data_numeric)
+        
+        feature_names = imputer.get_feature_names_out()
+        self.assertEqual(len(feature_names), imputer.n_features_in_)
+
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

From 7efb5fb46be4012ef41cf403b595d798136bb6f8 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 7 Aug 2026 18:03:50 +0200
Subject: [PATCH 10/35] Add tests for data preprocessing components: ffill
 imputer and scaler

- Enhanced `TestDataPreprocessorPipeline` to include tests for a preprocessor with a categorical encoder and forward fill imputer.
- Updated `TestForwardFillImputer` to validate fitting with non-declared categorical features and improved handling of NaN values.
- Introduced `TestScaler` to cover various scenarios including fitting, transforming, and inverse transforming with both standard and min-max scalers, as well as handling of categorical features, negative values, outliers, and constant columns.
---
 .../data_preprocessing/categorical_encoder.py |  32 +-
 .../data_preprocessing/data_preprocessor.py   |   7 +-
 .../data_preprocessing/ffill_imputer.py       |  20 +-
 .../data_preprocessing/imputer.py             |   1 +
 .../data_preprocessing/scaler.py              |  30 +-
 .../test_categorical_encoder.py               | 472 ++++++++----------
 .../test_data_preprocessor.py                 |  48 +-
 .../data_preprocessing/test_ffill_imputer.py  |  82 ++-
 tests/data_preprocessing/test_scaler.py       | 336 +++++++++++++
 9 files changed, 690 insertions(+), 338 deletions(-)
 create mode 100644 tests/data_preprocessing/test_scaler.py

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index 45fa57ce..59014444 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -104,7 +104,6 @@ def __init__(self, categorical_features: list = None):
         super().__init__()
         self.feature_names_out_ = None
         self.numerical_columns = None
-        self.n_features_in_ = None
         self.feature_names_in_ = None
         self.categorical_features = categorical_features if categorical_features else []
         self.one_hot_encoder = OneHotEncoder(sparse_output=False)
@@ -122,19 +121,24 @@ def fit(self, x: pd.DataFrame, y=None) -> "CategoricalEncoder":
         self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
 
         if self.non_declared_categorical_features:
-            logger.info(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
-                        f"They will be dropped. Consider adding them to the categorical_features list if they should be treated as categorical.")
-        
+            logger.info(
+                "Non-declared categorical features found in data: %s. "
+                "They will be dropped. Consider adding them to the categorical_features list if they should be treated as categorical.",
+                self.non_declared_categorical_features,
+            )
 
         self.n_features_in_ = len(self.feature_names_in_)
         categorical_data = x[self.categorical_columns]
 
-        # Do the one-hot-encode on categorical data
+        # # Do the one-hot-encode on categorical data
+        # if categorical_data.isna().any().any():
+        #     raise ValueError("Missing values detected in categorical features. Please handle missing values before fitting the encoder.")
+
         self.one_hot_encoder.fit(categorical_data)
         return self
     
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
-        check_is_fitted(self)
+        check_is_fitted(self, "n_features_in_")
 
         # Separate data into numerical and categorical features. Only categorical features are transformed
         try:
@@ -154,19 +158,23 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
             return numerical_data  # Returns input df if no categorical features are specified in config file
 
     def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
-        check_is_fitted(self)
+        check_is_fitted(self, "n_features_in_")
 
         # Get the one-hot encoded column names from the encoder
-        encoded_categorical_columns = list(self.one_hot_encoder.get_feature_names_out(self.categorical_features))
+        categorical_encoded_columns = list(self.one_hot_encoder.get_feature_names_out(self.categorical_features))
 
         # Separate numerical columns from one-hot-encoded categorical columns
-        categorical_encoded_columns = [col for col in x.columns if col in encoded_categorical_columns]
-        numerical_columns = [col for col in x.columns if col not in encoded_categorical_columns]
+        numerical_columns = [col for col in x.columns if col not in categorical_encoded_columns]
 
         numerical_data = x[numerical_columns]
 
         if categorical_encoded_columns:
-            categorical_data = x[categorical_encoded_columns]
+            # Check if the categorical_encoded_columns are present in the input DataFrame
+            try:
+                categorical_data = x[categorical_encoded_columns]
+            except KeyError as exc:
+                raise KeyError(f"The features {categorical_encoded_columns} are not present in the input data.") from exc
+
             x_categorical_ = self.one_hot_encoder.inverse_transform(categorical_data)
             categorical_original = pd.DataFrame(x_categorical_, columns=self.categorical_columns, index=x.index)
             return pd.concat([numerical_data, categorical_original], axis=1)
@@ -174,6 +182,6 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
             return numerical_data # Returns input df if no categorical features are specified in config file
     
     def get_feature_names_out(self, input_features=None) -> List[str]:
-        check_is_fitted(self)
+        check_is_fitted(self, "n_features_in_")
         self.feature_names_out_ = self.numerical_columns + list(self.one_hot_encoder.get_feature_names_out(self.categorical_columns))
         return self.feature_names_out_
\ No newline at end of file
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index 466263f0..2b31a172 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -42,10 +42,7 @@ class DataPreprocessor(Pipeline, SaveLoadMixin):
         "angle_transform": "angle_transformer",
         "counter_diff": "counter_diff_transformer",
         "counter_diff_transform": "counter_diff_transformer",
-        "standardize": "standard_scaler",
-        "standard": "standard_scaler",
-        "standardscaler": "standard_scaler",
-        "minmax": "minmax_scaler",
+        "scaler": "scaler",
         "imputer": "simple_imputer",
         "duplicate_value_to_nan": "duplicate_to_nan",
         "duplicate_values_to_nan": "duplicate_to_nan",
@@ -392,7 +389,7 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
         ordered.extend(others)
         # Imputation
         ordered.extend(imputer)
-        # Encoding categorical features before scaling
+        # Encoding categorical features before scaling and after imputation (encoder would crash if NaN values are present)
         ordered.extend(encoders)
         # Scaling
         ordered.extend(scalers)
diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py
index 93c94aba..ec712c90 100644
--- a/energy_fault_detector/data_preprocessing/ffill_imputer.py
+++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py
@@ -10,7 +10,9 @@
 
 class ForwardFillImputer(DataTransformer):
 
-    """Impute missing values using forward fill after resampling frequency."""
+    """Impute missing values using forward fill with limit after resampling frequency.
+    Duplicate rows are dropped and any remaining rows with NaN values are removed.
+    """
 
     def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_features: Optional[List[str]] = None):
         super().__init__()
@@ -18,7 +20,6 @@ def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_featur
         self.ffill_limit = ffill_limit
 
         # Attributes set during fit.
-        self.n_features_in_ = None
         self.feature_names_in_: List[str] = []
         self.feature_names_out_: List[str] = []
         self.categorical_features = categorical_features if categorical_features else []
@@ -28,17 +29,20 @@ def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_featur
 
     def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImputer":
         """Store input feature metadata required by the DataTransformer API."""
+        # TODO: at this point there is no handling of all-NaN columns, should be included in fit to drop them and log a warning.
+
         if not isinstance(x, pd.DataFrame):
             raise TypeError("x must be a pandas DataFrame.")
 
         self.feature_names_in_ = x.columns.tolist()
         self.n_features_in_ = len(self.feature_names_in_)
 
-        self.numerical_columns = [col for col in self.feature_names_in_ if not any(feature in col for feature in self.categorical_features)]
-        self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
+        self.numerical_columns = [col for col in self.feature_names_in_ if not any(feature in col for feature in self.categorical_features)] # Uses nested for loop in case categorical cols have been encoded already and contain the original categorical feature name as a substring.
+        self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)] # Uses nested for loop in case categorical cols have been encoded already and contain the original categorical feature name as a substring.
         # Clean numerical columns from non_declared categorical features
         numerical_data = x.loc[:, self.numerical_columns]
-        self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist()
+        self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist() 
+        # Keep numerical and booleans
         self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
 
         if self.non_declared_categorical_features:
@@ -49,7 +53,7 @@ def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImp
 
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         """Apply impute_by_row logic: asfreq -> ffill(limit) -> drop_duplicates -> dropna."""
-        check_is_fitted(self, attributes=["feature_names_in_", "n_features_in_"])
+        check_is_fitted(self, "n_features_in_")
         feature_names_in = self.feature_names_in_
         if feature_names_in is None:
             raise ValueError("ForwardFillImputer is not fitted.")
@@ -86,10 +90,10 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
         For compatibility, this method returns the DataFrame with the original column
         order but assumes no special inversions are necessary after imputation.
         """
-        check_is_fitted(self)
+        check_is_fitted(self, "n_features_in_")
         return pd.DataFrame(x, columns=self.feature_names_in_)
 
     def get_feature_names_out(self, input_features=None) -> List[str]:
         """Return output feature names for downstream transformers."""
-        check_is_fitted(self)
+        check_is_fitted(self, "n_features_in_")
         return self.numerical_columns + self.categorical_columns
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index 1b5b6322..d3630897 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -84,6 +84,7 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
             raise ValueError(f"Input is missing columns seen during fit: {missing_columns}")
 
         # Harmonize data types in numerical columns to avoid issues during concatenation after one-hot encoding
+        x = x.copy()  # Avoid modifying the original DataFrame
         for col in self.numerical_columns:
             if col in x.columns:
                 try:
diff --git a/energy_fault_detector/data_preprocessing/scaler.py b/energy_fault_detector/data_preprocessing/scaler.py
index 938ab715..824d5931 100644
--- a/energy_fault_detector/data_preprocessing/scaler.py
+++ b/energy_fault_detector/data_preprocessing/scaler.py
@@ -38,7 +38,6 @@ def __init__(self, scaler_type: str = 'standard', scale_categorical_features: bo
         self.scaler = self.SCALER_REGISTRY.get(scaler_type)
 
         # Attributes to be defined during fitting
-        self.n_features_in_ = None
         self.feature_names_in_ = None
         self.feature_names_out_ = None
         self.columns_dropped_ = []
@@ -60,6 +59,15 @@ def fit(self, x: pd.DataFrame, y: pd.Series = None) -> 'Scaler':
             Self.
         """
         logger.debug("Fitting Scaler transformer...")
+        x = x.copy()  # Avoid modifying the original DataFrame
+
+        # Check that all columns in input are numerical types.
+        for col in x.columns:
+            try:
+                x[col] = x[col].astype(float, errors='raise')
+            except ValueError as e:
+                raise ValueError(f"Column '{col}' cannot be converted to float.") from e
+
         self.feature_names_in_ = list(x.columns)
         self.n_features_in_ = len(self.feature_names_in_)
 
@@ -88,9 +96,20 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
             Transformed DataFrame.
         """
         logger.debug("Transforming data with Scaler transformer...")
-        check_is_fitted(self)
-        x_transformed = x.copy()
-        # TODO: not sure if this work when x has different columns as the ones during fit...
+        check_is_fitted(self, "n_features_in_")
+
+        x_transformed = x.copy()  # Avoid modifying the original DataFrame
+
+        # Check that all columns in input are numerical types.
+        for col in x_transformed.columns:
+            try:
+                x_transformed[col] = x_transformed[col].astype(float, errors='raise')
+            except ValueError as e:
+                raise ValueError(f"Column '{col}' cannot be converted to float.") from e
+                
+        # Check if input features match the features seen during fit
+        if list(x.columns) != self.feature_names_in_:
+            raise ValueError(f"Input features {list(x.columns)} do not match the features seen during fit {self.feature_names_in_}.")
         # Transform the appropriate features
         if self.scale_categorical_features:
             x_transformed.iloc[:, :] = self.scaler.transform(x_transformed)
@@ -111,6 +130,8 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
             Inversely transformed DataFrame.
         """
         logger.debug("Applying inverse transformation with Scaler transformer...")
+        check_is_fitted(self, "n_features_in_")
+
         x_inverse_transformed = x.copy()
 
         # Apply inverse transform to the appropriate features
@@ -132,4 +153,5 @@ def get_feature_names_out(self, input_features=None) -> list:
         Returns:
             List of output feature names.
         """
+        check_is_fitted(self, "n_features_in_")
         return self.feature_names_out_
diff --git a/tests/data_preprocessing/test_categorical_encoder.py b/tests/data_preprocessing/test_categorical_encoder.py
index 5be17a66..facef90e 100644
--- a/tests/data_preprocessing/test_categorical_encoder.py
+++ b/tests/data_preprocessing/test_categorical_encoder.py
@@ -1,277 +1,233 @@
-import pytest
+import unittest
 import pandas as pd
 import numpy as np
-from unittest.mock import MagicMock, patch
-from energy_fault_detector.data_preprocessing.categorical_encoder import CategoricalEncoder
-
-class TestCategoricalEncoder:
-    @pytest.fixture
-    def sample_data(self):
-        """Create a sample DataFrame with mixed categorical and numerical features."""
-        return pd.DataFrame({
-            'Category1': ['A', 'B', 'A', 'C', 'B'],
-            'Category2': ['X', 'Y', 'Z', 'X', 'Y'],
-            'Numerical1': [1.0, 2.0, 3.0, 4.0, 5.0],
-            'Numerical2': [10, 20, 30, 40, 50],
-        })
-
-    @pytest.fixture
-    def encoder_with_config(self):
-        """Create an encoder with declared categorical features."""
-        return CategoricalEncoder(categorical_features=['Category1', 'Category2'])
-
-    @pytest.fixture
-    def empty_encoder(self):
-        """Create an encoder without declared categorical features."""
-        return CategoricalEncoder(categorical_features=[])
-
-    def test_initialization(self, encoder_with_config, empty_encoder):
-        """Test the initialization of CategoricalEncoder."""
-        assert encoder_with_config.categorical_features == ['Category1', 'Category2']
-        assert empty_encoder.categorical_features == []
-        assert isinstance(encoder_with_config.one_hot_encoder, type(MagicMock().__class__))
-
-    def test_fit_with_valid_data(self, encoder_with_config, sample_data):
-        """Test that fitting works correctly on valid data."""
-        encoder_with_config.fit(sample_data)
-        
-        # Check fitted attributes
-        assert encoder_with_config.feature_names_in_ == sample_data.columns.tolist()
-        assert encoder_with_config.categorical_columns == ['Category1', 'Category2']
-        assert encoder_with_config.numerical_columns == ['Numerical1', 'Numerical2']
-        assert encoder_with_config.n_features_in_ == 4
-        assert len(encoder_with_config.categorical_columns) == 2
-        assert len(encoder_with_config.numerical_columns) == 2
-
-    def test_fit_with_non_declared_categorical_in_numerical_columns(self, encoder_with_config):
-        """Test handling of non-declared categorical features in numerical columns."""
-        data_with_mixed = pd.DataFrame({
-            'Category1': ['A', 'B', 'A'],
-            'Numerical': ['1', '2', '3'],  # String instead of numeric
-            'RealNumerical': [1.0, 2.0, 3.0]
-        })
-        
-        encoder_with_config.categorical_features = ['Category1']
-        with patch('energy_fault_detector.data_preprocessing.categorical_encoder.logger') as mock_logger:
-            encoder_with_config.fit(data_with_mixed)
-            
-            # Verify non-declared categorical features were identified and dropped
-            assert 'Numerical' in encoder_with_config.non_declared_categorical_features
-            assert 'Numerical' not in encoder_with_config.numerical_columns
-            assert 'Numerical' not in encoder_with_config.categorical_columns
-            mock_logger.info.assert_called_once()
-
-    def test_fit_with_no_categorical_features(self, empty_encoder, sample_data):
-        """Test fit behavior when no categorical features are declared."""
-        empty_encoder.fit(sample_data)
-        
-        assert empty_encoder.categorical_columns == []
-        assert empty_encoder.numerical_columns == sample_data.columns.tolist()
-        assert empty_encoder.n_features_in_ == 4
-
-    def test_transform_with_valid_data(self, encoder_with_config, sample_data):
-        """Test transformation with valid data."""
-        encoder_with_config.fit(sample_data)
-        transformed = encoder_with_config.transform(sample_data)
-        
-        # Verify output shape and column names
-        expected_columns = [
-            'Numerical1', 'Numerical2', 
-            'Category1_A', 'Category1_B', 'Category1_C', 
-            'Category2_X', 'Category2_Y', 'Category2_Z'
-        ]
-        assert list(transformed.columns) == expected_columns
-        assert transformed.shape == (5, len(expected_columns))
-        
-        # Check values are binary for one-hot columns
-        onehot_columns = [col for col in transformed.columns if col.startswith('Category')]
-        assert all(transformed[col].isin([0, 1]).all() for col in onehot_columns)
-
-    def test_transform_missing_features_raises_key_error(self, encoder_with_config, sample_data):
-        """Test that transform raises KeyError when input is missing required features."""
-        encoder_with_config.fit(sample_data)
-        
-        incomplete_data = sample_data.drop(columns=['Category1'])
-        with pytest.raises(KeyError, match="Category1"):
-            encoder_with_config.transform(incomplete_data)
-
-    def test_transform_with_no_categorical_features(self, empty_encoder, sample_data):
-        """Test transform behavior when no categorical features are declared."""
-        empty_encoder.fit(sample_data)
-        transformed = empty_encoder.transform(sample_data)
-        
-        # Should return only numerical data without transformation
-        pd.testing.assert_frame_equal(transformed, sample_data)
-
-    def test_inverse_transform_roundtrip(self, encoder_with_config, sample_data):
-        """Test inverse_transform correctly reverses the transformation."""
-        encoder_with_config.fit(sample_data)
-        transformed = encoder_with_config.transform(sample_data)
-        reconstructed = encoder_with_config.inverse_transform(transformed)
-        
-        # Sort columns before comparison to handle potential column order differences
-        pd.testing.assert_frame_equal(
-            sample_data.sort_index(axis=1), 
-            reconstructed.sort_index(axis=1)
-        )
-
-    def test_inverse_transform_with_missing_categorical_columns(self, encoder_with_config, sample_data):
-        """Test inverse_transform handles missing categorical columns gracefully."""
-        encoder_with_config.fit(sample_data)
-        transformed = encoder_with_config.transform(sample_data)
-        
-        # Create a DataFrame missing some one-hot encoded columns
-        incomplete_transformed = transformed.drop(columns=['Category1_C', 'Category2_Z'])
-        reconstructed = encoder_with_config.inverse_transform(incomplete_transformed)
-        
-        # Original data should still be reconstructed correctly
-        pd.testing.assert_frame_equal(
-            sample_data.sort_index(axis=1), 
-            reconstructed.sort_index(axis=1)
-        )
+from sklearn.exceptions import NotFittedError
 
-    def test_inverse_transform_with_no_categorical_features(self, empty_encoder, sample_data):
-        """Test inverse_transform behavior when no categorical features are declared."""
-        empty_encoder.fit(sample_data)
-        transformed = empty_encoder.transform(sample_data)
-        reconstructed = empty_encoder.inverse_transform(transformed)
-        
-        # Should return only numerical data without transformation
-        pd.testing.assert_frame_equal(reconstructed, sample_data)
-
-    def test_get_feature_names_out(self, encoder_with_config, sample_data):
-        """Test that get_feature_names_out returns correct feature names."""
-        encoder_with_config.fit(sample_data)
-        feature_names = encoder_with_config.get_feature_names_out()
-        
-        expected_names = [
-            'Numerical1', 'Numerical2',
-            'Category1_A', 'Category1_B', 'Category1_C',
-            'Category2_X', 'Category2_Y', 'Category2_Z'
-        ]
-        assert feature_names == expected_names
+from energy_fault_detector.data_preprocessing.categorical_encoder import CategoricalEncoder
 
-    def test_get_feature_names_out_with_empty_categorical_features(self, empty_encoder, sample_data):
-        """Test get_feature_names_out when no categorical features are declared."""
-        empty_encoder.fit(sample_data)
-        feature_names = empty_encoder.get_feature_names_out()
-        
-        assert feature_names == ['Category1', 'Category2', 'Numerical1', 'Numerical2']
 
-    def test_unfitted_transform_raises_error(self, sample_data):
-        """Test that transform raises error when called before fitting."""
+class TestCategoricalEncoder(unittest.TestCase):
+    def setUp(self):
+        """Set up common test fixtures."""
+        # Create sample data with numerical, categorical, and mixed features
+        self.data_clean = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, 58, 65, 67],
+            'category': ['A', 'B', 'A', 'C', 'B'],
+            'region': ['North', 'South', 'North', 'East', 'West']
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+
+        self.data_clean_encoded_features = ['temperature', 'humidity', 'category_A', 'category_B', 'category_C', 'region_North', 'region_South', 'region_East', 'region_West']
+        
+        # Create data with non-declared categorical features in numerical columns
+        self.data_with_non_declared = self.data_clean.copy()
+        self.data_with_non_declared['status'] = ['high', 'medium', 'low', 'high', 'medium']
+        
+        # Create data with only numerical columns
+        self.data_numerical_only = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, 58, 65, 67]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        self.data_numerical_only_encoded_features = ['temperature', 'humidity']
+        
+        # Create data with missing values in categorical columns
+        self.data_with_nan = self.data_clean.copy()
+        self.data_with_nan.iloc[2, 2] = np.nan
+        self.data_with_nan.iloc[3, 3] = np.nan
+        
+        # Create data with duplicate rows for testing deduplication
+        self.data_with_duplicates = pd.DataFrame({
+            'temperature': [20.5, 20.5, 21.3, 22.1],
+            'category': ['A', 'A', 'B', 'C']
+        }, index=pd.date_range('2024-01-01', periods=4, freq='1Min'))
+        
+        # Create data for testing inverse transform
+        self.data_for_inverse = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8],
+            'category': ['A', 'B', 'A'],
+            'region': ['North', 'South', 'North']
+        }, index=pd.date_range('2024-01-01', periods=3, freq='1Min'))
+
+    def tearDown(self):
+        """Clean up after each test."""
+        del (self.data_clean, self.data_with_non_declared, self.data_numerical_only,
+             self.data_with_nan, self.data_with_duplicates, self.data_for_inverse)
+
+    def test_init_defaults(self):
+        """Test default initialization."""
         encoder = CategoricalEncoder()
-        with pytest.raises(NotImplementedError):
-            encoder.transform(sample_data)
-
-    def test_unfitted_inverse_transform_raises_error(self, sample_data):
-        """Test that inverse_transform raises error when called before fitting."""
+        self.assertEqual(encoder.categorical_features, [])
+        self.assertIsNone(encoder.categorical_columns)
+        self.assertIsNone(encoder.numerical_columns)
+
+    def test_init_with_categorical_features(self):
+        """Test initialization with categorical features."""
+        encoder = CategoricalEncoder(categorical_features=['category', 'region'])
+        self.assertListEqual(encoder.categorical_features, ['category', 'region'])
+
+    def test_fit_with_categorical_features(self):
+        """Test fitting with categorical features."""
+        encoder = CategoricalEncoder(categorical_features=['category', 'region'])
+        encoder.fit(self.data_clean)
+        
+        self.assertEqual(encoder.n_features_in_, 4)
+        self.assertListEqual(encoder.feature_names_in_ or [], ['temperature', 'humidity', 'category', 'region'])
+        self.assertListEqual(encoder.numerical_columns or [], ['temperature', 'humidity'])
+        self.assertListEqual(encoder.categorical_columns or [], ['category', 'region'])
+
+    def test_fit_without_declaring_categorical_features(self):
+        """Test fitting without specifying categorical features."""
         encoder = CategoricalEncoder()
-        with pytest.raises(NotImplementedError):
-            encoder.inverse_transform(sample_data)
-
-    def test_unfitted_get_feature_names_out_raises_error(self, sample_data):
-        """Test that get_feature_names_out raises error when called before fitting."""
-        encoder = CategoricalEncoder()
-        with pytest.raises(NotImplementedError):
-            encoder.get_feature_names_out()
-
-    def test_handle_empty_categorical_data(self):
-        """Test behavior with empty categorical dataframes."""
-        data = pd.DataFrame({
-            'Numerical1': [1.0, 2.0, 3.0],
-            'Numerical2': [10, 20, 30]
-        })
-        encoder = CategoricalEncoder(categorical_features=[])
-        
-        encoder.fit(data)
-        transformed = encoder.transform(data)
-        
-        pd.testing.assert_frame_equal(transformed, data)
-
-    def test_handle_unknown_categories_in_transform(self, encoder_with_config):
-        """Test how transform handles categories not seen during fit."""
-        train_data = pd.DataFrame({
-            'Category1': ['A', 'B', 'A'],
-            'Category2': ['X', 'Y', 'Z'],
-            'Numerical1': [1.0, 2.0, 3.0]
-        })
-        test_data = pd.DataFrame({
-            'Category1': ['A', 'C', 'D'],  # 'D' is unknown category
-            'Category2': ['X', 'Y', 'Z'],
-            'Numerical1': [4.0, 5.0, 6.0]
-        })
-        
-        encoder_with_config.fit(train_data)
-        
-        # Note: With handle_unknown='ignore', unknown categories are encoded as all zeros
-        transformed = encoder_with_config.transform(test_data)
+        encoder.fit(self.data_clean)
         
-        # Verify unknown categories are encoded as zeros
-        assert transformed.loc[2, 'Category1_D'] == 0  # Unknown category encoded as 0
+        # Should ignore categorical columns
+        self.assertEqual(encoder.n_features_in_, 4)
+        self.assertListEqual(encoder.numerical_columns or [], ['temperature', 'humidity'])
+        self.assertListEqual(encoder.categorical_columns or [], [])
 
-    def test_pandas_dtype_preservation(self, encoder_with_config):
-        """Test that numerical dtypes are preserved in transformed data."""
-        data = pd.DataFrame({
-            'Category1': ['A', 'B', 'C'],
-            'Numerical1': [1, 2, 3],
-            'Numerical2': [1.1, 2.2, 3.3]
-        })
+    def test_fit_with_non_declared_categorical_features(self):
+        """Test detection of non-declared categorical features in numerical columns."""
+        encoder = CategoricalEncoder(categorical_features=['category', 'region'])
+        encoder.fit(self.data_with_non_declared)
         
-        encoder_with_config.fit(data)
-        transformed = encoder_with_config.transform(data)
-        
-        # Check dtypes for numerical columns in transformed data
-        assert transformed['Numerical1'].dtype == np.float64
-        assert transformed['Numerical2'].dtype == np.float64
+        # Should detect 'status' as non-declared categorical
+        self.assertIn('status', encoder.non_declared_categorical_features or [])
+        self.assertNotIn('status', encoder.numerical_columns or [])
+        self.assertNotIn('status', encoder.categorical_columns or [])
 
-    def test_index_preservation_in_transform(self, encoder_with_config):
-        """Test that index is preserved during transform."""
-        data = pd.DataFrame({
-            'Category1': ['A', 'B', 'C'],
-            'Numerical1': [1.0, 2.0, 3.0]
-        }, index=['row1', 'row2', 'row3'])
-        
-        encoder_with_config.fit(data)
-        transformed = encoder_with_config.transform(data)
+    def test_fit_with_only_numerical_data(self):
+        """Test fitting on numerical-only data."""
+        encoder = CategoricalEncoder()
+        encoder.fit(self.data_numerical_only)
+        
+        self.assertEqual(encoder.n_features_in_, 2)
+        self.assertListEqual(encoder.numerical_columns or [], ['temperature', 'humidity'])
+        self.assertEqual(len(encoder.categorical_columns or []), 0)
+
+    def test_transform_with_categorical_features(self):
+        """Test transformation with categorical features."""
+        encoder = CategoricalEncoder(categorical_features=['category', 'region'])
+        encoder.fit(self.data_clean)
+        result = encoder.transform(self.data_clean)
+        
+        # Should have transformed columns: temperature, humidity, category_A, category_B, category_C, region_*
+        for col in self.data_clean_encoded_features:
+            self.assertIn(col, result.columns)
+        # Should maintain same index
+        self.assertTrue(result.index.equals(self.data_clean.index))
+        # Should not contain NaN
+        self.assertFalse(result.isna().any().any())
+
+    def test_transform_with_only_numerical_data(self):
+        """Test transformation when no categorical features."""
+        encoder = CategoricalEncoder()
+        encoder.fit(self.data_numerical_only)
+        result = encoder.transform(self.data_numerical_only)
         
-        assert list(transformed.index) == ['row1', 'row2', 'row3']
+        # Should return same columns and shape
+        self.assertEqual(result.shape, self.data_numerical_only.shape)
+        self.assertListEqual(list(result.columns), self.data_numerical_only_encoded_features)
 
-    def test_index_preservation_in_inverse_transform(self, encoder_with_config):
-        """Test that index is preserved during inverse_transform."""
-        data = pd.DataFrame({
-            'Category1': ['A', 'B', 'C'],
-            'Numerical1': [1.0, 2.0, 3.0]
-        }, index=['row1', 'row2', 'row3'])
+    def test_transform_missing_columns_raises(self):
+        """Test that missing columns raise KeyError."""
+        encoder = CategoricalEncoder(categorical_features=['category'])
+        encoder.fit(self.data_clean)
         
-        encoder_with_config.fit(data)
-        transformed = encoder_with_config.transform(data)
-        reconstructed = encoder_with_config.inverse_transform(transformed)
+        # Create data missing one feature
+        partial_data = self.data_clean.drop(columns=['temperature'])
         
-        assert list(reconstructed.index) == ['row1', 'row2', 'row3']
+        with self.assertRaises(KeyError):
+            encoder.transform(partial_data)
 
-    def test_repeated_fit_and_transform(self, encoder_with_config, sample_data):
-        """Test that repeated fitting and transforming works correctly."""
-        encoder_with_config.fit(sample_data)
-        transformed1 = encoder_with_config.transform(sample_data)
-        
-        # Fit again with different subset
-        subset_data = sample_data.iloc[:3]
-        encoder_with_config.fit(subset_data)
-        transformed2 = encoder_with_config.transform(subset_data)
+    def test_transform_invalid_type_raises(self):
+        """Test that non-DataFrame input raises TypeError."""
+        encoder = CategoricalEncoder()
+        encoder.fit(self.data_clean)
         
-        # Check that results differ due to different fit
-        assert not transformed1.equals(transformed2)
-        assert transformed2.shape == (3, transformed1.shape[1])
+        with self.assertRaises(TypeError):
+            encoder.transform([1, 2, 3])
 
-    def test_get_feature_names_out_with_custom_input_features(self, encoder_with_config, sample_data):
-        """Test that input_features argument is ignored in get_feature_names_out."""
-        encoder_with_config.fit(sample_data)
-        
-        # Provide arbitrary input_features - should be ignored
-        custom_features = ['custom1', 'custom2']
-        feature_names = encoder_with_config.get_feature_names_out(custom_features)
+    def test_transform_unfitted_raises(self):
+        """Test that transform on unfitted model raises error."""
+        encoder = CategoricalEncoder()
         
-        # Result should not be affected by input_features
-        assert 'Category1' not in feature_names
-        assert feature_names == encoder_with_config.get_feature_names_out()
\ No newline at end of file
+        with self.assertRaises(NotFittedError):
+            encoder.transform(self.data_clean)
+
+    def test_inverse_transform(self):
+        """Test inverse transform returns original categorical values."""
+        encoder = CategoricalEncoder(categorical_features=['category', 'region'])
+        encoder.fit(self.data_for_inverse)
+        
+        transformed = encoder.transform(self.data_for_inverse)
+        inverse = encoder.inverse_transform(transformed)
+        
+        # Should have same columns and shape as original
+        self.assertListEqual(list(inverse.columns), list(self.data_for_inverse.columns))
+        self.assertEqual(inverse.shape, self.data_for_inverse.shape)
+        # Should match original data (except for any NaN handling)
+        pd.testing.assert_frame_equal(inverse, self.data_for_inverse)
+
+    def test_get_feature_names_out(self):
+        """Test feature names out method."""
+        encoder = CategoricalEncoder(categorical_features=['category', 'region'])
+        encoder.fit(self.data_clean)
+        
+        feature_names = encoder.get_feature_names_out()
+        
+        # Should include original numerical features
+        self.assertIn('temperature', feature_names)
+        self.assertIn('humidity', feature_names)
+        # Should include encoded categorical features (exact names depend on OneHotEncoder output)
+        categorical_feature_names = [name for name in feature_names if 'category' in name or 'region' in name]
+        self.assertGreater(len(categorical_feature_names), 0)
+        self.assertEqual(len(feature_names), encoder.n_features_in_ + len(categorical_feature_names) - 2)
+
+    def test_empty_dataframe_handling(self):
+        """Test behavior with empty DataFrame."""
+        empty_df = pd.DataFrame(columns=self.data_clean.columns, 
+                                index=pd.DatetimeIndex([]))
+        
+        encoder = CategoricalEncoder(categorical_features=['category', 'region'])
+        encoder.fit(self.data_clean)
+        
+        result = encoder.transform(empty_df)
+        
+        self.assertEqual(result.shape[0], 0)
+
+    def test_transform_preserves_index_type(self):
+        """Test that index type is preserved after transformation."""
+        encoder = CategoricalEncoder(categorical_features=['category'])
+        encoder.fit(self.data_clean)
+        result = encoder.transform(self.data_clean)
+        
+        self.assertIsInstance(result.index, pd.DatetimeIndex)
+
+    def test_transform_with_nan_in_categorical(self):
+        """Test transformation with NaN values in categorical columns."""
+        encoder = CategoricalEncoder(categorical_features=['category', 'region'])
+        encoder.fit(self.data_with_nan)
+        
+        # Should ignore NaN values in categorical columns during transformation
+        result = encoder.transform(self.data_with_nan)
+        self.assertFalse(result.isna().any().any())
+
+    def test_transform_with_new_categories(self):
+        """Test transformation with unseen category values."""
+        encoder = CategoricalEncoder(categorical_features=['category'])
+        encoder.fit(self.data_clean)
+        
+        # Create data with unseen category value
+        new_data = pd.DataFrame({
+            'temperature': [25.0],
+            'humidity': [70],
+            'category': ['D'],
+            'region': ['North']
+        }, index=pd.date_range('2024-01-01', periods=1, freq='1Min'))
+
+        # Should raise error as per OneHotEncoder default behavior (handle_unknown='error')
+        with self.assertRaises(ValueError):
+            encoder.transform(new_data)
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file
diff --git a/tests/data_preprocessing/test_data_preprocessor.py b/tests/data_preprocessing/test_data_preprocessor.py
index a4a1567d..7b8b51fd 100644
--- a/tests/data_preprocessing/test_data_preprocessor.py
+++ b/tests/data_preprocessing/test_data_preprocessor.py
@@ -43,6 +43,16 @@ def setUp(self) -> None:
             ]
         )
 
+        # Includes categorical encoder and ffill imputer
+        self.preprocessor_with_encoder = DataPreprocessor(
+            steps=[
+                {'name': 'column_selector', 'params': {'max_nan_frac_per_col': 0.2}},
+                {'name': 'ffill_imputer',
+                 'params': {'freq': '1Min', 'ffill_limit': 1, 'categorical_features': ['category', 'region']}},
+                {'name': 'categorical_encoder',
+                 'params': {'categorical_features': ['category', 'region']}},
+            ]
+        )
         # generate data for standard and feature consistent preprocessor tests
         length = 10  # choose an even number for simplicity
         time_index = pd.date_range(start='1/1/2021', end='10/1/2021', periods=length)
@@ -86,6 +96,14 @@ def setUp(self) -> None:
                 }
         self.test_data3 = pd.DataFrame(index=time_index, data=data)
 
+        # generate data for tests with categorical encoder and ffill imputer
+        self.test_data4 = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, np.nan, 65, 67],
+            'category': ['A', 'B', 'A', np.nan, 'B'],
+            'region': ['North', 'South', 'North', 'East', 'West']
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+
     def test_transform(self):
         # expected output
         exp_result = np.array([[-1.5666989, 0.],
@@ -202,22 +220,22 @@ def test_steps_mode_no_duplicate_imputer(self) -> None:
             steps=[
                 {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}},
                 {"name": "simple_imputer", "params": {"strategy": "median"}},
-                {"name": "standard_scaler"},
+                {"name": "scaler"},
             ]
         )
         # Count imputers by estimator type
         n_imputers = sum(
-            est.__class__.__name__ == "SimpleImputer" for _, est in dp.steps
+            est.__class__.__name__ == "Imputer" for _, est in dp.steps
         )
         self.assertEqual(n_imputers, 1, "There should be exactly one SimpleImputer.")
 
         # Ensure imputer precedes scaler
         imputer_idx = next(
-            i for i, (_, est) in enumerate(dp.steps) if est.__class__.__name__ == "SimpleImputer"
+            i for i, (_, est) in enumerate(dp.steps) if est.__class__.__name__ == "Imputer"
         )
         scaler_idx = next(
             i for i, (_, est) in enumerate(dp.steps)
-            if est.__class__.__name__ in {"StandardScaler", "MinMaxScaler"}
+            if est.__class__.__name__ in {"Scaler"}
         )
         self.assertLess(imputer_idx, scaler_idx, "Imputer must precede scaler.")
 
@@ -226,22 +244,22 @@ def test_steps_mode_default_imputer_inserted(self) -> None:
         dp = DataPreprocessor(
             steps=[
                 {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}},
-                {"name": "standard_scaler"},
+                {"name": "scaler"},
             ]
         )
         # Exactly one imputer should be present
         n_imputers = sum(
-            est.__class__.__name__ == "SimpleImputer" for _, est in dp.steps
+            est.__class__.__name__ == "Imputer" for _, est in dp.steps
         )
         self.assertEqual(n_imputers, 1, "A single default SimpleImputer should be added.")
 
         # Imputer must be before scaler
         imputer_idx = next(
-            i for i, (_, est) in enumerate(dp.steps) if est.__class__.__name__ == "SimpleImputer"
+            i for i, (_, est) in enumerate(dp.steps) if est.__class__.__name__ == "Imputer"
         )
         scaler_idx = next(
             i for i, (_, est) in enumerate(dp.steps)
-            if est.__class__.__name__ in {"StandardScaler", "MinMaxScaler"}
+            if est.__class__.__name__ in {"Scaler"}
         )
         self.assertLess(imputer_idx, scaler_idx, "Default imputer must be inserted before scaler.")
 
@@ -250,7 +268,7 @@ def test_steps_mode_alias_imputer_is_normalized(self) -> None:
         dp = DataPreprocessor(
             steps=[
                 {"name": "imputer", "params": {"strategy": "mean"}},  # alias
-                {"name": "standard_scaler"},
+                {"name": "scaler"},
             ]
         )
         # Named steps should include the canonical 'simple_imputer'
@@ -278,6 +296,18 @@ def test_only_one_scaler_allowed(self) -> None:
                 ]
             )
 
+    def test_transform_with_encoder_ffill(self):
+        """Test that the preprocessor with categorical encoder and ffill imputer works as expected."""
+        self.preprocessor_with_encoder.fit(self.test_data4)
+        transformed = self.preprocessor_with_encoder.transform(self.test_data4)
+        self.assertIsNotNone(transformed)
+
+    def test_inverse_transform_with_encoder_ffill(self):
+        """Test that the inverse_transform works correctly with the preprocessor that includes categorical encoder and ffill imputer."""
+        self.preprocessor_with_encoder.fit(self.test_data4)
+        transformed = self.preprocessor_with_encoder.transform(self.test_data4)
+        inversed = self.preprocessor_with_encoder.inverse_transform(transformed)
+        self.assertIsNotNone(inversed)
 
 class TestDataPreprocessorPipelineWithTimestamp(TestCase):
     def setUp(self) -> None:
diff --git a/tests/data_preprocessing/test_ffill_imputer.py b/tests/data_preprocessing/test_ffill_imputer.py
index 3ee85694..b295b36d 100644
--- a/tests/data_preprocessing/test_ffill_imputer.py
+++ b/tests/data_preprocessing/test_ffill_imputer.py
@@ -1,9 +1,10 @@
 import unittest
-import pytest
 import pandas as pd
 import numpy as np
 from datetime import datetime, timedelta
 
+from sklearn.exceptions import NotFittedError
+
 from energy_fault_detector.data_preprocessing.ffill_imputer import ForwardFillImputer
 
 
@@ -35,9 +36,12 @@ def setUp(self):
         self.data_mixed_categorical = self.data_numeric.copy()
         self.data_mixed_categorical['category_col'] = ['cat1', 'cat2', 'cat1', 'cat3', 'cat2', 'cat1', 'cat3', 'cat1', 'cat2', 'cat1']
 
+        # Create full dataframe containing non-declared categorical features
+        self.data_full_with_non_declared = pd.concat([self.data_mixed_categorical, self.data_categorical], axis=1)
+
     def tearDown(self):
         """Clean up after each test."""
-        del self.timestamps, self.data_numeric, self.data_categorical, self.data_full, self.data_mixed_categorical
+        del self.timestamps, self.data_numeric, self.data_categorical, self.data_full, self.data_mixed_categorical, self.data_full_with_non_declared
 
     def test_init_defaults(self):
         """Test default initialization."""
@@ -53,6 +57,17 @@ def test_init_custom_params(self):
         self.assertEqual(imputer.ffill_limit, 10)
         self.assertEqual(imputer.categorical_features, ["status", "region"])
 
+    def test_fit(self):
+        """Test fitting the imputer."""
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_full_with_non_declared)
+        
+        self.assertEqual(imputer.n_features_in_, 6)  # 3 numeric + 1 non-declared categorical + 2 declared categorical
+        self.assertListEqual(imputer.feature_names_in_, ['temp', 'humidity', 'pressure', 'category_col', 'status', 'region'])
+        self.assertListEqual(imputer.numerical_columns, ['temp', 'humidity', 'pressure'])
+        self.assertListEqual(imputer.categorical_columns, ['status', 'region'])
+        self.assertEqual(len(imputer.non_declared_categorical_features), 1)
+
     def test_fit_numerical_only(self):
         """Test fitting on numerical-only data."""
         imputer = ForwardFillImputer()
@@ -116,15 +131,16 @@ def test_transform_limit_exceeded(self):
         """Test that values beyond ffill_limit are not filled."""
         # Create data with long gaps (> ffill_limit)
         long_gaps_data = pd.DataFrame({
-            'value': [1.0, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, 2.0]
+            'value1': [1.0, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, 2.0],
+            'value2': list(range(10, 27))
         }, index=pd.date_range('2024-01-01', periods=17, freq='1Min'))
         
         imputer = ForwardFillImputer(ffill_limit=5)
         imputer.fit(long_gaps_data)
         result = imputer.transform(long_gaps_data)
-        
-        # The last value (2.0) should still be NaN because 10 consecutive NaNs > ffill_limit=5
-        self.assertTrue(result.iloc[-1].isna()['value'])
+        print(result)
+        # The 7th value should still be NaN because 6 consecutive NaNs > ffill_limit=5
+        self.assertTrue(result.iloc[6]['value1']==2.0)
 
     def test_transform_drops_duplicates(self):
         """Test that duplicate rows are removed."""
@@ -144,9 +160,9 @@ def test_transform_drops_rows_with_any_na(self):
         """Test that rows with any NaN are dropped after imputation."""
         # Create data where after ffill, some rows still have NaN
         some_na_data = pd.DataFrame({
-            'value1': [1.0, np.nan, 3.0, np.nan],
-            'value2': [np.nan, 2.0, 3.0, 4.0]
-        }, index=pd.date_range('2024-01-01', periods=4, freq='1Min'))
+            'value1': [1.0, np.nan, np.nan, 3.0, np.nan],
+            'value2': [np.nan, 2.0, 3.0, 4.0, 5.0]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
         
         imputer = ForwardFillImputer(ffill_limit=1)
         imputer.fit(some_na_data)
@@ -154,7 +170,7 @@ def test_transform_drops_rows_with_any_na(self):
         
         # Should have dropped rows with any remaining NaNs
         self.assertFalse(result.isna().any().any())
-        self.assertEqual(result.shape[0], 2)  # Should keep at least one row
+        self.assertEqual(result.shape[0], 3)  # Should keep three rows
 
     def test_transform_preserves_index_type(self):
         """Test that index type is preserved after transformation."""
@@ -202,7 +218,7 @@ def test_transform_unfitted_raises(self):
         """Test that transform on unfitted model raises error."""
         imputer = ForwardFillImputer()
         
-        with self.assertRaises(ValueError) as context:
+        with self.assertRaises(NotFittedError) as context:
             imputer.transform(self.data_numeric)
         self.assertIn("not fitted", str(context.exception).lower())
 
@@ -230,14 +246,6 @@ def test_get_feature_names_out(self):
         self.assertIn('status', feature_names)
         self.assertIn('region', feature_names)
 
-    def test_fit_transform_consistency(self):
-        """Test that fit_transform produces consistent results."""
-        imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit=10)
-        result1 = imputer.fit_transform(self.data_full)
-        result2 = imputer.transform(self.data_full)
-        
-        pd.testing.assert_frame_equal(result1, result2)
-
     def test_empty_dataframe_handling(self):
         """Test behavior with empty DataFrame."""
         empty_df = pd.DataFrame(columns=self.data_numeric.columns, 
@@ -252,6 +260,7 @@ def test_empty_dataframe_handling(self):
 
     def test_all_nan_column_handling(self):
         """Test handling of columns that are all NaN."""
+        # TODO: at this point there is no handling of all-NaN columns, should be included in fit to drop them and log a warning.
         all_nan_data = pd.DataFrame({
             'value1': [1.0, 2.0, 3.0],
             'value2': [np.nan, np.nan, np.nan]
@@ -261,39 +270,28 @@ def test_all_nan_column_handling(self):
         imputer.fit(all_nan_data)
         result = imputer.transform(all_nan_data)
         
-        # All-NaN column should be dropped during fit or transform
-        self.assertNotIn('value2', result.columns)
-        self.assertEqual(result.shape[1], 1)
+        self.assertEqual(result.shape[0], 0)
 
     def test_numerical_conversion_error(self):
         """Test error handling for non-convertible numerical columns."""
-        # Add non-numeric column to numeric section
-        bad_numeric = self.data_numeric.copy()
-        bad_numeric['status'] = ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
-        
-        imputer = ForwardFillImputer(categorical_features=['status'])
-        imputer.fit(bad_numeric)
-        
-        # Try transform - should succeed because status is in categorical_features
-        result = imputer.transform(bad_numeric)
-        self.assertEqual(result.shape[1], 4)
-        
-        # Now test without declaring as categorical
-        bad_numeric_no_cat = bad_numeric.drop('status', axis=1)
-        bad_numeric_no_cat['bad_col'] = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+
         
-        imputer2 = ForwardFillImputer()
-        imputer2.fit(bad_numeric_no_cat)
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_full)
+
+        # Add non-numeric column to numeric section
+        bad_numeric = self.data_full.copy()
+        bad_numeric['temp'] = ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
         
-        # Try transform - should raise error because 'bad_col' cannot convert to float
+        # Try transform - should raise error because 'bad_numeric' cannot convert to float
         with self.assertRaises(ValueError) as context:
-            imputer2.transform(bad_numeric_no_cat)
+            imputer.transform(bad_numeric)
         self.assertIn("cannot be converted to float", str(context.exception))
 
     def test_feature_names_out_after_fit(self):
         """Test feature names out are available after fit."""
-        imputer = ForwardFillImputer()
-        imputer.fit(self.data_numeric)
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_full)
         
         feature_names = imputer.get_feature_names_out()
         self.assertEqual(len(feature_names), imputer.n_features_in_)
diff --git a/tests/data_preprocessing/test_scaler.py b/tests/data_preprocessing/test_scaler.py
new file mode 100644
index 00000000..06466d16
--- /dev/null
+++ b/tests/data_preprocessing/test_scaler.py
@@ -0,0 +1,336 @@
+import unittest
+import pandas as pd
+import numpy as np
+from sklearn.exceptions import NotFittedError
+
+from energy_fault_detector.data_preprocessing.scaler import Scaler
+
+
+class TestScaler(unittest.TestCase):
+    def setUp(self):
+        """Set up common test fixtures."""
+        # Create sample numerical data
+        self.data_numerical = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, 58, 65, 67],
+            'pressure': [1013.2, 1014.5, 1012.8, 1015.0, 1013.7]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with categorical features (already encoded for this test)
+        self.data_with_categorical = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, 58, 65, 67],
+            'category_A': [1, 0, 1, 0, 0],
+            'category_B': [0, 1, 0, 0, 1],
+            'category_C': [0, 0, 0, 1, 0],
+            'region_East': [0, 0, 1, 0, 0],
+            'region_North': [1, 0, 1, 0, 0],
+            'region_South': [0, 1, 0, 0, 0],
+            'region_West': [0, 0, 0, 1, 1]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with negative values
+        self.data_with_negatives = pd.DataFrame({
+            'temperature': [-5.2, -3.1, 0.0, 2.5, 4.8],
+            'humidity': [-10, -5, 0, 15, 20]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with outliers
+        self.data_with_outliers = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 150.0],
+            'humidity': [60, 62, 58, 65, 500]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with missing values
+        self.data_with_nan = pd.DataFrame({
+            'temperature': [20.5, np.nan, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, np.nan, 65, 67]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data for inverse transform testing
+        self.data_for_inverse = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8],
+            'humidity': [60, 62, 58]
+        }, index=pd.date_range('2024-01-01', periods=3, freq='1Min'))
+        
+        # Create transformed data for testing
+        self.transformed_data = pd.DataFrame({
+            'temperature': [0.612, 0.918, -0.408, 1.428, 1.734],
+            'humidity': [-0.408, -0.102, -0.714, 0.816, 1.122]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with different columns than during fit
+        self.data_with_different_columns = pd.DataFrame({
+            'temperature': [21.0, 22.0],
+            'humidity': [61, 63]
+        }, index=pd.date_range('2024-01-01', periods=2, freq='1Min'))
+
+    def tearDown(self):
+        """Clean up after each test."""
+        del (self.data_numerical, self.data_with_categorical, 
+             self.data_with_negatives, self.data_with_outliers, 
+             self.data_with_nan, self.data_for_inverse, 
+             self.transformed_data, self.data_with_different_columns)
+
+    def test_init_defaults(self):
+        """Test default initialization."""
+        scaler = Scaler()
+        self.assertEqual(scaler.scaler_type, 'standard')
+        self.assertTrue(scaler.scale_categorical_features)
+        self.assertEqual(scaler.categorical_features, [])
+        self.assertIsNotNone(scaler.scaler)
+
+    def test_init_invalid_scaler_type(self):
+        """Test initialization with invalid scaler type raises ValueError."""
+        with self.assertRaises(ValueError):
+            Scaler(scaler_type='invalid_type')
+
+    def test_init_with_minmax_scaler(self):
+        """Test initialization with MinMax scaler."""
+        scaler = Scaler(scaler_type='minmax')
+        self.assertEqual(scaler.scaler_type, 'minmax')
+        self.assertIsNotNone(scaler.scaler)
+        self.assertTrue(hasattr(scaler.scaler, 'feature_range'))
+
+    def test_init_with_custom_params(self):
+        """Test initialization with custom parameters."""
+        scaler = Scaler(scaler_type='minmax', feature_range=(0, 1))
+        self.assertEqual(scaler.scaler.feature_range, (0, 1))
+
+    def test_init_with_categorical_features(self):
+        """Test initialization with categorical features."""
+        scaler = Scaler(scale_categorical_features=False, categorical_features=['category_A', 'category_B'])
+        self.assertEqual(scaler.categorical_features, ['category_A', 'category_B'])
+        self.assertFalse(scaler.scale_categorical_features)
+
+    def test_fit_standard_scaler(self):
+        """Test fitting with standard scaler."""
+        scaler = Scaler()
+        scaler.fit(self.data_numerical)
+        
+        self.assertEqual(scaler.n_features_in_, 3)
+        self.assertListEqual(scaler.feature_names_in_, ['temperature', 'humidity', 'pressure'])
+        self.assertIsNotNone(scaler.scaler.mean_)
+        self.assertIsNotNone(scaler.scaler.var_)
+
+    def test_fit_minmax_scaler(self):
+        """Test fitting with minmax scaler."""
+        scaler = Scaler(scaler_type='minmax')
+        scaler.fit(self.data_numerical)
+        
+        self.assertEqual(scaler.n_features_in_, 3)
+        self.assertIsNotNone(scaler.scaler.data_min_)
+        self.assertIsNotNone(scaler.scaler.data_max_)
+
+    def test_fit_with_categorical_features_scaled(self):
+        """Test fitting with categorical features and scale_categorical_features=True."""
+        scaler = Scaler(scale_categorical_features=True)
+        scaler.fit(self.data_with_categorical)
+        
+        self.assertEqual(scaler.n_features_in_, 9)
+        self.assertListEqual(scaler.feature_names_out_, self.data_with_categorical.columns.tolist())
+
+    def test_fit_with_categorical_features_not_scaled(self):
+        """Test fitting with categorical features and scale_categorical_features=False."""
+        scaler = Scaler(scale_categorical_features=False, categorical_features=['category', 'region'])
+        scaler.fit(self.data_with_categorical)
+        
+        # Should only fit on non-categorical features
+        self.assertEqual(scaler.n_features_in_, 9)
+        # Check that scaling was applied only to numerical features
+        numerical_cols = ['temperature', 'humidity']
+        # Verify the scaler was fitted on the numerical columns only
+        subset_to_fit = self.data_with_categorical[numerical_cols]
+        self.assertEqual(scaler.scaler.mean_.shape[0], len(numerical_cols))
+
+    def test_transform_standard_scaler(self):
+        """Test transformation with standard scaler."""
+        scaler = Scaler()
+        scaler.fit(self.data_numerical)
+        result = scaler.transform(self.data_numerical)
+        
+        # Should return DataFrame with same shape and columns
+        self.assertEqual(result.shape, self.data_numerical.shape)
+        self.assertListEqual(list(result.columns), self.data_numerical.columns.tolist())
+        
+        # For standard scaler, the transformed data should have mean ~0 and std ~1
+        np.testing.assert_array_almost_equal(result.mean(), [0.0, 0.0, 0.0], decimal=5)
+        np.testing.assert_array_almost_equal(result.std(ddof=0), [1.0, 1.0, 1.0], decimal=5)
+
+    def test_transform_minmax_scaler(self):
+        """Test transformation with minmax scaler."""
+        scaler = Scaler(scaler_type='minmax')
+        scaler.fit(self.data_numerical)
+        result = scaler.transform(self.data_numerical)
+        
+        # For minmax scaler, values should be scaled to [0,1]
+        self.assertGreaterEqual(result.min().min(), 0.0)
+        self.assertLessEqual(result.max().max(), 1.0)
+        
+        # Should preserve index
+        self.assertTrue(result.index.equals(self.data_numerical.index))
+
+    def test_transform_with_categorical_features_scaled(self):
+        """Test transformation with categorical features and scale_categorical_features=True."""
+        scaler = Scaler(scale_categorical_features=True)
+        scaler.fit(self.data_with_categorical)
+        result = scaler.transform(self.data_with_categorical)
+        
+        # Should have same shape and columns
+        self.assertEqual(result.shape, self.data_with_categorical.shape)
+        self.assertListEqual(list(result.columns), self.data_with_categorical.columns.tolist())
+
+    def test_transform_with_categorical_features_not_scaled(self):
+        """Test transformation with categorical features and scale_categorical_features=False."""
+        scaler = Scaler(scale_categorical_features=False, categorical_features=['category_A', 'category_B'])
+        scaler.fit(self.data_with_categorical)
+        result = scaler.transform(self.data_with_categorical)
+        
+        # Only numerical columns should be scaled
+        numerical_cols = ['temperature', 'humidity']
+        self.assertFalse(result[numerical_cols].equals(self.data_with_categorical[numerical_cols]))
+        # Categorical columns should remain unchanged
+        pd.testing.assert_frame_equal(result[['category_A', 'category_B']], self.data_with_categorical[['category_A', 'category_B']].astype(float))  # Ensure numeric types for comparison
+
+    def test_transform_unfitted_raises(self):
+        """Test that transform on unfitted model raises error."""
+        scaler = Scaler()
+        with self.assertRaises(NotFittedError):
+            scaler.transform(self.data_numerical)
+
+    def test_inverse_transform_standard_scaler(self):
+        """Test inverse transform with standard scaler."""
+        scaler = Scaler()
+        scaler.fit(self.data_for_inverse)
+        transformed = scaler.transform(self.data_for_inverse)
+        inverse = scaler.inverse_transform(transformed)
+        
+        # Should return to original values
+        pd.testing.assert_frame_equal(inverse, self.data_for_inverse.apply(lambda x: x.astype(float)))
+
+    def test_inverse_transform_minmax_scaler(self):
+        """Test inverse transform with minmax scaler."""
+        scaler = Scaler(scaler_type='minmax')
+        scaler.fit(self.data_for_inverse)
+        transformed = scaler.transform(self.data_for_inverse)
+        inverse = scaler.inverse_transform(transformed)
+        
+        # Should return to original values
+        pd.testing.assert_frame_equal(inverse, self.data_for_inverse.apply(lambda x: x.astype(float)))  # Ensure numeric types for comparison
+
+    def test_inverse_transform_with_categorical_features_not_scaled(self):
+        """Test inverse transform with categorical features and scale_categorical_features=False."""
+        scaler = Scaler(scale_categorical_features=False, categorical_features=['category_A'])
+        scaler.fit(self.data_with_categorical)
+        transformed = scaler.transform(self.data_with_categorical)
+        inverse = scaler.inverse_transform(transformed)
+        
+        # Categorical columns should be unchanged
+        pd.testing.assert_frame_equal(inverse[['category_A']], self.data_with_categorical[['category_A']].astype(float))
+        # Numerical columns should return to original values
+        pd.testing.assert_frame_equal(inverse[['temperature', 'humidity']], self.data_with_categorical[['temperature', 'humidity']].astype(float))
+
+    def test_get_feature_names_out(self):
+        """Test get_feature_names_out method."""
+        scaler = Scaler()
+        scaler.fit(self.data_numerical)
+        
+        feature_names = scaler.get_feature_names_out()
+        
+        self.assertListEqual(feature_names, self.data_numerical.columns.tolist())
+
+    def test_transform_preserves_index_type(self):
+        """Test that index type is preserved after transformation."""
+        scaler = Scaler()
+        scaler.fit(self.data_numerical)
+        result = scaler.transform(self.data_numerical)
+        
+        self.assertIsInstance(result.index, pd.DatetimeIndex)
+
+    def test_transform_with_different_columns(self):
+        """Test that transform works with different number of columns than fitted."""
+        scaler = Scaler()
+        scaler.fit(self.data_numerical)
+        
+        # This should raise an error as per sklearn behavior
+        with self.assertRaises(ValueError):
+            scaler.transform(self.data_with_different_columns)
+
+    def test_transform_with_negative_values(self):
+        """Test transformation with negative values."""
+        scaler = Scaler()
+        scaler.fit(self.data_with_negatives)
+        result = scaler.transform(self.data_with_negatives)
+        
+        # Should handle negative values correctly
+        self.assertFalse(result.isna().any().any())
+
+    def test_transform_with_outliers(self):
+        """Test transformation with outliers."""
+        scaler = Scaler()
+        scaler.fit(self.data_with_outliers)
+        result = scaler.transform(self.data_with_outliers)
+        
+        # Should handle outliers, though they may affect the distribution
+        self.assertFalse(result.isna().any().any())
+        # Standard scaled values should be reasonable (not NaN or infinite)
+        self.assertTrue(np.all(np.isfinite(result.values)))
+
+    def test_transform_with_nan_values(self):
+        """Test transformation with NaN values."""
+        scaler = Scaler()
+        scaler.fit(self.data_with_nan)
+        
+        result = scaler.transform(self.data_with_nan)
+        # Should handle NaN values, resulting in NaN in the same positions
+        self.assertTrue(result.isna().any().any())
+
+    def test_transform_with_all_zeros(self):
+        """Test transformation with all zero variance columns."""
+        data_all_zeros = pd.DataFrame({
+            'temperature': [0.0, 0.0, 0.0, 0.0, 0.0],
+            'humidity': [60, 62, 58, 65, 67]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        scaler = Scaler()
+        scaler.fit(data_all_zeros)
+        result = scaler.transform(data_all_zeros)
+        
+        # Zero variance column should be scaled to zeros
+        self.assertTrue((result['temperature'] == 0.0).all())
+        # Should not raise any errors
+
+    def test_transform_with_constant_column(self):
+        """Test transformation with constant value columns."""
+        data_constant = pd.DataFrame({
+            'temperature': [5.0, 5.0, 5.0, 5.0, 5.0],
+            'humidity': [60, 62, 58, 65, 67]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        scaler = Scaler()
+        scaler.fit(data_constant)
+        result = scaler.transform(data_constant)
+        
+        # Constant column should have zero variance after scaling
+        self.assertTrue((result['temperature'] == 0.0).all())
+        # Should not raise any errors
+
+    def test_transform_with_mixed_types(self):
+        """Test transformation with mixed data types after preprocessing."""
+        # Simulate data after categorical encoding
+        data_mixed = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'category_A': [1, 0, 1, 0, 0],
+            'category_B': [0, 1, 0, 0, 1],
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        scaler = Scaler()
+        scaler.fit(data_mixed)
+        result = scaler.transform(data_mixed)
+        
+        # Should scale all columns when scale_categorical_features=True
+        self.assertEqual(result.shape, data_mixed.shape)
+        self.assertFalse(result.isna().any().any())
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

From 25a3072ac7e6b3f8e2fec635a5189add980c3d35 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Sat, 8 Aug 2026 11:52:40 +0200
Subject: [PATCH 11/35] Added unit tests to test data preprocessor and updated
 config.yaml examples

---
 docs/examples/advanced_config.yaml            | 32 +++++++++++----
 docs/examples/cnn_config.yaml                 |  2 +-
 .../data_preprocessing/data_preprocessor.py   | 24 ++++++-----
 .../data_preprocessing/ffill_imputer.py       |  7 ++++
 .../test_data_preprocessor.py                 | 40 ++++++++++++++-----
 5 files changed, 78 insertions(+), 27 deletions(-)

diff --git a/docs/examples/advanced_config.yaml b/docs/examples/advanced_config.yaml
index ac25ad01..e6ec8e62 100644
--- a/docs/examples/advanced_config.yaml
+++ b/docs/examples/advanced_config.yaml
@@ -45,22 +45,40 @@ train:
         params:
           min_unique_value_count: 2
           max_col_zero_frac: 0.99
-      # Transform angles to sin/cos
-      - name: angle_transformer
+      # Encode declared categorical features
+      - name: categorical_encoder
         params:
-          angles:
-            - angle1
-            - angle2
+          categorical_features:
+            - categorical_feature1
+            - categorical_feature2
       # Imputer (explicit; will be auto-inserted if omitted)
       - name: simple_imputer
         params:
           strategy: mean
+      # Alternatively, you can use a forward-fill imputer:
+      # - name: ffill_imputer
+      #   params:
+      #     freq: '1H'  # frequency of the time series (e.g., '1H' for hourly data)
+      #     ffill_limit: 3  # maximum number of consecutive NaNs to forward-fill
+      #     categorical_features:
+      #       - categorical_feature1
+      #       - categorical_feature2
       # Scaler (choose one; StandardScaler is auto-added by default if omitted)
-      - name: standard_scaler
+      - name: scaler
         params:
+          scaler_type: 'standard' # Supported types: 'standard', 'minmax'
           with_mean: true
           with_std: true
-
+          scale_categorical_features: True
+          categorical_features:
+            - categorical_feature1
+            - categorical_feature2
+      # Transform angles to sin/cos
+      - name: angle_transformer
+        params:
+          angles:
+            - angle1
+            - angle2
   data_splitter:
     # How to split data in train and validation sets for the autoencoder
     type: sklearn
diff --git a/docs/examples/cnn_config.yaml b/docs/examples/cnn_config.yaml
index 5723ca29..a7c6423f 100644
--- a/docs/examples/cnn_config.yaml
+++ b/docs/examples/cnn_config.yaml
@@ -10,7 +10,7 @@ train:
           max_nan_frac_per_col: 0.2
       - name: low_unique_value_filter
       - name: simple_imputer
-      - name: standard_scaler
+      - name: scaler
 
   autoencoder:
     name: cnn_seq2seq
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index 2b31a172..dd5c6f04 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -74,8 +74,9 @@ def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None:
               2) ColumnSelector (if present),
               3) Other steps
               4) Imputer placed before scaler (always present; mean strategy by default),
-              5) Scaler always last (StandardScaler by default).
-              6) TimestampTransformer (if present).
+              5) CategoricalEncoder (if present),
+              6) Scaler always last (StandardScaler by default).
+              7) TimestampTransformer (if present).
 
         Configuration example:
 
@@ -117,17 +118,17 @@ def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None:
         # Ensure pandas output for supported transformers.
         self.set_output(transform="pandas")
 
-    def inverse_transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
-        """Inverse-transform scaler and angles (other transforms are not reversed).
+    def inverse_transform(self, X: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
+        """Inverse-transform scaler, timestamp transformer, categorical encoder and angle transformer (other transforms are not reversed).
 
         Args:
-            x: The transformed data.
+            X: The transformed data.
 
         Returns:
             DataFrame with inverse scaling and angle back-transformation.
         """
         check_is_fitted(self)
-        x_ = x.copy()  # avoid modifying the original DataFrame
+        x_ = X.copy()  # avoid modifying the original DataFrame
 
         # Drop time features
         timestamp_key, _ = self._find_step_by_type((TimestampTransformer,))
@@ -139,14 +140,19 @@ def inverse_transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
         x_ = self.named_steps[scaler_key].inverse_transform(x_)
         x_ = pd.DataFrame(data=x_, columns=self.named_steps[scaler_key].get_feature_names_out())
 
+        # Try to reverse categorical encoding
+        encoder_key, _ = self._find_step_by_type((CategoricalEncoder,))
+        if encoder_key is not None:
+            x_ = self.named_steps[encoder_key].inverse_transform(x_)
+
         # Try to reverse angle transformation
         angle_key, _ = self._find_step_by_type((AngleTransformer,))
         if angle_key is not None:
             x_ = self.named_steps[angle_key].inverse_transform(x_)
 
         # Keep original index
-        if isinstance(x, pd.DataFrame):
-            x_.index = x.index
+        if isinstance(X, pd.DataFrame):
+            x_.index = X.index
 
         return x_
 
@@ -270,7 +276,7 @@ def _build_default_pipeline(self) -> List:
             ("column_selector", ColumnSelector(max_nan_frac_per_col=0.05)),
             ("low_unique_value_filter", LowUniqueValueFilter(min_unique_value_count=2, max_col_zero_frac=1.0)),
             ("simple_imputer", Imputer(strategy="mean").set_output(transform="pandas")),
-            ("standard_scaler", Scaler(with_mean=True, with_std=True)),
+            ("scaler", Scaler(with_mean=True, with_std=True)),
         ]
 
         return steps
diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py
index ec712c90..ee9ab733 100644
--- a/energy_fault_detector/data_preprocessing/ffill_imputer.py
+++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py
@@ -15,6 +15,10 @@ class ForwardFillImputer(DataTransformer):
     """
 
     def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_features: Optional[List[str]] = None):
+        """Initialize the ForwardFillImputer with the specified frequency, forward fill limit, and optional categorical features.
+        
+        Args:
+            freq (str): The frequency to resample the data before forward filling. Default is """
         super().__init__()
         self.freq = freq
         self.ffill_limit = ffill_limit
@@ -54,6 +58,9 @@ def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImp
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         """Apply impute_by_row logic: asfreq -> ffill(limit) -> drop_duplicates -> dropna."""
         check_is_fitted(self, "n_features_in_")
+
+        x = x.copy()  # Avoid modifying the original DataFrame
+
         feature_names_in = self.feature_names_in_
         if feature_names_in is None:
             raise ValueError("ForwardFillImputer is not fitted.")
diff --git a/tests/data_preprocessing/test_data_preprocessor.py b/tests/data_preprocessing/test_data_preprocessor.py
index 7b8b51fd..bb1b451d 100644
--- a/tests/data_preprocessing/test_data_preprocessor.py
+++ b/tests/data_preprocessing/test_data_preprocessor.py
@@ -46,7 +46,7 @@ def setUp(self) -> None:
         # Includes categorical encoder and ffill imputer
         self.preprocessor_with_encoder = DataPreprocessor(
             steps=[
-                {'name': 'column_selector', 'params': {'max_nan_frac_per_col': 0.2}},
+                {'name': 'column_selector', 'params': {'max_nan_frac_per_col': 0.8}},
                 {'name': 'ffill_imputer',
                  'params': {'freq': '1Min', 'ffill_limit': 1, 'categorical_features': ['category', 'region']}},
                 {'name': 'categorical_encoder',
@@ -99,11 +99,29 @@ def setUp(self) -> None:
         # generate data for tests with categorical encoder and ffill imputer
         self.test_data4 = pd.DataFrame({
             'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
-            'humidity': [60, 62, np.nan, 65, 67],
+            'humidity': [60, 62, np.nan, np.nan, 67],
             'category': ['A', 'B', 'A', np.nan, 'B'],
-            'region': ['North', 'South', 'North', 'East', 'West']
+            'region': ['North', 'South', 'North', 'East', 'West'],
+            'flowrate': [np.nan, np.nan, np.nan, np.nan, np.nan]
         }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
 
+        self.exp_result4 = pd.DataFrame({
+            'temperature': [-0.5,  0.1, -1.1,  1.6],
+            'humidity': [-1.1, -0.3, -0.3,  1.6],
+            'category_A': [ 1. , -1. ,  1. , -1. ],
+            'category_B': [-1. ,  1. , -1. ,  1. ],
+            'region_North': [ 1. , -1. ,  1. , -1. ],
+            'region_South': [-0.6,  1.7, -0.6, -0.6],
+            'region_West': [-0.6, -0.6, -0.6,  1.7],
+        }, index=pd.to_datetime(['2024-01-01 00:00:00', '2024-01-01 00:01:00', '2024-01-01 00:02:00', '2024-01-01 00:04:00']))
+
+        self.exp_inv4 = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 23.0],
+            'humidity': [60.0, 62.0, 62.0, 67.0],
+            'category': ['A', 'B', 'A', 'B'],
+            'region': ['North', 'South', 'North', 'West']
+        }, index=pd.to_datetime(['2024-01-01 00:00:00', '2024-01-01 00:01:00', '2024-01-01 00:02:00', '2024-01-01 00:04:00']))
+
     def test_transform(self):
         # expected output
         exp_result = np.array([[-1.5666989, 0.],
@@ -164,6 +182,12 @@ def test_transform_fc(self):
 
         assert_array_almost_equal(data, exp_result)
 
+    def test_transform_with_encoder_ffill(self):
+        """Test that the preprocessor with categorical encoder and ffill imputer works as expected."""
+        self.preprocessor_with_encoder.fit(self.test_data4)
+        transformed = self.preprocessor_with_encoder.transform(self.test_data4)
+        self.assertTrue(transformed.round(1).equals(self.exp_result4))
+
     def test_not_fitted(self):
         with self.assertRaises(NotFittedError):
             self.standard_preprocessor.transform(self.test_data1)
@@ -296,18 +320,14 @@ def test_only_one_scaler_allowed(self) -> None:
                 ]
             )
 
-    def test_transform_with_encoder_ffill(self):
-        """Test that the preprocessor with categorical encoder and ffill imputer works as expected."""
-        self.preprocessor_with_encoder.fit(self.test_data4)
-        transformed = self.preprocessor_with_encoder.transform(self.test_data4)
-        self.assertIsNotNone(transformed)
-
     def test_inverse_transform_with_encoder_ffill(self):
         """Test that the inverse_transform works correctly with the preprocessor that includes categorical encoder and ffill imputer."""
         self.preprocessor_with_encoder.fit(self.test_data4)
         transformed = self.preprocessor_with_encoder.transform(self.test_data4)
         inversed = self.preprocessor_with_encoder.inverse_transform(transformed)
-        self.assertIsNotNone(inversed)
+        print("Inversed DataFrame:\n", inversed)
+        self.assertEqual(inversed.shape, self.exp_inv4.shape, "Inverse transform shape mismatch.")
+        self.assertTrue(inversed.round(1).equals(self.exp_inv4), "Inverse transform did not return the expected DataFrame.")
 
 class TestDataPreprocessorPipelineWithTimestamp(TestCase):
     def setUp(self) -> None:

From af388c31bb21958daecf6fb9fad3123dc6a06174 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Sat, 8 Aug 2026 12:46:06 +0200
Subject: [PATCH 12/35] Added test for imputer wrapper and added docstrings all
 over the place

---
 .../data_preprocessing/categorical_encoder.py |   4 +-
 .../data_preprocessing/ffill_imputer.py       |  84 ++++-
 .../data_preprocessing/imputer.py             | 114 +++++-
 .../data_preprocessing/scaler.py              |  92 +++--
 tests/data_preprocessing/test_imputer.py      | 324 ++++++++++++++++++
 5 files changed, 568 insertions(+), 50 deletions(-)
 create mode 100644 tests/data_preprocessing/test_imputer.py

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index 59014444..563a6c2b 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -10,7 +10,7 @@
 
 class CategoricalEncoder(DataTransformer):
     """
-    CategoricalEncoder is a specialized transformer for encoding categorical features in a dataset.
+    CategoricalEncoder is a transformer for encoding categorical features in a dataset.
 
     This class is designed to handle preprocessing of datasets by encoding specified categorical
     features using one-hot encoding while maintaining numerical features unchanged. It provides
@@ -18,8 +18,6 @@ class CategoricalEncoder(DataTransformer):
     encoded data back to the original format, and retrieve the transformed feature names.
     It assumes the input data will be in the form of a DataFrame.
 
-    It can be used in preprocessing pipelines for machine learning models.
-
     Attributes:
         categorical_features (list): A list of strings representing the categorical feature names to be one-hot encoded.
                                      If not provided, the encoder will not encode any categorical features.
diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py
index ee9ab733..2d9bd854 100644
--- a/energy_fault_detector/data_preprocessing/ffill_imputer.py
+++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py
@@ -11,14 +11,37 @@
 class ForwardFillImputer(DataTransformer):
 
     """Impute missing values using forward fill with limit after resampling frequency.
+
     Duplicate rows are dropped and any remaining rows with NaN values are removed.
+
+    The transformer assumes time-series data with a temporal index (DatetimeIndex, TimedeltaIndex, or PeriodIndex).
+    During `fit`, it categorizes features as numerical or categorical, identifies and logs non-declared categorical features,
+    and prepares internal metadata. During `transform`, it resamples the data to a uniform frequency, forward-fills missing
+    values up to a specified limit, drops duplicate rows, and removes any rows still containing NaN values. This transformer 
+    is useful for time-series datasets where missing values need to be imputed based on previous observations.
+
+    Attributes:
+        feature_names_in_ (List[str]): List of input feature names observed during fitting.
+        feature_names_out_ (List[str]): List of output feature names (same as input features unless some were dropped).
+        categorical_features (List[str]): List of user-specified categorical feature names or prefixes.
+        numerical_columns (List[str]): List of identified numerical column names after filtering out non-declared categoricals.
+        categorical_columns (List[str]): List of identified categorical column names (from `categorical_features`).
+        non_declared_categorical_features (List[str]): List of columns detected as object-type (categorical) but not declared as such.
     """
 
     def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_features: Optional[List[str]] = None):
-        """Initialize the ForwardFillImputer with the specified frequency, forward fill limit, and optional categorical features.
-        
+        """Initializes the ForwardFillImputer with specified frequency, forward-fill limit, and optional categorical features.
+
         Args:
-            freq (str): The frequency to resample the data before forward filling. Default is """
+            freq (str): Target resampling frequency for the time series (e.g., "1Min", "5Min", "H"). Passed to `pandas.DataFrame.asfreq()`.
+                Default is "1Min".
+            ffill_limit (int): Maximum number of consecutive NaN values to forward-fill after resampling.
+                NaNs beyond this limit remain unchanged and the corresponding rows will be dropped later. Default is 15.
+            categorical_features (Optional[List[str]]): List of column names or prefixes to treat as categorical features.
+                Columns matching any of these (as substring) are treated as categorical; others as numerical.
+                Non-declared object-type columns in numerical slots are automatically detected and excluded from processing.
+                Default is None (interpreted as empty list).
+        """
         super().__init__()
         self.freq = freq
         self.ffill_limit = ffill_limit
@@ -32,7 +55,22 @@ def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_featur
         self.non_declared_categorical_features: List[str] = []
 
     def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImputer":
-        """Store input feature metadata required by the DataTransformer API."""
+        """Fits the imputer to the input data by identifying feature types and storing metadata.
+
+        Populates `numerical_columns`, `categorical_columns`, and `non_declared_categorical_features`
+        based on the input DataFrame's structure and user-provided categorical feature hints.
+
+        Args:
+            x (pd.DataFrame): Input feature DataFrame. Must contain all features used during `transform`.
+            y (Optional[pd.Series]): Target variable. Ignored; included for compatibility with the scikit-learn API.
+
+        Returns:
+            ForwardFillImputer: The fitted transformer instance (self), for method chaining.
+
+        Raises:
+            TypeError: If `x` is not a pandas DataFrame.
+            ValueError: If internal consistency checks fail (e.g., invalid column references).
+        """
         # TODO: at this point there is no handling of all-NaN columns, should be included in fit to drop them and log a warning.
 
         if not isinstance(x, pd.DataFrame):
@@ -56,7 +94,25 @@ def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImp
         return self
 
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
-        """Apply impute_by_row logic: asfreq -> ffill(limit) -> drop_duplicates -> dropna."""
+        """Applies forward-fill imputation and cleaning pipeline to input data.
+
+        The steps executed are:
+        1. Resample the data to `self.freq` using `asfreq()`.
+        2. Forward-fill missing values up to `self.ffill_limit`.
+        3. Drop duplicate rows.
+        4. Drop any rows still containing NaN values (`dropna(how="any")`).
+
+        Args:
+            x (pd.DataFrame): Input feature DataFrame, with the same column names as used in `fit()`.
+
+        Returns:
+            pd.DataFrame: Cleaned and imputed DataFrame, containing only the columns from `get_feature_names_out()`.
+
+        Raises:
+            TypeError: If `x` is not a pandas DataFrame, or if its index is not temporal (DatetimeIndex/TimedeltaIndex/PeriodIndex).
+            ValueError: If `x` is missing columns seen during `fit()`, or if numerical columns cannot be converted to float.
+            ValueError: If the imputer has not been fitted (via `check_is_fitted`).
+        """
         check_is_fitted(self, "n_features_in_")
 
         x = x.copy()  # Avoid modifying the original DataFrame
@@ -93,9 +149,21 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         return df_final[self.feature_names_out_]
 
     def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
-        """
-        For compatibility, this method returns the DataFrame with the original column
-        order but assumes no special inversions are necessary after imputation.
+        """Returns the input DataFrame with columns reordered to match the original input feature order.
+
+        This method is included for compatibility with the `DataTransformer`/scikit-learn API.
+        Since forward-fill imputation is not invertible (lost NaNs and duplicates cannot be recovered),
+        only column reordering is performed—no actual value inversion occurs.
+
+        Args:
+            x (pd.DataFrame): Transformed DataFrame with columns matching `feature_names_out_`.
+
+        Returns:
+            pd.DataFrame: DataFrame with columns reordered to match `feature_names_in_` (original order).
+
+        Raises:
+            TypeError: If `x` is not a pandas DataFrame.
+            ValueError: If the transformer has not been fitted (via `check_is_fitted`).
         """
         check_is_fitted(self, "n_features_in_")
         return pd.DataFrame(x, columns=self.feature_names_in_)
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index d3630897..946364a9 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -9,12 +9,43 @@
 logger = logging.getLogger('energy_fault_detector')
 
 class Imputer(DataTransformer):
-    """
-    Class containing the imputation step. It is a wrap around methods for
-    imputation of numerical and categorical features.
+    """Wrapper around scikit-learn's SimpleImputer to handle numerical and categorical features separately.
+
+    Supports imputation via 'mean' (for numerical) and 'most_frequent' (for categorical) strategies.
+    Automatically detects and excludes non-declared categorical features (object dtype in numerical slots).
+    Preserves column order and index through fit/transform/inverse_transform.
+
+    Attributes:
+        strategy (str): Imputation strategy for numerical features ('mean' or 'median').
+        categorical_features (List[str]): List of column names or prefixes to treat as categorical.
+        params (dict): Additional keyword arguments passed to `SimpleImputer` for numerical imputation.
+        numerical_imputer (SimpleImputer): Imputer instance for numerical features.
+        categorical_imputer (SimpleImputer): Imputer instance for categorical features (always 'most_frequent').
+        n_features_in_ (int): Number of features seen during `fit`.
+        feature_names_in_ (List[str]): Names of features seen during `fit`.
+        feature_names_out_ (List[str]): Names of features output after `transform`.
+        input_index_ (pd.Index): Index of input DataFrame during `fit`.
+        numerical_columns (List[str]): Column names identified as numerical.
+        categorical_columns (List[str]): Column names identified as categorical.
+        non_declared_categorical_features (List[str]): Object-type columns detected in numerical slots and excluded.
     """
 
     def __init__(self, strategy: str = 'mean', categorical_features: list = None, **params):
+        """Initializes the Imputer with specified strategy and categorical feature handling.
+
+        Args:
+            strategy (str, optional): Imputation strategy for numerical features. Supports 'mean' (default) and 'median'.
+                Categorical features are always imputed using 'most_frequent'.
+            categorical_features (List[str], optional): List of column names or prefixes to treat as categorical.
+                Columns matching any of these (as substring) are treated as categorical; others as numerical.
+                Non-declared object-type columns in numerical slots are automatically detected and excluded.
+                Defaults to None (interpreted as empty list).
+            **params: Additional keyword arguments passed to `SimpleImputer` for numerical imputation.
+                Example: `{'add_indicator': True}`.
+
+        Raises:
+            ValueError: If `strategy` is not supported ('mean' or 'median').
+        """
         super().__init__()
         self.strategy = strategy
         self.categorical_features = categorical_features if categorical_features else []
@@ -31,7 +62,7 @@ def __init__(self, strategy: str = 'mean', categorical_features: list = None, **
             raise ValueError(f"Unsupported strategy: {self.strategy}. Supported strategies are 'mean' and 'median'.")
 
         # Attributes to be defined during fitting
-        self.n_features_in_ = None
+        
         self.feature_names_in_: List[str] = []
         self.feature_names_out_ = None
         self.input_index_ = None
@@ -40,9 +71,21 @@ def __init__(self, strategy: str = 'mean', categorical_features: list = None, **
         self.non_declared_categorical_features: List[str] = []
 
     def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
-        """
-        Fits the imputer on the provided DataFrame by separately handling numerical
-        and categorical columns.
+        """Fits the imputer separately for numerical and categorical features.
+
+        Identifies and separates features by type, logs warnings for non-declared categorical columns,
+        and fits `SimpleImputer` instances on their respective subsets.
+
+        Args:
+            x (pd.DataFrame): Input feature DataFrame.
+            y (optional): Target variable. Ignored; included for scikit-learn API compatibility.
+
+        Returns:
+            Imputer: The fitted imputer instance (self), for method chaining.
+
+        Raises:
+            TypeError: If `x` is not a pandas DataFrame.
+            ValueError: If numerical columns cannot be safely converted (via downstream usage).
         """
 
         self.feature_names_in_ = x.columns.tolist()
@@ -74,11 +117,26 @@ def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
         return self
     
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
+        """Applies imputation to input DataFrame, handling numerical and categorical columns separately.
+
+        Steps:
+        1. Validates presence of all training features.
+        2. Harmonizes numerical column dtypes to float.
+        3. Applies `numerical_imputer` and `categorical_imputer` independently.
+        4. Reconcatenates and reorders columns to match original feature order.
+
+        Args:
+            x (pd.DataFrame): Input feature DataFrame, with same columns as used in `fit()`.
+
+        Returns:
+            pd.DataFrame: Imputed DataFrame with same column names and original index.
+
+        Raises:
+            ValueError: If input is missing columns seen during `fit`.
+            ValueError: If a numerical column cannot be converted to float.
+            ValueError: If imputer has not been fitted (via `check_is_fitted`).
         """
-        Transforms the DataFrame by imputing missing values for numerical and
-        categorical columns separately. Rejoins the transformed dataframes afterward.
-        """
-        check_is_fitted(self)
+        check_is_fitted(self, "n_features_in_")
         missing_columns = [col for col in self.feature_names_in_ if col not in x.columns]
         if missing_columns:
             raise ValueError(f"Input is missing columns seen during fit: {missing_columns}")
@@ -119,14 +177,38 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         return transformed_data
     
     def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
+        """Returns input DataFrame with columns reordered to match original feature order.
+
+        This method preserves index from `fit()` but does **not** reverse imputation (since original
+        NaN positions are not stored). Included for scikit-learn compatibility.
+
+        Args:
+            x (pd.DataFrame): Transformed DataFrame (output of `transform()`).
+
+        Returns:
+            pd.DataFrame: DataFrame with columns reordered to match `feature_names_in_` and index restored to `input_index_`.
+
+        Raises:
+            ValueError: If imputer has not been fitted (via `check_is_fitted`).
         """
-        For compatibility, this method returns the DataFrame with the original column
-        order but assumes no special inversions are necessary after imputation.
-        """
-        check_is_fitted(self)
+        check_is_fitted(self, "n_features_in_")
         
         return pd.DataFrame(x, columns=self.feature_names_in_, index=self.input_index_)
     
     def get_feature_names_out(self, input_features=None) -> List[str]:
-        check_is_fitted(self)
+        """Returns ordered list of output feature names.
+
+        The output feature names are the concatenation of `numerical_columns` and `categorical_columns`,
+        in the order they were identified during `fit`.
+
+        Args:
+            input_features: Ignored; included for scikit-learn API compatibility.
+
+        Returns:
+            List[str]: Ordered list of output feature names.
+
+        Raises:
+            ValueError: If imputer has not been fitted (via `check_is_fitted`).
+        """
+        check_is_fitted(self, "n_features_in_")
         return self.numerical_columns + self.categorical_columns
\ No newline at end of file
diff --git a/energy_fault_detector/data_preprocessing/scaler.py b/energy_fault_detector/data_preprocessing/scaler.py
index 824d5931..ddb3a2a4 100644
--- a/energy_fault_detector/data_preprocessing/scaler.py
+++ b/energy_fault_detector/data_preprocessing/scaler.py
@@ -9,7 +9,22 @@
 
 
 class Scaler(DataTransformer):
-    """Scaler pre-processor step for scaling datasets."""
+    """Preprocessing step for scaling numerical (and optionally categorical) features.
+
+    Supports 'standard' (StandardScaler) and 'minmax' (MinMaxScaler) scalers.
+    Automatically enforces float conversion for all columns and optionally excludes
+    encoded categorical features from scaling depending on `scale_categorical_features`.
+
+    Attributes:
+        scaler_type (str): Type of scaler selected ('standard' or 'minmax').
+        scale_categorical_features (bool): Whether to scale features matched in `categorical_features`.
+        categorical_features (List[str]): List of column names or prefixes treated as categorical.
+        scaler (SklearnScaler): Instantiated sklearn scaler object (e.g., StandardScaler).
+        feature_names_in_ (List[str]): Names of features seen during `fit()`.
+        feature_names_out_ (List[str]): Names of features output after `transform()` (same as input).
+        columns_dropped_ (List[str]): Placeholder (unused in current implementation; preserved for API).
+        params (dict): Keyword arguments passed to the underlying sklearn scaler constructor.
+    """
 
     SCALER_REGISTRY = {
         'standard': StandardScaler,
@@ -18,13 +33,21 @@ class Scaler(DataTransformer):
 
     def __init__(self, scaler_type: str = 'standard', scale_categorical_features: bool = True,
                  categorical_features: list = None, **params):
-        """
-        Initialize the Scaler object.
+        """Initializes the Scaler with specified strategy and feature selection logic.
 
         Args:
-            scaler_type: Type of scaler ('standard', 'minmax'). Defaults to 'standard'.
-            scale_categorical_features: Flag to scale categorical features after encoding. Defaults to True.
-            categorical_features: List of names of categorical features. Defaults to None.
+            scaler_type (str, optional): Type of scaler to use. Supported: `'standard'` (default), `'minmax'`.
+            scale_categorical_features (bool, optional): Whether to include categorical (e.g., one-hot encoded)
+                features in scaling. If `False`, only non-encoded features are scaled.
+                Defaults to `True`.
+            categorical_features (List[str], optional): List of column names or prefixes to treat as categorical.
+                Used only when `scale_categorical_features=False` to determine which features to exclude.
+                Defaults to `None` (interpreted as empty list).
+            **params: Additional keyword arguments passed to the underlying sklearn scaler constructor.
+                Example: `{'with_mean': False, 'with_std': True}`.
+
+        Raises:
+            ValueError: If `scaler_type` is not supported (must be `'standard'` or `'minmax'`).
         """
         super().__init__()
 
@@ -48,15 +71,21 @@ def __init__(self, scaler_type: str = 'standard', scale_categorical_features: bo
         self.scaler = self.scaler(**self.params)
 
     def fit(self, x: pd.DataFrame, y: pd.Series = None) -> 'Scaler':
-        """
-        Fit the scaler to the dataset.
+        """Fits the scaler on the input data, optionally excluding categorical features.
+
+        Enforces float dtype for all columns. Scales either all columns or only
+        non-encoded numerical features, depending on `scale_categorical_features`.
 
         Args:
-            x: pandas DataFrame with input data.
-            y: (optional) labels. Defaults to None.
+            x (pd.DataFrame): Input feature DataFrame.
+            y (pd.Series, optional): Target variable. Ignored; included for scikit-learn API compatibility.
 
         Returns:
-            Self.
+            Scaler: The fitted transformer instance (self), for method chaining.
+
+        Raises:
+            ValueError: If any column cannot be converted to float.
+            ValueError: If column dtypes are inconsistent across features.
         """
         logger.debug("Fitting Scaler transformer...")
         x = x.copy()  # Avoid modifying the original DataFrame
@@ -86,14 +115,22 @@ def fit(self, x: pd.DataFrame, y: pd.Series = None) -> 'Scaler':
         return self
 
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
-        """
-        Apply the scaling transformation to the data.
+        """Applies the learned scaling transformation to input data.
+
+        Enforces float dtypes and validates feature alignment with `fit()`.
+        Scales either all columns or only numerical (non-encoded) columns,
+        depending on `scale_categorical_features`.
 
         Args:
-            x: pandas DataFrame of input data.
+            x (pd.DataFrame): Input feature DataFrame, with column order and names matching those seen in `fit()`.
 
         Returns:
-            Transformed DataFrame.
+            pd.DataFrame: Scaled DataFrame, same shape and index as input.
+
+        Raises:
+            ValueError: If input feature names/order do not match those seen during `fit()`.
+            ValueError: If any column cannot be converted to float.
+            ValueError: If scaler has not been fitted (via `check_is_fitted`).
         """
         logger.debug("Transforming data with Scaler transformer...")
         check_is_fitted(self, "n_features_in_")
@@ -120,14 +157,19 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         return x_transformed
 
     def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
-        """
-        Apply the inverse scaling transformation to the data.
+        """Applies the inverse scaling transformation to revert scaled data.
+
+        Requires that the scaler was fitted and that input column structure matches `fit()`.
 
         Args:
-            x: pandas DataFrame of scaled input data.
+            x (pd.DataFrame): Scaled input DataFrame, with same columns and order as `fit()` output.
 
         Returns:
-            Inversely transformed DataFrame.
+            pd.DataFrame: Inversely transformed (descaled) DataFrame, with same shape/index as input.
+
+        Raises:
+            ValueError: If scaler has not been fitted (via `check_is_fitted`).
+            ValueError: If input feature names/order do not match those seen during `fit()`.
         """
         logger.debug("Applying inverse transformation with Scaler transformer...")
         check_is_fitted(self, "n_features_in_")
@@ -144,14 +186,18 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
         return x_inverse_transformed
 
     def get_feature_names_out(self, input_features=None) -> list:
-        """
-        Get output feature names for the transformed data.
+        """Returns the list of output feature names (same as input feature names).
+
+        Preserves original column order and names seen during `fit()`.
 
         Args:
-            input_features: Optional list of input features.
+            input_features: Ignored; included for scikit-learn API compatibility.
 
         Returns:
-            List of output feature names.
+            List[str]: Ordered list of output feature names.
+
+        Raises:
+            ValueError: If scaler has not been fitted (via `check_is_fitted`).
         """
         check_is_fitted(self, "n_features_in_")
         return self.feature_names_out_
diff --git a/tests/data_preprocessing/test_imputer.py b/tests/data_preprocessing/test_imputer.py
new file mode 100644
index 00000000..c2e57df3
--- /dev/null
+++ b/tests/data_preprocessing/test_imputer.py
@@ -0,0 +1,324 @@
+import unittest
+import pandas as pd
+import numpy as np
+from sklearn.exceptions import NotFittedError
+
+from energy_fault_detector.data_preprocessing.imputer import Imputer
+
+
+class TestImputer(unittest.TestCase):
+    def setUp(self):
+        """Set up common test fixtures."""
+        # Create sample numerical data with missing values
+        self.data_numerical = pd.DataFrame({
+            'temperature': [20.5, np.nan, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, np.nan, 65, 67],
+            'pressure': [1013.2, 1014.5, 1012.8, np.nan, 1013.7]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with categorical features (object dtype)
+        self.data_with_categorical = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, 58, 65, 67],
+            'status': ['A', 'B', 'A', 'C', 'B'],
+            'region': ['North', 'South', 'North', 'East', 'West'],
+            'category': ['X', 'Y', 'X', 'Z', 'Y']
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with mixed numerical and categorical columns
+        self.data_mixed = pd.DataFrame({
+            'temperature': [20.5, np.nan, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, np.nan, 65, 67],
+            'status': ['A', 'B', 'A', 'C', 'B'],
+            'region': ['North', 'South', 'North', 'East', 'West'],
+            'voltage': [220.5, 221.3, np.nan, 219.8, 220.1]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with NaN values in categorical columns
+        self.data_with_nan_categorical = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, np.nan, 23.0],
+            'humidity': [60, 62, 58, 65, np.nan],
+            'status': ['A', 'B', np.nan, 'C', 'B'],
+            'region': ['North', 'South', 'North', 'East', 'West']
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with all NaN in a column
+        self.data_all_nan = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, 58, 65, 67],
+            'broken_sensor': [np.nan, np.nan, np.nan, np.nan, np.nan]
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data for inverse transform testing
+        self.data_for_inverse = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8],
+            'humidity': [60, 62, 58],
+            'status': ['A', 'B', 'A']
+        }, index=pd.date_range('2024-01-01', periods=3, freq='1Min'))
+        
+        # Create transformed data (with NaN values filled)
+        self.transformed_data = pd.DataFrame({
+            'temperature': [20.5, 21.1, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, 63.5, 65, 67],
+            'status': ['A', 'B', 'A', 'C', 'B'],
+            'region': ['North', 'South', 'North', 'East', 'West']
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with missing columns (for error testing)
+        self.data_missing_columns = pd.DataFrame({
+            'temperature': [21.0, 22.0],
+            'humidity': [61, 63]
+        }, index=pd.date_range('2024-01-01', periods=2, freq='1Min'))
+        
+        # Create data with non-declared categorical features (object dtype in numerical columns)
+        self.data_with_object_in_num = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'humidity': [60, 62, 58, 65, 67],
+            'sensor_id': ['S1', 'S2', 'S3', 'S4', 'S5']  # This should be detected as categorical
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with only categorical features
+        self.data_only_categorical = pd.DataFrame({
+            'status': ['A', 'B', 'A', 'C', 'B'],
+            'region': ['North', 'South', 'North', 'East', 'West']
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+        
+        # Create data with boolean columns
+        self.data_with_boolean = pd.DataFrame({
+            'temperature': [20.5, 21.3, 19.8, 22.1, 23.0],
+            'is_active': [True, False, True, False, True],
+            'status': ['A', 'B', 'A', 'C', 'B']
+        }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
+
+    def tearDown(self):
+        """Clean up after each test."""
+        del (self.data_numerical, self.data_with_categorical, 
+             self.data_mixed, self.data_with_nan_categorical,
+             self.data_all_nan, self.data_for_inverse,
+             self.transformed_data, self.data_missing_columns,
+             self.data_with_object_in_num, self.data_only_categorical,
+             self.data_with_boolean)
+
+    def test_init_defaults(self):
+        """Test default initialization."""
+        imputer = Imputer()
+        self.assertEqual(imputer.strategy, 'mean')
+        self.assertEqual(imputer.categorical_features, [])
+        self.assertIsNotNone(imputer.numerical_imputer)
+        self.assertIsNotNone(imputer.categorical_imputer)
+
+    def test_init_median_strategy(self):
+        """Test initialization with median strategy."""
+        imputer = Imputer(strategy='median')
+        self.assertEqual(imputer.strategy, 'median')
+        self.assertEqual(imputer.numerical_imputer.strategy, 'median')
+        self.assertEqual(imputer.categorical_imputer.strategy, 'most_frequent')
+
+    def test_init_invalid_strategy(self):
+        """Test initialization with invalid strategy raises ValueError."""
+        with self.assertRaises(ValueError):
+            Imputer(strategy='invalid_strategy')
+
+    def test_init_with_categorical_features(self):
+        """Test initialization with categorical features."""
+        imputer = Imputer(categorical_features=['status', 'region'])
+        self.assertEqual(imputer.categorical_features, ['status', 'region'])
+
+    def test_fit_numerical_data(self):
+        """Test fitting with numerical data containing NaN."""
+        imputer = Imputer()
+        imputer.fit(self.data_numerical)
+        
+        self.assertEqual(imputer.n_features_in_, 3)
+        self.assertEqual(len(imputer.feature_names_in_), 3)
+        self.assertEqual(len(imputer.numerical_columns), 3)
+        self.assertEqual(len(imputer.categorical_columns), 0)
+        self.assertIsNotNone(imputer.numerical_imputer.statistics_)
+
+    def test_fit_with_categorical_features(self):
+        """Test fitting with categorical features."""
+        imputer = Imputer(categorical_features=['status'])
+        imputer.fit(self.data_with_categorical)
+        
+        self.assertEqual(imputer.n_features_in_, 5)
+        self.assertIn('status', imputer.categorical_columns)
+        self.assertIn('temperature', imputer.numerical_columns)
+        self.assertEqual(len(imputer.numerical_columns), 2)  # temperature and humidity
+        self.assertEqual(len(imputer.categorical_columns), 1)  # status
+
+    def test_fit_mixed_data(self):
+        """Test fitting with mixed numerical and categorical data."""
+        imputer = Imputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_mixed)
+        
+        self.assertEqual(imputer.n_features_in_, 5)
+        self.assertEqual(len(imputer.numerical_columns), 3)  # temperature, humidity, voltage
+        self.assertEqual(len(imputer.categorical_columns), 2)  # status, region
+
+    def test_fit_detects_non_declared_categorical(self):
+        """Test that non-declared categorical features are detected."""
+        imputer = Imputer()
+        imputer.fit(self.data_with_object_in_num)
+        
+        # sensor_id should be detected as non-declared categorical
+        self.assertIn('sensor_id', imputer.non_declared_categorical_features)
+        # sensor_id should be excluded from numerical columns
+        self.assertNotIn('sensor_id', imputer.numerical_columns)
+        # sensor_id should be dropped entirely (not in categorical either)
+        self.assertEqual(len(imputer.numerical_columns), 2)  # temperature, humidity only
+
+    def test_fit_with_all_nan_column(self):
+        """Test fitting with a column that has all NaN values."""
+        imputer = Imputer()
+        imputer.fit(self.data_all_nan)
+        
+        # All-NaN column should be handled without errors
+        self.assertEqual(imputer.n_features_in_, 3)
+        # The numerical imputer will compute a default value (0.0 for mean)
+        self.assertIsNotNone(imputer.numerical_imputer.statistics_)
+
+    def test_fit_empty_categorical(self):
+        """Test fitting when categorical columns DataFrame is empty."""
+        imputer = Imputer(categorical_features=['nonexistent'])
+        imputer.fit(self.data_numerical)
+        
+        # Should handle empty categorical data gracefully
+        self.assertEqual(len(imputer.categorical_columns), 0)
+        self.assertIsNotNone(imputer.categorical_imputer)
+
+    def test_transform_with_numerical_data(self):
+        """Test transformation with numerical data."""
+        imputer = Imputer()
+        imputer.fit(self.data_numerical)
+        result = imputer.transform(self.data_numerical)
+        
+        # Should return DataFrame with same shape and columns
+        self.assertEqual(result.shape, self.data_numerical.shape)
+        self.assertListEqual(list(result.columns), self.data_numerical.columns.tolist())
+        # No NaN values should remain
+        self.assertFalse(result.isna().any().any())
+        # Index should be preserved
+        self.assertTrue(result.index.equals(self.data_numerical.index))
+
+    def test_transform_with_categorical_data(self):
+        """Test transformation with categorical data."""
+        imputer = Imputer(categorical_features=['status'])
+        imputer.fit(self.data_with_categorical)
+        result = imputer.transform(self.data_with_categorical)
+        
+        # Should return DataFrame without non-declared categorical features
+        self.assertEqual(result.shape[1], 3)  # temperature, humidity, status
+        # Categorical values should be imputed (most frequent)
+        self.assertFalse(result['status'].isna().any())
+        # Index should be preserved
+        self.assertTrue(result.index.equals(self.data_with_categorical.index))
+
+    def test_transform_mixed_data(self):
+        """Test transformation with mixed data."""
+        imputer = Imputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_mixed)
+        result = imputer.transform(self.data_mixed)
+        
+        # Should handle mixed data correctly
+        self.assertEqual(result.shape, self.data_mixed.shape)
+        # No NaN values should remain
+        self.assertFalse(result.isna().any().any())
+
+    def test_transform_unfitted_raises(self):
+        """Test that transform on unfitted model raises error."""
+        imputer = Imputer()
+        with self.assertRaises(NotFittedError):
+            imputer.transform(self.data_numerical)
+
+    def test_transform_with_nan_categorical(self):
+        """Test transformation with NaN values in categorical columns."""
+        imputer = Imputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_with_nan_categorical)
+        result = imputer.transform(self.data_with_nan_categorical)
+        
+        # Categorical NaN values should be imputed
+        self.assertFalse(result[['status', 'region']].isna().any().any())
+
+    def test_inverse_transform(self):
+        """Test inverse transform method."""
+        imputer = Imputer(categorical_features=['status'])
+        imputer.fit(self.data_for_inverse)
+        transformed = imputer.transform(self.data_for_inverse)
+        inverse = imputer.inverse_transform(transformed)
+        
+        # Should return DataFrame with same shape
+        self.assertEqual(inverse.shape, self.data_for_inverse.shape)
+        # Index should be restored to original
+        self.assertTrue(inverse.index.equals(self.data_for_inverse.index))
+        # Column names should match original
+        self.assertListEqual(list(inverse.columns), self.data_for_inverse.columns.tolist())
+
+    def test_get_feature_names_out(self):
+        """Test get_feature_names_out method."""
+        imputer = Imputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_mixed)
+        
+        feature_names = imputer.get_feature_names_out()
+        
+        # Should return numerical columns first, then categorical
+        expected_order = ['temperature', 'humidity', 'voltage', 'status', 'region']
+        self.assertListEqual(feature_names, expected_order)
+
+    def test_transform_with_missing_columns(self):
+        """Test that transform raises error for missing columns."""
+        imputer = Imputer()
+        imputer.fit(self.data_numerical)
+        
+        with self.assertRaises(ValueError):
+            imputer.transform(self.data_missing_columns)
+
+    def test_transform_with_boolean_columns(self):
+        """Test transformation with boolean columns."""
+        imputer = Imputer(categorical_features=['status'])
+        imputer.fit(self.data_with_boolean)
+        result = imputer.transform(self.data_with_boolean)
+        
+        # Boolean column should be handled as numerical
+        self.assertFalse(result.isna().any().any())
+
+    def test_transform_preserves_index_type(self):
+        """Test that index type is preserved after transformation."""
+        imputer = Imputer()
+        imputer.fit(self.data_numerical)
+        result = imputer.transform(self.data_numerical)
+        
+        self.assertIsInstance(result.index, pd.DatetimeIndex)
+
+    def test_transform_with_median_strategy(self):
+        """Test transformation with median strategy."""
+        imputer = Imputer(strategy='median')
+        imputer.fit(self.data_numerical)
+        result = imputer.transform(self.data_numerical)
+        
+        # Should handle NaN values correctly
+        self.assertFalse(result.isna().any().any())
+        # Should return DataFrame with same shape
+        self.assertEqual(result.shape, self.data_numerical.shape)
+
+    def test_transform_with_empty_categorical_data(self):
+        """Test transformation when no categorical columns match."""
+        imputer = Imputer(categorical_features=['nonexistent'])
+        imputer.fit(self.data_numerical)
+        result = imputer.transform(self.data_numerical)
+        
+        # Should handle empty categorical data gracefully
+        self.assertEqual(result.shape, self.data_numerical.shape)
+
+    # TODO: Imputer doesnt work if no numerical data in the input.
+    # def test_transform_with_only_categorical(self):
+    #     """Test transformation with only categorical features."""
+    #     imputer = Imputer(categorical_features=['status', 'region'])
+    #     imputer.fit(self.data_only_categorical)
+    #     result = imputer.transform(self.data_only_categorical)
+        
+    #     # Should handle categorical-only data
+    #     self.assertEqual(result.shape, self.data_only_categorical.shape)
+    #     self.assertFalse(result.isna().any().any())
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

From f50690c62e6ef9156acd1f6b0e6cedc098b807cd Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Tue, 25 Aug 2026 16:39:57 +0200
Subject: [PATCH 13/35] Added adavanced_config.yaml, that contains the added
 preprocessing steps and other advanced parameters. Refactored
 base_config.yaml to reflect changes in the data_preprocessor.

---
 energy_fault_detector/advanced_config.yaml | 94 ++++++++++++++++++++++
 energy_fault_detector/base_config.yaml     | 12 +--
 2 files changed, 100 insertions(+), 6 deletions(-)
 create mode 100644 energy_fault_detector/advanced_config.yaml

diff --git a/energy_fault_detector/advanced_config.yaml b/energy_fault_detector/advanced_config.yaml
new file mode 100644
index 00000000..5df58d30
--- /dev/null
+++ b/energy_fault_detector/advanced_config.yaml
@@ -0,0 +1,94 @@
+train:
+  data_clipping:
+    lower_percentile: 0.001
+    upper_percentile: 0.999
+
+  data_preprocessor:
+    steps:
+      # 1) Drop columns with too many NaNs and exclude specific features
+      - name: column_selector
+        params:
+          max_nan_frac_per_col: 0.9
+          features_to_exclude: [] # Inlcude features to exclude in the list
+      # 2) Drop columns with low number of unique values
+      - name: low_unique_value_filter
+        params:
+          min_unique_value_count: 2
+      # 3) Transform counter readings to differences
+      - name: counter_diff_transformer
+        params:
+          counters: [] # Include counter features in the list
+          compute_rate: True
+          reset_strategy: 'nan'
+          fill_first: 'nan'
+          keep_original: False
+          max_gap_seconds: 3600
+      # 4) Imputation
+      - name: ffill_imputer
+        params:
+          freq: '1Min' # Frequency used to determine limit to forward fill.
+          ffill_limit: 60 # Limit number of consecutive NaNs tu forward fill.
+          categorical_features: [] # include categorical features in the list
+
+      # 5) Encode categorical features
+      - name: categorical_encoder
+        params:
+          categorical_features: [] # include categorical features in the list
+            
+      # 6) Scaling (standardize)
+      - name: scaler
+        params:
+          scaler_type: 'standard' # Supported types are 'standard' and 'minmax'
+          with_mean: True
+          with_std: True
+          scale_categorical_features: False
+          categorical_features: [] # include categorical features in the list
+
+      # 7) Add time-based features
+      - name: timestamp_transformer
+        params:
+          features: ['minute_of_hour', 'hour_of_day', 'day_of_week']
+  
+  data_splitter:
+    shuffle: true
+    type: sklearn
+    validation_split: 0.2
+
+  autoencoder:
+    name: MultilayerAutoencoder
+    params:
+      act: prelu
+      batch_size: 256
+      code_size: 9
+      early_stopping: true
+      epochs: 1000
+      last_act: linear
+      layers:
+      - 64
+      - 32
+      learning_rate: 0.0001
+      loss_name: mean_squared_error
+      min_delta: 0.0001
+      noise: 0.0
+      patience: 5
+    verbose: 0
+
+  anomaly_score:
+    name: rmse
+    params:
+      scale: false
+   
+  threshold_selector:
+    fit_on_val: true
+    name: QuantileThresholdSelector
+    params:
+      quantile: 0.98
+  
+  protect_conditional_features: true # Protect conditional features from being dropped during preprocessing
+
+root_cause_analysis:
+  alpha: 0.8
+  init_x_bias: recon
+  num_iter: 200
+  max_sample_threshold: 1000
+  verbose: true
\ No newline at end of file
diff --git a/energy_fault_detector/base_config.yaml b/energy_fault_detector/base_config.yaml
index aebcc2b0..5267352b 100644
--- a/energy_fault_detector/base_config.yaml
+++ b/energy_fault_detector/base_config.yaml
@@ -10,7 +10,12 @@ train:
           max_nan_frac_per_col: 0.2
       - name: low_unique_value_filter
       - name: simple_imputer
-      - name: standard_scaler
+      - name: scaler
+
+  data_splitter:
+    type: sklearn
+    validation_split: 0.2
+    shuffle: true
 
   autoencoder:
     name: default
@@ -29,11 +34,6 @@ train:
   anomaly_score:
     name: rmse
 
-  data_splitter:
-    type: sklearn
-    validation_split: 0.2
-    shuffle: true
-
   threshold_selector:
     fit_on_val: true
     name: quantile

From 3fab24e3c09910a220c783784bc49bd1b74b2fd9 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Wed, 26 Aug 2026 13:27:52 +0200
Subject: [PATCH 14/35] Refactor conditional feature checks in FaultDetector to
 handle encoded categorical columns

---
 energy_fault_detector/fault_detector.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index d0dc9539..7fc98032 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -132,7 +132,6 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
         self._resolve_conditional_features(x_prepped)
 
         # Post-preprocessing check: conditionals may have been dropped
-        # TODO: consider adding the encoded categorical features to conditional_features at this point. Declared categorical features can be used as conditions after encoding.
         if not self.config.protect_conditional_features and self.autoencoder.is_conditional:
             # Check if conditionals survived preprocessing
             surviving = [
@@ -480,8 +479,8 @@ def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]:
         if not configured:
             return []
 
-        available = [f for f in configured if f in sensor_data.columns]
-        missing = set(configured) - set(available)
+        available = [col for col in sensor_data.columns if any(declared_condition in col for declared_condition in configured)] # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring.
+        missing = [col for col in configured if not any(col in available_condition for available_condition in available)] # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring.
 
         if missing:
             logger.warning(f"Conditional features not found in sensor_data and will be ignored: "

From ca7d7bcbac7fa4fb05042eff03e0a196545336cb Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Thu, 27 Aug 2026 11:10:39 +0200
Subject: [PATCH 15/35] Refactor conditional feature checks in FaultDetector.
 Declared conditions are now resolved properly even if the declared name is a
 substring on any column in data, which is the case of encoded categorical
 featrues. Improve clarity and logging for available, missing and surviving
 conditions during preprocessing.

---
 energy_fault_detector/fault_detector.py | 42 +++++++++++--------------
 1 file changed, 19 insertions(+), 23 deletions(-)

diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index 7fc98032..05c4639d 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -128,24 +128,25 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
             sensor_data=sensor_data, normal_index=normal_index, fit_preprocessor=fit_preprocessor
         )
 
-        # --- Resolve conditional features against available data ---
+        # Resolve declared conditions against preprocessed data
         self._resolve_conditional_features(x_prepped)
 
-        # Post-preprocessing check: conditionals may have been dropped
-        if not self.config.protect_conditional_features and self.autoencoder.is_conditional:
-            # Check if conditionals survived preprocessing
-            surviving = [
-                f for f in (self.autoencoder.conditional_features or [])
-                if f in x_prepped.columns
-            ]
-            dropped_by_pipeline = set(self.autoencoder.conditional_features or []) - set(surviving)
-            if dropped_by_pipeline:
-                logger.warning(f"Conditional features dropped by preprocessing pipeline: "
-                               f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}")
-                if surviving:
-                    self.autoencoder.conditional_features = surviving
-                else:
-                    self._fallback_if_no_conditionals()
+        # Check conditionals: avilable conditions in original data and surviving conditions during preprocessing.
+        if self.autoencoder.is_conditional:
+            configured = self.autoencoder.conditional_features or []
+            available = [declared_condition for declared_condition in configured if any(declared_condition in col for col in sensor_data.columns)] # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring.
+            missing = [declared_condition for declared_condition in configured if declared_condition not in available]
+            if missing:
+                logger.warning(f"Declared conditions not found in sensor_data will be ignored: "
+                               f"{sorted(missing)}. Using: {available or 'none'}")
+
+            if not self.config.protect_conditional_features:
+                surviving = [declared_condition for declared_condition in available if any(declared_condition in col for col in x_prepped.columns)] # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring.
+                dropped_by_pipeline = set(available or []) - set(surviving)
+                
+                if dropped_by_pipeline:
+                    logger.warning(f"Declared conditions dropped by preprocessing pipeline: "
+                                f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}")
 
         train_recon_error, val_recon_error = None, None
         x_train, x_val = self.train_val_split(x_prepped)
@@ -463,9 +464,9 @@ def _fit_threshold(self, x: pd.DataFrame, y: pd.Series, x_val: pd.DataFrame, fit
             self.threshold_selector.fit(x=scores, y=y.loc[scores.index])
 
     def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]:
-        """Resolve which conditional features are actually available in the data.
+        """Resolve which declared conditions are actually available in the data.
 
-        If all conditional features are missing and the autoencoder is a ConditionalAE,
+        If all conditions are missing and the autoencoder is a ConditionalAE,
         falls back to MultilayerAutoencoder. For sequence models, simply clears the
         conditional_features list (they handle None gracefully).
 
@@ -480,11 +481,6 @@ def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]:
             return []
 
         available = [col for col in sensor_data.columns if any(declared_condition in col for declared_condition in configured)] # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring.
-        missing = [col for col in configured if not any(col in available_condition for available_condition in available)] # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring.
-
-        if missing:
-            logger.warning(f"Conditional features not found in sensor_data and will be ignored: "
-                           f"{sorted(missing)}. Using: {available or 'none'}")
 
         if not available:
             self._fallback_if_no_conditionals()

From 0c40916a4afda8686a08e1f398a06060970e5a4c Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 28 Aug 2026 12:01:35 +0200
Subject: [PATCH 16/35] Refactor protected feature validation in
 DataPreprocessor and update FaultDetector comments. Improved handling of
 available protected features and clarified potential issues with conditional
 feature name substrings.

---
 .../data_preprocessing/data_preprocessor.py           | 11 ++++-------
 energy_fault_detector/fault_detector.py               |  6 +++---
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index dd5c6f04..9d492278 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -195,17 +195,14 @@ def fit(self, X: pd.DataFrame, y=None, **fit_params):
         result = super().fit(X=X, y=y, **fit_params)
 
         # Validate that protected features are in the output
+        # TODO: protected features shouldnt be dropped. Include this in a test.
         if protected_features:
             output_features = self.get_feature_names_out()
-            missing_features = [f for f in protected_features if f not in output_features]
+            available_protected = [f for f in protected_features if f in X.columns]
+            missing_features = [col for col in output_features if not any(available_feature in col for available_feature in available_protected)]
             if missing_features:
                 raise ValueError(
-                    f"Protected features were dropped by the preprocessing pipeline: {missing_features}. "
-                    f"This may be caused by AngleTransformer or CounterDiffTransformer modifying these features. "
-                    f"Please ensure protected features (e.g., conditional features for autoencoders) are not "
-                    f"transformed by these steps or consider the transformed version of these features (e.g. "
-                    f"<feature_name>_sin, <feature_name>_cos, <feature_name>_rate, <feature_name>_diff) as "
-                    f"conditional features."
+                    f"Protected features were dropped by the preprocessing pipeline: {missing_features}."
                 )
 
         return result
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index 05c4639d..20f7d565 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -68,7 +68,6 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
         ) if protect else []
 
         # Data clipping (outlier clipping)
-        # TODO: what happens in DataClipper if data contains non-numerical features?
         if self.config.data_clipping:
             logger.debug('Clip data before scaling.')
             clipper_params = self.config.data_clipping_params.copy()
@@ -479,8 +478,9 @@ def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]:
         configured = self.autoencoder.conditional_features or []
         if not configured:
             return []
-
-        available = [col for col in sensor_data.columns if any(declared_condition in col for declared_condition in configured)] # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring.
+        # TODO: this can produce undesired results if any column in the original data contains a substring of a declared conditional feature name.
+        # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring. The applies for timestamp and column_diff transformers.
+        available = [col for col in sensor_data.columns if any(declared_condition in col for declared_condition in configured)] 
 
         if not available:
             self._fallback_if_no_conditionals()

From c32d35a5d7abe03c7e65ad59bfc6d55b4be1d8c5 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 28 Aug 2026 15:31:23 +0200
Subject: [PATCH 17/35] Update advanced_config.yaml and enhance test coverage
 for preprocessing pipeline

- Changed autoencoder name to ConditionalAutoencoder in advanced_config.yaml.
- Added comprehensive tests for preprocessing pipeline, including dtype conversion, expected columns, NaN handling, and integration with FaultDetector.
- Ensured protected features are retained during preprocessing and validated conditional feature handling.
---
 energy_fault_detector/advanced_config.yaml    |   3 +-
 .../test_data_preprocessor.py                 | 290 +++++++++++++++++-
 2 files changed, 275 insertions(+), 18 deletions(-)

diff --git a/energy_fault_detector/advanced_config.yaml b/energy_fault_detector/advanced_config.yaml
index 5df58d30..3e1eeb87 100644
--- a/energy_fault_detector/advanced_config.yaml
+++ b/energy_fault_detector/advanced_config.yaml
@@ -55,7 +55,7 @@ train:
     validation_split: 0.2
 
   autoencoder:
-    name: MultilayerAutoencoder
+    name: ConditionalAutoencoder
     params:
       act: prelu
       batch_size: 256
@@ -71,6 +71,7 @@ train:
       min_delta: 0.0001
       noise: 0.0
       patience: 5
+      conditional_features: [] # Include conditional features in the list
     verbose: 0
 
   anomaly_score:
diff --git a/tests/data_preprocessing/test_data_preprocessor.py b/tests/data_preprocessing/test_data_preprocessor.py
index bb1b451d..ec4345b2 100644
--- a/tests/data_preprocessing/test_data_preprocessor.py
+++ b/tests/data_preprocessing/test_data_preprocessor.py
@@ -1,3 +1,4 @@
+import tempfile
 from unittest import TestCase
 
 import numpy as np
@@ -6,7 +7,9 @@
 from pandas.testing import assert_frame_equal
 from sklearn.utils.validation import check_is_fitted, NotFittedError
 
+from energy_fault_detector.config.config import Config
 from energy_fault_detector.data_preprocessing.data_preprocessor import DataPreprocessor
+from energy_fault_detector.fault_detector import FaultDetector
 
 
 class TestDataPreprocessorPipeline(TestCase):
@@ -466,23 +469,276 @@ def test_protected_features_with_both_filters(self):
         self.assertIn('high_nan_feature', transformed.columns,
                       "High-NaN protected feature should be kept")
 
-    def test_validation_fails_when_protected_feature_missing_from_output(self):
-        """Test that fit raises ValueError if a protected feature is missing from output."""
-        from energy_fault_detector.data_preprocessing.angle_transformer import AngleTransformer
+class TestComprehensivePipelineFlow(TestCase):
+    """Test comprehensive preprocessing pipeline"""
+
+    def setUp(self):
+        """Set up test fixtures for comprehensive pipeline testing."""
+        self.n_samples = 500
+        self.time_index = pd.date_range('2025-01-01', periods=self.n_samples, freq='5min')
+        
+        # Create realistic test data matching the updated advanced_config.yaml scenario
+        np.random.seed(42)
+        self.test_data = pd.DataFrame({
+            # Numerical features
+            'temp_sensor_1': np.random.normal(100, 5, self.n_samples),
+            'temp_sensor_2': np.random.normal(85, 3, self.n_samples),
+            'pressure_sensor': np.random.normal(50, 2, self.n_samples),
+            'flow_rate': np.random.exponential(10, self.n_samples),
+            # Categorical features
+            'category_col': np.random.choice(['A', 'B', 'C', 'D'], self.n_samples),
+            'equipment_type': np.random.choice(['Type1', 'Type2', 'Type3'], self.n_samples),
+            # Conditional feature (numerical)
+            'operating_condition': np.random.normal(0, 1, self.n_samples),
+        }, index=self.time_index)
+        
+        # Add some NaN values to test imputation
+        self.test_data.loc[50:100, 'temp_sensor_1'] = np.nan
+        self.test_data.loc[200:250, 'pressure_sensor'] = np.nan
+        self.test_data.loc[150:170, 'category_col'] = np.nan
+
+    def _create_config_dict(self):
+        """Create configuration matching updated advanced_config.yaml structure."""
+        return {
+            'train': {
+                'data_clipping': {
+                    'lower_percentile': 0.001,
+                    'upper_percentile': 0.999,
+                },
+                'data_preprocessor': {
+                    'steps': [
+                        {'name': 'column_selector', 'params': {'max_nan_frac_per_col': 0.9}},
+                        {'name': 'low_unique_value_filter', 'params': {'min_unique_value_count': 2}},
+                        {'name': 'ffill_imputer', 'params': {'freq': '1Min', 'ffill_limit': 60, 'categorical_features': ['category_col', 'equipment_type']}},
+                        {'name': 'categorical_encoder', 'params': {'categorical_features': ['category_col', 'equipment_type']}},
+                        {'name': 'scaler', 'params': {'scaler_type': 'standard', 'with_mean': True, 'with_std': True, 'scale_categorical_features': False, 'categorical_features': ['category_col', 'equipment_type']}},
+                        {'name': 'timestamp_transformer', 'params': {'features': ['minute_of_hour', 'hour_of_day', 'day_of_week']}}
+                    ]
+                },
+                'autoencoder': {
+                    'name': 'ConditionalAutoencoder',
+                    'params': {
+                        'act': 'prelu',
+                        'batch_size': 256,
+                        'code_size': 9,
+                        'early_stopping': True,
+                        'epochs': 10,
+                        'last_act': 'linear',
+                        'layers': [64, 32],
+                        'learning_rate': 0.0001,
+                        'loss_name': 'mean_squared_error',
+                        'min_delta': 0.0001,
+                        'noise': 0.0,
+                        'patience': 5,
+                        'conditional_features': ['operating_condition', 'category_col'],
+                        'verbose': 0
+                    }
+                },
+                'protect_conditional_features': True
+            }
+        }
 
-        # Create a pipeline that transforms a protected feature
-        preprocessor = DataPreprocessor(
-            steps=[
-                {'name': 'angle_transformer',
-                 'params': {'angles': ['conditional_feature']}},  # Transforms conditional_feature
-            ]
-        )
+    def test_pipeline_dtype_conversion(self):
+        """Test that all dtypes are numerical after preprocessing."""
+        config_dict = self._create_config_dict()
+        preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps'])
+        
+        preprocessor.fit(self.test_data)
+        transformed_data = preprocessor.transform(self.test_data)
+        
+        # Verify all dtypes are numerical after preprocessing
+        for dtype in transformed_data.dtypes:
+            self.assertTrue(pd.api.types.is_numeric_dtype(dtype),
+                          f"Column {dtype.name} should be numeric after preprocessing")
+
+    def test_pipeline_expected_columns(self):
+        """Test that expected columns exist after preprocessing."""
+        config_dict = self._create_config_dict()
+        preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps'])
+        
+        preprocessor.fit(self.test_data)
+        transformed_data = preprocessor.transform(self.test_data)
+        
+        # Check expected numerical columns
+        expected_numerical = ['temp_sensor_1', 'temp_sensor_2', 'pressure_sensor', 'flow_rate', 'operating_condition']
+        for col in expected_numerical:
+            self.assertIn(col, transformed_data.columns, f"Expected numerical column {col}")
+        
+        # Check categorical features were one-hot encoded
+        expected_categorical = ['category_col_A', 'category_col_B', 'category_col_C', 'category_col_D',
+                               'equipment_type_Type1', 'equipment_type_Type2', 'equipment_type_Type3']
+        for col in expected_categorical:
+            self.assertIn(col, transformed_data.columns, f"Expected encoded categorical column {col}")
+        
+        # Check timestamp features
+        timestamp_features = ['minute_of_hour_sine', 'minute_of_hour_cosine',
+                             'hour_of_day_sine', 'hour_of_day_cosine',
+                             'day_of_week_sine', 'day_of_week_cosine']
+        for col in timestamp_features:
+            self.assertIn(col, transformed_data.columns, f"Expected timestamp feature {col}")
+
+    def test_pipeline_nan_handling(self):
+        """Test that NaN values are properly handled."""
+        config_dict = self._create_config_dict()
+        preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps'])
+        
+        preprocessor.fit(self.test_data)
+        transformed_data = preprocessor.transform(self.test_data)
+        
+        # Verify no NaN values remain after preprocessing
+        self.assertFalse(transformed_data.isna().any().any(),
+                        "Transformed data should not contain NaN values after preprocessing")
+
+    def test_pipeline_shape_transformation(self):
+        """Test that data shape is appropriate after preprocessing."""
+        config_dict = self._create_config_dict()
+        preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps'])
+        
+        preprocessor.fit(self.test_data)
+        transformed_data = preprocessor.transform(self.test_data)
+        
+        # Verify data shape (should have more columns than original due to encoding)
+        self.assertGreater(transformed_data.shape[1], self.test_data.shape[1],
+                          "Transformed data should have more columns due to categorical encoding and timestamp features")
+
+    def test_pipeline_inverse_transform(self):
+        """Test that inverse transform works correctly."""
+        config_dict = self._create_config_dict()
+        preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps'])
+        
+        preprocessor.fit(self.test_data)
+        transformed_data = preprocessor.transform(self.test_data)
+        inverse_data = preprocessor.inverse_transform(transformed_data)
+        
+        # Verify inverse transform preserves row count
+        self.assertEqual(inverse_data.shape[0], self.test_data.shape[0],
+                        "Inverse transform should preserve row count")
+
+    def test_pipeline_fault_detector_integration(self):
+        """Test integration with FaultDetector."""
+        config_dict = self._create_config_dict()
+        
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = Config(config_dict=config_dict)
+            fault_detector = FaultDetector(config=config, model_directory=tmpdir)
+            
+            # Create normal index (all normal for training)
+            normal_index = pd.Series([True] * self.n_samples, index=self.time_index)
+            
+            # Fit the model
+            result = fault_detector.fit(
+                sensor_data=self.test_data,
+                normal_index=normal_index,
+                save_models=False
+            )
+            
+            # Verify model was trained successfully
+            self.assertIsNotNone(fault_detector.autoencoder,
+                               "Autoencoder should be created and trained")
+            self.assertEqual(fault_detector.autoencoder.__class__.__name__, 'ConditionalAE',
+                           "Autoencoder should be ConditionalAE as specified in config")
+            self.assertIsNotNone(result.train_recon_error,
+                               "Training reconstruction error should be available")
+
+    def test_resolution_of_conditions(self):
+        """Test pipeline handling of missing conditional features."""
+        config_dict = self._create_config_dict()
+        
+        # Create data without the conditional feature
+        test_data_no_cond = self.test_data.drop(columns=['category_col'])
+        
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = Config(config_dict=config_dict)
+            fault_detector = FaultDetector(config=config, model_directory=tmpdir)
+        
+            # Create normal index (all normal for training)
+            normal_index = pd.Series([True] * self.n_samples, index=self.time_index)
+        
+            # Fit the model
+            result = fault_detector.fit(
+                sensor_data=test_data_no_cond,
+                normal_index=normal_index,
+                save_models=False
+            )
+
+        # Verify that autoencoder conditions are updated with available feature names
+        self.assertIn('operating_condition', fault_detector.autoencoder.conditional_features,
+                      "Autoencoder should have 'operating_condition' as a conditional feature")
+        self.assertNotIn('category_col', fault_detector.autoencoder.conditional_features,
+                         "Autoencoder should not have 'category_col' as a conditional feature since it's missing")
+
+    def test_declared_category_not_as_condition(self):
+        """Test pipeline handling of declared categorical features not being declared as conditional features."""
+        config_dict = self._create_config_dict()
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = Config(config_dict=config_dict)
+            fault_detector = FaultDetector(config=config, model_directory=tmpdir)
+
+            # Create normal index (all normal for training)
+            normal_index = pd.Series([True] * self.n_samples, index=self.time_index)
 
-        # This should raise an error because 'conditional_feature' will be transformed to
-        # 'conditional_feature_sin' and 'conditional_feature_cos'
-        with self.assertRaises(ValueError) as context:
-            preprocessor.fit(self.test_data, protected_features=['conditional_feature'])
+            # Fit the model
+            result = fault_detector.fit(
+                sensor_data=test_data_no_cond,
+                normal_index=normal_index,
+                save_models=False
+            )
+
+        # Verify that autoencoder conditions are updated with available feature names
+        self.assertIn('equipment_type_Type_1', fault_detector.data_preprocessor.get_feature_names_out(),
+                      "Data preprocessor should have 'equipment_type' as a feature")
+        self.assertNotIn('equipment_type', fault_detector.autoencoder.conditional_features,
+                            "Autoencoder should not have 'equipment_type' as a conditional feature since it's not declared as such.")
+
+    def test_declared_condition_not_as_category(self):
+        """Test pipeline handling of declared conditional features not being declared as categorical features."""
+        config_dict = self._create_config_dict()
+
+        # Remove 'category_col' from categorical features in the config to simulate it being declared as conditional but not categorical
+        config_dict['train']['data_preprocessor']['steps'][2]['params']['categorical_features'] = ['equipment_type']  # Only 'equipment_type' is categorical
+        config_dict['train']['data_preprocessor']['steps'][3]['params']['categorical_features'] = ['equipment_type']  # Only 'equipment_type' is categorical
+        config_dict['train']['data_preprocessor']['steps'][4]['params']['categorical_features'] = ['equipment_type']  # Only 'equipment_type' is categorical
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = Config(config_dict=config_dict)
+            fault_detector = FaultDetector(config=config, model_directory=tmpdir)
+
+            # Create normal index (all normal for training)
+            normal_index = pd.Series([True] * self.n_samples, index=self.time_index)
+
+            # Fit the model
+            result = fault_detector.fit(
+                sensor_data=self.test_data,
+                normal_index=normal_index,
+                save_models=False
+            )
+
+        # Verify that autoencoder conditions are updated properly.
+        self.assertFalse(any('category_col' in col for col in fault_detector.data_preprocessor.get_feature_names_out()),
+                        "Data preprocessor should not have any column containing substring 'category_col' since it's not declared as categorical.")
+
+    def test_protected_features_are_not_dropped_in_pipeline(self):
+        """Test that protected features are not dropped in the comprehensive pipeline."""
+        config_dict = self._create_config_dict()
+        config = Config(config_dict=config_dict)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            fault_detector = FaultDetector(config=config, model_directory=tmpdir)
+        
+            # Create normal index (all normal for training)
+            normal_index = pd.Series([True] * self.n_samples, index=self.time_index)
+        
+            # Fit the model
+            result = fault_detector.fit(
+                sensor_data=self.test_data,
+                normal_index=normal_index,
+                save_models=False
+            )
 
-        self.assertIn('Protected features were dropped', str(context.exception))
-        self.assertIn('conditional_feature', str(context.exception))
-        self.assertIn('AngleTransformer or CounterDiffTransformer', str(context.exception))
+        protected_features = config_dict['train']['autoencoder']['params']['conditional_features']
+    
+        output_features = fault_detector.data_preprocessor.get_feature_names_out()
+        available_protected = [f for f in protected_features if f in self.test_data.columns]
+        missing_features = [col for col in output_features if not any(available_feature in col for available_feature in available_protected)]
+        self.assertEqual(len(missing_features), 0, f"Protected features {available_protected} should not be dropped in the pipeline.")

From 7796df649986856b7502366e0aebc06fbb5ec63e Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Wed, 2 Sep 2026 09:01:37 +0200
Subject: [PATCH 18/35] Fix preprocessing unit tests and finalize categorical
 feature handling

Unit tests:
- Fix TestComprehensivePipelineFlow.setUp: use iloc instead of positional
  .loc slices (rejected by newer pandas on DatetimeIndex).
- Fix inverted protected-features check in DataPreprocessor.fit and the
  corresponding test assertion: iterate over protected features, not output
  columns, so normal columns are no longer flagged as dropped.
- Adjust test_pipeline_inverse_transform to compare against the transformed
  row count, since ForwardFillImputer drops rows with remaining NaNs.

Categorical feature handling:
- CategoricalEncoder, Imputer, ForwardFillImputer now match
  categorical_features by exact name (they run pre-encoding); Scaler keeps
  substring matching to cover one-hot encoded columns.
- Imputer.inverse_transform: preserve the input index instead of overwriting
  it with the fit-time index (fixes wrong row alignment at inference).
- CategoricalEncoder.inverse_transform: use categorical_columns (the fitted
  names) instead of categorical_features, consistent with get_feature_names_out.
- DataPreprocessor._build_from_steps_spec: deepcopy the steps spec so name
  normalization and step_name assignment no longer mutate the caller's config.
- Scaler.inverse_transform: add the same column-order validation as transform.
- CategoricalEncoder.transform: report only the actually-missing columns in
  the KeyError message, and guard non-DataFrame input with a TypeError.

Other:
- Promote non-declared categorical features dropped log messages from INFO
  to WARNING across CategoricalEncoder, Imputer, and ForwardFillImputer.
- Document ffill_limit as a time delta (count x freq) in ForwardFillImputer.
- Update docstrings to reflect exact-name vs substring matching.
---
 .../data_preprocessing/categorical_encoder.py | 40 ++++++++++---------
 .../data_preprocessing/data_preprocessor.py   | 14 ++++---
 .../data_preprocessing/ffill_imputer.py       | 24 ++++++-----
 .../data_preprocessing/imputer.py             | 23 ++++++-----
 .../data_preprocessing/scaler.py              | 13 ++++--
 .../test_data_preprocessor.py                 | 33 +++++++--------
 6 files changed, 85 insertions(+), 62 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index 563a6c2b..43923f6d 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -1,4 +1,4 @@
-from typing import Optional, List, Union, Callable
+from typing import List
 import logging
 import numpy as np
 import pandas as pd
@@ -8,6 +8,7 @@
 
 logger = logging.getLogger('energy_fault_detector')
 
+
 class CategoricalEncoder(DataTransformer):
     """
     CategoricalEncoder is a transformer for encoding categorical features in a dataset.
@@ -19,7 +20,7 @@ class CategoricalEncoder(DataTransformer):
     It assumes the input data will be in the form of a DataFrame.
 
     Attributes:
-        categorical_features (list): A list of strings representing the categorical feature names to be one-hot encoded.
+        categorical_features (list): A list of strings representing the categorical feature names to be one-hot encoded (matched by exact name).
                                      If not provided, the encoder will not encode any categorical features.
         feature_names_out_ (list): The list of output feature names after transformation.
         numerical_columns (list): The list of numerical feature names identified from the input data.
@@ -110,16 +111,18 @@ def __init__(self, categorical_features: list = None):
 
     def fit(self, x: pd.DataFrame, y=None) -> "CategoricalEncoder":
         self.feature_names_in_ = x.columns.tolist()
-        self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
+        self.categorical_columns = [col for col in self.feature_names_in_
+                                    if col in self.categorical_features]
         self.numerical_columns = [col for col in self.feature_names_in_ if col not in self.categorical_columns]
 
         # Clean numerical columns from non_declared categorical features
         numerical_data = x.loc[:, self.numerical_columns]
         self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist()
-        self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
+        self.numerical_columns = [col for col in self.numerical_columns
+                                  if col not in self.non_declared_categorical_features]
 
         if self.non_declared_categorical_features:
-            logger.info(
+            logger.warning(
                 "Non-declared categorical features found in data: %s. "
                 "They will be dropped. Consider adding them to the categorical_features list if they should be treated as categorical.",
                 self.non_declared_categorical_features,
@@ -128,22 +131,21 @@ def fit(self, x: pd.DataFrame, y=None) -> "CategoricalEncoder":
         self.n_features_in_ = len(self.feature_names_in_)
         categorical_data = x[self.categorical_columns]
 
-        # # Do the one-hot-encode on categorical data
-        # if categorical_data.isna().any().any():
-        #     raise ValueError("Missing values detected in categorical features. Please handle missing values before fitting the encoder.")
-
         self.one_hot_encoder.fit(categorical_data)
         return self
     
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         check_is_fitted(self, "n_features_in_")
 
+        if not isinstance(x, pd.DataFrame):
+            raise TypeError(f"x must be a pandas DataFrame, got {type(x).__name__}.")
+
         # Separate data into numerical and categorical features. Only categorical features are transformed
-        try:
-            numerical_data = x[self.numerical_columns]
-            categorical_data = x[self.categorical_columns]
-        except KeyError:
-            raise KeyError(f"The features {self.feature_names_in_} are not present in the input data.")
+        missing = [c for c in self.numerical_columns + self.categorical_columns if c not in x.columns]
+        if missing:
+            raise KeyError(f"The following features are not present in the input data: {missing}")
+        numerical_data = x[self.numerical_columns]
+        categorical_data = x[self.categorical_columns]
 
         if not categorical_data.empty:
             x_categorical_ = self.one_hot_encoder.transform(categorical_data)
@@ -159,7 +161,7 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
         check_is_fitted(self, "n_features_in_")
 
         # Get the one-hot encoded column names from the encoder
-        categorical_encoded_columns = list(self.one_hot_encoder.get_feature_names_out(self.categorical_features))
+        categorical_encoded_columns = list(self.one_hot_encoder.get_feature_names_out(self.categorical_columns))
 
         # Separate numerical columns from one-hot-encoded categorical columns
         numerical_columns = [col for col in x.columns if col not in categorical_encoded_columns]
@@ -177,9 +179,11 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
             categorical_original = pd.DataFrame(x_categorical_, columns=self.categorical_columns, index=x.index)
             return pd.concat([numerical_data, categorical_original], axis=1)
         else:
-            return numerical_data # Returns input df if no categorical features are specified in config file
+            return numerical_data  # Returns input df if no categorical features are specified in config file
     
     def get_feature_names_out(self, input_features=None) -> List[str]:
         check_is_fitted(self, "n_features_in_")
-        self.feature_names_out_ = self.numerical_columns + list(self.one_hot_encoder.get_feature_names_out(self.categorical_columns))
-        return self.feature_names_out_
\ No newline at end of file
+        self.feature_names_out_ = self.numerical_columns + list(
+            self.one_hot_encoder.get_feature_names_out(self.categorical_columns)
+        )
+        return self.feature_names_out_
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index 9d492278..38daf64a 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -1,5 +1,6 @@
 """Generic class for building a preprocessing pipeline."""
 
+import copy
 from collections import Counter, defaultdict
 from typing import List, Optional, Dict, Any, Tuple
 
@@ -199,7 +200,8 @@ def fit(self, X: pd.DataFrame, y=None, **fit_params):
         if protected_features:
             output_features = self.get_feature_names_out()
             available_protected = [f for f in protected_features if f in X.columns]
-            missing_features = [col for col in output_features if not any(available_feature in col for available_feature in available_protected)]
+            missing_features = [f for f in available_protected
+                                if not any(f in col for col in output_features)]
             if missing_features:
                 raise ValueError(
                     f"Protected features were dropped by the preprocessing pipeline: {missing_features}."
@@ -243,7 +245,6 @@ def _validate_singletons(steps_spec: List[Dict[str, Any]]) -> None:
             "simple_imputer",
             "ffill_imputer",
             "timestamp_transformer",
-            # scaler handled separately (standard_scaler/minmax_scaler) in your code
         }
         counts: List[Tuple[str, int]] = []
         for name in singleton_names:
@@ -294,10 +295,13 @@ def _build_from_steps_spec(self) -> List:
             ValueError: If a step lacks 'name' or references an unknown step.
         """
 
-        self._validate_step_spec_keys(self.steps_spec_)
+        # Work on a copy so the caller's config dict is not mutated by name normalization / step name assignment.
+        steps_spec = copy.deepcopy(self.steps_spec_)
+
+        self._validate_step_spec_keys(steps_spec)
 
         # Filter disabled steps first to simplify ordering.
-        enabled_spec = [s for s in self.steps_spec_ if s.get("enabled", True)]
+        enabled_spec = [s for s in steps_spec if s.get("enabled", True)]
         # Order the steps
         ordered_spec = self._order_steps_spec(enabled_spec)
         self._validate_singletons(enabled_spec)
@@ -392,7 +396,7 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str,
         ordered.extend(others)
         # Imputation
         ordered.extend(imputer)
-        # Encoding categorical features before scaling and after imputation (encoder would crash if NaN values are present)
+        # Encoding categorical features before scaling and after imputation
         ordered.extend(encoders)
         # Scaling
         ordered.extend(scalers)
diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py
index 2d9bd854..d1740322 100644
--- a/energy_fault_detector/data_preprocessing/ffill_imputer.py
+++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py
@@ -23,7 +23,7 @@ class ForwardFillImputer(DataTransformer):
     Attributes:
         feature_names_in_ (List[str]): List of input feature names observed during fitting.
         feature_names_out_ (List[str]): List of output feature names (same as input features unless some were dropped).
-        categorical_features (List[str]): List of user-specified categorical feature names or prefixes.
+        categorical_features (List[str]): List of user-specified categorical feature names (matched by exact name).
         numerical_columns (List[str]): List of identified numerical column names after filtering out non-declared categoricals.
         categorical_columns (List[str]): List of identified categorical column names (from `categorical_features`).
         non_declared_categorical_features (List[str]): List of columns detected as object-type (categorical) but not declared as such.
@@ -34,11 +34,14 @@ def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_featur
 
         Args:
             freq (str): Target resampling frequency for the time series (e.g., "1Min", "5Min", "H"). Passed to `pandas.DataFrame.asfreq()`.
-                Default is "1Min".
-            ffill_limit (int): Maximum number of consecutive NaN values to forward-fill after resampling.
+                The data is regridded onto this frequency before forward filling, which means missing timestamps are
+                introduced as NaN rows. Default is "1Min".
+            ffill_limit (int): Maximum number of consecutive NaN values to forward-fill after resampling. Because the data
+                is resampled to `freq` first, this count acts as a time delta: e.g. `freq="1Min"` with `ffill_limit=30`
+                fills gaps of up to 30 minutes, while `freq="5min"` with `ffill_limit=6` also fills up to 30 minutes.
                 NaNs beyond this limit remain unchanged and the corresponding rows will be dropped later. Default is 15.
-            categorical_features (Optional[List[str]]): List of column names or prefixes to treat as categorical features.
-                Columns matching any of these (as substring) are treated as categorical; others as numerical.
+            categorical_features (Optional[List[str]]): List of column names to treat as categorical features.
+                Columns matching any of these (by exact name) are treated as categorical; others as numerical.
                 Non-declared object-type columns in numerical slots are automatically detected and excluded from processing.
                 Default is None (interpreted as empty list).
         """
@@ -79,8 +82,10 @@ def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImp
         self.feature_names_in_ = x.columns.tolist()
         self.n_features_in_ = len(self.feature_names_in_)
 
-        self.numerical_columns = [col for col in self.feature_names_in_ if not any(feature in col for feature in self.categorical_features)] # Uses nested for loop in case categorical cols have been encoded already and contain the original categorical feature name as a substring.
-        self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)] # Uses nested for loop in case categorical cols have been encoded already and contain the original categorical feature name as a substring.
+        self.numerical_columns = [col for col in self.feature_names_in_
+                                  if col not in self.categorical_features]
+        self.categorical_columns = [col for col in self.feature_names_in_
+                                    if col in self.categorical_features]
         # Clean numerical columns from non_declared categorical features
         numerical_data = x.loc[:, self.numerical_columns]
         self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist() 
@@ -88,8 +93,9 @@ def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImp
         self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
 
         if self.non_declared_categorical_features:
-            logger.info(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
-                        f"They will be dropped. Consider adding them to the categorical_features list if they should be treated as categorical.")
+            logger.warning(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
+                        f"They will be dropped. Consider adding them to the categorical_features list if they should be"
+                        f" treated as categorical.")
 
         return self
 
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index 946364a9..bce38d5e 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -17,7 +17,7 @@ class Imputer(DataTransformer):
 
     Attributes:
         strategy (str): Imputation strategy for numerical features ('mean' or 'median').
-        categorical_features (List[str]): List of column names or prefixes to treat as categorical.
+        categorical_features (List[str]): List of column names to treat as categorical (matched by exact name).
         params (dict): Additional keyword arguments passed to `SimpleImputer` for numerical imputation.
         numerical_imputer (SimpleImputer): Imputer instance for numerical features.
         categorical_imputer (SimpleImputer): Imputer instance for categorical features (always 'most_frequent').
@@ -36,8 +36,8 @@ def __init__(self, strategy: str = 'mean', categorical_features: list = None, **
         Args:
             strategy (str, optional): Imputation strategy for numerical features. Supports 'mean' (default) and 'median'.
                 Categorical features are always imputed using 'most_frequent'.
-            categorical_features (List[str], optional): List of column names or prefixes to treat as categorical.
-                Columns matching any of these (as substring) are treated as categorical; others as numerical.
+            categorical_features (List[str], optional): List of column names to treat as categorical.
+                Columns matching any of these (by exact name) are treated as categorical; others as numerical.
                 Non-declared object-type columns in numerical slots are automatically detected and excluded.
                 Defaults to None (interpreted as empty list).
             **params: Additional keyword arguments passed to `SimpleImputer` for numerical imputation.
@@ -93,8 +93,10 @@ def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
         self.input_index_ = x.index
 
         # Split data into numerical (and boolean) and categorical features
-        self.numerical_columns = [col for col in self.feature_names_in_ if not any(feature in col for feature in self.categorical_features)]
-        self.categorical_columns = [col for col in self.feature_names_in_ if any(feature in col for feature in self.categorical_features)]
+        self.numerical_columns = [col for col in self.feature_names_in_
+                                  if col not in self.categorical_features]
+        self.categorical_columns = [col for col in self.feature_names_in_
+                                    if col in self.categorical_features]
         numerical_data = x.loc[:, self.numerical_columns]
         categorical_data = x.loc[:, self.categorical_columns]
 
@@ -103,8 +105,9 @@ def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
         self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
         numerical_data = numerical_data.loc[:, self.numerical_columns]
         if self.non_declared_categorical_features:
-            logger.info(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
-                        f"They will be dropped. Consider adding them to the categorical_features list if they should be treated as categorical.")
+            logger.warning(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
+                        f"They will be dropped. Consider adding them to the categorical_features list if they should be"
+                        f" treated as categorical.")
 
         logger.debug(f"Numerical columns: {self.numerical_columns}")
         logger.debug(f"Categorical columns: {self.categorical_columns}")
@@ -179,21 +182,21 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
     def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
         """Returns input DataFrame with columns reordered to match original feature order.
 
-        This method preserves index from `fit()` but does **not** reverse imputation (since original
+        This method preserves the index of the input `x` and does **not** reverse imputation (since original
         NaN positions are not stored). Included for scikit-learn compatibility.
 
         Args:
             x (pd.DataFrame): Transformed DataFrame (output of `transform()`).
 
         Returns:
-            pd.DataFrame: DataFrame with columns reordered to match `feature_names_in_` and index restored to `input_index_`.
+            pd.DataFrame: DataFrame with columns reordered to match `feature_names_in_`. The index of `x` is preserved.
 
         Raises:
             ValueError: If imputer has not been fitted (via `check_is_fitted`).
         """
         check_is_fitted(self, "n_features_in_")
         
-        return pd.DataFrame(x, columns=self.feature_names_in_, index=self.input_index_)
+        return pd.DataFrame(x, columns=self.feature_names_in_)
     
     def get_feature_names_out(self, input_features=None) -> List[str]:
         """Returns ordered list of output feature names.
diff --git a/energy_fault_detector/data_preprocessing/scaler.py b/energy_fault_detector/data_preprocessing/scaler.py
index ddb3a2a4..f23dbff0 100644
--- a/energy_fault_detector/data_preprocessing/scaler.py
+++ b/energy_fault_detector/data_preprocessing/scaler.py
@@ -18,7 +18,7 @@ class Scaler(DataTransformer):
     Attributes:
         scaler_type (str): Type of scaler selected ('standard' or 'minmax').
         scale_categorical_features (bool): Whether to scale features matched in `categorical_features`.
-        categorical_features (List[str]): List of column names or prefixes treated as categorical.
+        categorical_features (List[str]): List of column names treated as categorical (matched by substring, to cover one-hot encoded columns).
         scaler (SklearnScaler): Instantiated sklearn scaler object (e.g., StandardScaler).
         feature_names_in_ (List[str]): Names of features seen during `fit()`.
         feature_names_out_ (List[str]): Names of features output after `transform()` (same as input).
@@ -40,7 +40,8 @@ def __init__(self, scaler_type: str = 'standard', scale_categorical_features: bo
             scale_categorical_features (bool, optional): Whether to include categorical (e.g., one-hot encoded)
                 features in scaling. If `False`, only non-encoded features are scaled.
                 Defaults to `True`.
-            categorical_features (List[str], optional): List of column names or prefixes to treat as categorical.
+            categorical_features (List[str], optional): List of column names to treat as categorical (matched by substring,
+                to cover one-hot encoded columns such as `category_col_A`).
                 Used only when `scale_categorical_features=False` to determine which features to exclude.
                 Defaults to `None` (interpreted as empty list).
             **params: Additional keyword arguments passed to the underlying sklearn scaler constructor.
@@ -58,7 +59,7 @@ def __init__(self, scaler_type: str = 'standard', scale_categorical_features: bo
         self.scaler_type = scaler_type
         self.scale_categorical_features = scale_categorical_features
         self.categorical_features = categorical_features if categorical_features else []
-        self.scaler = self.SCALER_REGISTRY.get(scaler_type)
+        scaler = self.SCALER_REGISTRY.get(scaler_type)
 
         # Attributes to be defined during fitting
         self.feature_names_in_ = None
@@ -68,7 +69,7 @@ def __init__(self, scaler_type: str = 'standard', scale_categorical_features: bo
         # Parameters of nested estimators
         self.params = params
         # Initialize nested estimators
-        self.scaler = self.scaler(**self.params)
+        self.scaler = scaler(**self.params)
 
     def fit(self, x: pd.DataFrame, y: pd.Series = None) -> 'Scaler':
         """Fits the scaler on the input data, optionally excluding categorical features.
@@ -176,6 +177,10 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
 
         x_inverse_transformed = x.copy()
 
+        # Check if input features match the features seen during fit
+        if list(x.columns) != self.feature_names_in_:
+            raise ValueError(f"Input features {list(x.columns)} do not match the features seen during fit {self.feature_names_in_}.")
+
         # Apply inverse transform to the appropriate features
         if self.scale_categorical_features:
             x_inverse_transformed.iloc[:, :] = self.scaler.inverse_transform(x_inverse_transformed)
diff --git a/tests/data_preprocessing/test_data_preprocessor.py b/tests/data_preprocessing/test_data_preprocessor.py
index ec4345b2..c2e8e39e 100644
--- a/tests/data_preprocessing/test_data_preprocessor.py
+++ b/tests/data_preprocessing/test_data_preprocessor.py
@@ -469,6 +469,7 @@ def test_protected_features_with_both_filters(self):
         self.assertIn('high_nan_feature', transformed.columns,
                       "High-NaN protected feature should be kept")
 
+
 class TestComprehensivePipelineFlow(TestCase):
     """Test comprehensive preprocessing pipeline"""
 
@@ -493,11 +494,11 @@ def setUp(self):
         }, index=self.time_index)
         
         # Add some NaN values to test imputation
-        self.test_data.loc[50:100, 'temp_sensor_1'] = np.nan
-        self.test_data.loc[200:250, 'pressure_sensor'] = np.nan
-        self.test_data.loc[150:170, 'category_col'] = np.nan
+        self.test_data.iloc[50:100, self.test_data.columns.get_loc('temp_sensor_1')] = np.nan
+        self.test_data.iloc[200:250, self.test_data.columns.get_loc('pressure_sensor')] = np.nan
+        self.test_data.iloc[150:170, self.test_data.columns.get_loc('category_col')] = np.nan
 
-    def _create_config_dict(self):
+    def _create_config_dict(self, protect_conditional_features: bool = True):
         """Create configuration matching updated advanced_config.yaml structure."""
         return {
             'train': {
@@ -521,20 +522,19 @@ def _create_config_dict(self):
                         'act': 'prelu',
                         'batch_size': 256,
                         'code_size': 9,
-                        'early_stopping': True,
                         'epochs': 10,
                         'last_act': 'linear',
                         'layers': [64, 32],
                         'learning_rate': 0.0001,
                         'loss_name': 'mean_squared_error',
-                        'min_delta': 0.0001,
                         'noise': 0.0,
-                        'patience': 5,
                         'conditional_features': ['operating_condition', 'category_col'],
                         'verbose': 0
                     }
                 },
-                'protect_conditional_features': True
+                'protect_conditional_features': protect_conditional_features,
+                'anomaly_score': {'name': 'rmse'},
+                'threshold_selector': {'name': 'quantile'},
             }
         }
 
@@ -610,9 +610,10 @@ def test_pipeline_inverse_transform(self):
         transformed_data = preprocessor.transform(self.test_data)
         inverse_data = preprocessor.inverse_transform(transformed_data)
         
-        # Verify inverse transform preserves row count
-        self.assertEqual(inverse_data.shape[0], self.test_data.shape[0],
-                        "Inverse transform should preserve row count")
+        # Verify inverse transform preserves row count of the transformed data
+        # (the ffill_imputer may drop rows that still contain NaNs after filling)
+        self.assertEqual(inverse_data.shape[0], transformed_data.shape[0],
+                        "Inverse transform should preserve the transformed data's row count")
 
     def test_pipeline_fault_detector_integration(self):
         """Test integration with FaultDetector."""
@@ -680,20 +681,19 @@ def test_declared_category_not_as_condition(self):
 
             # Fit the model
             result = fault_detector.fit(
-                sensor_data=test_data_no_cond,
+                sensor_data=self.test_data,
                 normal_index=normal_index,
                 save_models=False
             )
 
-        # Verify that autoencoder conditions are updated with available feature names
-        self.assertIn('equipment_type_Type_1', fault_detector.data_preprocessor.get_feature_names_out(),
+        self.assertIn('equipment_type_Type1', fault_detector.data_preprocessor.get_feature_names_out(),
                       "Data preprocessor should have 'equipment_type' as a feature")
         self.assertNotIn('equipment_type', fault_detector.autoencoder.conditional_features,
                             "Autoencoder should not have 'equipment_type' as a conditional feature since it's not declared as such.")
 
     def test_declared_condition_not_as_category(self):
         """Test pipeline handling of declared conditional features not being declared as categorical features."""
-        config_dict = self._create_config_dict()
+        config_dict = self._create_config_dict(False)
 
         # Remove 'category_col' from categorical features in the config to simulate it being declared as conditional but not categorical
         config_dict['train']['data_preprocessor']['steps'][2]['params']['categorical_features'] = ['equipment_type']  # Only 'equipment_type' is categorical
@@ -740,5 +740,6 @@ def test_protected_features_are_not_dropped_in_pipeline(self):
     
         output_features = fault_detector.data_preprocessor.get_feature_names_out()
         available_protected = [f for f in protected_features if f in self.test_data.columns]
-        missing_features = [col for col in output_features if not any(available_feature in col for available_feature in available_protected)]
+        missing_features = [f for f in available_protected
+                            if not any(f in col for col in output_features)]
         self.assertEqual(len(missing_features), 0, f"Protected features {available_protected} should not be dropped in the pipeline.")

From 818b01f3a8823e3fce36fbc9694d2a028402abba Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Wed, 2 Sep 2026 11:24:22 +0200
Subject: [PATCH 19/35] Rework ForwardFillImputer to time-based ffill and
 harden Imputer

ForwardFillImputer:
- Replace resampling-based ffill with time-based ffill: remove asfreq()
  resampling, preserve original (possibly irregular) timestamps.
- Replace freq + int ffill_limit with a single Timedelta ffill_limit
  (string or pd.Timedelta). Forward-fill all NaNs, then invalidate
  fills where elapsed time from the last valid observation exceeds the
  threshold.
- Drop PeriodIndex support (only DatetimeIndex/TimedeltaIndex now).
- Drop all-NaN columns in fit and log a warning, preventing dropna from
  wiping the entire dataset when a column is unusable.

Imputer:
- Guard numerical_imputer.fit/transform when no numerical columns are
  present (categorical-only data no longer crashes).
- Drop all-NaN columns in fit and log a warning, preventing the
  SimpleImputer silent-skip that caused a column-count mismatch on
  transform.

Tests:
- Update ffill_imputer tests for Timedelta-based ffill_limit.
- Update test_all_nan_column_handling to assert the column is dropped.
- Enable test_transform_with_only_categorical (was TODO/commented out).
- Update test_fit_with_all_nan_column to assert the column is dropped
  and transform succeeds.
---
 energy_fault_detector/advanced_config.yaml    |   3 +-
 .../data_preprocessing/ffill_imputer.py       | 112 ++++++++++++------
 .../data_preprocessing/imputer.py             |  28 ++++-
 .../test_data_preprocessor.py                 |   4 +-
 .../data_preprocessing/test_ffill_imputer.py  |  34 +++---
 tests/data_preprocessing/test_imputer.py      |  33 +++---
 6 files changed, 140 insertions(+), 74 deletions(-)

diff --git a/energy_fault_detector/advanced_config.yaml b/energy_fault_detector/advanced_config.yaml
index 3e1eeb87..68a5d5c0 100644
--- a/energy_fault_detector/advanced_config.yaml
+++ b/energy_fault_detector/advanced_config.yaml
@@ -26,8 +26,7 @@ train:
       # 4) Imputation
       - name: ffill_imputer
         params:
-          freq: '1Min' # Frequency used to determine limit to forward fill.
-          ffill_limit: 60 # Limit number of consecutive NaNs tu forward fill.
+          ffill_limit: '60min' # Maximum time gap to forward-fill (pandas Timedelta string).
           categorical_features: [] # include categorical features in the list
 
       # 5) Encode categorical features
diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py
index d1740322..0b8b6c06 100644
--- a/energy_fault_detector/data_preprocessing/ffill_imputer.py
+++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py
@@ -1,6 +1,7 @@
-from typing import List, Optional
+from typing import List, Optional, Union
 
 import logging
+import numpy as np
 import pandas as pd
 from sklearn.utils.validation import check_is_fitted
 
@@ -8,50 +9,54 @@
 
 logger = logging.getLogger('energy_fault_detector')
 
+
 class ForwardFillImputer(DataTransformer):
 
-    """Impute missing values using forward fill with limit after resampling frequency.
+    """Impute missing values using time-based forward fill with a maximum gap duration.
 
-    Duplicate rows are dropped and any remaining rows with NaN values are removed.
+    Forward-fills NaN values from the last valid observation, but only when the elapsed
+    time between the last valid value and the current row does not exceed ``ffill_limit``.
+    Rows where the gap is too large remain NaN and are subsequently dropped.
 
-    The transformer assumes time-series data with a temporal index (DatetimeIndex, TimedeltaIndex, or PeriodIndex).
-    During `fit`, it categorizes features as numerical or categorical, identifies and logs non-declared categorical features,
-    and prepares internal metadata. During `transform`, it resamples the data to a uniform frequency, forward-fills missing
-    values up to a specified limit, drops duplicate rows, and removes any rows still containing NaN values. This transformer 
-    is useful for time-series datasets where missing values need to be imputed based on previous observations.
+    The transformer assumes time-series data with a temporal index (DatetimeIndex or
+    TimedeltaIndex). During ``fit``, it categorizes features as numerical or categorical,
+    identifies and logs non-declared categorical features, and converts ``ffill_limit``
+    to a :class:`~pandas.Timedelta`. During ``transform``, it forward-fills missing values
+    (invalidating fills that exceed the time threshold), drops duplicate rows, and removes
+    any rows still containing NaN values.
 
     Attributes:
         feature_names_in_ (List[str]): List of input feature names observed during fitting.
         feature_names_out_ (List[str]): List of output feature names (same as input features unless some were dropped).
+        ffill_limit_ (pd.Timedelta): Fitted ``ffill_limit`` converted to a Timedelta.
         categorical_features (List[str]): List of user-specified categorical feature names (matched by exact name).
         numerical_columns (List[str]): List of identified numerical column names after filtering out non-declared categoricals.
-        categorical_columns (List[str]): List of identified categorical column names (from `categorical_features`).
+        categorical_columns (List[str]): List of identified categorical column names (from ``categorical_features``).
         non_declared_categorical_features (List[str]): List of columns detected as object-type (categorical) but not declared as such.
     """
 
-    def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_features: Optional[List[str]] = None):
-        """Initializes the ForwardFillImputer with specified frequency, forward-fill limit, and optional categorical features.
+    def __init__(self, ffill_limit: Union[str, pd.Timedelta] = "15min",
+                 categorical_features: Optional[List[str]] = None):
+        """Initializes the ForwardFillImputer with a forward-fill time limit and optional categorical features.
 
         Args:
-            freq (str): Target resampling frequency for the time series (e.g., "1Min", "5Min", "H"). Passed to `pandas.DataFrame.asfreq()`.
-                The data is regridded onto this frequency before forward filling, which means missing timestamps are
-                introduced as NaN rows. Default is "1Min".
-            ffill_limit (int): Maximum number of consecutive NaN values to forward-fill after resampling. Because the data
-                is resampled to `freq` first, this count acts as a time delta: e.g. `freq="1Min"` with `ffill_limit=30`
-                fills gaps of up to 30 minutes, while `freq="5min"` with `ffill_limit=6` also fills up to 30 minutes.
-                NaNs beyond this limit remain unchanged and the corresponding rows will be dropped later. Default is 15.
+            ffill_limit (str | pd.Timedelta): Maximum time gap to forward-fill, expressed as a
+                pandas-compatible Timedelta string (e.g. ``"15min"``, ``"1h"``) or a
+                :class:`~pandas.Timedelta` instance.  NaNs whose elapsed time from the last valid
+                observation exceeds this limit are left unfilled and the corresponding rows are
+                dropped. Default is ``"15min"``.
             categorical_features (Optional[List[str]]): List of column names to treat as categorical features.
                 Columns matching any of these (by exact name) are treated as categorical; others as numerical.
                 Non-declared object-type columns in numerical slots are automatically detected and excluded from processing.
                 Default is None (interpreted as empty list).
         """
         super().__init__()
-        self.freq = freq
         self.ffill_limit = ffill_limit
 
         # Attributes set during fit.
         self.feature_names_in_: List[str] = []
         self.feature_names_out_: List[str] = []
+        self.ffill_limit_: pd.Timedelta = pd.Timedelta(0)
         self.categorical_features = categorical_features if categorical_features else []
         self.numerical_columns: List[str] = []
         self.categorical_columns: List[str] = []
@@ -60,11 +65,12 @@ def __init__(self, freq: str = "1Min", ffill_limit: int = 15, categorical_featur
     def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImputer":
         """Fits the imputer to the input data by identifying feature types and storing metadata.
 
-        Populates `numerical_columns`, `categorical_columns`, and `non_declared_categorical_features`
+        Populates ``numerical_columns``, ``categorical_columns``, and ``non_declared_categorical_features``
         based on the input DataFrame's structure and user-provided categorical feature hints.
+        Converts ``ffill_limit`` to a :class:`~pandas.Timedelta` and stores it as ``ffill_limit_``.
 
         Args:
-            x (pd.DataFrame): Input feature DataFrame. Must contain all features used during `transform`.
+            x (pd.DataFrame): Input feature DataFrame. Must contain all features used during ``transform``.
             y (Optional[pd.Series]): Target variable. Ignored; included for compatibility with the scikit-learn API.
 
         Returns:
@@ -72,16 +78,22 @@ def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImp
 
         Raises:
             TypeError: If `x` is not a pandas DataFrame.
-            ValueError: If internal consistency checks fail (e.g., invalid column references).
+            ValueError: If `ffill_limit` cannot be converted to a Timedelta.
         """
-        # TODO: at this point there is no handling of all-NaN columns, should be included in fit to drop them and log a warning.
-
         if not isinstance(x, pd.DataFrame):
             raise TypeError("x must be a pandas DataFrame.")
 
         self.feature_names_in_ = x.columns.tolist()
         self.n_features_in_ = len(self.feature_names_in_)
 
+        try:
+            self.ffill_limit_ = pd.Timedelta(self.ffill_limit)
+        except ValueError as e:
+            raise ValueError(
+                f"ffill_limit must be a pandas-compatible Timedelta string or Timedelta "
+                f"(e.g. '15min', '1h'), got: {self.ffill_limit!r}"
+            ) from e
+
         self.numerical_columns = [col for col in self.feature_names_in_
                                   if col not in self.categorical_features]
         self.categorical_columns = [col for col in self.feature_names_in_
@@ -97,25 +109,39 @@ def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImp
                         f"They will be dropped. Consider adding them to the categorical_features list if they should be"
                         f" treated as categorical.")
 
+        # Drop columns that are entirely NaN — they cannot be forward-filled and would otherwise
+        # cause every row to be dropped by dropna(how="any").
+        all_nan_cols = [col for col in self.numerical_columns + self.categorical_columns
+                        if x[col].isna().all()]
+        if all_nan_cols:
+            logger.warning(f"Columns containing only NaN values found: {all_nan_cols}. "
+                        f"They will be dropped as they cannot be forward-filled.")
+            self.numerical_columns = [col for col in self.numerical_columns if col not in all_nan_cols]
+            self.categorical_columns = [col for col in self.categorical_columns if col not in all_nan_cols]
+
         return self
 
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
-        """Applies forward-fill imputation and cleaning pipeline to input data.
+        """Applies time-based forward-fill imputation and cleaning pipeline to input data.
 
         The steps executed are:
-        1. Resample the data to `self.freq` using `asfreq()`.
-        2. Forward-fill missing values up to `self.ffill_limit`.
+        1. Forward-fill all NaN values.
+        2. Invalidate fills where the elapsed time from the last valid observation exceeds
+           ``ffill_limit_`` (set those values back to NaN).
         3. Drop duplicate rows.
-        4. Drop any rows still containing NaN values (`dropna(how="any")`).
+        4. Drop any rows still containing NaN values (``dropna(how="any")``).
+
+        Unlike the previous resampling-based approach, the original (possibly irregular)
+        timestamps are preserved — no synthetic rows are introduced.
 
         Args:
-            x (pd.DataFrame): Input feature DataFrame, with the same column names as used in `fit()`.
+            x (pd.DataFrame): Input feature DataFrame, with the same column names as used in ``fit()``.
 
         Returns:
-            pd.DataFrame: Cleaned and imputed DataFrame, containing only the columns from `get_feature_names_out()`.
+            pd.DataFrame: Cleaned and imputed DataFrame, containing only the columns from ``get_feature_names_out()``.
 
         Raises:
-            TypeError: If `x` is not a pandas DataFrame, or if its index is not temporal (DatetimeIndex/TimedeltaIndex/PeriodIndex).
+            TypeError: If `x` is not a pandas DataFrame, or if its index is not temporal (DatetimeIndex/TimedeltaIndex).
             ValueError: If `x` is missing columns seen during `fit()`, or if numerical columns cannot be converted to float.
             ValueError: If the imputer has not been fitted (via `check_is_fitted`).
         """
@@ -134,9 +160,9 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         if missing_columns:
             raise ValueError(f"Input is missing columns seen during fit: {missing_columns}")
 
-        if not isinstance(x.index, (pd.DatetimeIndex, pd.TimedeltaIndex, pd.PeriodIndex)):
+        if not isinstance(x.index, (pd.DatetimeIndex, pd.TimedeltaIndex)):
             raise TypeError(
-                "x index must be a DatetimeIndex, TimedeltaIndex, or PeriodIndex to use asfreq()."
+                "x index must be a DatetimeIndex or TimedeltaIndex for time-based forward fill."
             )
         # Harmonize data types in numerical columns to avoid issues during concatenation after one-hot encoding
         for col in self.numerical_columns:
@@ -146,8 +172,24 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
                 except ValueError as e:
                     raise ValueError(f"Column '{col}' cannot be converted to float.") from e
         x_selected = x[self.numerical_columns + self.categorical_columns]
-        df_resampled = x_selected.asfreq(self.freq)
-        df_filled = df_resampled.ffill(limit=self.ffill_limit)
+
+        # --- Time-based forward fill ---
+        idx_series = x.index.to_series()
+        df_filled = x_selected.copy()
+        for col in df_filled.columns:
+            col_series = df_filled[col]
+            nan_mask = col_series.isna()
+            if not nan_mask.any():
+                continue
+            filled = col_series.ffill()
+            # Time elapsed since the last valid observation for each row
+            last_valid = idx_series.where(col_series.notna()).ffill()
+            elapsed = idx_series - last_valid
+            # Invalidate fills where the gap exceeds the threshold
+            invalid = nan_mask & (elapsed > self.ffill_limit_)
+            filled[invalid] = np.nan
+            df_filled[col] = filled
+
         df_cleaned = df_filled.drop_duplicates(keep="first")
         df_final = df_cleaned.dropna(how="any")
 
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index bce38d5e..3b9905f1 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -109,11 +109,24 @@ def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
                         f"They will be dropped. Consider adding them to the categorical_features list if they should be"
                         f" treated as categorical.")
 
+        # Drop columns that are entirely NaN — they cannot be imputed and would cause
+        # SimpleImputer to silently skip them, leading to a column-count mismatch on transform.
+        all_nan_cols = [col for col in self.numerical_columns + self.categorical_columns
+                        if x[col].isna().all()]
+        if all_nan_cols:
+            logger.warning(f"Columns containing only NaN values found: {all_nan_cols}. "
+                        f"They will be dropped as they cannot be imputed.")
+            self.numerical_columns = [col for col in self.numerical_columns if col not in all_nan_cols]
+            self.categorical_columns = [col for col in self.categorical_columns if col not in all_nan_cols]
+            numerical_data = numerical_data.loc[:, self.numerical_columns]
+            categorical_data = categorical_data.loc[:, self.categorical_columns]
+
         logger.debug(f"Numerical columns: {self.numerical_columns}")
         logger.debug(f"Categorical columns: {self.categorical_columns}")
 
         # Fit the imputers
-        self.numerical_imputer.fit(numerical_data)
+        if not numerical_data.empty:
+            self.numerical_imputer.fit(numerical_data)
         if not categorical_data.empty:
             self.categorical_imputer.fit(categorical_data)
 
@@ -158,11 +171,14 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         categorical_data = x.loc[:, self.categorical_columns]
 
         # Transform the data
-        numerical_transformed = pd.DataFrame(
-            self.numerical_imputer.transform(numerical_data),
-            columns=self.numerical_columns,
-            index=x.index
-        )
+        if not numerical_data.empty:
+            numerical_transformed = pd.DataFrame(
+                self.numerical_imputer.transform(numerical_data),
+                columns=self.numerical_columns,
+                index=x.index
+            )
+        else:
+            numerical_transformed = pd.DataFrame(index=x.index)
         if not categorical_data.empty:
             categorical_transformed = pd.DataFrame(
                 self.categorical_imputer.transform(categorical_data),
diff --git a/tests/data_preprocessing/test_data_preprocessor.py b/tests/data_preprocessing/test_data_preprocessor.py
index c2e8e39e..cc6cc8eb 100644
--- a/tests/data_preprocessing/test_data_preprocessor.py
+++ b/tests/data_preprocessing/test_data_preprocessor.py
@@ -51,7 +51,7 @@ def setUp(self) -> None:
             steps=[
                 {'name': 'column_selector', 'params': {'max_nan_frac_per_col': 0.8}},
                 {'name': 'ffill_imputer',
-                 'params': {'freq': '1Min', 'ffill_limit': 1, 'categorical_features': ['category', 'region']}},
+                 'params': {'ffill_limit': '1min', 'categorical_features': ['category', 'region']}},
                 {'name': 'categorical_encoder',
                  'params': {'categorical_features': ['category', 'region']}},
             ]
@@ -510,7 +510,7 @@ def _create_config_dict(self, protect_conditional_features: bool = True):
                     'steps': [
                         {'name': 'column_selector', 'params': {'max_nan_frac_per_col': 0.9}},
                         {'name': 'low_unique_value_filter', 'params': {'min_unique_value_count': 2}},
-                        {'name': 'ffill_imputer', 'params': {'freq': '1Min', 'ffill_limit': 60, 'categorical_features': ['category_col', 'equipment_type']}},
+                        {'name': 'ffill_imputer', 'params': {'ffill_limit': '60min', 'categorical_features': ['category_col', 'equipment_type']}},
                         {'name': 'categorical_encoder', 'params': {'categorical_features': ['category_col', 'equipment_type']}},
                         {'name': 'scaler', 'params': {'scaler_type': 'standard', 'with_mean': True, 'with_std': True, 'scale_categorical_features': False, 'categorical_features': ['category_col', 'equipment_type']}},
                         {'name': 'timestamp_transformer', 'params': {'features': ['minute_of_hour', 'hour_of_day', 'day_of_week']}}
diff --git a/tests/data_preprocessing/test_ffill_imputer.py b/tests/data_preprocessing/test_ffill_imputer.py
index b295b36d..1aa76701 100644
--- a/tests/data_preprocessing/test_ffill_imputer.py
+++ b/tests/data_preprocessing/test_ffill_imputer.py
@@ -46,15 +46,13 @@ def tearDown(self):
     def test_init_defaults(self):
         """Test default initialization."""
         imputer = ForwardFillImputer()
-        self.assertEqual(imputer.freq, "1Min")
-        self.assertEqual(imputer.ffill_limit, 15)
+        self.assertEqual(imputer.ffill_limit, "15min")
         self.assertEqual(imputer.categorical_features, [])
 
     def test_init_custom_params(self):
         """Test initialization with custom parameters."""
-        imputer = ForwardFillImputer(freq="5Min", ffill_limit=10, categorical_features=["status", "region"])
-        self.assertEqual(imputer.freq, "5Min")
-        self.assertEqual(imputer.ffill_limit, 10)
+        imputer = ForwardFillImputer(ffill_limit="10min", categorical_features=["status", "region"])
+        self.assertEqual(imputer.ffill_limit, "10min")
         self.assertEqual(imputer.categorical_features, ["status", "region"])
 
     def test_fit(self):
@@ -101,7 +99,7 @@ def test_fit_non_declared_categorical_in_numerical_columns(self):
 
     def test_transform_basic(self):
         """Test basic forward fill transformation."""
-        imputer = ForwardFillImputer(ffill_limit=10)
+        imputer = ForwardFillImputer(ffill_limit="10min")
         imputer.fit(self.data_numeric)
         result = imputer.transform(self.data_numeric)
         
@@ -114,7 +112,7 @@ def test_transform_basic(self):
 
     def test_transform_with_categorical(self):
         """Test transformation with categorical features."""
-        imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit=10)
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit="10min")
         imputer.fit(self.data_full)
         result = imputer.transform(self.data_full)
         
@@ -135,11 +133,11 @@ def test_transform_limit_exceeded(self):
             'value2': list(range(10, 27))
         }, index=pd.date_range('2024-01-01', periods=17, freq='1Min'))
         
-        imputer = ForwardFillImputer(ffill_limit=5)
+        imputer = ForwardFillImputer(ffill_limit="5min")
         imputer.fit(long_gaps_data)
         result = imputer.transform(long_gaps_data)
         print(result)
-        # The 7th value should still be NaN because 6 consecutive NaNs > ffill_limit=5
+        # The 7th row should be 2.0 because elapsed time (6min) exceeds the 5min limit
         self.assertTrue(result.iloc[6]['value1']==2.0)
 
     def test_transform_drops_duplicates(self):
@@ -164,7 +162,7 @@ def test_transform_drops_rows_with_any_na(self):
             'value2': [np.nan, 2.0, 3.0, 4.0, 5.0]
         }, index=pd.date_range('2024-01-01', periods=5, freq='1Min'))
         
-        imputer = ForwardFillImputer(ffill_limit=1)
+        imputer = ForwardFillImputer(ffill_limit="1min")
         imputer.fit(some_na_data)
         result = imputer.transform(some_na_data)
         
@@ -224,7 +222,7 @@ def test_transform_unfitted_raises(self):
 
     def test_inverse_transform(self):
         """Test inverse transform maintains column order."""
-        imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit=10)
+        imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit="10min")
         imputer.fit(self.data_full)
         transformed = imputer.transform(self.data_full)
         inverse = imputer.inverse_transform(transformed)
@@ -259,8 +257,7 @@ def test_empty_dataframe_handling(self):
         self.assertEqual(result.shape[0], 0)
 
     def test_all_nan_column_handling(self):
-        """Test handling of columns that are all NaN."""
-        # TODO: at this point there is no handling of all-NaN columns, should be included in fit to drop them and log a warning.
+        """Test that all-NaN columns are dropped during fit."""
         all_nan_data = pd.DataFrame({
             'value1': [1.0, 2.0, 3.0],
             'value2': [np.nan, np.nan, np.nan]
@@ -268,9 +265,16 @@ def test_all_nan_column_handling(self):
         
         imputer = ForwardFillImputer()
         imputer.fit(all_nan_data)
+
+        # All-NaN column should be dropped from numerical_columns
+        self.assertNotIn('value2', imputer.numerical_columns)
+        self.assertIn('value1', imputer.numerical_columns)
+
+        # Transform should keep rows from the valid column, not drop everything
         result = imputer.transform(all_nan_data)
-        
-        self.assertEqual(result.shape[0], 0)
+        self.assertEqual(result.shape[1], 1)
+        self.assertListEqual(list(result.columns), ['value1'])
+        self.assertEqual(result.shape[0], 3)
 
     def test_numerical_conversion_error(self):
         """Test error handling for non-convertible numerical columns."""
diff --git a/tests/data_preprocessing/test_imputer.py b/tests/data_preprocessing/test_imputer.py
index c2e57df3..51a980fd 100644
--- a/tests/data_preprocessing/test_imputer.py
+++ b/tests/data_preprocessing/test_imputer.py
@@ -171,11 +171,17 @@ def test_fit_with_all_nan_column(self):
         """Test fitting with a column that has all NaN values."""
         imputer = Imputer()
         imputer.fit(self.data_all_nan)
-        
-        # All-NaN column should be handled without errors
+
+        # All-NaN column should be dropped from numerical_columns
         self.assertEqual(imputer.n_features_in_, 3)
-        # The numerical imputer will compute a default value (0.0 for mean)
-        self.assertIsNotNone(imputer.numerical_imputer.statistics_)
+        self.assertNotIn('broken_sensor', imputer.numerical_columns)
+        self.assertListEqual(imputer.numerical_columns, ['temperature', 'humidity'])
+
+        # Transform should work and return only the valid columns
+        result = imputer.transform(self.data_all_nan)
+        self.assertEqual(result.shape[1], 2)
+        self.assertListEqual(list(result.columns), ['temperature', 'humidity'])
+        self.assertFalse(result.isna().any().any())
 
     def test_fit_empty_categorical(self):
         """Test fitting when categorical columns DataFrame is empty."""
@@ -309,16 +315,15 @@ def test_transform_with_empty_categorical_data(self):
         # Should handle empty categorical data gracefully
         self.assertEqual(result.shape, self.data_numerical.shape)
 
-    # TODO: Imputer doesnt work if no numerical data in the input.
-    # def test_transform_with_only_categorical(self):
-    #     """Test transformation with only categorical features."""
-    #     imputer = Imputer(categorical_features=['status', 'region'])
-    #     imputer.fit(self.data_only_categorical)
-    #     result = imputer.transform(self.data_only_categorical)
-        
-    #     # Should handle categorical-only data
-    #     self.assertEqual(result.shape, self.data_only_categorical.shape)
-    #     self.assertFalse(result.isna().any().any())
+    def test_transform_with_only_categorical(self):
+        """Test transformation with only categorical features."""
+        imputer = Imputer(categorical_features=['status', 'region'])
+        imputer.fit(self.data_only_categorical)
+        result = imputer.transform(self.data_only_categorical)
+
+        # Should handle categorical-only data
+        self.assertEqual(result.shape, self.data_only_categorical.shape)
+        self.assertFalse(result.isna().any().any())
 
 if __name__ == '__main__':
     unittest.main()
\ No newline at end of file

From 102f9c6c30020c626e16cea8fa346e8d35d8350a Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Wed, 2 Sep 2026 13:19:08 +0200
Subject: [PATCH 20/35] Fix Sphinx warnings and improve API docs readability

Fix RST formatting issues in docstrings (numbered lists, bullet alignment,
indentation) that caused Sphinx build warnings. Re-enable run_apidoc in
conf.py with templatedir, and strip 'energy_fault_detector.' prefix from
auto-generated API page titles for a cleaner sidebar. Exclude sklearn
boilerplate methods (set_fit_request, set_output, __sklearn_is_fitted__,
etc.) from autodoc via exclude-members and a high-priority skip-member
handler. Document generate_quickstart_config by un-excluding config/
__init__.py and using :no-index: to prevent duplicate Config object
warnings.
---
 docs/_templates/module.rst_t                  |   2 +-
 docs/_templates/package.rst_t                 |   9 +-
 docs/_templates/toc.rst_t                     |   2 +-
 docs/conf.py                                  | 168 ++++++++++--------
 .../data_preprocessing/categorical_encoder.py | 141 +++++++--------
 .../data_preprocessing/data_preprocessor.py   |   6 +-
 .../data_preprocessing/ffill_imputer.py       |   1 +
 .../timestamp_transformer.py                  |   4 +-
 energy_fault_detector/fault_detector.py       |  20 ++-
 energy_fault_detector/utils/visualisation.py  |   2 +-
 10 files changed, 187 insertions(+), 168 deletions(-)

diff --git a/docs/_templates/module.rst_t b/docs/_templates/module.rst_t
index 53cfbac8..0a5b6b27 100644
--- a/docs/_templates/module.rst_t
+++ b/docs/_templates/module.rst_t
@@ -1,5 +1,5 @@
 {%- if show_headings %}
-{{- [basename] | join(' ') | e | heading }}
+{{- [basename | replace('energy_fault_detector.', '')] | join(' ') | e | heading }}
 
 {% endif -%}
 .. automodule:: {{ qualname }}
diff --git a/docs/_templates/package.rst_t b/docs/_templates/package.rst_t
index 7e5b87ea..ba7e0a1d 100644
--- a/docs/_templates/package.rst_t
+++ b/docs/_templates/package.rst_t
@@ -14,13 +14,14 @@
 {%- endmacro %}
 
 {%- if is_namespace %}
-{{- [pkgname, "namespace"] | join(" ") | e | heading }}
+{{- pkgname | replace('energy_fault_detector.', '') | e | heading }}
 {% else %}
-{{- pkgname | e | heading }}
+{{- pkgname | replace('energy_fault_detector.', '') | e | heading }}
 {% endif %}
 
 {%- if modulefirst and not is_namespace %}
-{{ automodule(pkgname, automodule_options) }}
+{{ automodule(pkgname, automodule_options) }}{% if pkgname == 'energy_fault_detector.config' %}
+   :no-index:{% endif %}
 {% endif %}
 
 {%- if subpackages %}
@@ -35,7 +36,7 @@
 {% else %}
 {%- for submodule in submodules %}
 {% if show_headings %}
-{{- submodule | e | heading(2) }}
+{{- submodule | replace('energy_fault_detector.', '') | e | heading(2) }}
 {% endif %}
 {{ automodule(submodule, automodule_options) }}
 {% endfor %}
diff --git a/docs/_templates/toc.rst_t b/docs/_templates/toc.rst_t
index f0877eeb..7835fff8 100644
--- a/docs/_templates/toc.rst_t
+++ b/docs/_templates/toc.rst_t
@@ -1,4 +1,4 @@
-{{ header | heading }}
+{{ header | replace('energy_fault_detector.', '') | heading }}
 
 .. toctree::
    :maxdepth: {{ maxdepth }}
diff --git a/docs/conf.py b/docs/conf.py
index 5ced0440..ed69f7f7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -53,6 +53,16 @@
 
 autoclass_content = 'both'
 
+# Exclude sklearn boilerplate methods (metadata routing, set_output) that are
+# not re-implemented in our classes.
+autodoc_default_options = {
+    'exclude-members': (
+        'set_fit_request, set_transform_request, set_inverse_transform_request, '
+        'set_predict_request, set_score_request, set_output, get_metadata_routing, '
+        '__sklearn_is_fitted__'
+    ),
+}
+
 # ======================================================================================
 # No configurable options below this line
 
@@ -209,78 +219,92 @@
 ]
 
 
-# def run_apidoc(app):
-#     """Generate API .rst files with sphinx-apidoc, like in CI:
-
-#        sphinx-apidoc -o docs energy_fault_detector/ --module-first --force --separate --templatedir docs/_templates
-#     """
-
-#     # Only run this step if we're building the HTML docs
-#     if app.builder.name != "html":
-#         return
-
-#     from sphinx.ext.apidoc import main as apidoc_main
-#     from pathlib import Path
-
-#     here = Path(__file__).parent
-#     pkg_dir = here.parent / IMPORT_NAME
-#     out_dir = here
-#     templates = here / "_templates"
-
-#     apidoc_main([
-#         "-o", str(out_dir),
-#         str(pkg_dir),
-#         "--module-first",
-#         "--force",
-#         "--separate",
-#         "--templatedir", str(templates),
-#         # Following files are skipped
-#         # Quick fault detection internals
-#         str(pkg_dir / "main.py"),
-#         str(pkg_dir / "quick_fault_detection" / "configuration.py"),
-#         str(pkg_dir / "quick_fault_detection" / "data_loading.py"),
-#         str(pkg_dir / "quick_fault_detection" / "optimization.py"),
-#         str(pkg_dir / "quick_fault_detection" / "output.py"),
-#         str(pkg_dir / "quick_fault_detection" / "pipeline.py"),
-#         str(pkg_dir / "quick_fault_detection" / "quick_fault_detector.py"),
-#         # core internals
-#         str(pkg_dir / "core" / "anomaly_score.py"),
-#         str(pkg_dir / "core" / "data_transformer.py"),
-#         str(pkg_dir / "core" / "threshold_selector.py"),
-#         str(pkg_dir / "core" / "fault_detection_result.py"),
-#         str(pkg_dir / "core" / "fault_detection_model.py"),
-#         str(pkg_dir / "core" / "model_factory.py"),
-#         str(pkg_dir / "core" / "save_load_mixin.py"),
-#         # internal utilities
-#         str(pkg_dir / "utils" / "index_utils.py"),
-#         # config internals
-#         str(pkg_dir / "config" / "base_config.py"),
-#         str(pkg_dir / "config" / "config.py"),
-#         str(pkg_dir / "config" / "quickstart_config.py"),
-#         # Class specific files (documented directly under the top module)
-#         # autoencoder
-#         str(pkg_dir / "autoencoders" / "multilayer_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "conditional_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "lstm_seq2one_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "cnn_seq2one_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "bidirectional_lstm_seq2one_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "cnn_seq_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "lstm_seq2seq_autoencoder.py"),
-#         # anomaly score
-#         str(pkg_dir / "anomaly_scores" / "rmse_score.py"),
-#         str(pkg_dir / "anomaly_scores" / "mahalanobis_score.py"),
-#         # threshold selector
-#         str(pkg_dir / "threshold_selectors" / "adaptive_threshold.py"),
-#         str(pkg_dir / "threshold_selectors" / "fdr_threshold.py"),
-#         str(pkg_dir / "threshold_selectors" / "fbeta_threshold.py"),
-#         str(pkg_dir / "threshold_selectors" / "quantile_threshold.py"),
-#         # main class (documented directly under the top module)
-#         str(pkg_dir / "fault_detector.py"),
-#     ])
-
-
-# def setup(app):
-#     app.connect("builder-inited", run_apidoc)
+def run_apidoc(app):
+    """Generate API .rst files with sphinx-apidoc, like in CI:
+
+       sphinx-apidoc -o docs energy_fault_detector/ --module-first --force --separate --templatedir docs/_templates
+    """
+
+    # Only run this step if we're building the HTML docs
+    if app.builder.name != "html":
+        return
+
+    from sphinx.ext.apidoc import main as apidoc_main
+    from pathlib import Path
+
+    here = Path(__file__).parent
+    pkg_dir = here.parent / IMPORT_NAME
+    out_dir = here
+    templates = here / "_templates"
+
+    apidoc_main([
+        "-o", str(out_dir),
+        str(pkg_dir),
+        "--module-first",
+        "--force",
+        "--separate",
+        "--templatedir", str(templates),
+        # Following files are skipped
+        # Quick fault detection internals
+        str(pkg_dir / "main.py"),
+        str(pkg_dir / "quick_fault_detection" / "configuration.py"),
+        str(pkg_dir / "quick_fault_detection" / "data_loading.py"),
+        str(pkg_dir / "quick_fault_detection" / "optimization.py"),
+        str(pkg_dir / "quick_fault_detection" / "output.py"),
+        str(pkg_dir / "quick_fault_detection" / "pipeline.py"),
+        str(pkg_dir / "quick_fault_detection" / "quick_fault_detector.py"),
+        # core internals
+        str(pkg_dir / "core" / "anomaly_score.py"),
+        str(pkg_dir / "core" / "data_transformer.py"),
+        str(pkg_dir / "core" / "threshold_selector.py"),
+        str(pkg_dir / "core" / "fault_detection_result.py"),
+        str(pkg_dir / "core" / "fault_detection_model.py"),
+        str(pkg_dir / "core" / "model_factory.py"),
+        str(pkg_dir / "core" / "save_load_mixin.py"),
+        # internal utilities
+        str(pkg_dir / "utils" / "index_utils.py"),
+        # config internals
+        str(pkg_dir / "config" / "base_config.py"),
+        str(pkg_dir / "config" / "config.py"),
+        str(pkg_dir / "config" / "quickstart_config.py"),
+        # Class specific files (documented directly under the top module)
+        # autoencoder
+        str(pkg_dir / "autoencoders" / "multilayer_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "conditional_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "lstm_seq2one_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "cnn_seq2one_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "bidirectional_lstm_seq2one_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "cnn_seq_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "lstm_seq2seq_autoencoder.py"),
+        # anomaly score
+        str(pkg_dir / "anomaly_scores" / "rmse_score.py"),
+        str(pkg_dir / "anomaly_scores" / "mahalanobis_score.py"),
+        # threshold selector
+        str(pkg_dir / "threshold_selectors" / "adaptive_threshold.py"),
+        str(pkg_dir / "threshold_selectors" / "fdr_threshold.py"),
+        str(pkg_dir / "threshold_selectors" / "fbeta_threshold.py"),
+        str(pkg_dir / "threshold_selectors" / "quantile_threshold.py"),
+        # main class (documented directly under the top module)
+        str(pkg_dir / "fault_detector.py"),
+    ])
+
+
+def _skip_sklearn_dunders(app, what, name, obj, skip, options):
+    """Skip sklearn boilerplate dunder methods that have docstrings.
+
+    ``napoleon_include_special_with_doc = True`` forces inclusion of special
+    methods with docstrings via ``emit_first_result``, overriding
+    ``exclude-members``. This handler runs with higher priority (lower number)
+    so it is called before Napoleon's handler and takes precedence.
+    """
+    if name == '__sklearn_is_fitted__':
+        return True
+    return None
+
+
+def setup(app):
+    app.connect("builder-inited", run_apidoc)
+    app.connect("autodoc-skip-member", _skip_sklearn_dunders, priority=100)
 
 
 def linkcode_resolve(domain, info):
diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index 43923f6d..842d5d24 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -10,14 +10,11 @@
 
 
 class CategoricalEncoder(DataTransformer):
-    """
-    CategoricalEncoder is a transformer for encoding categorical features in a dataset.
+    """Transformer for encoding categorical features using one-hot encoding.
 
-    This class is designed to handle preprocessing of datasets by encoding specified categorical
-    features using one-hot encoding while maintaining numerical features unchanged. It provides
-    methods to fit to the dataset, transform it into an encoded format, inversely transform
-    encoded data back to the original format, and retrieve the transformed feature names.
-    It assumes the input data will be in the form of a DataFrame.
+    Encodes specified categorical features using one-hot encoding while maintaining numerical
+    features unchanged. Non-declared object-type columns in numerical slots are automatically
+    detected and excluded. Column matching for ``categorical_features`` is by exact name.
 
     Attributes:
         categorical_features (list): A list of strings representing the categorical feature names to be one-hot encoded (matched by exact name).
@@ -26,79 +23,11 @@ class CategoricalEncoder(DataTransformer):
         numerical_columns (list): The list of numerical feature names identified from the input data.
         n_features_in_ (int): The total number of input features in the dataset.
         feature_names_in_ (list): The list of input feature names identified during the fit process.
-        one_hot_encoder (OneHotEncoder): An instance of `OneHotEncoder` used for transforming categorical features.
+        one_hot_encoder (OneHotEncoder): An instance of ``OneHotEncoder`` used for transforming categorical features.
         categorical_columns (list): The list of categorical columns identified from the input data.
-
-    Methods:
-        fit(x: pd.DataFrame, y=None):
-            Fits the OneHotEncoder to the categorical features in the provided DataFrame.
-
-            Args:
-                x (pd.DataFrame): The input data containing all features.
-                                  Only the specified categorical features will be fitted.
-                y: Ignored. Retained for compatibility with scikit-learn API.
-
-            Returns:
-                self: The fitted CategoricalEncoder instance.
-
-        transform(x: pd.DataFrame) -> pd.DataFrame:
-            Transforms the input DataFrame by applying one-hot encoding to categorical features
-            and combining them with numerical features.
-
-            Args:
-                x (pd.DataFrame): The input data to be transformed.
-
-            Returns:
-                pd.DataFrame: The transformed DataFrame with one-hot encoded categorical features
-                              and numerical features.
-
-            Raises:
-                KeyError: If the input DataFrame is missing any of the features fitted during the `fit` step.
-
-        inverse_transform(x: pd.DataFrame) -> pd.DataFrame:
-            Reverts the transformed data back to its original form by mapping one-hot encoded
-            categorical features back to the original categorical values.
-
-            Args:
-                x (pd.DataFrame): The transformed input data.
-
-            Returns:
-                pd.DataFrame: The original data with categorical features restored to their original values.
-
-        get_feature_names_out(input_features=None) -> list:
-            Returns the names of features after the transformation.
-
-            Args:
-                input_features (list, optional): Unused. Retained for compatibility with scikit-learn API.
-
-            Returns:
-                list: List of output feature names including both numerical and one-hot encoded features.
-
-    Example:
-        ```python
-        import pandas as pd
-        from categorical_encoder import CategoricalEncoder
-
-        # Example dataset
-        data = pd.DataFrame({
-            'Category1': ['A', 'B', 'A'],
-            'Category2': ['X', 'Y', 'Z'],
-            'Numerical': [1, 2, 3]
-        })
-
-        # Initialize and fit the encoder
-        encoder = CategoricalEncoder(categorical_features=['Category1', 'Category2'])
-        encoder.fit(data)
-
-        # Transform the data
-        transformed_data = encoder.transform(data)
-        print(transformed_data)
-
-        # Inverse transform the data
-        original_data = encoder.inverse_transform(transformed_data)
-        print(original_data)
-
+        non_declared_categorical_features (list): Object-type columns detected in numerical slots and excluded.
     """
+
     def __init__(self, categorical_features: list = None):
         super().__init__()
         self.feature_names_out_ = None
@@ -110,6 +39,19 @@ def __init__(self, categorical_features: list = None):
         self.non_declared_categorical_features = None
 
     def fit(self, x: pd.DataFrame, y=None) -> "CategoricalEncoder":
+        """Fits the OneHotEncoder to the categorical features in the provided DataFrame.
+
+        Identifies numerical and categorical columns, detects and excludes non-declared
+        object-type columns from numerical slots, and fits the internal ``OneHotEncoder``.
+
+        Args:
+            x (pd.DataFrame): The input data containing all features.
+                Only the specified categorical features will be fitted.
+            y: Ignored. Retained for compatibility with scikit-learn API.
+
+        Returns:
+            CategoricalEncoder: The fitted encoder instance (self).
+        """
         self.feature_names_in_ = x.columns.tolist()
         self.categorical_columns = [col for col in self.feature_names_in_
                                     if col in self.categorical_features]
@@ -135,6 +77,22 @@ def fit(self, x: pd.DataFrame, y=None) -> "CategoricalEncoder":
         return self
     
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
+        """Transforms the input DataFrame by applying one-hot encoding to categorical features.
+
+        Combines one-hot encoded categorical features with unchanged numerical features.
+        If no categorical features are specified, returns the numerical data unchanged.
+
+        Args:
+            x (pd.DataFrame): The input data to be transformed.
+
+        Returns:
+            pd.DataFrame: The transformed DataFrame with one-hot encoded categorical features
+                and numerical features.
+
+        Raises:
+            KeyError: If the input DataFrame is missing any of the features fitted during ``fit``.
+            TypeError: If ``x`` is not a pandas DataFrame.
+        """
         check_is_fitted(self, "n_features_in_")
 
         if not isinstance(x, pd.DataFrame):
@@ -158,6 +116,21 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
             return numerical_data  # Returns input df if no categorical features are specified in config file
 
     def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
+        """Reverts transformed data back to original form by decoding one-hot encoded features.
+
+        Maps one-hot encoded categorical columns back to their original categorical values
+        and concatenates them with the numerical columns. If no categorical features were
+        encoded, returns the input unchanged.
+
+        Args:
+            x (pd.DataFrame): The transformed input data with one-hot encoded columns.
+
+        Returns:
+            pd.DataFrame: The data with categorical features restored to their original values.
+
+        Raises:
+            KeyError: If the one-hot encoded columns are not present in the input data.
+        """
         check_is_fitted(self, "n_features_in_")
 
         # Get the one-hot encoded column names from the encoder
@@ -182,6 +155,18 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
             return numerical_data  # Returns input df if no categorical features are specified in config file
     
     def get_feature_names_out(self, input_features=None) -> List[str]:
+        """Returns the names of features after the transformation.
+
+        The output names are the concatenation of numerical column names and one-hot
+        encoded categorical column names (e.g. ``category_A``, ``category_B``).
+
+        Args:
+            input_features: Unused. Retained for scikit-learn API compatibility.
+
+        Returns:
+            List[str]: List of output feature names including both numerical and
+                one-hot encoded features.
+        """
         check_is_fitted(self, "n_features_in_")
         self.feature_names_out_ = self.numerical_columns + list(
             self.one_hot_encoder.get_feature_names_out(self.categorical_columns)
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index 38daf64a..75d033eb 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -195,8 +195,10 @@ def fit(self, X: pd.DataFrame, y=None, **fit_params):
         # Call parent fit
         result = super().fit(X=X, y=y, **fit_params)
 
-        # Validate that protected features are in the output
-        # TODO: protected features shouldnt be dropped. Include this in a test.
+        # Validate that protected features survive the pipeline. ColumnSelector and LowUniqueValueFilter
+        # receive protected_features via fit_params and skip dropping them, but other steps (e.g. imputers
+        # dropping all-NaN columns) do not. This check enforces that protected features are still present
+        # in the output and raises if they were dropped (tested in test_data_preprocessor.py).
         if protected_features:
             output_features = self.get_feature_names_out()
             available_protected = [f for f in protected_features if f in X.columns]
diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py
index 0b8b6c06..a8cbaa60 100644
--- a/energy_fault_detector/data_preprocessing/ffill_imputer.py
+++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py
@@ -125,6 +125,7 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         """Applies time-based forward-fill imputation and cleaning pipeline to input data.
 
         The steps executed are:
+
         1. Forward-fill all NaN values.
         2. Invalidate fills where the elapsed time from the last valid observation exceeds
            ``ffill_limit_`` (set those values back to NaN).
diff --git a/energy_fault_detector/data_preprocessing/timestamp_transformer.py b/energy_fault_detector/data_preprocessing/timestamp_transformer.py
index 39807a04..17d3d0d0 100644
--- a/energy_fault_detector/data_preprocessing/timestamp_transformer.py
+++ b/energy_fault_detector/data_preprocessing/timestamp_transformer.py
@@ -107,8 +107,8 @@ class TimestampTransformer(DataTransformer):
 
     Args:
         features (Optional[List[str]]): List of feature names to generate. Supported:
-            ["second_of_minute", "minute_of_hour", "hour_of_day", "day_of_week", "day_of_month", "month_of_year",
-             "is_weekend", "year"]
+            ``second_of_minute``, ``minute_of_hour``, ``hour_of_day``, ``day_of_week``,
+            ``day_of_month``, ``day_of_year``, ``month_of_year``, ``is_weekend``, ``year``.
         timestamp_col (Optional[str]): The column name of the DataFrame containing timestamps.
             If None, the index is assumed to be the timestamp.
         groupby_level (Optional[str]): Optional index level name or position for grouping (e.g., 'device_id' or 0).
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index 20f7d565..2d9e578b 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -7,7 +7,6 @@
 
 import pandas as pd
 import numpy as np
-from sklearn.utils.validation import check_is_fitted
 
 from energy_fault_detector.core.fault_detection_model import FaultDetectionModel
 from energy_fault_detector.core.fault_detection_result import FaultDetectionResult, ModelMetadata
@@ -130,17 +129,23 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
         # Resolve declared conditions against preprocessed data
         self._resolve_conditional_features(x_prepped)
 
-        # Check conditionals: avilable conditions in original data and surviving conditions during preprocessing.
+        # Check conditionals: available conditions in original data and surviving conditions during preprocessing.
         if self.autoencoder.is_conditional:
             configured = self.autoencoder.conditional_features or []
-            available = [declared_condition for declared_condition in configured if any(declared_condition in col for col in sensor_data.columns)] # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring.
+            # Uses nested for loop in case categorical cols have been encoded already and contain the original
+            # conditional feature name as a substring.
+            available = [declared_condition for declared_condition in configured
+                         if any(declared_condition in col for col in sensor_data.columns)]
             missing = [declared_condition for declared_condition in configured if declared_condition not in available]
             if missing:
                 logger.warning(f"Declared conditions not found in sensor_data will be ignored: "
                                f"{sorted(missing)}. Using: {available or 'none'}")
 
             if not self.config.protect_conditional_features:
-                surviving = [declared_condition for declared_condition in available if any(declared_condition in col for col in x_prepped.columns)] # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring.
+                # Uses nested for loop in case categorical cols have been encoded already and contain the original
+                # conditional feature name as a substring.
+                surviving = [declared_condition for declared_condition in available
+                             if any(declared_condition in col for col in x_prepped.columns)]
                 dropped_by_pipeline = set(available or []) - set(surviving)
                 
                 if dropped_by_pipeline:
@@ -478,9 +483,10 @@ def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]:
         configured = self.autoencoder.conditional_features or []
         if not configured:
             return []
-        # TODO: this can produce undesired results if any column in the original data contains a substring of a declared conditional feature name.
-        # Uses nested for loop in case categorical cols have been encoded already and contain the original conditional feature name as a substring. The applies for timestamp and column_diff transformers.
-        available = [col for col in sensor_data.columns if any(declared_condition in col for declared_condition in configured)] 
+
+        # TODO: this can produce undesired results if any column in the original data contains a substring of a declared
+        #  conditional feature name.
+        available = [col for col in sensor_data.columns if any(declared_condition in col for declared_condition in configured)]
 
         if not available:
             self._fallback_if_no_conditionals()
diff --git a/energy_fault_detector/utils/visualisation.py b/energy_fault_detector/utils/visualisation.py
index 7bc5aac6..b605d763 100644
--- a/energy_fault_detector/utils/visualisation.py
+++ b/energy_fault_detector/utils/visualisation.py
@@ -59,7 +59,7 @@ def plot_reconstruction(data: pd.DataFrame, reconstruction: pd.DataFrame, featur
 
     Notes:
         - Can result in a very large plot, if the dataset contains many columns/features. Use the
-         `features_to_plot` parameter to specify which columns to plot.
+          ``features_to_plot`` parameter to specify which columns to plot.
         - For MultiIndex data (e.g. multiple devices), pass data for a single
           group/device, i.e. ``df.loc[device_id]`` to select one device before plotting.
 

From 06137022fd30f4d4ecaba29c8cc213852ad4ffac Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Thu, 3 Sep 2026 16:00:01 +0200
Subject: [PATCH 21/35] fix: correct scaler step names, config mutation, unseen
 categories, and dead code

- Emit canonical 'scaler' step name with 'scaler_type' param in
  generate_quickstart_config() and _data_preprocessor_params_to_steps()
  instead of unregistered 'standard_scaler'/'minmax_scaler' (BUG #1)
- Add _migrate_legacy_scaler_names() to DataPreprocessor that auto-converts
  old names with a DeprecationWarning showing the correct config format
- Fix tune() mutating the user's Config object by using a local variable
- Remove unreachable dead code in QuantileThresholdSelector.fit
- Add empty-data guard in preprocess_train_data raising ValueError
- Set OneHotEncoder handle_unknown='ignore' in CategoricalEncoder and log
  warning for unseen categories instead of crashing
- Fix tests that passed for the wrong reason (unknown-step-name error masked
  the intended singleton/duplicate-scaler checks)
- Add test_legacy_scaler_names_migrated
- Update notebook configs and docs to use canonical scaler name
---
 docs/configuration.rst                        |  4 +-
 docs/examples/advanced_config.yaml            |  3 +-
 energy_fault_detector/config/config.py        |  4 +-
 .../config/quickstart_config.py               |  4 +-
 .../data_preprocessing/categorical_encoder.py | 11 ++++-
 .../data_preprocessing/data_preprocessor.py   | 34 +++++++++++++++
 .../timestamp_transformer.py                  |  4 +-
 energy_fault_detector/fault_detector.py       | 14 ++++--
 .../threshold_selectors/quantile_threshold.py |  5 ---
 .../c2c_configs/windfarm_A.yaml               |  4 +-
 .../c2c_configs/windfarm_B.yaml               |  4 +-
 .../c2c_configs/windfarm_C.yaml               |  4 +-
 ...xample - Hyperparameter Optimization.ipynb |  6 +--
 notebooks/HDD Failure/base_model.yaml         |  4 +-
 tests/config/test_quickstart_config.py        | 10 +----
 .../test_categorical_encoder.py               | 14 +++---
 .../test_data_preprocessor.py                 | 43 ++++++++++++++++++-
 17 files changed, 130 insertions(+), 42 deletions(-)

diff --git a/docs/configuration.rst b/docs/configuration.rst
index 0abe2f86..5991c6a4 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -106,9 +106,7 @@ Allowed step names and aliases:
 +-------------------------+-----------------------------------------------+------------------------------------------------+
 | simple_imputer          | Impute missing values                         | imputer                                        |
 +-------------------------+-----------------------------------------------+------------------------------------------------+
-| standard_scaler         | Standardize features (z-score)                | standardize, standardscaler, standard          |
-+-------------------------+-----------------------------------------------+------------------------------------------------+
-| minmax_scaler           | Scale to [0, 1]                               | minmax                                         |
+| scaler                  | Standardize (standard) or scale to [0,1] (minmax)| standard_scaler, minmax_scaler (deprecated)  |
 +-------------------------+-----------------------------------------------+------------------------------------------------+
 | duplicate_to_nan        | Replace consecutive duplicate values with NaN | duplicate_value_to_nan, duplicate_values_to_nan|
 +-------------------------+-----------------------------------------------+------------------------------------------------+
diff --git a/docs/examples/advanced_config.yaml b/docs/examples/advanced_config.yaml
index e6ec8e62..b79496f1 100644
--- a/docs/examples/advanced_config.yaml
+++ b/docs/examples/advanced_config.yaml
@@ -58,8 +58,7 @@ train:
       # Alternatively, you can use a forward-fill imputer:
       # - name: ffill_imputer
       #   params:
-      #     freq: '1H'  # frequency of the time series (e.g., '1H' for hourly data)
-      #     ffill_limit: 3  # maximum number of consecutive NaNs to forward-fill
+      #     ffill_limit: '1h'  # maximum time gap to forward-fill as a pandas Timedelta string (e.g., '15min', '1h')
       #     categorical_features:
       #       - categorical_feature1
       #       - categorical_feature2
diff --git a/energy_fault_detector/config/config.py b/energy_fault_detector/config/config.py
index 8fb87e63..998099a4 100644
--- a/energy_fault_detector/config/config.py
+++ b/energy_fault_detector/config/config.py
@@ -351,8 +351,8 @@ def _data_preprocessor_params_to_steps(params: Dict[str, Any]) -> List[Dict[str,
     # 6. Scaler
     scale = p.get("scale", "standardize")
     if scale in ["standardize", "standard", "standardscaler"]:
-        steps.append({"name": "standard_scaler", "step_name": "scaler"})
+        steps.append({"name": "scaler", "step_name": "scaler", "params": {"scaler_type": "standard"}})
     else:
-        steps.append({"name": "minmax_scaler", "step_name": "scaler"})
+        steps.append({"name": "scaler", "step_name": "scaler", "params": {"scaler_type": "minmax"}})
 
     return steps
diff --git a/energy_fault_detector/config/quickstart_config.py b/energy_fault_detector/config/quickstart_config.py
index 93fe0e70..57506f42 100644
--- a/energy_fault_detector/config/quickstart_config.py
+++ b/energy_fault_detector/config/quickstart_config.py
@@ -93,9 +93,9 @@ def _build_preprocessor_steps(
     # Final scaler with aliases supported for convenience.
     scaler_key = scaler.lower()
     if scaler_key in ("standard", "standardize", "standard_scaler"):
-        steps.append({"name": "standard_scaler"})
+        steps.append({"name": "scaler", "params": {"scaler_type": "standard"}})
     elif scaler_key in ("minmax", "minmax_scaler", "normalize"):
-        steps.append({"name": "minmax_scaler"})
+        steps.append({"name": "scaler", "params": {"scaler_type": "minmax"}})
     else:
         raise ValueError(
             f"Unknown scaler '{scaler}'. Use 'standard' (aka 'standardize') or 'minmax'."
diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index 842d5d24..ff446768 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -34,7 +34,7 @@ def __init__(self, categorical_features: list = None):
         self.numerical_columns = None
         self.feature_names_in_ = None
         self.categorical_features = categorical_features if categorical_features else []
-        self.one_hot_encoder = OneHotEncoder(sparse_output=False)
+        self.one_hot_encoder = OneHotEncoder(sparse_output=False, handle_unknown='ignore')
         self.categorical_columns = None
         self.non_declared_categorical_features = None
 
@@ -106,6 +106,15 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         categorical_data = x[self.categorical_columns]
 
         if not categorical_data.empty:
+            for i, col in enumerate(self.categorical_columns):
+                fitted_cats = set(self.one_hot_encoder.categories_[i])
+                current_cats = set(categorical_data[col].dropna().unique())
+                new_cats = current_cats - fitted_cats
+                if new_cats:
+                    logger.warning(
+                        "Unseen categories found in feature '%s': %s.",
+                        col, sorted(new_cats),
+                    )
             x_categorical_ = self.one_hot_encoder.transform(categorical_data)
             x_numerical_ = numerical_data.values
             x_ = np.concatenate((x_numerical_, x_categorical_), axis=1)
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index 75d033eb..c637c860 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -1,6 +1,7 @@
 """Generic class for building a preprocessing pipeline."""
 
 import copy
+import warnings
 from collections import Counter, defaultdict
 from typing import List, Optional, Dict, Any, Tuple
 
@@ -300,6 +301,7 @@ def _build_from_steps_spec(self) -> List:
         # Work on a copy so the caller's config dict is not mutated by name normalization / step name assignment.
         steps_spec = copy.deepcopy(self.steps_spec_)
 
+        self._migrate_legacy_scaler_names(steps_spec)
         self._validate_step_spec_keys(steps_spec)
 
         # Filter disabled steps first to simplify ordering.
@@ -448,6 +450,38 @@ def _assign_unique_step_names(specs: List[Dict[str, Any]]) -> List[Dict[str, Any
 
         return specs
 
+    @staticmethod
+    def _migrate_legacy_scaler_names(steps_spec: List[Dict[str, Any]]) -> None:
+        """Migrate legacy 'standard_scaler'/'minmax_scaler' step names to the canonical 'scaler' name.
+
+        Legacy configs used ``standard_scaler`` or ``minmax_scaler`` as step names. This method rewrites
+        them in-place to the canonical ``scaler`` name with the appropriate ``scaler_type`` parameter and
+        emits a ``DeprecationWarning`` showing the correct configuration format.
+
+        Args:
+            steps_spec: List of step spec dicts (mutated in place).
+        """
+        legacy_map = {"standard_scaler": "standard", "minmax_scaler": "minmax"}
+        for spec in steps_spec:
+            name = spec.get("name")
+            if name in legacy_map:
+                scaler_type = legacy_map[name]
+                params = spec.get("params") or {}
+                params["scaler_type"] = scaler_type
+                spec["name"] = "scaler"
+                spec["params"] = params
+                warnings.warn(
+                    f"Step name '{name}' is deprecated and has been automatically migrated to "
+                    f"'scaler' with params {{'scaler_type': '{scaler_type}'}}. "
+                    f"Please update your configuration to use:\n"
+                    f"  - name: scaler\n"
+                    f"    params:\n"
+                    f"      scaler_type: {scaler_type}\n"
+                    f"This automatic migration will be removed in a future version.",
+                    DeprecationWarning,
+                    stacklevel=3,
+                )
+
     @staticmethod
     def _validate_step_spec_keys(steps_spec: List[Dict[str, Any]]) -> None:
         """Validate that each step spec uses only allowed keys and includes 'name'.
diff --git a/energy_fault_detector/data_preprocessing/timestamp_transformer.py b/energy_fault_detector/data_preprocessing/timestamp_transformer.py
index 17d3d0d0..bd219a97 100644
--- a/energy_fault_detector/data_preprocessing/timestamp_transformer.py
+++ b/energy_fault_detector/data_preprocessing/timestamp_transformer.py
@@ -124,7 +124,9 @@ class TimestampTransformer(DataTransformer):
         train:
           data_preprocessor:
             steps:
-              - name: standard_scaler
+              - name: scaler
+                params:
+                  scaler_type: standard
               - name: timestamp_transformer
                 params:
                   features:
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index 2d9e578b..bf67fd40 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -93,6 +93,13 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
         x_prepped = self.data_preprocessor.transform(x_normal)
         x_prepped = x_prepped.astype(self.config.dtype)
 
+        if x_prepped.empty:
+            raise ValueError(
+                "Preprocessed training data is empty after filtering for normal behaviour and applying the "
+                "preprocessing pipeline. Check that 'normal_index' marks enough rows as normal and that the "
+                "preprocessor (e.g. ForwardFillImputer, ColumnSelector) is not dropping all rows."
+            )
+
         return x_prepped, x, y
 
     def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_models: bool = True,
@@ -220,10 +227,11 @@ def tune(self, sensor_data: pd.DataFrame, normal_index: Optional[pd.Series] = No
         if tune_method not in ['threshold', 'decoder', 'full']:
             raise ValueError(f'Unknown tune method {tune_method}.')
 
-        if tune_method == 'threshold' and self.config.fit_threshold_on_val:
+        fit_on_val = self.config.fit_threshold_on_val
+        if tune_method == 'threshold' and fit_on_val:
             logger.warning('Fine-tuning using only validation data for threshold does not make sense if only the'
                            ' threshold is tuned! Setting fit_threshold_on_val to False.')
-            self.config['train']['threshold_selector']['fit_on_val'] = False
+            fit_on_val = False
 
         if pretrained_model_path is not None:
             self._load_from_path(model_path=pretrained_model_path)
@@ -270,7 +278,7 @@ def tune(self, sensor_data: pd.DataFrame, normal_index: Optional[pd.Series] = No
             )
 
         # tune/fit threshold
-        self._fit_threshold(x=x, y=y, x_val=x_val, fit_on_validation=self.config.fit_threshold_on_val)
+        self._fit_threshold(x=x, y=y, x_val=x_val, fit_on_validation=fit_on_val)
 
         model_path = None
         if save_models:
diff --git a/energy_fault_detector/threshold_selectors/quantile_threshold.py b/energy_fault_detector/threshold_selectors/quantile_threshold.py
index 4a60f3e2..fdb7d9bc 100644
--- a/energy_fault_detector/threshold_selectors/quantile_threshold.py
+++ b/energy_fault_detector/threshold_selectors/quantile_threshold.py
@@ -64,9 +64,4 @@ def fit(self, x: Array1D, y: pd.Series = None) -> 'QuantileThresholdSelector':
             x_ = x
 
         self.threshold = float(np.quantile(x_, self.quantile))
-
-        if self.threshold is None:
-            import warnings
-            warnings.warn('Could not find suitable threshold, `threshold` is set to max score.', UserWarning)
-            self.threshold = float(np.sort(x)[-1])
         return self
diff --git a/notebooks/CARE to Compare/c2c_configs/windfarm_A.yaml b/notebooks/CARE to Compare/c2c_configs/windfarm_A.yaml
index 5ef3264a..74c65f54 100644
--- a/notebooks/CARE to Compare/c2c_configs/windfarm_A.yaml	
+++ b/notebooks/CARE to Compare/c2c_configs/windfarm_A.yaml	
@@ -12,7 +12,9 @@ train:
       params:
         min_unique_value_count: 10
         max_col_zero_frac: 0.99
-    - name: minmax_scaler
+    - name: scaler
+      params:
+        scaler_type: minmax
 
   data_splitter:
     shuffle: true
diff --git a/notebooks/CARE to Compare/c2c_configs/windfarm_B.yaml b/notebooks/CARE to Compare/c2c_configs/windfarm_B.yaml
index 68320b93..320cf2b2 100644
--- a/notebooks/CARE to Compare/c2c_configs/windfarm_B.yaml	
+++ b/notebooks/CARE to Compare/c2c_configs/windfarm_B.yaml	
@@ -12,7 +12,9 @@ train:
         params:
           min_unique_value_count: 10
           max_col_zero_frac: 0.8
-      - name: minmax_scaler
+      - name: scaler
+        params:
+          scaler_type: minmax
 
   data_splitter:
     shuffle: true
diff --git a/notebooks/CARE to Compare/c2c_configs/windfarm_C.yaml b/notebooks/CARE to Compare/c2c_configs/windfarm_C.yaml
index 3f7b08a6..0a38f942 100644
--- a/notebooks/CARE to Compare/c2c_configs/windfarm_C.yaml	
+++ b/notebooks/CARE to Compare/c2c_configs/windfarm_C.yaml	
@@ -12,7 +12,9 @@ train:
         params:
           min_unique_value_count: 10
           max_col_zero_frac: 0.99
-      - name: minmax_scaler
+      - name: scaler
+        params:
+          scaler_type: minmax
 
   data_splitter:
     shuffle: true
diff --git a/notebooks/Example - Hyperparameter Optimization.ipynb b/notebooks/Example - Hyperparameter Optimization.ipynb
index 344db94e..bc4e52ac 100644
--- a/notebooks/Example - Hyperparameter Optimization.ipynb	
+++ b/notebooks/Example - Hyperparameter Optimization.ipynb	
@@ -199,14 +199,14 @@
     "    steps = dp.get('steps')\n",
     "\n",
     "    # Remove any existing scaler step(s)\n",
-    "    scaler_names = {'standard_scaler', 'minmax_scaler'}\n",
+    "    scaler_names = {'scaler'}\n",
     "    steps = [s for s in steps if s.get('name') not in scaler_names]\n",
     "    # Add the chosen scaler step\n",
     "    if choice == 'minmax':\n",
-    "        steps.append({'name': 'minmax_scaler'})\n",
+    "        steps.append({'name': 'scaler', 'params': {'scaler_type': 'minmax'}})\n",
     "    else:\n",
     "        # 'standardize'\n",
-    "        steps.append({'name': 'standard_scaler'})\n",
+    "        steps.append({'name': 'scaler', 'params': {'scaler_type': 'standard'}})\n",
     "\n",
     "    dp['steps'] = steps\n",
     "    return cfg\n",
diff --git a/notebooks/HDD Failure/base_model.yaml b/notebooks/HDD Failure/base_model.yaml
index cffd17ec..e8d06138 100644
--- a/notebooks/HDD Failure/base_model.yaml	
+++ b/notebooks/HDD Failure/base_model.yaml	
@@ -9,7 +9,9 @@ train:
         params:
           max_nan_frac_per_col: 0.2
       - name: simple_imputer
-      - name: standard_scaler
+      - name: scaler
+        params:
+          scaler_type: standard
 
   autoencoder:
     name: default
diff --git a/tests/config/test_quickstart_config.py b/tests/config/test_quickstart_config.py
index 4e93506f..393c98a1 100644
--- a/tests/config/test_quickstart_config.py
+++ b/tests/config/test_quickstart_config.py
@@ -40,15 +40,7 @@ def test_generate_quickstart_config_valid_dict(self) -> None:
         step_names = [s["name"] for s in train["data_preprocessor"]["steps"]]
         self.assertIn("column_selector", step_names)
         self.assertIn("simple_imputer", step_names)
-        self.assertTrue(
-            any(n in ("standard_scaler", "minmax_scaler") for n in step_names),
-            "Expected a scaler step in the pipeline."
-        )
-
-        self.assertTrue(
-            any(n in ("standard_scaler", "minmax_scaler") for n in step_names),
-            "Expected a scaler step in the pipeline."
-        )
+        self.assertIn("scaler", step_names)
 
     def test_generate_quickstart_config_validation_split_guard(self) -> None:
         """If validation split not in (0, 1) it should raise ValueError."""
diff --git a/tests/data_preprocessing/test_categorical_encoder.py b/tests/data_preprocessing/test_categorical_encoder.py
index facef90e..9ca86d53 100644
--- a/tests/data_preprocessing/test_categorical_encoder.py
+++ b/tests/data_preprocessing/test_categorical_encoder.py
@@ -213,10 +213,10 @@ def test_transform_with_nan_in_categorical(self):
         self.assertFalse(result.isna().any().any())
 
     def test_transform_with_new_categories(self):
-        """Test transformation with unseen category values."""
+        """Test transformation with unseen category values — should not crash, encoded as all-zeros."""
         encoder = CategoricalEncoder(categorical_features=['category'])
         encoder.fit(self.data_clean)
-        
+
         # Create data with unseen category value
         new_data = pd.DataFrame({
             'temperature': [25.0],
@@ -225,9 +225,13 @@ def test_transform_with_new_categories(self):
             'region': ['North']
         }, index=pd.date_range('2024-01-01', periods=1, freq='1Min'))
 
-        # Should raise error as per OneHotEncoder default behavior (handle_unknown='error')
-        with self.assertRaises(ValueError):
-            encoder.transform(new_data)
+        # With handle_unknown='ignore', unseen categories are encoded as all-zeros (no error)
+        transformed = encoder.transform(new_data)
+        # No new column for 'D' — only fitted categories A, B, C have columns
+        category_cols = [c for c in transformed.columns if c.startswith('category_')]
+        self.assertEqual(sorted(category_cols), ['category_A', 'category_B', 'category_C'])
+        self.assertTrue((transformed[category_cols] == 0).all().all(),
+                        "Unseen category should be encoded as all-zeros.")
 
 if __name__ == '__main__':
     unittest.main()
\ No newline at end of file
diff --git a/tests/data_preprocessing/test_data_preprocessor.py b/tests/data_preprocessing/test_data_preprocessor.py
index cc6cc8eb..18375762 100644
--- a/tests/data_preprocessing/test_data_preprocessor.py
+++ b/tests/data_preprocessing/test_data_preprocessor.py
@@ -5,7 +5,7 @@
 import pandas as pd
 from numpy.testing import assert_array_almost_equal
 from pandas.testing import assert_frame_equal
-from sklearn.utils.validation import check_is_fitted, NotFittedError
+from sklearn.utils.validation import NotFittedError
 
 from energy_fault_detector.config.config import Config
 from energy_fault_detector.data_preprocessing.data_preprocessor import DataPreprocessor
@@ -308,7 +308,7 @@ def test_singleton_violation_raises(self) -> None:
                 steps=[
                     {"name": "simple_imputer", "params": {"strategy": "mean"}},
                     {"name": "simple_imputer", "params": {"strategy": "median"}},
-                    {"name": "standard_scaler"},
+                    {"name": "scaler", "params": {"scaler_type": "standard"}},
                 ]
             )
 
@@ -316,13 +316,52 @@ def test_only_one_scaler_allowed(self) -> None:
         """Defining more than one scaler should raise a ValueError."""
         with self.assertRaises(ValueError):
             _ = DataPreprocessor(
+                steps=[
+                    {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}},
+                    {"name": "scaler", "params": {"scaler_type": "standard"}},
+                    {"name": "scaler", "params": {"scaler_type": "minmax"}},
+                ]
+            )
+
+    def test_legacy_scaler_names_migrated(self) -> None:
+        """Legacy 'standard_scaler'/'minmax_scaler' names should be migrated with a DeprecationWarning."""
+        import warnings as _warnings
+
+        with _warnings.catch_warnings(record=True) as caught:
+            _warnings.simplefilter("always")
+            dp = DataPreprocessor(
                 steps=[
                     {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}},
                     {"name": "standard_scaler"},
+                ]
+            )
+
+        deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)]
+        self.assertTrue(
+            any("standard_scaler" in str(w.message) for w in deprecation_warnings),
+            "Expected a DeprecationWarning mentioning 'standard_scaler'.",
+        )
+        # The migrated pipeline should have a 'scaler' step with a Scaler instance
+        self.assertIn("scaler", dp.named_steps)
+        self.assertEqual(dp.named_steps["scaler"].scaler_type, "standard")
+
+        with _warnings.catch_warnings(record=True) as caught:
+            _warnings.simplefilter("always")
+            dp = DataPreprocessor(
+                steps=[
+                    {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}},
                     {"name": "minmax_scaler"},
                 ]
             )
 
+        deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)]
+        self.assertTrue(
+            any("minmax_scaler" in str(w.message) for w in deprecation_warnings),
+            "Expected a DeprecationWarning mentioning 'minmax_scaler'.",
+        )
+        self.assertIn("scaler", dp.named_steps)
+        self.assertEqual(dp.named_steps["scaler"].scaler_type, "minmax")
+
     def test_inverse_transform_with_encoder_ffill(self):
         """Test that the inverse_transform works correctly with the preprocessor that includes categorical encoder and ffill imputer."""
         self.preprocessor_with_encoder.fit(self.test_data4)

From ccf24c52a172247ad09f5d1788ef7b5b8b9162a2 Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Fri, 4 Sep 2026 09:48:10 +0200
Subject: [PATCH 22/35] fix: handle duplicate indices and clarify ffill imputer
 docstrings

- Add `_handle_duplicate_index` to FaultDetectionModel base class: drops
  true duplicates (same index + same values), raises ValueError for
  contradicting duplicates (same index, different values)
- Replace blanket duplicate-index rejection in FaultDetector with the new
  helper, applied in preprocess_train_data and predict
- Update ForwardFillImputer docstrings to explain drop_duplicates removes
  exact duplicates created by forward-filling
- Add TestHandleDuplicateIndex covering DataFrame and Series variants
- Silence decoder-not-loaded warning (many models don't save the decoder)
---
 energy_fault_detector/core/autoencoder.py     |  5 +-
 .../core/fault_detection_model.py             | 46 +++++++++++++++++++
 .../data_preprocessing/ffill_imputer.py       |  9 +++-
 energy_fault_detector/fault_detector.py       |  9 ++--
 tests/test_fault_detector.py                  | 38 +++++++++++++++
 5 files changed, 99 insertions(+), 8 deletions(-)

diff --git a/energy_fault_detector/core/autoencoder.py b/energy_fault_detector/core/autoencoder.py
index 73321324..4d7a9c22 100644
--- a/energy_fault_detector/core/autoencoder.py
+++ b/energy_fault_detector/core/autoencoder.py
@@ -496,8 +496,9 @@ def load(self, directory: str, **kwargs) -> "Autoencoder":  # pylint: disable=W0
             warnings.warn("No fitted autoencoder model was found.")
         if not encoder_loaded:
             warnings.warn("No fitted encoder model was found.")
-        if not decoder_loaded:
-            warnings.warn("No fitted decoder model was found.")
+        # Uncommented for now, since many models do not save the decoder.
+        # if not decoder_loaded:
+        #    warnings.warn("No fitted decoder model was found.")
 
         return self
 
diff --git a/energy_fault_detector/core/fault_detection_model.py b/energy_fault_detector/core/fault_detection_model.py
index 5ce45798..d4b752b0 100644
--- a/energy_fault_detector/core/fault_detection_model.py
+++ b/energy_fault_detector/core/fault_detection_model.py
@@ -139,6 +139,52 @@ def predict(self, sensor_data: pd.DataFrame, model_path: Optional[str] = None, a
             FaultDetectionResult object.
         """
 
+    @staticmethod
+    def _handle_duplicate_index(data: Union[pd.DataFrame, pd.Series]) -> Union[pd.DataFrame, pd.Series]:
+        """Drop true duplicate indices and raise on contradicting ones.
+
+        Rows sharing the same index but with identical values across all columns are
+        treated as redundant copies and the first occurrence is kept. Rows sharing the
+        same index but with differing values indicate a data integrity problem and cause
+        a ``ValueError``.
+
+        Args:
+            data: A pandas DataFrame or Series with a potentially duplicated index.
+
+        Returns:
+            The input with true duplicates removed (keep first).
+
+        Raises:
+            ValueError: If contradicting duplicates (same index, different values) are found.
+        """
+        dup_mask = data.index.duplicated(keep=False)
+        if not dup_mask.any():
+            return data
+
+        dup_indices = data.index[dup_mask].unique()
+        contradicting = []
+        for idx in dup_indices:
+            rows = data.loc[idx]
+            if isinstance(rows, pd.DataFrame):
+                if len(rows.drop_duplicates()) > 1:
+                    contradicting.append(idx)
+            else:
+                if rows.nunique() > 1:
+                    contradicting.append(idx)
+
+        if contradicting:
+            raise ValueError(
+                f"Found contradicting duplicate indices (same index, different values): "
+                f"{list(contradicting)[:10]}. Please check your input data for data integrity issues."
+            )
+
+        n_before = len(data)
+        data = data[~data.index.duplicated(keep='first')]
+        n_dropped = n_before - len(data)
+        if n_dropped:
+            logger.info(f"Dropped {n_dropped} duplicate rows (identical index and values).")
+        return data
+
     def train_val_split(self, x: DataType) -> Tuple[DataType, DataType]:
         """Split data in train and validation data.
 
diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py
index a8cbaa60..c7e291ee 100644
--- a/energy_fault_detector/data_preprocessing/ffill_imputer.py
+++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py
@@ -25,6 +25,12 @@ class ForwardFillImputer(DataTransformer):
     (invalidating fills that exceed the time threshold), drops duplicate rows, and removes
     any rows still containing NaN values.
 
+    Dropping duplicate rows after forward-filling is necessary because forward-filling
+    propagates the last valid value into consecutive NaN rows. If the underlying signal did
+    not change between observations, the filled rows become exact duplicates of the source
+    row. Removing these redundant rows avoids introducing identical training samples that
+    carry no additional information and would otherwise bias the reconstruction error.
+
     Attributes:
         feature_names_in_ (List[str]): List of input feature names observed during fitting.
         feature_names_out_ (List[str]): List of output feature names (same as input features unless some were dropped).
@@ -129,7 +135,8 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         1. Forward-fill all NaN values.
         2. Invalidate fills where the elapsed time from the last valid observation exceeds
            ``ffill_limit_`` (set those values back to NaN).
-        3. Drop duplicate rows.
+        3. Drop duplicate rows (exact duplicates created by forward-filling where the
+           underlying signal did not change).
         4. Drop any rows still containing NaN values (``dropna(how="any")``).
 
         Unlike the previous resampling-based approach, the original (possibly irregular)
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index bf67fd40..060773cf 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -55,9 +55,9 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
         else:
             y = pd.Series(np.full(len(x), True), index=x.index)
 
-        if not x.loc[x.index.duplicated()].empty or not y.loc[y.index.duplicated()].empty:
-            raise ValueError('There are duplicated indices in the input dataframe `sensor_data` and/or in the '
-                             '`normal_index`, please check your input data.')
+        x = self._handle_duplicate_index(x)
+        if normal_index is not None:
+            y = self._handle_duplicate_index(y)
 
         # TODO: add list of relevant features to config and protect relevant features (if present) as well.
         # Determine which features to protect based on config flag
@@ -330,8 +330,7 @@ def predict(self, sensor_data: pd.DataFrame, model_path: Optional[str] = None,
                 raise ValueError('No models loaded and no model_path provided!')
             logger.debug('No model_path provided; using existing model instances.')
 
-        if not x.loc[x.index.duplicated()].empty:
-            raise ValueError('There are duplicated indices in the input dataframe `sensor_data`.')
+        x = self._handle_duplicate_index(x)
 
         x_prepped = self.data_preprocessor.transform(x).sort_index()
         x_prepped = x_prepped.astype(self.config.dtype)
diff --git a/tests/test_fault_detector.py b/tests/test_fault_detector.py
index 60c1d32c..7cd8d3b5 100644
--- a/tests/test_fault_detector.py
+++ b/tests/test_fault_detector.py
@@ -740,3 +740,41 @@ def test_explicit_true(self):
             }
         })
         self.assertTrue(config.protect_conditional_features)
+
+
+class TestHandleDuplicateIndex(unittest.TestCase):
+    """Test _handle_duplicate_index on the FaultDetectionModel base class."""
+
+    def test_no_duplicates_unchanged(self):
+        """Data without duplicate indices should be returned unchanged."""
+        df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]}, index=[0, 1, 2])
+        result = FaultDetector._handle_duplicate_index(df)
+        self.assertEqual(len(result), 3)
+        pd.testing.assert_frame_equal(result, df)
+
+    def test_true_duplicates_dropped(self):
+        """Rows with same index and same values should be collapsed to one."""
+        df = pd.DataFrame({'a': [1, 2, 2, 3], 'b': [4, 5, 5, 6]}, index=[0, 1, 1, 2])
+        result = FaultDetector._handle_duplicate_index(df)
+        self.assertEqual(len(result), 3)
+        self.assertEqual(list(result.index), [0, 1, 2])
+
+    def test_contradicting_duplicates_raise(self):
+        """Rows with same index but different values should raise ValueError."""
+        df = pd.DataFrame({'a': [1, 2, 3, 3], 'b': [4, 5, 6, 7]}, index=[0, 1, 2, 2])
+        with self.assertRaises(ValueError) as ctx:
+            FaultDetector._handle_duplicate_index(df)
+        self.assertIn('contradicting', str(ctx.exception))
+
+    def test_series_true_duplicates_dropped(self):
+        """Series with same index and same values should be collapsed."""
+        s = pd.Series([True, True, True], index=[0, 1, 1])
+        result = FaultDetector._handle_duplicate_index(s)
+        self.assertEqual(len(result), 2)
+
+    def test_series_contradicting_raise(self):
+        """Series with same index but different values should raise ValueError."""
+        s = pd.Series([True, True, False], index=[0, 1, 1])
+        with self.assertRaises(ValueError) as ctx:
+            FaultDetector._handle_duplicate_index(s)
+        self.assertIn('contradicting', str(ctx.exception))

From 0005e698caf1154721e6eaf9beab0d1ab3795dca Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 17 Jul 2026 09:35:34 +0200
Subject: [PATCH 23/35] Enhance data preprocessing components: - Update Imputer
 to drop non-declared categorical features and log details.. - Improve
 DataPreprocessor docstring and fix scaler_type parameter name. - Validate
 scaler_type in Scaler initialization. - Update FaultDetector to ensure
 preprocessor is fitted before use and log warnings for non-numeric data.

---
 energy_fault_detector/data_preprocessing/imputer.py | 12 ++++++++++--
 energy_fault_detector/fault_detector.py             |  5 ++++-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index 3b9905f1..2e34dcf6 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -62,7 +62,7 @@ def __init__(self, strategy: str = 'mean', categorical_features: list = None, **
             raise ValueError(f"Unsupported strategy: {self.strategy}. Supported strategies are 'mean' and 'median'.")
 
         # Attributes to be defined during fitting
-        
+
         self.feature_names_in_: List[str] = []
         self.feature_names_out_ = None
         self.input_index_ = None
@@ -124,6 +124,14 @@ def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
         logger.debug(f"Numerical columns: {self.numerical_columns}")
         logger.debug(f"Categorical columns: {self.categorical_columns}")
 
+        # Clean numerical columns from non_declared categorical features
+        self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist()
+        self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
+        numerical_data = numerical_data[self.numerical_columns]
+
+        logger.debug(f"Numerical columns: {self.numerical_columns}")
+        logger.debug(f"Categorical columns: {self.categorical_columns}")
+
         # Fit the imputers
         if not numerical_data.empty:
             self.numerical_imputer.fit(numerical_data)
@@ -211,7 +219,7 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
             ValueError: If imputer has not been fitted (via `check_is_fitted`).
         """
         check_is_fitted(self, "n_features_in_")
-        
+
         return pd.DataFrame(x, columns=self.feature_names_in_)
     
     def get_feature_names_out(self, input_features=None) -> List[str]:
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index 060773cf..203024da 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -7,6 +7,7 @@
 
 import pandas as pd
 import numpy as np
+from sklearn.utils.validation import check_is_fitted
 
 from energy_fault_detector.core.fault_detection_model import FaultDetectionModel
 from energy_fault_detector.core.fault_detection_result import FaultDetectionResult, ModelMetadata
@@ -106,6 +107,8 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
             overwrite_models: bool = False, fit_autoencoder_only: bool = False, fit_preprocessor: bool = True,
             **kwargs) -> ModelMetadata:
         """Fit models on the given sensor_data and save them locally and return the metadata."""
+        if not check_is_fitted(self.data_preprocessor) and not fit_preprocessor:
+            raise ValueError("Data preprocessor is not fitted. Consider setting `fit_preprocessor=True`.")
 
         try:
             from keras.backend import clear_session
@@ -154,7 +157,7 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
                 surviving = [declared_condition for declared_condition in available
                              if any(declared_condition in col for col in x_prepped.columns)]
                 dropped_by_pipeline = set(available or []) - set(surviving)
-                
+
                 if dropped_by_pipeline:
                     logger.warning(f"Declared conditions dropped by preprocessing pipeline: "
                                 f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}")

From 1ba686f9168e52d0eeab5b366c5793e1b71552f5 Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Fri, 24 Jul 2026 11:44:08 +0200
Subject: [PATCH 24/35] Enhance CategoricalEncoder and Imputer: Added logging
 for non-declared categorical features and harmonized data types in numerical
 columns converting numerical columns, which might inlcude booleans,  into
 float. Updated FaultDetector docstrings for clarity.

---
 energy_fault_detector/data_preprocessing/categorical_encoder.py | 1 -
 energy_fault_detector/fault_detector.py                         | 2 --
 2 files changed, 3 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index ff446768..c602dbe3 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -8,7 +8,6 @@
 
 logger = logging.getLogger('energy_fault_detector')
 
-
 class CategoricalEncoder(DataTransformer):
     """Transformer for encoding categorical features using one-hot encoding.
 
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index 203024da..ec64462e 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -107,8 +107,6 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
             overwrite_models: bool = False, fit_autoencoder_only: bool = False, fit_preprocessor: bool = True,
             **kwargs) -> ModelMetadata:
         """Fit models on the given sensor_data and save them locally and return the metadata."""
-        if not check_is_fitted(self.data_preprocessor) and not fit_preprocessor:
-            raise ValueError("Data preprocessor is not fitted. Consider setting `fit_preprocessor=True`.")
 
         try:
             from keras.backend import clear_session

From d1fcc69aa83a361ee406fbbd009759de9b62d77c Mon Sep 17 00:00:00 2001
From: edi44495 <edison.guevara@iee.fraunhofer.de>
Date: Tue, 4 Aug 2026 17:25:20 +0200
Subject: [PATCH 25/35] Added new test cases for CategoricalEncoder and
 ForwardFillImputer, updated .gitignore to include generated documentation
 files, and added TODO comments in FaultDetector for handling of encoded
 categorical features as conditions.

---
 docs/conf.py                            | 136 ++++++++++++------------
 energy_fault_detector/fault_detector.py |   1 +
 2 files changed, 69 insertions(+), 68 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index ed69f7f7..54303d3c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -219,74 +219,74 @@
 ]
 
 
-def run_apidoc(app):
-    """Generate API .rst files with sphinx-apidoc, like in CI:
-
-       sphinx-apidoc -o docs energy_fault_detector/ --module-first --force --separate --templatedir docs/_templates
-    """
-
-    # Only run this step if we're building the HTML docs
-    if app.builder.name != "html":
-        return
-
-    from sphinx.ext.apidoc import main as apidoc_main
-    from pathlib import Path
-
-    here = Path(__file__).parent
-    pkg_dir = here.parent / IMPORT_NAME
-    out_dir = here
-    templates = here / "_templates"
-
-    apidoc_main([
-        "-o", str(out_dir),
-        str(pkg_dir),
-        "--module-first",
-        "--force",
-        "--separate",
-        "--templatedir", str(templates),
-        # Following files are skipped
-        # Quick fault detection internals
-        str(pkg_dir / "main.py"),
-        str(pkg_dir / "quick_fault_detection" / "configuration.py"),
-        str(pkg_dir / "quick_fault_detection" / "data_loading.py"),
-        str(pkg_dir / "quick_fault_detection" / "optimization.py"),
-        str(pkg_dir / "quick_fault_detection" / "output.py"),
-        str(pkg_dir / "quick_fault_detection" / "pipeline.py"),
-        str(pkg_dir / "quick_fault_detection" / "quick_fault_detector.py"),
-        # core internals
-        str(pkg_dir / "core" / "anomaly_score.py"),
-        str(pkg_dir / "core" / "data_transformer.py"),
-        str(pkg_dir / "core" / "threshold_selector.py"),
-        str(pkg_dir / "core" / "fault_detection_result.py"),
-        str(pkg_dir / "core" / "fault_detection_model.py"),
-        str(pkg_dir / "core" / "model_factory.py"),
-        str(pkg_dir / "core" / "save_load_mixin.py"),
-        # internal utilities
-        str(pkg_dir / "utils" / "index_utils.py"),
-        # config internals
-        str(pkg_dir / "config" / "base_config.py"),
-        str(pkg_dir / "config" / "config.py"),
-        str(pkg_dir / "config" / "quickstart_config.py"),
-        # Class specific files (documented directly under the top module)
-        # autoencoder
-        str(pkg_dir / "autoencoders" / "multilayer_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "conditional_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "lstm_seq2one_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "cnn_seq2one_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "bidirectional_lstm_seq2one_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "cnn_seq_autoencoder.py"),
-        str(pkg_dir / "autoencoders" / "lstm_seq2seq_autoencoder.py"),
-        # anomaly score
-        str(pkg_dir / "anomaly_scores" / "rmse_score.py"),
-        str(pkg_dir / "anomaly_scores" / "mahalanobis_score.py"),
-        # threshold selector
-        str(pkg_dir / "threshold_selectors" / "adaptive_threshold.py"),
-        str(pkg_dir / "threshold_selectors" / "fdr_threshold.py"),
-        str(pkg_dir / "threshold_selectors" / "fbeta_threshold.py"),
-        str(pkg_dir / "threshold_selectors" / "quantile_threshold.py"),
-        # main class (documented directly under the top module)
-        str(pkg_dir / "fault_detector.py"),
-    ])
+# def run_apidoc(app):
+#     """Generate API .rst files with sphinx-apidoc, like in CI:
+
+#        sphinx-apidoc -o docs energy_fault_detector/ --module-first --force --separate --templatedir docs/_templates
+#     """
+
+#     # Only run this step if we're building the HTML docs
+#     if app.builder.name != "html":
+#         return
+
+#     from sphinx.ext.apidoc import main as apidoc_main
+#     from pathlib import Path
+
+#     here = Path(__file__).parent
+#     pkg_dir = here.parent / IMPORT_NAME
+#     out_dir = here
+#     templates = here / "_templates"
+
+#     apidoc_main([
+#         "-o", str(out_dir),
+#         str(pkg_dir),
+#         "--module-first",
+#         "--force",
+#         "--separate",
+#         "--templatedir", str(templates),
+#         # Following files are skipped
+#         # Quick fault detection internals
+#         str(pkg_dir / "main.py"),
+#         str(pkg_dir / "quick_fault_detection" / "configuration.py"),
+#         str(pkg_dir / "quick_fault_detection" / "data_loading.py"),
+#         str(pkg_dir / "quick_fault_detection" / "optimization.py"),
+#         str(pkg_dir / "quick_fault_detection" / "output.py"),
+#         str(pkg_dir / "quick_fault_detection" / "pipeline.py"),
+#         str(pkg_dir / "quick_fault_detection" / "quick_fault_detector.py"),
+#         # core internals
+#         str(pkg_dir / "core" / "anomaly_score.py"),
+#         str(pkg_dir / "core" / "data_transformer.py"),
+#         str(pkg_dir / "core" / "threshold_selector.py"),
+#         str(pkg_dir / "core" / "fault_detection_result.py"),
+#         str(pkg_dir / "core" / "fault_detection_model.py"),
+#         str(pkg_dir / "core" / "model_factory.py"),
+#         str(pkg_dir / "core" / "save_load_mixin.py"),
+#         # internal utilities
+#         str(pkg_dir / "utils" / "index_utils.py"),
+#         # config internals
+#         str(pkg_dir / "config" / "base_config.py"),
+#         str(pkg_dir / "config" / "config.py"),
+#         str(pkg_dir / "config" / "quickstart_config.py"),
+#         # Class specific files (documented directly under the top module)
+#         # autoencoder
+#         str(pkg_dir / "autoencoders" / "multilayer_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "conditional_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "lstm_seq2one_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "cnn_seq2one_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "bidirectional_lstm_seq2one_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "cnn_seq_autoencoder.py"),
+#         str(pkg_dir / "autoencoders" / "lstm_seq2seq_autoencoder.py"),
+#         # anomaly score
+#         str(pkg_dir / "anomaly_scores" / "rmse_score.py"),
+#         str(pkg_dir / "anomaly_scores" / "mahalanobis_score.py"),
+#         # threshold selector
+#         str(pkg_dir / "threshold_selectors" / "adaptive_threshold.py"),
+#         str(pkg_dir / "threshold_selectors" / "fdr_threshold.py"),
+#         str(pkg_dir / "threshold_selectors" / "fbeta_threshold.py"),
+#         str(pkg_dir / "threshold_selectors" / "quantile_threshold.py"),
+#         # main class (documented directly under the top module)
+#         str(pkg_dir / "fault_detector.py"),
+#     ])
 
 
 def _skip_sklearn_dunders(app, what, name, obj, skip, options):
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index ec64462e..9614e0db 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -68,6 +68,7 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
         ) if protect else []
 
         # Data clipping (outlier clipping)
+        # TODO: what happens in DataClipper if data contains non-numerical features?
         if self.config.data_clipping:
             logger.debug('Clip data before scaling.')
             clipper_params = self.config.data_clipping_params.copy()

From 4e9d86dbc0e2bec690721cb91de10d395accbe79 Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Wed, 2 Sep 2026 13:19:08 +0200
Subject: [PATCH 26/35] Fix Sphinx warnings and improve API docs readability

Fix RST formatting issues in docstrings (numbered lists, bullet alignment,
indentation) that caused Sphinx build warnings. Re-enable run_apidoc in
conf.py with templatedir, and strip 'energy_fault_detector.' prefix from
auto-generated API page titles for a cleaner sidebar. Exclude sklearn
boilerplate methods (set_fit_request, set_output, __sklearn_is_fitted__,
etc.) from autodoc via exclude-members and a high-priority skip-member
handler. Document generate_quickstart_config by un-excluding config/
__init__.py and using :no-index: to prevent duplicate Config object
warnings.
---
 docs/conf.py                            | 136 ++++++++++++------------
 energy_fault_detector/fault_detector.py |   1 -
 2 files changed, 68 insertions(+), 69 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 54303d3c..ed69f7f7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -219,74 +219,74 @@
 ]
 
 
-# def run_apidoc(app):
-#     """Generate API .rst files with sphinx-apidoc, like in CI:
-
-#        sphinx-apidoc -o docs energy_fault_detector/ --module-first --force --separate --templatedir docs/_templates
-#     """
-
-#     # Only run this step if we're building the HTML docs
-#     if app.builder.name != "html":
-#         return
-
-#     from sphinx.ext.apidoc import main as apidoc_main
-#     from pathlib import Path
-
-#     here = Path(__file__).parent
-#     pkg_dir = here.parent / IMPORT_NAME
-#     out_dir = here
-#     templates = here / "_templates"
-
-#     apidoc_main([
-#         "-o", str(out_dir),
-#         str(pkg_dir),
-#         "--module-first",
-#         "--force",
-#         "--separate",
-#         "--templatedir", str(templates),
-#         # Following files are skipped
-#         # Quick fault detection internals
-#         str(pkg_dir / "main.py"),
-#         str(pkg_dir / "quick_fault_detection" / "configuration.py"),
-#         str(pkg_dir / "quick_fault_detection" / "data_loading.py"),
-#         str(pkg_dir / "quick_fault_detection" / "optimization.py"),
-#         str(pkg_dir / "quick_fault_detection" / "output.py"),
-#         str(pkg_dir / "quick_fault_detection" / "pipeline.py"),
-#         str(pkg_dir / "quick_fault_detection" / "quick_fault_detector.py"),
-#         # core internals
-#         str(pkg_dir / "core" / "anomaly_score.py"),
-#         str(pkg_dir / "core" / "data_transformer.py"),
-#         str(pkg_dir / "core" / "threshold_selector.py"),
-#         str(pkg_dir / "core" / "fault_detection_result.py"),
-#         str(pkg_dir / "core" / "fault_detection_model.py"),
-#         str(pkg_dir / "core" / "model_factory.py"),
-#         str(pkg_dir / "core" / "save_load_mixin.py"),
-#         # internal utilities
-#         str(pkg_dir / "utils" / "index_utils.py"),
-#         # config internals
-#         str(pkg_dir / "config" / "base_config.py"),
-#         str(pkg_dir / "config" / "config.py"),
-#         str(pkg_dir / "config" / "quickstart_config.py"),
-#         # Class specific files (documented directly under the top module)
-#         # autoencoder
-#         str(pkg_dir / "autoencoders" / "multilayer_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "conditional_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "lstm_seq2one_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "cnn_seq2one_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "bidirectional_lstm_seq2one_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "cnn_seq_autoencoder.py"),
-#         str(pkg_dir / "autoencoders" / "lstm_seq2seq_autoencoder.py"),
-#         # anomaly score
-#         str(pkg_dir / "anomaly_scores" / "rmse_score.py"),
-#         str(pkg_dir / "anomaly_scores" / "mahalanobis_score.py"),
-#         # threshold selector
-#         str(pkg_dir / "threshold_selectors" / "adaptive_threshold.py"),
-#         str(pkg_dir / "threshold_selectors" / "fdr_threshold.py"),
-#         str(pkg_dir / "threshold_selectors" / "fbeta_threshold.py"),
-#         str(pkg_dir / "threshold_selectors" / "quantile_threshold.py"),
-#         # main class (documented directly under the top module)
-#         str(pkg_dir / "fault_detector.py"),
-#     ])
+def run_apidoc(app):
+    """Generate API .rst files with sphinx-apidoc, like in CI:
+
+       sphinx-apidoc -o docs energy_fault_detector/ --module-first --force --separate --templatedir docs/_templates
+    """
+
+    # Only run this step if we're building the HTML docs
+    if app.builder.name != "html":
+        return
+
+    from sphinx.ext.apidoc import main as apidoc_main
+    from pathlib import Path
+
+    here = Path(__file__).parent
+    pkg_dir = here.parent / IMPORT_NAME
+    out_dir = here
+    templates = here / "_templates"
+
+    apidoc_main([
+        "-o", str(out_dir),
+        str(pkg_dir),
+        "--module-first",
+        "--force",
+        "--separate",
+        "--templatedir", str(templates),
+        # Following files are skipped
+        # Quick fault detection internals
+        str(pkg_dir / "main.py"),
+        str(pkg_dir / "quick_fault_detection" / "configuration.py"),
+        str(pkg_dir / "quick_fault_detection" / "data_loading.py"),
+        str(pkg_dir / "quick_fault_detection" / "optimization.py"),
+        str(pkg_dir / "quick_fault_detection" / "output.py"),
+        str(pkg_dir / "quick_fault_detection" / "pipeline.py"),
+        str(pkg_dir / "quick_fault_detection" / "quick_fault_detector.py"),
+        # core internals
+        str(pkg_dir / "core" / "anomaly_score.py"),
+        str(pkg_dir / "core" / "data_transformer.py"),
+        str(pkg_dir / "core" / "threshold_selector.py"),
+        str(pkg_dir / "core" / "fault_detection_result.py"),
+        str(pkg_dir / "core" / "fault_detection_model.py"),
+        str(pkg_dir / "core" / "model_factory.py"),
+        str(pkg_dir / "core" / "save_load_mixin.py"),
+        # internal utilities
+        str(pkg_dir / "utils" / "index_utils.py"),
+        # config internals
+        str(pkg_dir / "config" / "base_config.py"),
+        str(pkg_dir / "config" / "config.py"),
+        str(pkg_dir / "config" / "quickstart_config.py"),
+        # Class specific files (documented directly under the top module)
+        # autoencoder
+        str(pkg_dir / "autoencoders" / "multilayer_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "conditional_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "lstm_seq2one_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "cnn_seq2one_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "bidirectional_lstm_seq2one_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "cnn_seq_autoencoder.py"),
+        str(pkg_dir / "autoencoders" / "lstm_seq2seq_autoencoder.py"),
+        # anomaly score
+        str(pkg_dir / "anomaly_scores" / "rmse_score.py"),
+        str(pkg_dir / "anomaly_scores" / "mahalanobis_score.py"),
+        # threshold selector
+        str(pkg_dir / "threshold_selectors" / "adaptive_threshold.py"),
+        str(pkg_dir / "threshold_selectors" / "fdr_threshold.py"),
+        str(pkg_dir / "threshold_selectors" / "fbeta_threshold.py"),
+        str(pkg_dir / "threshold_selectors" / "quantile_threshold.py"),
+        # main class (documented directly under the top module)
+        str(pkg_dir / "fault_detector.py"),
+    ])
 
 
 def _skip_sklearn_dunders(app, what, name, obj, skip, options):
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index 9614e0db..7976f156 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -7,7 +7,6 @@
 
 import pandas as pd
 import numpy as np
-from sklearn.utils.validation import check_is_fitted
 
 from energy_fault_detector.core.fault_detection_model import FaultDetectionModel
 from energy_fault_detector.core.fault_detection_result import FaultDetectionResult, ModelMetadata

From a41b6b0864833636691743bc3f41b42f6fcd2ab9 Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Fri, 17 Jul 2026 17:18:09 +0200
Subject: [PATCH 27/35] CARE2Compare guide and FAQ set up

---
 docs/care2compare_faq.rst      | 335 +++++++++++++++++++++++++++++++++
 docs/care2compare_guide.rst    | 200 ++++++++++++++++++++
 docs/index.rst                 |   2 +
 docs/quick_fault_detection.rst |   6 -
 4 files changed, 537 insertions(+), 6 deletions(-)
 create mode 100644 docs/care2compare_faq.rst
 create mode 100644 docs/care2compare_guide.rst

diff --git a/docs/care2compare_faq.rst b/docs/care2compare_faq.rst
new file mode 100644
index 00000000..5c07f927
--- /dev/null
+++ b/docs/care2compare_faq.rst
@@ -0,0 +1,335 @@
+CARE2Compare FAQ
+================
+
+.. note::
+
+    This page summarizes recurring interpretation questions and caveats around the
+    CARE2Compare dataset and the CARE score as supported in EnergyFaultDetector.
+    For official release notes and dataset updates, also consult the Zenodo record
+    and the companion publication.
+
+General label semantics
+-----------------------
+
+What is the difference between ``status_type_id`` and ``event_label``?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``status_type_id`` is a timestamp-level label that describes the recorded operating mode of the turbine.
+
+``event_label`` is an event-level label indicating whether the prediction section of an event dataset
+contains an anomalous event.
+
+They serve different purposes:
+
+- ``status_type_id`` helps interpret turbine operation and filter data,
+- ``event_label`` is the target for event-level anomaly evaluation.
+
+Should models predict ``status_type_id`` or ``event_label``?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+That depends on the modeling approach.
+
+For early fault detection, a practical strategy is often:
+
+- predict anomaly or normality at timestamp level,
+- then derive an event-level decision from those timestamp-level predictions.
+
+This is also consistent with CARE-style evaluation, where pointwise predictions can be aggregated into
+event-wise decisions.
+
+Why can anomalous events contain timestamps with ``status_type_id = 0``?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This is expected.
+
+The anomaly window between ``event_start`` and ``event_end`` represents an estimated anomaly time frame
+leading up to a fault. During that time, the operator may still have considered the turbine to be in normal
+operation. Therefore, timestamps with ``status_type_id = 0`` can appear inside anomalous events.
+
+This is particularly important for early fault detection, because these timestamps are often the most relevant
+ones from the operator's perspective.
+
+Training and preprocessing
+--------------------------
+
+Should abnormal timestamps be removed from the training data?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Usually yes, if you are training a normal-behavior model.
+
+The dataset provides ``status_type_id`` so users can filter training data. In practice, filtering by status is
+an intended use of the dataset.
+
+Depending on the model, it can also be reasonable to apply additional cleaning such as power-curve-based
+filtering.
+
+Has the dataset already been preprocessed?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+No.
+
+The data was anonymized, but it has not undergone full preprocessing such as:
+
+- missing-value removal,
+- invalid-measurement filtering,
+- normalization or scaling.
+
+Additional preprocessing is generally required before modeling.
+
+What preprocessing is typically needed?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This depends on the wind farm and the model, but common steps include:
+
+- feature selection,
+- NaN imputation,
+- angle transformation,
+- filtering invalid measurements,
+- scaling.
+
+The EnergyFaultDetector package can be used as an example implementation of such preprocessing, but it is
+not the only possible approach.
+
+Does using the dataset require domain knowledge?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Not always to the same extent.
+
+For purely statistical anomaly detection experiments, general data analysis methods may be sufficient up to a
+point. However, domain knowledge becomes increasingly important for:
+
+- feature selection,
+- validating detected anomalies,
+- interpreting event behavior,
+- root-cause-oriented analysis.
+
+Pointwise CARE evaluation
+-------------------------
+
+Which timestamps are used for pointwise evaluation?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For CARE-style pointwise measures, timestamps with normal status are the important ones.
+
+In practice, timestamps with abnormal status labels are excluded from pointwise evaluation, because it is not
+useful to reward detection of a timestamp as anomalous when the operator already knew it was not in normal
+operation.
+
+How is the ground truth for Coverage interpreted?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Coverage is a pointwise F-score computed on anomalous events.
+
+The intended interpretation is:
+
+- timestamps with anomalous status are anomalous,
+- timestamps between ``event_start`` and ``event_end`` of anomalous events are also considered anomalous,
+- all other timestamps are considered normal.
+
+At the same time, abnormal-status timestamps can be omitted from the pointwise evaluation set, so true
+positives can still arise from timestamps with normal status inside the anomalous event window.
+
+Why can true positives still exist after excluding abnormal-status timestamps?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Because anomalous events can contain timestamps with normal status labels.
+
+After excluding abnormal-status timestamps, the remaining timestamps may still lie inside the anomalous event
+window. Those timestamps are still treated as anomalous ground truth for Coverage and can therefore produce
+true positives.
+
+Should samples with ``status_type_id = 0`` inside the event window be excluded from evaluation?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+No.
+
+Those timestamps are often exactly the timestamps of interest for early fault detection. In CARE-style
+evaluation, they are typically retained and used for pointwise evaluation.
+
+Reliability and event decisions
+-------------------------------
+
+How is Reliability computed?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Reliability is an event-wise F-score.
+
+For each event dataset, a criticality measure is accumulated over timestamp-level predictions. In the described
+CARE logic:
+
+- criticality increases when an anomaly is detected on a timestamp with normal status,
+- criticality decreases when no anomaly is detected on a timestamp with normal status,
+- criticality does not change on timestamps with abnormal status.
+
+If the criticality reaches the threshold, the whole event is treated as predicted anomalous; otherwise it is
+treated as predicted normal. Those event-level predictions are then compared with ``event_label`` values.
+
+What is the default criticality threshold?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The default threshold used in the CARE description and package discussions is ``72``.
+
+Why is Reliability not averaged like the other CARE components?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Because Reliability is already computed as an event-wise score across events.
+
+By contrast:
+
+- Coverage,
+- Accuracy,
+- Earliness
+
+are first computed per event and then averaged over the relevant events.
+
+Wind Farm A special handling
+----------------------------
+
+Can ``status_type_id`` be used normally for Wind Farm A?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Not in the same way as for Wind Farms B and C.
+
+For Wind Farm A, the status labels are based on available failure-log information and should mainly be used
+for filtering training data. For prediction-time CARE evaluation, they should largely be ignored.
+
+Why is Wind Farm A different?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For Wind Farms B and C, the status labels reflect anonymized operator-recorded turbine states.
+
+For Wind Farm A, the status labels were derived differently and do not provide the same interpretation value
+for prediction-time evaluation.
+
+Are there known status-label issues in Wind Farm A?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Yes, there have been reported issues in earlier dataset versions.
+
+These included observations such as:
+
+- problematic status assignments in some events,
+- inconsistencies in the expected relation between status IDs 3 and 4,
+- status ID 5 being an internal derived label that may later be removed from some descriptions.
+
+When discussing such issues, it is best to make the statement version-specific.
+
+Wind Farms B and C
+------------------
+
+How should ``status_type_id`` be interpreted for Wind Farms B and C?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For Wind Farms B and C, ``status_type_id`` is an anonymized version of operator-provided SCADA status
+information.
+
+This means a timestamp with ``status_type_id = 0`` reflects that the operator considered the turbine to be in
+normal operation at that time, even if later retrospective analysis identified that timestamp as part of an
+anomaly window.
+
+Model training strategy
+-----------------------
+
+Should I train one model per turbine or one model per wind farm?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There is no universally correct answer.
+
+Possible strategies include:
+
+- one model per turbine,
+- one model per wind farm,
+- hybrid strategies combining local and shared information.
+
+In the referenced benchmark work, individual autoencoder models per turbine were used.
+
+Features and interpretation
+---------------------------
+
+Can the feature names be mapped to more detailed physical sensor identities?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Not fully.
+
+Because of anonymization agreements with data providers, no additional sensor metadata beyond the published
+dataset description may be available.
+
+Are all feature groups equally trustworthy?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Not necessarily.
+
+Version-specific notes indicate implausible values in some Min, Max, and Std features. For practical work,
+Avg features are often the safest starting point, especially in Wind Farm B.
+
+Root-cause analysis and ARCANA
+------------------------------
+
+Was ARCANA used to create the root-cause labels?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+No.
+
+ARCANA was used by the authors to analyze which features contribute to reconstruction error, i.e. for
+model-dependent feature-importance analysis. The root-cause information included in the dataset is based on
+operator feedback and service reports, not on ARCANA-generated labels.
+
+Benchmark reproducibility
+-------------------------
+
+Is the exact original benchmark code available?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Not exactly.
+
+The EnergyFaultDetector repository provides open-source implementations related to the CARE workflow,
+including CARE score support and example notebooks. However, the exact original code used for the paper is
+not guaranteed to be available unchanged.
+
+Can benchmark scores vary when reproduced?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Yes.
+
+Even when following the provided examples and configurations, results can vary due to factors such as model
+training randomness and later package updates.
+
+Timestamps and anonymization
+----------------------------
+
+Are duplicate timestamps within a file expected?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+No, duplicate timestamps within a single CSV file were reported as an issue and later fixed in newer dataset
+versions.
+
+Can different event files still contain overlapping timestamps?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Yes.
+
+Overlaps across different event CSV files are an expected side effect of the anonymization procedure. Event
+files may therefore share timestamp ranges even when they represent different benchmark events.
+
+Can I reconstruct the true chronological order of all events for one asset?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Generally no.
+
+Because timestamps were anonymized per file, the true chronological order across event files is not preserved.
+
+Does this create a data-leakage risk across events?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Potentially yes, if data from multiple events of the same asset are combined without care.
+
+The intended benchmark usage is event-based. If you aggregate across events, you should be aware that
+overlaps introduced by anonymization may make leakage hard to rule out completely.
+
+
+See also
+--------
+
+- :doc:`care2compare_guide`
+- :class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset`
+- :class:`energy_fault_detector.evaluation.care_score.CAREScore`
\ No newline at end of file
diff --git a/docs/care2compare_guide.rst b/docs/care2compare_guide.rst
new file mode 100644
index 00000000..b178d47d
--- /dev/null
+++ b/docs/care2compare_guide.rst
@@ -0,0 +1,200 @@
+CARE2Compare guide
+==================
+
+.. note::
+
+    This page documents how CARE2Compare support is exposed in EnergyFaultDetector.
+    For generated API reference pages, see the corresponding module documentation.
+    For interpretation caveats and recurring dataset questions, see :doc:`care2compare_faq`.
+
+EnergyFaultDetector provides support for:
+
+- loading CARE2Compare event datasets,
+- accessing event metadata,
+- formatting normal-operation masks based on ``status_type_id``,
+- evaluating predictions with the CARE score.
+
+Relevant classes include:
+
+- :class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset`
+- :class:`energy_fault_detector.evaluation.care_score.CAREScore`
+
+Background
+----------
+
+The CARE2Compare dataset and CARE score are introduced in:
+
+`CARE to Compare: A Real-World Benchmark Dataset for Early Fault Detection in Wind Turbine Data <https://doi.org/10.3390/data9120138>`_
+
+In the package, CARE2Compare support is intended to help with two tasks:
+
+#. loading event-based benchmark datasets,
+#. evaluating anomaly predictions in a way that is aligned with the CARE benchmark.
+
+Conceptual overview
+-------------------
+
+The dataset contains two related but distinct kinds of labels:
+
+- ``status_type_id``:
+  timestamp-level operating-status information,
+- ``event_label``:
+  event-level label indicating whether the prediction section contains an anomalous event.
+
+These labels should not be treated as interchangeable.
+
+In particular, anomalous events may still contain timestamps with ``status_type_id = 0``.
+This is expected for CARE2Compare and reflects the difference between:
+
+- the operator's recorded turbine status at a timestamp, and
+- the retrospectively defined anomaly window used for early fault detection.
+
+For details, see :doc:`care2compare_faq`.
+
+Dataset loading
+---------------
+
+The :class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset` helper can be used to
+load local CARE2Compare data or download the dataset automatically, depending on package configuration.
+
+Typical usage:
+
+.. code-block:: python
+
+    from energy_fault_detector.evaluation.care2compare import Care2CompareDataset
+
+    dataset = Care2CompareDataset(
+        path="./CARE_To_Compare",
+        download_dataset=False,
+    )
+
+    x_train, x_test = dataset.load_event_dataset(event_id=53)
+    info = dataset.get_event_info(53)
+
+The returned event metadata can then be used for evaluation, filtering, or reporting.
+
+Formatted loading
+-----------------
+
+If you want a convenience split into values and normal-operation masks, use the formatted loader.
+
+.. code-block:: python
+
+    x_train, train_normal, x_test, test_normal = dataset.load_and_format_event_dataset(event_id=53)
+
+In the package, the normal masks are derived from:
+
+.. code-block:: text
+
+    status_type_id == 0
+
+This is mainly useful when training normal-behavior models or when applying pointwise evaluation logic.
+
+Training-data filtering
+-----------------------
+
+For normal-behavior modeling, it is generally recommended to filter out timestamps that are clearly not
+representative of normal operation.
+
+In practice, this often means using ``status_type_id`` to exclude abnormal operating modes from the
+training data.
+
+However, CARE2Compare should not be understood as fully pre-cleaned data. Depending on the model and
+wind farm, additional preprocessing may still be needed, such as:
+
+- missing-value handling,
+- invalid-measurement filtering,
+- feature selection,
+- angle transformation,
+- scaling.
+
+Example workflow
+----------------
+
+A typical workflow looks like this:
+
+#. load an event dataset,
+#. separate training and prediction sections,
+#. derive or load model predictions for prediction timestamps,
+#. evaluate those predictions with :class:`energy_fault_detector.evaluation.care_score.CAREScore`.
+
+Example:
+
+.. code-block:: python
+
+    from energy_fault_detector.evaluation.care2compare import Care2CompareDataset
+    from energy_fault_detector.evaluation.care_score import CAREScore
+
+    dataset = Care2CompareDataset(path="./CARE_To_Compare", download_dataset=False)
+
+    x_train, train_normal, x_test, test_normal = dataset.load_and_format_event_dataset(event_id=53)
+    info = dataset.get_event_info(53)
+
+    # Example placeholder prediction:
+    # one boolean anomaly prediction per timestamp in the prediction section
+    predicted_anomalies = [False] * len(x_test)
+
+    scorer = CAREScore()
+
+    scorer.evaluate_event(
+        event_start=info["event_start"],
+        event_end=info["event_end"],
+        event_label=info["event_label"],
+        predicted_anomalies=predicted_anomalies,
+        normal_index=test_normal,
+        event_id=53,
+    )
+
+    final_score = scorer.get_final_score()
+
+CARE score overview
+-------------------
+
+The CARE score combines four aspects of detection quality:
+
+- **Coverage**
+- **Accuracy**
+- **Reliability**
+- **Earliness**
+
+At a high level:
+
+- Coverage and Earliness are computed on anomalous events,
+- Accuracy is computed on normal events,
+- Reliability is an event-wise score based on event decisions.
+
+The package implementation follows the CARE benchmark logic provided in the publication and subsequent
+package support. For detailed interpretation notes, see :doc:`care2compare_faq`.
+
+Wind-farm-specific interpretation
+---------------------------------
+
+The meaning and usefulness of ``status_type_id`` differs slightly between wind farms.
+
+For Wind Farms B and C, status labels reflect anonymized operator-provided status information and are
+useful both for training-data filtering and for parts of CARE-style evaluation.
+
+For Wind Farm A, the status labels should mainly be used for training-data filtering. For prediction-time
+CARE evaluation, they should largely be ignored.
+
+This distinction is important when interpreting evaluation results.
+
+Known practical limitations
+---------------------------
+
+Users should be aware of several dataset and benchmark limitations:
+
+- the dataset is anonymized,
+- the data is not fully preprocessed for modeling,
+- exact benchmark code from the original paper is not guaranteed to match the current package state,
+- overlapping timestamps across different event files may occur due to anonymization,
+- some version-specific data-quality notes apply.
+
+These are discussed in more detail in :doc:`care2compare_faq`.
+
+See also
+--------
+
+- :doc:`care2compare_faq`
+- :class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset`
+- :class:`energy_fault_detector.evaluation.care_score.CAREScore`
diff --git a/docs/index.rst b/docs/index.rst
index b75ddfef..0db887b6 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -37,6 +37,8 @@ Installation
     arcana
     models_overview
     sequence_models
+    care2compare_guide
+    care2compare_faq
 
 .. toctree::
     :caption: Advanced usage
diff --git a/docs/quick_fault_detection.rst b/docs/quick_fault_detection.rst
index e34a09fd..383e3c59 100644
--- a/docs/quick_fault_detection.rst
+++ b/docs/quick_fault_detection.rst
@@ -120,12 +120,6 @@ dataclass. An example options file:
    automatic_optimization: true
    enable_debug_plots: false
 
-The underlying helper functions are implemented in:
-
-- :mod:`energy_fault_detector.quick_fault_detection.data_loading`
-- :mod:`energy_fault_detector.quick_fault_detection.configuration`
-- :mod:`energy_fault_detector.quick_fault_detection.pipeline`
-
 Output
 ------
 The CLI writes:

From 304b97d70ee441d6e403b4bedcea3aa2d269413b Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Wed, 2 Sep 2026 13:51:32 +0200
Subject: [PATCH 28/35] Replace 'code-block:: text' with 'code-block:: yaml'
 where necessary.

---
 energy_fault_detector/anomaly_scores/mahalanobis_score.py     | 2 +-
 energy_fault_detector/anomaly_scores/rmse_score.py            | 2 +-
 .../autoencoders/bidirectional_lstm_seq2one_autoencoder.py    | 2 +-
 energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py | 2 +-
 energy_fault_detector/autoencoders/cnn_seq_autoencoder.py     | 2 +-
 energy_fault_detector/autoencoders/conditional_autoencoder.py | 4 ++--
 .../autoencoders/lstm_seq2one_autoencoder.py                  | 2 +-
 .../autoencoders/lstm_seq2seq_autoencoder.py                  | 2 +-
 energy_fault_detector/autoencoders/multilayer_autoencoder.py  | 4 ++--
 energy_fault_detector/data_preprocessing/data_clipper.py      | 2 +-
 energy_fault_detector/data_preprocessing/data_preprocessor.py | 4 ++--
 energy_fault_detector/data_splitting/data_splitter.py         | 2 +-
 energy_fault_detector/root_cause_analysis/arcana.py           | 2 +-
 .../threshold_selectors/adaptive_threshold.py                 | 2 +-
 energy_fault_detector/threshold_selectors/fbeta_threshold.py  | 2 +-
 energy_fault_detector/threshold_selectors/fdr_threshold.py    | 2 +-
 .../threshold_selectors/quantile_threshold.py                 | 2 +-
 17 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/energy_fault_detector/anomaly_scores/mahalanobis_score.py b/energy_fault_detector/anomaly_scores/mahalanobis_score.py
index 556e898f..329b9b65 100644
--- a/energy_fault_detector/anomaly_scores/mahalanobis_score.py
+++ b/energy_fault_detector/anomaly_scores/mahalanobis_score.py
@@ -25,7 +25,7 @@ class MahalanobisScore(AnomalyScore):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           anomaly_score:
diff --git a/energy_fault_detector/anomaly_scores/rmse_score.py b/energy_fault_detector/anomaly_scores/rmse_score.py
index 5c29a0d6..ea3f6850 100644
--- a/energy_fault_detector/anomaly_scores/rmse_score.py
+++ b/energy_fault_detector/anomaly_scores/rmse_score.py
@@ -19,7 +19,7 @@ class RMSEScore(AnomalyScore):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           anomaly_score:
diff --git a/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py b/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py
index 8f31ac93..e00e4f52 100644
--- a/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py
+++ b/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py
@@ -50,7 +50,7 @@ class BidirectionalLSTMSeq2OneAutoencoder(Seq2OneAutoencoder):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           autoencoder:
diff --git a/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py b/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py
index 0cda7e3b..f525729d 100644
--- a/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py
+++ b/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py
@@ -64,7 +64,7 @@ class CNNSeq2OneAutoencoder(Seq2OneAutoencoder):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           autoencoder:
diff --git a/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py b/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py
index 0c7b9c1d..bccb9b20 100644
--- a/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py
+++ b/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py
@@ -38,7 +38,7 @@ class CNNAutoencoder(Seq2SeqAutoencoder):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           autoencoder:
diff --git a/energy_fault_detector/autoencoders/conditional_autoencoder.py b/energy_fault_detector/autoencoders/conditional_autoencoder.py
index e00357a1..4f3119a5 100644
--- a/energy_fault_detector/autoencoders/conditional_autoencoder.py
+++ b/energy_fault_detector/autoencoders/conditional_autoencoder.py
@@ -35,7 +35,7 @@ class ConditionalAE(Autoencoder):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           autoencoder:
@@ -44,7 +44,7 @@ class ConditionalAE(Autoencoder):
               layers: [200]
               code_size: 40
               learning_rate: 0.001
-              batch_size: 128,
+              batch_size: 128
               epochs: 15
               loss_name: mse
               conditional_features:
diff --git a/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py b/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py
index c771f4ee..f02b5cb2 100644
--- a/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py
+++ b/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py
@@ -47,7 +47,7 @@ class LSTMSeq2OneAutoencoder(Seq2OneAutoencoder):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           autoencoder:
diff --git a/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py b/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py
index 3d1fba66..410b33fe 100644
--- a/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py
+++ b/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py
@@ -42,7 +42,7 @@ class LSTMSeqAutoencoder(Seq2SeqAutoencoder):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           autoencoder:
diff --git a/energy_fault_detector/autoencoders/multilayer_autoencoder.py b/energy_fault_detector/autoencoders/multilayer_autoencoder.py
index d5cb1b15..18f38536 100644
--- a/energy_fault_detector/autoencoders/multilayer_autoencoder.py
+++ b/energy_fault_detector/autoencoders/multilayer_autoencoder.py
@@ -35,7 +35,7 @@ class MultilayerAutoencoder(Autoencoder):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           autoencoder:
@@ -44,7 +44,7 @@ class MultilayerAutoencoder(Autoencoder):
               layers: [200]
               code_size: 40
               learning_rate: 0.001
-              batch_size: 128,
+              batch_size: 128
               epochs: 15
               loss_name: mse
     """
diff --git a/energy_fault_detector/data_preprocessing/data_clipper.py b/energy_fault_detector/data_preprocessing/data_clipper.py
index e735d2dc..a0683a32 100644
--- a/energy_fault_detector/data_preprocessing/data_clipper.py
+++ b/energy_fault_detector/data_preprocessing/data_clipper.py
@@ -25,7 +25,7 @@ class DataClipper(DataTransformer):
 
     Configuration example:
 
-        .. code-block:: text
+        .. code-block:: yaml
 
             train:
               data_clipping:
diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py
index c637c860..31399bf7 100644
--- a/energy_fault_detector/data_preprocessing/data_preprocessor.py
+++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py
@@ -82,7 +82,7 @@ def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None:
 
         Configuration example:
 
-            .. code-block:: text
+            .. code-block:: yaml
 
                 train:
                   data_preprocessor:
@@ -103,7 +103,7 @@ def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None:
                         counters: ['energy_total_kwh']
                         compute_rate: False
                         fill_first: 'zero'
-                        reset_strategy: 'rollover',
+                        reset_strategy: 'rollover'
                         rollover_values:
                           'energy_total_kwh': 100000.0
         """
diff --git a/energy_fault_detector/data_splitting/data_splitter.py b/energy_fault_detector/data_splitting/data_splitter.py
index 3fb9463e..1ad7fb0a 100644
--- a/energy_fault_detector/data_splitting/data_splitter.py
+++ b/energy_fault_detector/data_splitting/data_splitter.py
@@ -25,7 +25,7 @@ class BlockDataSplitter:
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           data_splitter:
diff --git a/energy_fault_detector/root_cause_analysis/arcana.py b/energy_fault_detector/root_cause_analysis/arcana.py
index 11387af5..3e5d69da 100644
--- a/energy_fault_detector/root_cause_analysis/arcana.py
+++ b/energy_fault_detector/root_cause_analysis/arcana.py
@@ -65,7 +65,7 @@ class Arcana:
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         root_cause_analysis:
           alpha: 0.8
diff --git a/energy_fault_detector/threshold_selectors/adaptive_threshold.py b/energy_fault_detector/threshold_selectors/adaptive_threshold.py
index 837b10bc..bcae66a1 100644
--- a/energy_fault_detector/threshold_selectors/adaptive_threshold.py
+++ b/energy_fault_detector/threshold_selectors/adaptive_threshold.py
@@ -35,7 +35,7 @@ class AdaptiveThresholdSelector(ThresholdSelector):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           threshold_selector:
diff --git a/energy_fault_detector/threshold_selectors/fbeta_threshold.py b/energy_fault_detector/threshold_selectors/fbeta_threshold.py
index 34bdc676..c7e22651 100644
--- a/energy_fault_detector/threshold_selectors/fbeta_threshold.py
+++ b/energy_fault_detector/threshold_selectors/fbeta_threshold.py
@@ -31,7 +31,7 @@ class FbetaSelector(ThresholdSelector):
 
     Configuration example:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           threshold_selector:
diff --git a/energy_fault_detector/threshold_selectors/fdr_threshold.py b/energy_fault_detector/threshold_selectors/fdr_threshold.py
index beef183f..d4989d89 100644
--- a/energy_fault_detector/threshold_selectors/fdr_threshold.py
+++ b/energy_fault_detector/threshold_selectors/fdr_threshold.py
@@ -24,7 +24,7 @@ class FDRSelector(ThresholdSelector):
 
     Example Configuration:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           threshold_selector:
diff --git a/energy_fault_detector/threshold_selectors/quantile_threshold.py b/energy_fault_detector/threshold_selectors/quantile_threshold.py
index fdb7d9bc..bf7a191f 100644
--- a/energy_fault_detector/threshold_selectors/quantile_threshold.py
+++ b/energy_fault_detector/threshold_selectors/quantile_threshold.py
@@ -21,7 +21,7 @@ class QuantileThresholdSelector(ThresholdSelector):
 
     Example Configuration:
 
-    .. code-block:: text
+    .. code-block:: yaml
 
         train:
           threshold_selector:

From c37ab7177bbb9ef35f230def82bc6c4ecb4d0bc0 Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Wed, 2 Sep 2026 18:38:55 +0200
Subject: [PATCH 29/35] Fix CARE2Compare guide example and refine FAQ

---
 docs/care2compare_faq.rst   | 214 ++++++++++++++++--------------------
 docs/care2compare_guide.rst |  49 ++++++---
 2 files changed, 128 insertions(+), 135 deletions(-)

diff --git a/docs/care2compare_faq.rst b/docs/care2compare_faq.rst
index 5c07f927..f45ee34a 100644
--- a/docs/care2compare_faq.rst
+++ b/docs/care2compare_faq.rst
@@ -6,7 +6,10 @@ CARE2Compare FAQ
     This page summarizes recurring interpretation questions and caveats around the
     CARE2Compare dataset and the CARE score as supported in EnergyFaultDetector.
     For official release notes and dataset updates, also consult the Zenodo record
-    and the companion publication.
+    and the companion publication:
+
+    CARE to Compare: A Real-World Benchmark Dataset for Early Fault Detection in Wind Turbine Data.
+    Data. 2024; 9(12):138. https://doi.org/10.3390/data9120138
 
 General label semantics
 -----------------------
@@ -24,10 +27,18 @@ They serve different purposes:
 - ``status_type_id`` helps interpret turbine operation and filter data,
 - ``event_label`` is the target for event-level anomaly evaluation.
 
+
 Should models predict ``status_type_id`` or ``event_label``?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-That depends on the modeling approach.
+That depends on the goal. For early fault detection, you want to predict the ``event_label``. On the timestamp level,
+this means that you will want to detect anomalies within the provided ``event_start`` and ``event_end`` for events with
+``event_label == 'anomaly'`` and no anomalies if the ``event_label == 'normal'``.
+
+The ``status_type_id`` only indicates whether the wind turbine has an operationally normal state or not. Once a fault is
+known, this state is often already nor normal. Therefore, it is not interesting to detect whether the state is
+anomalous, but it is interesting to find anomalies during expected normal operation before a fault is known or becomes
+critical.
 
 For early fault detection, a practical strategy is often:
 
@@ -37,11 +48,10 @@ For early fault detection, a practical strategy is often:
 This is also consistent with CARE-style evaluation, where pointwise predictions can be aggregated into
 event-wise decisions.
 
+
 Why can anomalous events contain timestamps with ``status_type_id = 0``?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-This is expected.
-
 The anomaly window between ``event_start`` and ``event_end`` represents an estimated anomaly time frame
 leading up to a fault. During that time, the operator may still have considered the turbine to be in normal
 operation. Therefore, timestamps with ``status_type_id = 0`` can appear inside anomalous events.
@@ -49,6 +59,27 @@ operation. Therefore, timestamps with ``status_type_id = 0`` can appear inside a
 This is particularly important for early fault detection, because these timestamps are often the most relevant
 ones from the operator's perspective.
 
+
+How do the status labels of Wind Farm A differ from those of Wind Farms B and C?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+For Wind Farms B and C, the status labels reflect anonymized operator-recorded turbine states.
+
+For Wind Farm A, the status labels were derived differently and do not provide the same interpretation value
+for prediction-time evaluation. The status labels are based on available failure-log information and should mainly be
+used for filtering training data. For prediction-time evaluation, they should largely be ignored.
+
+
+How should ``status_type_id`` be interpreted for Wind Farms B and C?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For Wind Farms B and C, ``status_type_id`` is an anonymized version of operator-provided SCADA status
+information.
+
+This means a timestamp with ``status_type_id = 0`` reflects that the operator considered the turbine to be in
+normal operation at that time, even if later retrospective analysis identified that timestamp as part of an
+anomaly window.
+
+
 Training and preprocessing
 --------------------------
 
@@ -60,21 +91,23 @@ Usually yes, if you are training a normal-behavior model.
 The dataset provides ``status_type_id`` so users can filter training data. In practice, filtering by status is
 an intended use of the dataset.
 
-Depending on the model, it can also be reasonable to apply additional cleaning such as power-curve-based
+Depending on the model and the goal, it can also be reasonable to apply additional cleaning such as power-curve-based
 filtering.
 
+
 Has the dataset already been preprocessed?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-No.
-
-The data was anonymized, but it has not undergone full preprocessing such as:
+No. The data was anonymized, but it has not undergone full preprocessing such as:
 
 - missing-value removal,
 - invalid-measurement filtering,
 - normalization or scaling.
 
 Additional preprocessing is generally required before modeling.
+Please check the dataset description and dataset README on `Zenodo <https://doi.org/10.5281/zenodo.14958989>`_
+for details on known data quality issues.
+
 
 What preprocessing is typically needed?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -87,14 +120,12 @@ This depends on the wind farm and the model, but common steps include:
 - filtering invalid measurements,
 - scaling.
 
-The EnergyFaultDetector package can be used as an example implementation of such preprocessing, but it is
-not the only possible approach.
+An example is found in the notebook `CARE to Compare.ipynb <https://github.com/AEFDI/EnergyFaultDetector/blob/main/notebooks/CARE%20to%20Compare/CARE%20to%20Compare.ipynb>`_
+
 
 Does using the dataset require domain knowledge?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Not always to the same extent.
-
 For purely statistical anomaly detection experiments, general data analysis methods may be sufficient up to a
 point. However, domain knowledge becomes increasingly important for:
 
@@ -103,31 +134,40 @@ point. However, domain knowledge becomes increasingly important for:
 - interpreting event behavior,
 - root-cause-oriented analysis.
 
-Pointwise CARE evaluation
--------------------------
+
+Evaluation
+----------
 
 Which timestamps are used for pointwise evaluation?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 For CARE-style pointwise measures, timestamps with normal status are the important ones.
 
-In practice, timestamps with abnormal status labels are excluded from pointwise evaluation, because it is not
+We consider a timestamp to be correctly detected as anomalous, if the timestamp is part of an event with
+``event_label == 'anomaly'``. It is correctly detected as normal behaviour if the timestamp is part of an event with
+``event_label == 'normal'``.
+
+Timestamps with abnormal status labels are excluded from pointwise evaluation, because it is not
 useful to reward detection of a timestamp as anomalous when the operator already knew it was not in normal
-operation.
+operation. Note that we only apply this rule to wind farm B and C. For WF A, the status information can only be used
+for filtering training data and not for evaluation.
+
 
 How is the ground truth for Coverage interpreted?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Coverage is a pointwise F-score computed on anomalous events.
 
-The intended interpretation is:
+The ground truth is as follows:
 
 - timestamps with anomalous status are anomalous,
 - timestamps between ``event_start`` and ``event_end`` of anomalous events are also considered anomalous,
 - all other timestamps are considered normal.
 
-At the same time, abnormal-status timestamps can be omitted from the pointwise evaluation set, so true
-positives can still arise from timestamps with normal status inside the anomalous event window.
+For wind farm B and C the first type of true anomalies are ignored, because it is not
+useful to reward detection of a timestamp as anomalous when the operator already knew it was not in normal
+operation.
+
 
 Why can true positives still exist after excluding abnormal-status timestamps?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -138,24 +178,14 @@ After excluding abnormal-status timestamps, the remaining timestamps may still l
 window. Those timestamps are still treated as anomalous ground truth for Coverage and can therefore produce
 true positives.
 
-Should samples with ``status_type_id = 0`` inside the event window be excluded from evaluation?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-No.
-
-Those timestamps are often exactly the timestamps of interest for early fault detection. In CARE-style
-evaluation, they are typically retained and used for pointwise evaluation.
-
-Reliability and event decisions
--------------------------------
 
 How is Reliability computed?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Reliability is an event-wise F-score.
+Reliability is an event-wise F-score. The model used should provide a decision for a complete event. This decision can
+then be compared with the ``event_label``.
 
-For each event dataset, a criticality measure is accumulated over timestamp-level predictions. In the described
-CARE logic:
+As an example, in the paper we used a criticality measure, which is accumulated over timestamp-level predictions:
 
 - criticality increases when an anomaly is detected on a timestamp with normal status,
 - criticality decreases when no anomaly is detected on a timestamp with normal status,
@@ -164,68 +194,21 @@ CARE logic:
 If the criticality reaches the threshold, the whole event is treated as predicted anomalous; otherwise it is
 treated as predicted normal. Those event-level predictions are then compared with ``event_label`` values.
 
+
 What is the default criticality threshold?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The default threshold used in the CARE description and package discussions is ``72``.
+You should of course tune this threshold for your dataset.
+
 
 Why is Reliability not averaged like the other CARE components?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Because Reliability is already computed as an event-wise score across events.
 
-By contrast:
-
-- Coverage,
-- Accuracy,
-- Earliness
-
-are first computed per event and then averaged over the relevant events.
-
-Wind Farm A special handling
-----------------------------
-
-Can ``status_type_id`` be used normally for Wind Farm A?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Not in the same way as for Wind Farms B and C.
+By contrast, Coverage, Accuracy, Earliness are first computed per event and then averaged over the relevant events.
 
-For Wind Farm A, the status labels are based on available failure-log information and should mainly be used
-for filtering training data. For prediction-time CARE evaluation, they should largely be ignored.
-
-Why is Wind Farm A different?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-For Wind Farms B and C, the status labels reflect anonymized operator-recorded turbine states.
-
-For Wind Farm A, the status labels were derived differently and do not provide the same interpretation value
-for prediction-time evaluation.
-
-Are there known status-label issues in Wind Farm A?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Yes, there have been reported issues in earlier dataset versions.
-
-These included observations such as:
-
-- problematic status assignments in some events,
-- inconsistencies in the expected relation between status IDs 3 and 4,
-- status ID 5 being an internal derived label that may later be removed from some descriptions.
-
-When discussing such issues, it is best to make the statement version-specific.
-
-Wind Farms B and C
-------------------
-
-How should ``status_type_id`` be interpreted for Wind Farms B and C?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-For Wind Farms B and C, ``status_type_id`` is an anonymized version of operator-provided SCADA status
-information.
-
-This means a timestamp with ``status_type_id = 0`` reflects that the operator considered the turbine to be in
-normal operation at that time, even if later retrospective analysis identified that timestamp as part of an
-anomaly window.
 
 Model training strategy
 -----------------------
@@ -241,7 +224,17 @@ Possible strategies include:
 - one model per wind farm,
 - hybrid strategies combining local and shared information.
 
-In the referenced benchmark work, individual autoencoder models per turbine were used.
+
+For early fault detection, a practical strategy is often:
+
+- predict anomaly or normality at timestamp level,
+- then derive an event-level decision from those timestamp-level predictions.
+
+This is also consistent with CARE-style evaluation, where pointwise predictions can be aggregated into
+event-wise decisions.
+
+In the dataset paper, individual autoencoder models per turbine were used.
+
 
 Features and interpretation
 ---------------------------
@@ -249,30 +242,33 @@ Features and interpretation
 Can the feature names be mapped to more detailed physical sensor identities?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Not fully.
-
 Because of anonymization agreements with data providers, no additional sensor metadata beyond the published
 dataset description may be available.
 
+The feature descriptions are the most detailed available for this dataset. If you load the data using the
+:class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset`, the feature names are based on these
+descriptions (instead of the enumerated feature names of the csv files).
+
+
 Are all feature groups equally trustworthy?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Not necessarily.
-
-Version-specific notes indicate implausible values in some Min, Max, and Std features. For practical work,
+There are implausible values in some Min, Max, and Std features. For practical work,
 Avg features are often the safest starting point, especially in Wind Farm B.
 
+Please check the dataset description and dataset README on `Zenodo <https://doi.org/10.5281/zenodo.14958989>`_
+for details on known data quality issues.
+
+
 Root-cause analysis and ARCANA
 ------------------------------
 
 Was ARCANA used to create the root-cause labels?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-No.
+No. The root-cause information included in the dataset is based on operator feedback and service reports,
+not on ARCANA-generated labels.
 
-ARCANA was used by the authors to analyze which features contribute to reconstruction error, i.e. for
-model-dependent feature-importance analysis. The root-cause information included in the dataset is based on
-operator feedback and service reports, not on ARCANA-generated labels.
 
 Benchmark reproducibility
 -------------------------
@@ -284,47 +280,31 @@ Not exactly.
 
 The EnergyFaultDetector repository provides open-source implementations related to the CARE workflow,
 including CARE score support and example notebooks. However, the exact original code used for the paper is
-not guaranteed to be available unchanged.
+not available.
+
 
 Can benchmark scores vary when reproduced?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Yes.
+Yes. Even when following the provided examples and configurations, results can vary due to factors such as random
+initialization of the models and later package updates.
 
-Even when following the provided examples and configurations, results can vary due to factors such as model
-training randomness and later package updates.
 
 Timestamps and anonymization
 ----------------------------
 
-Are duplicate timestamps within a file expected?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-No, duplicate timestamps within a single CSV file were reported as an issue and later fixed in newer dataset
-versions.
-
 Can different event files still contain overlapping timestamps?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Yes.
+Yes. Overlaps across different event CSV files are an expected side effect of the anonymization procedure. Event
+files may therefore share timestamp ranges even when they represent different events.
 
-Overlaps across different event CSV files are an expected side effect of the anonymization procedure. Event
-files may therefore share timestamp ranges even when they represent different benchmark events.
 
 Can I reconstruct the true chronological order of all events for one asset?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Generally no.
-
-Because timestamps were anonymized per file, the true chronological order across event files is not preserved.
-
-Does this create a data-leakage risk across events?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Potentially yes, if data from multiple events of the same asset are combined without care.
-
-The intended benchmark usage is event-based. If you aggregate across events, you should be aware that
-overlaps introduced by anonymization may make leakage hard to rule out completely.
+Generally no. Because timestamps were anonymized per file, the true chronological order across event files is
+not preserved.
 
 
 See also
diff --git a/docs/care2compare_guide.rst b/docs/care2compare_guide.rst
index b178d47d..795f5e62 100644
--- a/docs/care2compare_guide.rst
+++ b/docs/care2compare_guide.rst
@@ -68,7 +68,7 @@ Typical usage:
         download_dataset=False,
     )
 
-    x_train, x_test = dataset.load_event_dataset(event_id=53)
+    x_train, x_test = dataset.load_event_dataset(event_id=53, index_column="time_stamp")
     info = dataset.get_event_info(53)
 
 The returned event metadata can then be used for evaluation, filtering, or reporting.
@@ -122,31 +122,42 @@ Example:
 
 .. code-block:: python
 
+    import pandas as pd
     from energy_fault_detector.evaluation.care2compare import Care2CompareDataset
     from energy_fault_detector.evaluation.care_score import CAREScore
 
     dataset = Care2CompareDataset(path="./CARE_To_Compare", download_dataset=False)
-
-    x_train, train_normal, x_test, test_normal = dataset.load_and_format_event_dataset(event_id=53)
-    info = dataset.get_event_info(53)
-
-    # Example placeholder prediction:
-    # one boolean anomaly prediction per timestamp in the prediction section
-    predicted_anomalies = [False] * len(x_test)
-
     scorer = CAREScore()
 
-    scorer.evaluate_event(
-        event_start=info["event_start"],
-        event_end=info["event_end"],
-        event_label=info["event_label"],
-        predicted_anomalies=predicted_anomalies,
-        normal_index=test_normal,
-        event_id=53,
-    )
+    # Iterate over the events of a chosen wind farm. The data is loaded with
+    # timestamps as the index so they match the event_start / event_end metadata.
+    for x_train, train_normal, x_test, test_normal, event_id in dataset.iter_formatted_datasets(
+        wind_farm="B", index_column="time_stamp",
+    ):
+        info = dataset.get_event_info(event_id)
+
+        # Example placeholder prediction: one boolean anomaly prediction per
+        # timestamp in the prediction section, indexed like x_test.
+        predicted_anomalies = pd.Series(False, index=x_test.index)
+
+        scorer.evaluate_event(
+            event_start=info["event_start"],
+            event_end=info["event_end"],
+            event_label=info["event_label"],
+            predicted_anomalies=predicted_anomalies,
+            normal_index=test_normal,
+            event_id=event_id,
+        )
 
     final_score = scorer.get_final_score()
 
+.. note::
+
+    ``get_final_score`` requires at least one evaluated ``anomaly`` event *and* one
+    evaluated ``normal`` event. A single ``evaluate_event`` call is fine for a quick
+    smoke test, but the final CARE score can only be computed once events of both
+    labels have been added.
+
 CARE score overview
 -------------------
 
@@ -175,7 +186,9 @@ For Wind Farms B and C, status labels reflect anonymized operator-provided statu
 useful both for training-data filtering and for parts of CARE-style evaluation.
 
 For Wind Farm A, the status labels should mainly be used for training-data filtering. For prediction-time
-CARE evaluation, they should largely be ignored.
+CARE evaluation, they should largely be ignored; pass ``ignore_normal_index=True`` to
+:meth:`~energy_fault_detector.evaluation.care_score.CAREScore.evaluate_event` so that every timestamp is
+evaluated regardless of ``status_type_id``.
 
 This distinction is important when interpreting evaluation results.
 

From 0d9e0d58380fab990bc38e388b9dea0135300b2e Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Wed, 2 Sep 2026 19:09:41 +0200
Subject: [PATCH 30/35] Use Zenodo concept DOI and fix CARE2Compare notebook
 API calls

Replace version-specific Zenodo DOIs (14958989, 15846963) with the concept
DOI 10958774 across docs, source, and notebooks so links always point to
the latest version. The concept ID resolves correctly via Zenodo's REST
API, so download_zenodo_data and Care2CompareDataset now default to it.

Fix broken evaluate_event kwargs in the CARE to Compare notebook
(event_start_id/event_end_id -> event_start/event_end), switch all
examples to index_column='time_stamp', and update the data directory name
to CARE_To_Compare.
---
 README.md                                     |  2 +-
 docs/care2compare_faq.rst                     |  4 ++--
 .../evaluation/care2compare.py                |  4 ++--
 energy_fault_detector/utils/data_downloads.py |  6 ++---
 .../CARE to Compare/CARE to Compare.ipynb     | 22 +++++++++----------
 .../Example - Create new model classes.ipynb  |  2 +-
 notebooks/Example - FaultDetector Usage.ipynb |  2 +-
 ...xample - Hyperparameter Optimization.ipynb |  2 +-
 notebooks/Example - Model Finetuning.ipynb    |  2 +-
 .../Example - Quick Fault Detection.ipynb     |  2 +-
 10 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/README.md b/README.md
index 84e90eeb..0a0042d5 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,7 @@ Run the full pipeline (train → predict → events → ARCANA) in a single comm
 ```bash
 quick_fault_detector path/to/data.csv
 ```
-For [CARE2Compare](https://doi.org/10.5281/zenodo.14958989) data:
+For [CARE2Compare](https://doi.org/10.5281/zenodo.10958774) data:
 ```bash
 quick_fault_detector path/to/c2c_dataset.csv --c2c_example
 ```
diff --git a/docs/care2compare_faq.rst b/docs/care2compare_faq.rst
index f45ee34a..cb1fc8cd 100644
--- a/docs/care2compare_faq.rst
+++ b/docs/care2compare_faq.rst
@@ -105,7 +105,7 @@ No. The data was anonymized, but it has not undergone full preprocessing such as
 - normalization or scaling.
 
 Additional preprocessing is generally required before modeling.
-Please check the dataset description and dataset README on `Zenodo <https://doi.org/10.5281/zenodo.14958989>`_
+Please check the dataset description and dataset README on `Zenodo <https://doi.org/10.5281/zenodo.10958774>`_
 for details on known data quality issues.
 
 
@@ -256,7 +256,7 @@ Are all feature groups equally trustworthy?
 There are implausible values in some Min, Max, and Std features. For practical work,
 Avg features are often the safest starting point, especially in Wind Farm B.
 
-Please check the dataset description and dataset README on `Zenodo <https://doi.org/10.5281/zenodo.14958989>`_
+Please check the dataset description and dataset README on `Zenodo <https://doi.org/10.5281/zenodo.10958774>`_
 for details on known data quality issues.
 
 
diff --git a/energy_fault_detector/evaluation/care2compare.py b/energy_fault_detector/evaluation/care2compare.py
index 9e1afb03..37ebe18d 100644
--- a/energy_fault_detector/evaluation/care2compare.py
+++ b/energy_fault_detector/evaluation/care2compare.py
@@ -15,7 +15,7 @@
 class Care2CompareDataset:
     """Loads the Care to Compare Dataset (accompanying paper https://doi.org/10.3390/data9120138).
 
-    The data can be downloaded either manually from https://doi.org/10.5281/zenodo.14958989 (in this case specify
+    The data can be downloaded either manually from https://doi.org/10.5281/zenodo.10958774 (in this case specify
     `path`) or it can be downloaded automatically by setting download_dataset to True.
 
     By default, only the averages are read. See statistics argument of the data loading methods.
@@ -46,7 +46,7 @@ def __init__(self, path: Union[Path, str] = "./CARE_To_Compare", download_datase
             logger.info("Downloading CARE to Compare dataset (~5 GB) from zenodo. Depending on your internet "
                         "connection this can take some time. Additionally the downloaded zip-file will be unzipped "
                         "(~20 GB) which can also take longer.")
-            path = download_zenodo_data(identifier="10.5281/zenodo.15846963", dest=path, overwrite=False)
+            path = download_zenodo_data(identifier="10.5281/zenodo.10958774", dest=path, overwrite=False)
         self.path: Path = Path(path)
 
         self.wind_farms: Dict[str, Path] = {
diff --git a/energy_fault_detector/utils/data_downloads.py b/energy_fault_detector/utils/data_downloads.py
index 0d403535..ce50a697 100644
--- a/energy_fault_detector/utils/data_downloads.py
+++ b/energy_fault_detector/utils/data_downloads.py
@@ -22,7 +22,7 @@ def parse_record_id(identifier: str) -> str:
 
     Accepts:
       - Numeric ID (e.g., "15846963")
-      - DOI (e.g., "10.5281/zenodo.15846963")
+      - DOI (e.g., "10.5281/zenodo.10958774")
       - Record URL (e.g., "https://zenodo.org/records/15846963")
 
     Args:
@@ -197,7 +197,7 @@ def prepare_output_dir(out_dir: Path, overwrite: bool) -> None:
     out_dir.mkdir(parents=True, exist_ok=True)
 
 
-def download_zenodo_data(identifier: str = "10.5281/zenodo.15846963", dest: Path = "./downloads",
+def download_zenodo_data(identifier: str = "10.5281/zenodo.10958774", dest: Path = "./downloads",
                          remove_zip: bool = True, overwrite: bool = False, flatten_file_structure: bool = True,
                          expected_file_types: Union[List[str], str] = "*.csv") -> Path:
     """ Download a Zenodo record via API and unzip any .zip files.
@@ -207,7 +207,7 @@ def download_zenodo_data(identifier: str = "10.5281/zenodo.15846963", dest: Path
     that result from extracting ZIP archives.
 
     Args:
-        identifier (str): Zenodo record ID, DOI (e.g., 10.5281/zenodo.15846963), or record URL.
+        identifier (str): Zenodo record ID, DOI (e.g., 10.5281/zenodo.10958774), or record URL.
             Defaults to the CARE2Compare dataset.
         dest (Path): Local output directory to save downloaded files. (default: downloads)
         remove_zip (bool): If True, ZIP archives will be removed after extraction.
diff --git a/notebooks/CARE to Compare/CARE to Compare.ipynb b/notebooks/CARE to Compare/CARE to Compare.ipynb
index 9303b422..f5fa4b21 100644
--- a/notebooks/CARE to Compare/CARE to Compare.ipynb	
+++ b/notebooks/CARE to Compare/CARE to Compare.ipynb	
@@ -50,7 +50,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "data_dir = Path('..') / '..' / 'Care_To_Compare_v6'"
+    "data_dir = Path('..') / '..' / 'CARE_To_Compare'"
    ]
   },
   {
@@ -81,7 +81,7 @@
    "outputs": [],
    "source": [
     "# select data for a specific event\n",
-    "x, y = c2c.load_event_dataset(0, statistics=['average', 'std_dev'])\n",
+    "x, y = c2c.load_event_dataset(0, statistics=['average', 'std_dev'], index_column='time_stamp')\n",
     "x.head()"
    ]
   },
@@ -101,7 +101,7 @@
    "outputs": [],
    "source": [
     "c2c = Care2CompareDataset(data_dir)\n",
-    "index_column = 'id'  # us time_stamp as index column if you are using the TimestampTransformer\n",
+    "index_column = 'time_stamp'\n",
     "suffix = ''\n",
     "\n",
     "configs = {\n",
@@ -163,8 +163,8 @@
     "        \n",
     "        # evaluate event\n",
     "        event_info = c2c.event_info_all[c2c.event_info_all['event_id'] == event_id]\n",
-    "        event_start = event_info['event_start_id'].iloc[0] if index_column == 'id' else event_info['event_start'].iloc[0]\n",
-    "        event_end = event_info['event_end_id'].iloc[0] if index_column == 'id' else event_info['event_end'].iloc[0]\n",
+    "        event_start = event_info['event_start'].iloc[0]\n",
+    "        event_end = event_info['event_end'].iloc[0]\n",
     "        care_score.evaluate_event(\n",
     "            event_id=event_id,\n",
     "            event_start=event_start,\n",
@@ -240,13 +240,13 @@
     "c2c.update_c2c_config(config, wf)\n",
     "care_score = CAREScore(coverage_beta=0.5, reliability_beta=0.5, anomaly_detection_method='criticality')\n",
     "\n",
-    "for x_train, asset_id, event_ids in c2c.iter_train_datasets_per_asset(wf):\n",
+    "for x_train, asset_id, event_ids in c2c.iter_train_datasets_per_asset(wf, index_column='time_stamp'):\n",
     "    x_train = x_train.reset_index(drop=True)\n",
     "\n",
     "    # create normal index\n",
     "    y_train = x_train['status_type_id'] == 0\n",
     "    # drop unnecessary features (keep asset ID as a sort of condition)\n",
-    "    x_train = x_train.drop(['time_stamp', 'status_type_id'], axis=1)\n",
+    "    x_train = x_train.drop(['time_stamp', 'status_type_id'], axis=1, errors='ignore')\n",
     "    \n",
     "    # create model\n",
     "    model = FaultDetector(config)\n",
@@ -257,17 +257,17 @@
     "    for event_id in event_ids:\n",
     "        print(event_id)\n",
     "        # test model\n",
-    "        x_test = c2c.load_event_dataset(event_id=event_id, test_only=True)\n",
+    "        x_test = c2c.load_event_dataset(event_id=event_id, test_only=True, index_column='time_stamp')\n",
     "        y_test = x_test['status_type_id'] == 0\n",
-    "        x_test = x_test.drop(['time_stamp', 'asset_id', 'status_type_id'], axis=1)\n",
+    "        x_test = x_test.drop(['time_stamp', 'asset_id', 'status_type_id'], axis=1, errors='ignore')\n",
     "        prediction = model.predict(x_test)\n",
     "        \n",
     "        # evaluate event\n",
     "        event_info = c2c.event_info_all[c2c.event_info_all['event_id'] == event_id]\n",
     "        care_score.evaluate_event(\n",
     "            event_id=event_id,\n",
-    "            event_start_id=event_info['event_start_id'].iloc[0],\n",
-    "            event_end_id=event_info['event_end_id'].iloc[0],\n",
+    "            event_start=event_info['event_start'].iloc[0],\n",
+    "            event_end=event_info['event_end'].iloc[0],\n",
     "            event_label=event_info['event_label'].iloc[0],\n",
     "            normal_index=y_test,\n",
     "            predicted_anomalies=prediction.predicted_anomalies,\n",
diff --git a/notebooks/Example - Create new model classes.ipynb b/notebooks/Example - Create new model classes.ipynb
index fe54108a..f87fee34 100644
--- a/notebooks/Example - Create new model classes.ipynb	
+++ b/notebooks/Example - Create new model classes.ipynb	
@@ -16,7 +16,7 @@
     "- A simple non-symmetric autoencoder.\n",
     "- A Threshold selection method based on the telemanom model from the paper [Detecting Spacecraft Anomalies Using LSTMs and Nonparametric Dynamic Thresholding](https://doi.org/10.1145/3219819.3219845).\n",
     "\n",
-    "For the usage example of the new classes the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.14958989) is used."
+    "For the usage example of the new classes the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774) is used."
    ],
    "id": "7d0933b25f2a9b46"
   },
diff --git a/notebooks/Example - FaultDetector Usage.ipynb b/notebooks/Example - FaultDetector Usage.ipynb
index cd8271a1..d25cb45a 100644
--- a/notebooks/Example - FaultDetector Usage.ipynb	
+++ b/notebooks/Example - FaultDetector Usage.ipynb	
@@ -13,7 +13,7 @@
     "5. Evaluation of results\n",
     "6. Root cause analysis using ARCANA\n",
     "\n",
-    "Get the dataset which is used in this notebook here: [CARE to Compare dataset](https://doi.org/10.5281/zenodo.15846963)"
+    "Get the dataset which is used in this notebook here: [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774)"
    ],
    "id": "687630b4d4423666"
   },
diff --git a/notebooks/Example - Hyperparameter Optimization.ipynb b/notebooks/Example - Hyperparameter Optimization.ipynb
index bc4e52ac..e77392c1 100644
--- a/notebooks/Example - Hyperparameter Optimization.ipynb	
+++ b/notebooks/Example - Hyperparameter Optimization.ipynb	
@@ -11,7 +11,7 @@
     "2. Optimizing the FaultDetector classification performance using the Fbeta score\n",
     "3. Optimizing the FaultDetector classification performance using the CARE-score\n",
     "\n",
-    "The optimization is done using the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.14958989)"
+    "The optimization is done using the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774)"
    ],
    "id": "acc177b6ece47b21"
   },
diff --git a/notebooks/Example - Model Finetuning.ipynb b/notebooks/Example - Model Finetuning.ipynb
index 9b6ff27e..aa129eb5 100644
--- a/notebooks/Example - Model Finetuning.ipynb	
+++ b/notebooks/Example - Model Finetuning.ipynb	
@@ -5,7 +5,7 @@
    "source": [
     "# Model Finetuning\n",
     "\n",
-    "This notebook explains how to use the fine-tuning function of an FaultDetector on the example use-case of transfer learning, demonstrated on the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.14958989)."
+    "This notebook explains how to use the fine-tuning function of an FaultDetector on the example use-case of transfer learning, demonstrated on the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774)."
    ],
    "metadata": {
     "collapsed": false
diff --git a/notebooks/Example - Quick Fault Detection.ipynb b/notebooks/Example - Quick Fault Detection.ipynb
index 6628592f..ff742720 100644
--- a/notebooks/Example - Quick Fault Detection.ipynb	
+++ b/notebooks/Example - Quick Fault Detection.ipynb	
@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "source": [
     "# Example: Quick fault detection in one line of code\n",
-    "Shows how to use quick fault detection module using example data from the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.14958989)\n",
+    "Shows how to use quick fault detection module using example data from the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774)\n",
     "Structure:\n",
     "1. Importing\n",
     "2. Data format preparation\n",

From e20c76ccc5af48a93b8975d2440705ee26c8fd87 Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Wed, 2 Sep 2026 19:22:10 +0200
Subject: [PATCH 31/35] Fix outdated API calls in example notebooks

- Model Finetuning: get_formatted_event_dataset -> load_and_format_event_dataset
- Create new model classes: get_dataset_for_event -> load_event_dataset;
  migrate data_preprocessor config from deprecated params style to steps
- Hyperparameter Optimization: save_model -> save_models (plural);
  eventwise_f_score_beta -> reliability_beta; remove ineffective
  decay_rate sampling (decay_steps was never set)
---
 docs/care2compare_guide.rst                   |  7 +++--
 .../Example - Create new model classes.ipynb  | 26 ++++++++++++-------
 ...xample - Hyperparameter Optimization.ipynb |  6 ++---
 notebooks/Example - Model Finetuning.ipynb    |  4 +--
 4 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/docs/care2compare_guide.rst b/docs/care2compare_guide.rst
index 795f5e62..1217d359 100644
--- a/docs/care2compare_guide.rst
+++ b/docs/care2compare_guide.rst
@@ -4,8 +4,7 @@ CARE2Compare guide
 .. note::
 
     This page documents how CARE2Compare support is exposed in EnergyFaultDetector.
-    For generated API reference pages, see the corresponding module documentation.
-    For interpretation caveats and recurring dataset questions, see :doc:`care2compare_faq`.
+    For interpretation and recurring dataset questions, see :doc:`care2compare_faq`.
 
 EnergyFaultDetector provides support for:
 
@@ -161,7 +160,7 @@ Example:
 CARE score overview
 -------------------
 
-The CARE score combines four aspects of detection quality:
+The CARE score combines four aspects of fault detection quality:
 
 - **Coverage**
 - **Accuracy**
@@ -174,7 +173,7 @@ At a high level:
 - Accuracy is computed on normal events,
 - Reliability is an event-wise score based on event decisions.
 
-The package implementation follows the CARE benchmark logic provided in the publication and subsequent
+The package implementation follows the CARE logic provided in the publication and subsequent
 package support. For detailed interpretation notes, see :doc:`care2compare_faq`.
 
 Wind-farm-specific interpretation
diff --git a/notebooks/Example - Create new model classes.ipynb b/notebooks/Example - Create new model classes.ipynb
index f87fee34..f8cfa9b3 100644
--- a/notebooks/Example - Create new model classes.ipynb	
+++ b/notebooks/Example - Create new model classes.ipynb	
@@ -246,7 +246,7 @@
     "c2c = Care2CompareDataset(data_path)\n",
     "\n",
     "event_id = 44\n",
-    "train, test = c2c.get_dataset_for_event(event_id, index_column='time_stamp')\n",
+    "train, test = c2c.load_event_dataset(event_id, index_column='time_stamp')\n",
     "\n",
     "y_train = train['status_type_id'] == 0\n",
     "x_train = train.drop(['asset_id', 'id', 'time_stamp', 'status_type_id'], axis=1, errors='ignore')\n",
@@ -540,15 +540,21 @@
     "        'upper_percentile': 0.999\n",
     "    },\n",
     "    'data_preprocessor': {\n",
-    "        'params': {\n",
-    "            'include_column_selector': True,\n",
-    "            'include_low_unique_value_filter': True,\n",
-    "            'include_duplicate_value_to_nan': False,\n",
-    "            'max_col_zero_frac': 0.99,\n",
-    "            'max_nan_frac_per_col': 0.05,\n",
-    "            'min_unique_value_count': 10,\n",
-    "            'scale': 'minmax'\n",
-    "        }\n",
+    "        'steps': [\n",
+    "            {\n",
+    "                'name': 'column_selector',\n",
+    "                'params': {\n",
+    "                    'max_nan_frac_per_col': 0.05,\n",
+    "                }\n",
+    "            },\n",
+    "            {\n",
+    "                'name': 'low_unique_value_filter',\n",
+    "                'params': {\n",
+    "                    'min_unique_value_count': 10,\n",
+    "                    'max_col_zero_frac': 0.99,\n",
+    "                }\n",
+    "            }\n",
+    "        ]\n",
     "    },\n",
     "    'data_splitter': {\n",
     "        'shuffle': True,\n",
diff --git a/notebooks/Example - Hyperparameter Optimization.ipynb b/notebooks/Example - Hyperparameter Optimization.ipynb
index e77392c1..b1f89be0 100644
--- a/notebooks/Example - Hyperparameter Optimization.ipynb	
+++ b/notebooks/Example - Hyperparameter Optimization.ipynb	
@@ -87,7 +87,6 @@
     "    # sample new parameters\n",
     "    autoencoder_params['batch_size'] = int(trial.suggest_categorical(name='batch_size', choices=[32, 64, 128]))\n",
     "    autoencoder_params['learning_rate'] = trial.suggest_float(name='learning_rate', low=1e-5, high=0.01, log=True)\n",
-    "    autoencoder_params['decay_rate'] = trial.suggest_float(name='decay_rate', low=0.8, high=0.99)\n",
     "\n",
     "    # architecture\n",
     "    autoencoder_params['layers'][0] = trial.suggest_int(name='layers_0', low=100, high=400)\n",
@@ -97,7 +96,7 @@
     "    # create a new model using our new configuration and train the model\n",
     "    model = FaultDetector(Config(config_dict=cfg))\n",
     "    # For autoencoder optimization, we do not need to fit a threshold\n",
-    "    training_result = model.fit(train_data, normal_index=normal_index, fit_autoencoder_only=True, save_model=False)\n",
+    "    training_result = model.fit(train_data, normal_index=normal_index, fit_autoencoder_only=True, save_models=False)\n",
     "\n",
     "    # Calculate the MSE of the reconstruction errors of the validation data - this is minimized\n",
     "    deviations = training_result.val_recon_error\n",
@@ -232,7 +231,6 @@
     "    autoencoder_params = cfg['train']['autoencoder']['params']\n",
     "    autoencoder_params['batch_size'] = int(trial.suggest_categorical(name='batch_size', choices=[32, 64, 128]))\n",
     "    autoencoder_params['learning_rate'] = trial.suggest_float(name='learning_rate', low=1e-5, high=0.01, log=True)\n",
-    "    autoencoder_params['decay_rate'] = trial.suggest_float(name='decay_rate', low=0.8, high=0.99)\n",
     "\n",
     "    # architecture\n",
     "    autoencoder_params['layers'][0] = trial.suggest_int(name='layers_0', low=100, high=400)\n",
@@ -330,7 +328,7 @@
     "    threshold_params['nn_size'] = trial.suggest_int(name='nn_size', low=20, high=50)\n",
     "\n",
     "    # Create a CAREScore object and train+evaluate each dataset for this wind farm\n",
-    "    care_score = CAREScore(coverage_beta=0.5, eventwise_f_score_beta=0.5, anomaly_detection_method='criticality')\n",
+    "    care_score = CAREScore(coverage_beta=0.5, reliability_beta=0.5, anomaly_detection_method='criticality')\n",
     "    i = 1\n",
     "    for x_train, y_train, x_test, y_test, event_id in c2c.iter_formatted_datasets(wind_farm=wind_farm, index_column='time_stamp'):\n",
     "        print(f\"event {i}/{len(c2c.event_info_all[c2c.event_info_all['wind_farm'] == wind_farm])}\")\n",
diff --git a/notebooks/Example - Model Finetuning.ipynb b/notebooks/Example - Model Finetuning.ipynb
index aa129eb5..4786e5a9 100644
--- a/notebooks/Example - Model Finetuning.ipynb	
+++ b/notebooks/Example - Model Finetuning.ipynb	
@@ -58,11 +58,11 @@
     "# select two events from different assets\n",
     "event_id = 44  # anomaly event of asset_id 44\n",
     "event_info_1 = c2c.get_event_info(event_id=event_id)\n",
-    "train_sensor_data_1, train_status_1, test_sensor_data_1, test_status_1 = c2c.get_formatted_event_dataset(event_id=event_id, index_column='time_stamp')\n",
+    "train_sensor_data_1, train_status_1, test_sensor_data_1, test_status_1 = c2c.load_and_format_event_dataset(event_id=event_id, index_column='time_stamp')\n",
     "\n",
     "event_id = 5  # anomaly event of asset_id 32\n",
     "event_info_2 = c2c.get_event_info(event_id=event_id)\n",
-    "train_sensor_data_2, train_status_2, test_sensor_data_2, test_status_2 = c2c.get_formatted_event_dataset(event_id=event_id, index_column='time_stamp')"
+    "train_sensor_data_2, train_status_2, test_sensor_data_2, test_status_2 = c2c.load_and_format_event_dataset(event_id=event_id, index_column='time_stamp')"
    ],
    "metadata": {
     "collapsed": false,

From d12e6c029468a19f77149215771c842837007945 Mon Sep 17 00:00:00 2001
From: chr39552 <Etrusker9@>
Date: Fri, 4 Sep 2026 13:55:36 +0200
Subject: [PATCH 32/35] Reviewed and edited C2C FAQ and guide docs.

---
 docs/care2compare_faq.rst   | 74 +++++++++++++++++++------------------
 docs/care2compare_guide.rst |  2 +-
 2 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/docs/care2compare_faq.rst b/docs/care2compare_faq.rst
index cb1fc8cd..21610163 100644
--- a/docs/care2compare_faq.rst
+++ b/docs/care2compare_faq.rst
@@ -17,10 +17,15 @@ General label semantics
 What is the difference between ``status_type_id`` and ``event_label``?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-``status_type_id`` is a timestamp-level label that describes the recorded operating mode of the turbine.
+``status_type_id`` is a timestamp-level label that describes the **recorded** operating mode of the turbine. 
+The intended use of this label is to help with filtering training data for normal behavior. 
+Please note: The meaning and usefulness of ``status_type_id`` differs slightly between wind farms.
+
 
 ``event_label`` is an event-level label indicating whether the prediction section of an event dataset
-contains an anomalous event.
+contains an anomalous event. The intended use of this label is to define the ground truth for evaluation of anomaly detection.
+For Wind Farm A the event labels were created based on available failure-log information. For Wind Farms B and C, the event labels were created 
+and validated based on operator feedback and service reports.
 
 They serve different purposes:
 
@@ -28,25 +33,14 @@ They serve different purposes:
 - ``event_label`` is the target for event-level anomaly evaluation.
 
 
-Should models predict ``status_type_id`` or ``event_label``?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-That depends on the goal. For early fault detection, you want to predict the ``event_label``. On the timestamp level,
-this means that you will want to detect anomalies within the provided ``event_start`` and ``event_end`` for events with
-``event_label == 'anomaly'`` and no anomalies if the ``event_label == 'normal'``.
-
-The ``status_type_id`` only indicates whether the wind turbine has an operationally normal state or not. Once a fault is
-known, this state is often already nor normal. Therefore, it is not interesting to detect whether the state is
-anomalous, but it is interesting to find anomalies during expected normal operation before a fault is known or becomes
-critical.
-
-For early fault detection, a practical strategy is often:
-
-- predict anomaly or normality at timestamp level,
-- then derive an event-level decision from those timestamp-level predictions.
+How do the status labels of Wind Farm A differ from those of Wind Farms B and C?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+For Wind Farms B and C, the status labels reflect anonymized operator-recorded turbine states and are
+useful both for training-data filtering and for parts of CARE-style evaluation.
 
-This is also consistent with CARE-style evaluation, where pointwise predictions can be aggregated into
-event-wise decisions.
+For Wind Farm A, the status labels were derived differently and do not provide the same interpretation value
+for prediction-time evaluation. The status labels are based on available failure-log information and should mainly be
+used for filtering training data. For prediction-time evaluation, they should largely be ignored.
 
 
 Why can anomalous events contain timestamps with ``status_type_id = 0``?
@@ -60,15 +54,6 @@ This is particularly important for early fault detection, because these timestam
 ones from the operator's perspective.
 
 
-How do the status labels of Wind Farm A differ from those of Wind Farms B and C?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-For Wind Farms B and C, the status labels reflect anonymized operator-recorded turbine states.
-
-For Wind Farm A, the status labels were derived differently and do not provide the same interpretation value
-for prediction-time evaluation. The status labels are based on available failure-log information and should mainly be
-used for filtering training data. For prediction-time evaluation, they should largely be ignored.
-
-
 How should ``status_type_id`` be interpreted for Wind Farms B and C?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -80,6 +65,27 @@ normal operation at that time, even if later retrospective analysis identified t
 anomaly window.
 
 
+Should models predict ``status_type_id`` or ``event_label``?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+That depends on the goal. For early fault detection, you want to predict the ``event_label``. On the timestamp level,
+this means that you will want to detect anomalies within the provided ``event_start`` and ``event_end`` for events with
+``event_label == 'anomaly'`` and no anomalies if the ``event_label == 'normal'``.
+
+The ``status_type_id`` only indicates whether the wind turbine has an operationally normal state or not. Once a fault is
+known, this state is often already not normal. Therefore, it is not interesting to detect whether the state is
+anomalous, but it is interesting to find anomalies during expected normal operation before a fault is known or becomes
+critical.
+
+For early fault detection, a practical strategy is often:
+
+- predict anomaly or normality at timestamp level,
+- then derive an event-level decision from those timestamp-level predictions.
+
+This is also consistent with CARE-style evaluation, where pointwise predictions can be aggregated into
+event-wise decisions.
+
+
 Training and preprocessing
 --------------------------
 
@@ -169,14 +175,12 @@ useful to reward detection of a timestamp as anomalous when the operator already
 operation.
 
 
-Why can true positives still exist after excluding abnormal-status timestamps?
+Why can detectable anomalies still exist after excluding timestamps with abnormal ``status_type_id``?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Because anomalous events can contain timestamps with normal status labels.
-
-After excluding abnormal-status timestamps, the remaining timestamps may still lie inside the anomalous event
-window. Those timestamps are still treated as anomalous ground truth for Coverage and can therefore produce
-true positives.
+Because anomalous events contain timestamps with normal status labels. These timestamps represent the time window 
+leading up to a fault, where the operator still considered the turbine to be in normal operation. Those timestamps 
+are still treated as anomalous ground truth for Coverage and can therefore produce true positives.
 
 
 How is Reliability computed?
diff --git a/docs/care2compare_guide.rst b/docs/care2compare_guide.rst
index 1217d359..54df8532 100644
--- a/docs/care2compare_guide.rst
+++ b/docs/care2compare_guide.rst
@@ -199,7 +199,7 @@ Users should be aware of several dataset and benchmark limitations:
 - the dataset is anonymized,
 - the data is not fully preprocessed for modeling,
 - exact benchmark code from the original paper is not guaranteed to match the current package state,
-- overlapping timestamps across different event files may occur due to anonymization,
+- overlapping timestamps across different event files and and no temporal order of events from the same asset may occur due to anonymization,
 - some version-specific data-quality notes apply.
 
 These are discussed in more detail in :doc:`care2compare_faq`.

From d48e688d2ff98ba3c75a7fec02c6d4f12bf3337c Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:07:48 +0200
Subject: [PATCH 33/35] Rebase fix

---
 .../data_preprocessing/categorical_encoder.py |  1 +
 .../data_preprocessing/imputer.py             | 32 ++++++++-----------
 2 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py
index c602dbe3..ff446768 100644
--- a/energy_fault_detector/data_preprocessing/categorical_encoder.py
+++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py
@@ -8,6 +8,7 @@
 
 logger = logging.getLogger('energy_fault_detector')
 
+
 class CategoricalEncoder(DataTransformer):
     """Transformer for encoding categorical features using one-hot encoding.
 
diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py
index 2e34dcf6..e4490fae 100644
--- a/energy_fault_detector/data_preprocessing/imputer.py
+++ b/energy_fault_detector/data_preprocessing/imputer.py
@@ -1,4 +1,4 @@
-from typing import Optional, List, Union, Callable
+from typing import List
 import pandas as pd
 from sklearn.utils.validation import check_is_fitted
 from sklearn.impute import SimpleImputer
@@ -8,6 +8,7 @@
 
 logger = logging.getLogger('energy_fault_detector')
 
+
 class Imputer(DataTransformer):
     """Wrapper around scikit-learn's SimpleImputer to handle numerical and categorical features separately.
 
@@ -62,7 +63,6 @@ def __init__(self, strategy: str = 'mean', categorical_features: list = None, **
             raise ValueError(f"Unsupported strategy: {self.strategy}. Supported strategies are 'mean' and 'median'.")
 
         # Attributes to be defined during fitting
-
         self.feature_names_in_: List[str] = []
         self.feature_names_out_ = None
         self.input_index_ = None
@@ -102,12 +102,14 @@ def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
 
         # Clean numerical columns from non_declared categorical features
         self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist()
-        self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
+        self.numerical_columns = [col for col in self.numerical_columns
+                                  if col not in self.non_declared_categorical_features]
         numerical_data = numerical_data.loc[:, self.numerical_columns]
         if self.non_declared_categorical_features:
-            logger.warning(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
-                        f"They will be dropped. Consider adding them to the categorical_features list if they should be"
-                        f" treated as categorical.")
+            logger.warning(
+                f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. "
+                f"They will be dropped. Consider adding them to the categorical_features list if they should be"
+                f" treated as categorical.")
 
         # Drop columns that are entirely NaN — they cannot be imputed and would cause
         # SimpleImputer to silently skip them, leading to a column-count mismatch on transform.
@@ -115,7 +117,7 @@ def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
                         if x[col].isna().all()]
         if all_nan_cols:
             logger.warning(f"Columns containing only NaN values found: {all_nan_cols}. "
-                        f"They will be dropped as they cannot be imputed.")
+                           f"They will be dropped as they cannot be imputed.")
             self.numerical_columns = [col for col in self.numerical_columns if col not in all_nan_cols]
             self.categorical_columns = [col for col in self.categorical_columns if col not in all_nan_cols]
             numerical_data = numerical_data.loc[:, self.numerical_columns]
@@ -124,14 +126,6 @@ def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
         logger.debug(f"Numerical columns: {self.numerical_columns}")
         logger.debug(f"Categorical columns: {self.categorical_columns}")
 
-        # Clean numerical columns from non_declared categorical features
-        self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist()
-        self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features]
-        numerical_data = numerical_data[self.numerical_columns]
-
-        logger.debug(f"Numerical columns: {self.numerical_columns}")
-        logger.debug(f"Categorical columns: {self.categorical_columns}")
-
         # Fit the imputers
         if not numerical_data.empty:
             self.numerical_imputer.fit(numerical_data)
@@ -139,7 +133,7 @@ def fit(self, x: pd.DataFrame, y=None) -> "Imputer":
             self.categorical_imputer.fit(categorical_data)
 
         return self
-    
+
     def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         """Applies imputation to input DataFrame, handling numerical and categorical columns separately.
 
@@ -202,7 +196,7 @@ def transform(self, x: pd.DataFrame) -> pd.DataFrame:
         transformed_data = transformed_data[self.feature_names_out_]  # Ensure original column order
 
         return transformed_data
-    
+
     def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
         """Returns input DataFrame with columns reordered to match original feature order.
 
@@ -221,7 +215,7 @@ def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame:
         check_is_fitted(self, "n_features_in_")
 
         return pd.DataFrame(x, columns=self.feature_names_in_)
-    
+
     def get_feature_names_out(self, input_features=None) -> List[str]:
         """Returns ordered list of output feature names.
 
@@ -238,4 +232,4 @@ def get_feature_names_out(self, input_features=None) -> List[str]:
             ValueError: If imputer has not been fitted (via `check_is_fitted`).
         """
         check_is_fitted(self, "n_features_in_")
-        return self.numerical_columns + self.categorical_columns
\ No newline at end of file
+        return self.numerical_columns + self.categorical_columns

From 3bef76cddce84ee4957683953762052da8c1f198 Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Tue, 8 Sep 2026 14:16:26 +0200
Subject: [PATCH 34/35] Fix sphinx warnings and remove the unnecessary TODO in
 FaultDetector

---
 docs/care2compare_faq.rst               |  2 +-
 docs/configuration.rst                  | 38 ++++++++++++-------------
 energy_fault_detector/fault_detector.py |  1 -
 3 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/docs/care2compare_faq.rst b/docs/care2compare_faq.rst
index 21610163..108d240c 100644
--- a/docs/care2compare_faq.rst
+++ b/docs/care2compare_faq.rst
@@ -176,7 +176,7 @@ operation.
 
 
 Why can detectable anomalies still exist after excluding timestamps with abnormal ``status_type_id``?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Because anomalous events contain timestamps with normal status labels. These timestamps represent the time window 
 leading up to a fault, where the operator still considered the turbine to be in normal operation. Those timestamps 
diff --git a/docs/configuration.rst b/docs/configuration.rst
index 5991c6a4..de3eec02 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -91,25 +91,25 @@ with the following keys:
 
 Allowed step names and aliases:
 
-+-------------------------+-----------------------------------------------+------------------------------------------------+
-| Step name               | Purpose                                       | Aliases                                        |
-+=========================+===============================================+================================================+
-| column_selector         | Drop columns with too many NaNs               | \-                                             |
-+-------------------------+-----------------------------------------------+------------------------------------------------+
-| low_unique_value_filter | Drop columns with low variance/many zeros     | \-                                             |
-+-------------------------+-----------------------------------------------+------------------------------------------------+
-| angle_transformer       | Convert angles to sin/cos pairs               | angle_transform                                |
-+-------------------------+-----------------------------------------------+------------------------------------------------+
-| counter_diff_transformer| Convert counters to differences/rates         | counter_diff, counter_diff_transform           |
-+-------------------------+-----------------------------------------------+------------------------------------------------+
-| timestamp_transformer   | Extract time features (hour, day, etc.)       | timestamp_transform,timestamp_features         |
-+-------------------------+-----------------------------------------------+------------------------------------------------+
-| simple_imputer          | Impute missing values                         | imputer                                        |
-+-------------------------+-----------------------------------------------+------------------------------------------------+
-| scaler                  | Standardize (standard) or scale to [0,1] (minmax)| standard_scaler, minmax_scaler (deprecated)  |
-+-------------------------+-----------------------------------------------+------------------------------------------------+
-| duplicate_to_nan        | Replace consecutive duplicate values with NaN | duplicate_value_to_nan, duplicate_values_to_nan|
-+-------------------------+-----------------------------------------------+------------------------------------------------+
++--------------------------+---------------------------------------------------+-------------------------------------------------+
+| Step name                | Purpose                                           | Aliases                                         |
++==========================+===================================================+=================================================+
+| column_selector          | Drop columns with too many NaNs                   | \-                                              |
++--------------------------+---------------------------------------------------+-------------------------------------------------+
+| low_unique_value_filter  | Drop columns with low variance/many zeros         | \-                                              |
++--------------------------+---------------------------------------------------+-------------------------------------------------+
+| angle_transformer        | Convert angles to sin/cos pairs                   | angle_transform                                 |
++--------------------------+---------------------------------------------------+-------------------------------------------------+
+| counter_diff_transformer | Convert counters to differences/rates             | counter_diff, counter_diff_transform            |
++--------------------------+---------------------------------------------------+-------------------------------------------------+
+| timestamp_transformer    | Extract time features (hour, day, etc.)           | timestamp_transform, timestamp_features         |
++--------------------------+---------------------------------------------------+-------------------------------------------------+
+| simple_imputer           | Impute missing values                             | imputer                                         |
++--------------------------+---------------------------------------------------+-------------------------------------------------+
+| scaler                   | Standardize (standard) or scale to [0,1] (minmax) | standard_scaler, minmax_scaler (deprecated)     |
++--------------------------+---------------------------------------------------+-------------------------------------------------+
+| duplicate_to_nan         | Replace consecutive duplicate values with NaN     | duplicate_value_to_nan, duplicate_values_to_nan |
++--------------------------+---------------------------------------------------+-------------------------------------------------+
 
 For detailed documentation of the data preprocessor pipeline, refer to the
 :py:obj:`DataPreprocessor <energy_fault_detector.data_preprocessing.data_preprocessor.DataPreprocessor>` docs.
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index 7976f156..b742a914 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -67,7 +67,6 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri
         ) if protect else []
 
         # Data clipping (outlier clipping)
-        # TODO: what happens in DataClipper if data contains non-numerical features?
         if self.config.data_clipping:
             logger.debug('Clip data before scaling.')
             clipper_params = self.config.data_clipping_params.copy()

From 5dc70d4988442a2ecc7512bd37fafc02d0bd577e Mon Sep 17 00:00:00 2001
From: croelofs <25582572+roelofsc@users.noreply.github.com>
Date: Wed, 9 Sep 2026 11:43:49 +0200
Subject: [PATCH 35/35] Improve package docstrings, fix conditional-features
 handling, and remove redundant config

- Add concise module-level docstrings to all subpackage __init__.py files so
  the API reference and code navigation describe each submodule's purpose at a
  glance, without duplicating the class lists Sphinx generates.

- fault_detector: capture declared conditional features before the preprocessor
  expands categorical condition names to one-hot columns, so the
  available/missing checks operate on the original names.

- registration: drop the stale 'data_preprocessor' from the class_type docstring
  (DataTransformer steps are registered in DataPreprocessor, not the global
  registry).

- docs/sequence_models: rename section headings from 'Seq2One/Seq2Seq models'
  to the more readable 'Sequence-to-one/Sequence-to-sequence models'.

- docs/index: reorder the 'Models and methods' toctree entries.

- Remove energy_fault_detector/advanced_config.yaml: it was neither shipped
  (package-data lists only base_config.yaml and c2c_options.yaml) nor used by
  code, and had diverged from the canonical docs/examples/advanced_config.yaml
  shown in configuration.rst.
---
 docs/index.rst                                |  4 +-
 docs/sequence_models.rst                      |  8 +-
 energy_fault_detector/__init__.py             |  6 +-
 energy_fault_detector/advanced_config.yaml    | 94 -------------------
 .../anomaly_scores/__init__.py                | 10 +-
 .../autoencoders/__init__.py                  |  9 +-
 energy_fault_detector/config/__init__.py      |  2 +-
 energy_fault_detector/core/__init__.py        |  7 +-
 .../data_preprocessing/__init__.py            |  9 +-
 .../data_splitting/__init__.py                |  2 +-
 energy_fault_detector/evaluation/__init__.py  |  2 +-
 energy_fault_detector/fault_detector.py       | 12 ++-
 .../quick_fault_detection/__init__.py         |  7 ++
 energy_fault_detector/registration.py         |  2 +-
 .../root_cause_analysis/__init__.py           |  2 +-
 .../threshold_selectors/__init__.py           |  8 +-
 energy_fault_detector/utils/__init__.py       |  2 +-
 17 files changed, 69 insertions(+), 117 deletions(-)
 delete mode 100644 energy_fault_detector/advanced_config.yaml

diff --git a/docs/index.rst b/docs/index.rst
index 0db887b6..fc1cd0a2 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -35,10 +35,10 @@ Installation
     :maxdepth: 1
 
     arcana
-    models_overview
-    sequence_models
     care2compare_guide
     care2compare_faq
+    models_overview
+    sequence_models
 
 .. toctree::
     :caption: Advanced usage
diff --git a/docs/sequence_models.rst b/docs/sequence_models.rst
index 91574ef8..bcb9d726 100644
--- a/docs/sequence_models.rst
+++ b/docs/sequence_models.rst
@@ -133,8 +133,8 @@ Training and prediction are identical to the dense autoencoder case:
    print(results.reconstruction.shape)
 
 
-Seq2One models (sequence-to-one)
-================================
+Sequence-to-one models
+======================
 
 Seq2One models take a window of length ``sequence_length`` and reconstruct the
 **last timestep** of that window:
@@ -210,8 +210,8 @@ The ``merge_mode`` controls how forward and backward encoder outputs are merged.
 recommended setting is ``"sum"``.
 
 
-Seq2Seq models (sequence-to-sequence)
-=====================================
+Sequence-to-sequence models
+===========================
 
 Seq2Seq autoencoders reconstruct the **entire input window**:
 
diff --git a/energy_fault_detector/__init__.py b/energy_fault_detector/__init__.py
index 68145b36..41c62847 100644
--- a/energy_fault_detector/__init__.py
+++ b/energy_fault_detector/__init__.py
@@ -1,4 +1,8 @@
-"""The energy-fault-detector package"""
+"""Autoencoder-based anomaly and fault detection for renewable energy assets and power grids.
+
+The package learns a normal-behaviour model from operational data and flags deviations as
+anomalies, and includes the ARCANA method for root-cause analysis of detected anomalies.
+"""
 
 from .__about__ import __version__
 from energy_fault_detector.core._logs import setup_logging
diff --git a/energy_fault_detector/advanced_config.yaml b/energy_fault_detector/advanced_config.yaml
deleted file mode 100644
index 68a5d5c0..00000000
--- a/energy_fault_detector/advanced_config.yaml
+++ /dev/null
@@ -1,94 +0,0 @@
-train:
-  data_clipping:
-    lower_percentile: 0.001
-    upper_percentile: 0.999
-
-  data_preprocessor:
-    steps:
-      # 1) Drop columns with too many NaNs and exclude specific features
-      - name: column_selector
-        params:
-          max_nan_frac_per_col: 0.9
-          features_to_exclude: [] # Inlcude features to exclude in the list
-      # 2) Drop columns with low number of unique values
-      - name: low_unique_value_filter
-        params:
-          min_unique_value_count: 2
-      # 3) Transform counter readings to differences
-      - name: counter_diff_transformer
-        params:
-          counters: [] # Include counter features in the list
-          compute_rate: True
-          reset_strategy: 'nan'
-          fill_first: 'nan'
-          keep_original: False
-          max_gap_seconds: 3600
-      # 4) Imputation
-      - name: ffill_imputer
-        params:
-          ffill_limit: '60min' # Maximum time gap to forward-fill (pandas Timedelta string).
-          categorical_features: [] # include categorical features in the list
-
-      # 5) Encode categorical features
-      - name: categorical_encoder
-        params:
-          categorical_features: [] # include categorical features in the list
-            
-      # 6) Scaling (standardize)
-      - name: scaler
-        params:
-          scaler_type: 'standard' # Supported types are 'standard' and 'minmax'
-          with_mean: True
-          with_std: True
-          scale_categorical_features: False
-          categorical_features: [] # include categorical features in the list
-
-      # 7) Add time-based features
-      - name: timestamp_transformer
-        params:
-          features: ['minute_of_hour', 'hour_of_day', 'day_of_week']
-  
-  data_splitter:
-    shuffle: true
-    type: sklearn
-    validation_split: 0.2
-
-  autoencoder:
-    name: ConditionalAutoencoder
-    params:
-      act: prelu
-      batch_size: 256
-      code_size: 9
-      early_stopping: true
-      epochs: 1000
-      last_act: linear
-      layers:
-      - 64
-      - 32
-      learning_rate: 0.0001
-      loss_name: mean_squared_error
-      min_delta: 0.0001
-      noise: 0.0
-      patience: 5
-      conditional_features: [] # Include conditional features in the list
-    verbose: 0
-
-  anomaly_score:
-    name: rmse
-    params:
-      scale: false
-   
-  threshold_selector:
-    fit_on_val: true
-    name: QuantileThresholdSelector
-    params:
-      quantile: 0.98
-  
-  protect_conditional_features: true # Protect conditional features from being dropped during preprocessing
-
-root_cause_analysis:
-  alpha: 0.8
-  init_x_bias: recon
-  num_iter: 200
-  max_sample_threshold: 1000
-  verbose: true
\ No newline at end of file
diff --git a/energy_fault_detector/anomaly_scores/__init__.py b/energy_fault_detector/anomaly_scores/__init__.py
index 4148e466..4aea53a1 100644
--- a/energy_fault_detector/anomaly_scores/__init__.py
+++ b/energy_fault_detector/anomaly_scores/__init__.py
@@ -1,4 +1,12 @@
-"""Anomaly score classes."""
+"""Anomaly scores that reduce per-feature reconstruction errors to one score per sample.
+
+After an autoencoder reconstructs its input, an :class:`~energy_fault_detector.core.AnomalyScore`
+aggregates the per-feature reconstruction errors into a single scalar per sample.
+Samples with high scores are candidate anomalies.
+
+Each scorer subclasses :class:`~energy_fault_detector.core.AnomalyScore` and is selected
+via the ``train.anomaly_score`` config key.
+"""
 
 from energy_fault_detector.anomaly_scores.mahalanobis_score import MahalanobisScore
 from energy_fault_detector.anomaly_scores.rmse_score import RMSEScore
diff --git a/energy_fault_detector/autoencoders/__init__.py b/energy_fault_detector/autoencoders/__init__.py
index 3aa56ce5..f7e81d98 100644
--- a/energy_fault_detector/autoencoders/__init__.py
+++ b/energy_fault_detector/autoencoders/__init__.py
@@ -1,4 +1,11 @@
-"""Autoencoder model classes."""
+"""Autoencoder normal-behaviour models trained to reconstruct normal operation.
+
+Large reconstruction errors at prediction time indicate anomalies. Models range from dense
+(point-wise) autoencoders to sequence models (seq2one and seq2seq) for time-series input.
+
+Every autoencoder model here subclasses :class:`~energy_fault_detector.core.autoencoder.Autoencoder`
+and is referenced from the configuration under the ``train.autoencoder`` key.
+"""
 
 # Dense
 from .multilayer_autoencoder import MultilayerAutoencoder
diff --git a/energy_fault_detector/config/__init__.py b/energy_fault_detector/config/__init__.py
index 887e38f4..a7b6bc61 100644
--- a/energy_fault_detector/config/__init__.py
+++ b/energy_fault_detector/config/__init__.py
@@ -1,4 +1,4 @@
-"""Configuration classes."""
+"""Configuration of the fault-detection pipeline, loaded and validated from YAML."""
 
 from energy_fault_detector.config.config import Config
 from energy_fault_detector.config.base_config import InvalidConfigFile
diff --git a/energy_fault_detector/core/__init__.py b/energy_fault_detector/core/__init__.py
index e0b6d92a..7fbd9160 100644
--- a/energy_fault_detector/core/__init__.py
+++ b/energy_fault_detector/core/__init__.py
@@ -1,5 +1,8 @@
-"""This module contains class templates for most of the anomaly detection classes, such as
-anomaly scores, threshold selectors and data classes."""
+"""Abstract base classes and shared infrastructure for the fault-detection pipeline.
+
+Concrete implementations of these contracts live in their own subpackages
+(autoencoders, anomaly_scores, threshold_selectors, data_preprocessing).
+"""
 
 from .anomaly_score import AnomalyScore
 from .data_transformer import DataTransformer
diff --git a/energy_fault_detector/data_preprocessing/__init__.py b/energy_fault_detector/data_preprocessing/__init__.py
index 29ee04c9..c658be00 100644
--- a/energy_fault_detector/data_preprocessing/__init__.py
+++ b/energy_fault_detector/data_preprocessing/__init__.py
@@ -1,4 +1,11 @@
-"""Data preprocessing classes and utilities."""
+"""Configurable preprocessing pipeline and outlier clipping for sensor data.
+
+The :class:`~energy_fault_detector.data_preprocessing.data_preprocessor.DataPreprocessor` is an
+sklearn-style ``Pipeline`` assembled from configurable :class:`~energy_fault_detector.core.DataTransformer`
+steps (see ``DataPreprocessor.STEP_REGISTRY``).
+The pipeline is configured via the ``train.data_preprocessor.steps`` config key.
+
+"""
 
 from energy_fault_detector.data_preprocessing.data_preprocessor import DataPreprocessor
 from energy_fault_detector.data_preprocessing.data_clipper import DataClipper
diff --git a/energy_fault_detector/data_splitting/__init__.py b/energy_fault_detector/data_splitting/__init__.py
index 1033f7b2..e6c334bb 100644
--- a/energy_fault_detector/data_splitting/__init__.py
+++ b/energy_fault_detector/data_splitting/__init__.py
@@ -1,3 +1,3 @@
-"""Data splitter classes and functions."""
+"""Splitting and windowing of time-series data for training and validation."""
 
 from .data_splitter import BlockDataSplitter
diff --git a/energy_fault_detector/evaluation/__init__.py b/energy_fault_detector/evaluation/__init__.py
index 499c600f..97c28717 100644
--- a/energy_fault_detector/evaluation/__init__.py
+++ b/energy_fault_detector/evaluation/__init__.py
@@ -1,4 +1,4 @@
-"""Evaluation classes and methods, including the CARE-Score and Care2CompareDataset."""
+"""Benchmark metrics and dataset loaders for evaluating fault-detection models."""
 
 from .care_score import CAREScore
 from .care2compare import Care2CompareDataset
diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py
index b742a914..a9ea5c61 100644
--- a/energy_fault_detector/fault_detector.py
+++ b/energy_fault_detector/fault_detector.py
@@ -133,17 +133,21 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo
             sensor_data=sensor_data, normal_index=normal_index, fit_preprocessor=fit_preprocessor
         )
 
-        # Resolve declared conditions against preprocessed data
+        # Capture conditions available in the raw data before the preprocessor may expand
+        # categorical conditions into one-hot encoded column names (e.g. 'state' -> 'state_A').
+        declared_conditions = list(self.autoencoder.conditional_features or [])
+
+        # Resolve declared conditions against preprocessed data (may expand categorical names)
         self._resolve_conditional_features(x_prepped)
 
         # Check conditionals: available conditions in original data and surviving conditions during preprocessing.
         if self.autoencoder.is_conditional:
-            configured = self.autoencoder.conditional_features or []
             # Uses nested for loop in case categorical cols have been encoded already and contain the original
             # conditional feature name as a substring.
-            available = [declared_condition for declared_condition in configured
+            available = [declared_condition for declared_condition in declared_conditions
                          if any(declared_condition in col for col in sensor_data.columns)]
-            missing = [declared_condition for declared_condition in configured if declared_condition not in available]
+            missing = [declared_condition for declared_condition in declared_conditions
+                       if declared_condition not in available]
             if missing:
                 logger.warning(f"Declared conditions not found in sensor_data will be ignored: "
                                f"{sorted(missing)}. Using: {available or 'none'}")
diff --git a/energy_fault_detector/quick_fault_detection/__init__.py b/energy_fault_detector/quick_fault_detection/__init__.py
index eef4053e..283df610 100644
--- a/energy_fault_detector/quick_fault_detection/__init__.py
+++ b/energy_fault_detector/quick_fault_detection/__init__.py
@@ -1,3 +1,10 @@
+"""One-call end-to-end fault detection for testing the
+:class:`~energy_fault_detector.fault_detector.FaultDetector` on a single dataset.
+
+The function :func:`~energy_fault_detector.quick_fault_detector` is also exposed as
+the ``quick_fault_detector`` command-line entry point.
+
+"""
 
 from .pipeline import quick_fault_detector
 
diff --git a/energy_fault_detector/registration.py b/energy_fault_detector/registration.py
index c4adcb94..3ee874e2 100644
--- a/energy_fault_detector/registration.py
+++ b/energy_fault_detector/registration.py
@@ -18,7 +18,7 @@ def register(self, module_path: str, class_type: str, class_names: List[str]) ->
 
         Args:
             module_path (str): Path to the module containing the class.
-            class_type (str): type of the class (anomaly_score, autoencoder, threshold_selector or data_preprocessor)
+            class_type (str): type of the class (anomaly_score, autoencoder, threshold_selector)
             class_names (List[str], optional): a list of names which may be used to refer to this class. Default is None
         """
 
diff --git a/energy_fault_detector/root_cause_analysis/__init__.py b/energy_fault_detector/root_cause_analysis/__init__.py
index 7eb9cfdf..38a11152 100644
--- a/energy_fault_detector/root_cause_analysis/__init__.py
+++ b/energy_fault_detector/root_cause_analysis/__init__.py
@@ -1 +1 @@
-"""Root cause analysis."""
+"""Root-cause analysis explaining which sensors caused a detected anomaly."""
diff --git a/energy_fault_detector/threshold_selectors/__init__.py b/energy_fault_detector/threshold_selectors/__init__.py
index f9dc6e7b..27d1f41c 100644
--- a/energy_fault_detector/threshold_selectors/__init__.py
+++ b/energy_fault_detector/threshold_selectors/__init__.py
@@ -1,4 +1,10 @@
-"""Threshold selection methods"""
+"""Threshold selectors that turn anomaly scores into normal/anomaly labels.
+
+A :class:`~energy_fault_detector.core.ThresholdSelector` learns a threshold from
+(optionally labelled) anomaly scores. Samples above the threshold are flagged as anomalous.
+Each selector subclasses :class:`~energy_fault_detector.core.ThresholdSelector` and is selected
+via the ``train.threshold_selector`` config key.
+"""
 
 from energy_fault_detector.threshold_selectors.fdr_threshold import FDRSelector
 from energy_fault_detector.threshold_selectors.fbeta_threshold import FbetaSelector
diff --git a/energy_fault_detector/utils/__init__.py b/energy_fault_detector/utils/__init__.py
index 23eda90a..f1930beb 100644
--- a/energy_fault_detector/utils/__init__.py
+++ b/energy_fault_detector/utils/__init__.py
@@ -1 +1 @@
-"""Helper utilities for post-processing, data downloading, index handling, and visualization."""
+"""Helper utilities shared across the package."""