diff --git a/.gitignore b/.gitignore index 2283addc..6c605a1e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ coverage.xml /docs/modules.rst /docs/anomaly_detection.rst /docs/anomaly_detection.*.rst +/docs/energy_fault_detector.*.rst +/docs/energy_fault_detector.rst public/ # builds @@ -45,3 +47,6 @@ notebooks/PreDist/predist_data/ # logging *.log + +# coding agents +.continue/ diff --git a/README.md b/README.md index 84e90eeb..0a0042d5 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Run the full pipeline (train → predict → events → ARCANA) in a single comm ```bash quick_fault_detector path/to/data.csv ``` -For [CARE2Compare](https://doi.org/10.5281/zenodo.14958989) data: +For [CARE2Compare](https://doi.org/10.5281/zenodo.10958774) data: ```bash quick_fault_detector path/to/c2c_dataset.csv --c2c_example ``` diff --git a/docs/_templates/module.rst_t b/docs/_templates/module.rst_t index 53cfbac8..0a5b6b27 100644 --- a/docs/_templates/module.rst_t +++ b/docs/_templates/module.rst_t @@ -1,5 +1,5 @@ {%- if show_headings %} -{{- [basename] | join(' ') | e | heading }} +{{- [basename | replace('energy_fault_detector.', '')] | join(' ') | e | heading }} {% endif -%} .. automodule:: {{ qualname }} diff --git a/docs/_templates/package.rst_t b/docs/_templates/package.rst_t index 7e5b87ea..ba7e0a1d 100644 --- a/docs/_templates/package.rst_t +++ b/docs/_templates/package.rst_t @@ -14,13 +14,14 @@ {%- endmacro %} {%- if is_namespace %} -{{- [pkgname, "namespace"] | join(" ") | e | heading }} +{{- pkgname | replace('energy_fault_detector.', '') | e | heading }} {% else %} -{{- pkgname | e | heading }} +{{- pkgname | replace('energy_fault_detector.', '') | e | heading }} {% endif %} {%- if modulefirst and not is_namespace %} -{{ automodule(pkgname, automodule_options) }} +{{ automodule(pkgname, automodule_options) }}{% if pkgname == 'energy_fault_detector.config' %} + :no-index:{% endif %} {% endif %} {%- if subpackages %} @@ -35,7 +36,7 @@ {% else %} {%- for submodule in submodules %} {% if show_headings %} -{{- submodule | e | heading(2) }} +{{- submodule | replace('energy_fault_detector.', '') | e | heading(2) }} {% endif %} {{ automodule(submodule, automodule_options) }} {% endfor %} diff --git a/docs/_templates/toc.rst_t b/docs/_templates/toc.rst_t index f0877eeb..7835fff8 100644 --- a/docs/_templates/toc.rst_t +++ b/docs/_templates/toc.rst_t @@ -1,4 +1,4 @@ -{{ header | heading }} +{{ header | replace('energy_fault_detector.', '') | heading }} .. toctree:: :maxdepth: {{ maxdepth }} diff --git a/docs/care2compare_faq.rst b/docs/care2compare_faq.rst new file mode 100644 index 00000000..108d240c --- /dev/null +++ b/docs/care2compare_faq.rst @@ -0,0 +1,319 @@ +CARE2Compare FAQ +================ + +.. note:: + + This page summarizes recurring interpretation questions and caveats around the + CARE2Compare dataset and the CARE score as supported in EnergyFaultDetector. + For official release notes and dataset updates, also consult the Zenodo record + and the companion publication: + + CARE to Compare: A Real-World Benchmark Dataset for Early Fault Detection in Wind Turbine Data. + Data. 2024; 9(12):138. https://doi.org/10.3390/data9120138 + +General label semantics +----------------------- + +What is the difference between ``status_type_id`` and ``event_label``? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``status_type_id`` is a timestamp-level label that describes the **recorded** operating mode of the turbine. +The intended use of this label is to help with filtering training data for normal behavior. +Please note: The meaning and usefulness of ``status_type_id`` differs slightly between wind farms. + + +``event_label`` is an event-level label indicating whether the prediction section of an event dataset +contains an anomalous event. The intended use of this label is to define the ground truth for evaluation of anomaly detection. +For Wind Farm A the event labels were created based on available failure-log information. For Wind Farms B and C, the event labels were created +and validated based on operator feedback and service reports. + +They serve different purposes: + +- ``status_type_id`` helps interpret turbine operation and filter data, +- ``event_label`` is the target for event-level anomaly evaluation. + + +How do the status labels of Wind Farm A differ from those of Wind Farms B and C? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +For Wind Farms B and C, the status labels reflect anonymized operator-recorded turbine states and are +useful both for training-data filtering and for parts of CARE-style evaluation. + +For Wind Farm A, the status labels were derived differently and do not provide the same interpretation value +for prediction-time evaluation. The status labels are based on available failure-log information and should mainly be +used for filtering training data. For prediction-time evaluation, they should largely be ignored. + + +Why can anomalous events contain timestamps with ``status_type_id = 0``? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The anomaly window between ``event_start`` and ``event_end`` represents an estimated anomaly time frame +leading up to a fault. During that time, the operator may still have considered the turbine to be in normal +operation. Therefore, timestamps with ``status_type_id = 0`` can appear inside anomalous events. + +This is particularly important for early fault detection, because these timestamps are often the most relevant +ones from the operator's perspective. + + +How should ``status_type_id`` be interpreted for Wind Farms B and C? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For Wind Farms B and C, ``status_type_id`` is an anonymized version of operator-provided SCADA status +information. + +This means a timestamp with ``status_type_id = 0`` reflects that the operator considered the turbine to be in +normal operation at that time, even if later retrospective analysis identified that timestamp as part of an +anomaly window. + + +Should models predict ``status_type_id`` or ``event_label``? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +That depends on the goal. For early fault detection, you want to predict the ``event_label``. On the timestamp level, +this means that you will want to detect anomalies within the provided ``event_start`` and ``event_end`` for events with +``event_label == 'anomaly'`` and no anomalies if the ``event_label == 'normal'``. + +The ``status_type_id`` only indicates whether the wind turbine has an operationally normal state or not. Once a fault is +known, this state is often already not normal. Therefore, it is not interesting to detect whether the state is +anomalous, but it is interesting to find anomalies during expected normal operation before a fault is known or becomes +critical. + +For early fault detection, a practical strategy is often: + +- predict anomaly or normality at timestamp level, +- then derive an event-level decision from those timestamp-level predictions. + +This is also consistent with CARE-style evaluation, where pointwise predictions can be aggregated into +event-wise decisions. + + +Training and preprocessing +-------------------------- + +Should abnormal timestamps be removed from the training data? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Usually yes, if you are training a normal-behavior model. + +The dataset provides ``status_type_id`` so users can filter training data. In practice, filtering by status is +an intended use of the dataset. + +Depending on the model and the goal, it can also be reasonable to apply additional cleaning such as power-curve-based +filtering. + + +Has the dataset already been preprocessed? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +No. The data was anonymized, but it has not undergone full preprocessing such as: + +- missing-value removal, +- invalid-measurement filtering, +- normalization or scaling. + +Additional preprocessing is generally required before modeling. +Please check the dataset description and dataset README on `Zenodo `_ +for details on known data quality issues. + + +What preprocessing is typically needed? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This depends on the wind farm and the model, but common steps include: + +- feature selection, +- NaN imputation, +- angle transformation, +- filtering invalid measurements, +- scaling. + +An example is found in the notebook `CARE to Compare.ipynb `_ + + +Does using the dataset require domain knowledge? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For purely statistical anomaly detection experiments, general data analysis methods may be sufficient up to a +point. However, domain knowledge becomes increasingly important for: + +- feature selection, +- validating detected anomalies, +- interpreting event behavior, +- root-cause-oriented analysis. + + +Evaluation +---------- + +Which timestamps are used for pointwise evaluation? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For CARE-style pointwise measures, timestamps with normal status are the important ones. + +We consider a timestamp to be correctly detected as anomalous, if the timestamp is part of an event with +``event_label == 'anomaly'``. It is correctly detected as normal behaviour if the timestamp is part of an event with +``event_label == 'normal'``. + +Timestamps with abnormal status labels are excluded from pointwise evaluation, because it is not +useful to reward detection of a timestamp as anomalous when the operator already knew it was not in normal +operation. Note that we only apply this rule to wind farm B and C. For WF A, the status information can only be used +for filtering training data and not for evaluation. + + +How is the ground truth for Coverage interpreted? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Coverage is a pointwise F-score computed on anomalous events. + +The ground truth is as follows: + +- timestamps with anomalous status are anomalous, +- timestamps between ``event_start`` and ``event_end`` of anomalous events are also considered anomalous, +- all other timestamps are considered normal. + +For wind farm B and C the first type of true anomalies are ignored, because it is not +useful to reward detection of a timestamp as anomalous when the operator already knew it was not in normal +operation. + + +Why can detectable anomalies still exist after excluding timestamps with abnormal ``status_type_id``? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Because anomalous events contain timestamps with normal status labels. These timestamps represent the time window +leading up to a fault, where the operator still considered the turbine to be in normal operation. Those timestamps +are still treated as anomalous ground truth for Coverage and can therefore produce true positives. + + +How is Reliability computed? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Reliability is an event-wise F-score. The model used should provide a decision for a complete event. This decision can +then be compared with the ``event_label``. + +As an example, in the paper we used a criticality measure, which is accumulated over timestamp-level predictions: + +- criticality increases when an anomaly is detected on a timestamp with normal status, +- criticality decreases when no anomaly is detected on a timestamp with normal status, +- criticality does not change on timestamps with abnormal status. + +If the criticality reaches the threshold, the whole event is treated as predicted anomalous; otherwise it is +treated as predicted normal. Those event-level predictions are then compared with ``event_label`` values. + + +What is the default criticality threshold? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default threshold used in the CARE description and package discussions is ``72``. +You should of course tune this threshold for your dataset. + + +Why is Reliability not averaged like the other CARE components? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Because Reliability is already computed as an event-wise score across events. + +By contrast, Coverage, Accuracy, Earliness are first computed per event and then averaged over the relevant events. + + +Model training strategy +----------------------- + +Should I train one model per turbine or one model per wind farm? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There is no universally correct answer. + +Possible strategies include: + +- one model per turbine, +- one model per wind farm, +- hybrid strategies combining local and shared information. + + +For early fault detection, a practical strategy is often: + +- predict anomaly or normality at timestamp level, +- then derive an event-level decision from those timestamp-level predictions. + +This is also consistent with CARE-style evaluation, where pointwise predictions can be aggregated into +event-wise decisions. + +In the dataset paper, individual autoencoder models per turbine were used. + + +Features and interpretation +--------------------------- + +Can the feature names be mapped to more detailed physical sensor identities? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Because of anonymization agreements with data providers, no additional sensor metadata beyond the published +dataset description may be available. + +The feature descriptions are the most detailed available for this dataset. If you load the data using the +:class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset`, the feature names are based on these +descriptions (instead of the enumerated feature names of the csv files). + + +Are all feature groups equally trustworthy? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are implausible values in some Min, Max, and Std features. For practical work, +Avg features are often the safest starting point, especially in Wind Farm B. + +Please check the dataset description and dataset README on `Zenodo `_ +for details on known data quality issues. + + +Root-cause analysis and ARCANA +------------------------------ + +Was ARCANA used to create the root-cause labels? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +No. The root-cause information included in the dataset is based on operator feedback and service reports, +not on ARCANA-generated labels. + + +Benchmark reproducibility +------------------------- + +Is the exact original benchmark code available? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Not exactly. + +The EnergyFaultDetector repository provides open-source implementations related to the CARE workflow, +including CARE score support and example notebooks. However, the exact original code used for the paper is +not available. + + +Can benchmark scores vary when reproduced? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Yes. Even when following the provided examples and configurations, results can vary due to factors such as random +initialization of the models and later package updates. + + +Timestamps and anonymization +---------------------------- + +Can different event files still contain overlapping timestamps? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Yes. Overlaps across different event CSV files are an expected side effect of the anonymization procedure. Event +files may therefore share timestamp ranges even when they represent different events. + + +Can I reconstruct the true chronological order of all events for one asset? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Generally no. Because timestamps were anonymized per file, the true chronological order across event files is +not preserved. + + +See also +-------- + +- :doc:`care2compare_guide` +- :class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset` +- :class:`energy_fault_detector.evaluation.care_score.CAREScore` \ No newline at end of file diff --git a/docs/care2compare_guide.rst b/docs/care2compare_guide.rst new file mode 100644 index 00000000..54df8532 --- /dev/null +++ b/docs/care2compare_guide.rst @@ -0,0 +1,212 @@ +CARE2Compare guide +================== + +.. note:: + + This page documents how CARE2Compare support is exposed in EnergyFaultDetector. + For interpretation and recurring dataset questions, see :doc:`care2compare_faq`. + +EnergyFaultDetector provides support for: + +- loading CARE2Compare event datasets, +- accessing event metadata, +- formatting normal-operation masks based on ``status_type_id``, +- evaluating predictions with the CARE score. + +Relevant classes include: + +- :class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset` +- :class:`energy_fault_detector.evaluation.care_score.CAREScore` + +Background +---------- + +The CARE2Compare dataset and CARE score are introduced in: + +`CARE to Compare: A Real-World Benchmark Dataset for Early Fault Detection in Wind Turbine Data `_ + +In the package, CARE2Compare support is intended to help with two tasks: + +#. loading event-based benchmark datasets, +#. evaluating anomaly predictions in a way that is aligned with the CARE benchmark. + +Conceptual overview +------------------- + +The dataset contains two related but distinct kinds of labels: + +- ``status_type_id``: + timestamp-level operating-status information, +- ``event_label``: + event-level label indicating whether the prediction section contains an anomalous event. + +These labels should not be treated as interchangeable. + +In particular, anomalous events may still contain timestamps with ``status_type_id = 0``. +This is expected for CARE2Compare and reflects the difference between: + +- the operator's recorded turbine status at a timestamp, and +- the retrospectively defined anomaly window used for early fault detection. + +For details, see :doc:`care2compare_faq`. + +Dataset loading +--------------- + +The :class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset` helper can be used to +load local CARE2Compare data or download the dataset automatically, depending on package configuration. + +Typical usage: + +.. code-block:: python + + from energy_fault_detector.evaluation.care2compare import Care2CompareDataset + + dataset = Care2CompareDataset( + path="./CARE_To_Compare", + download_dataset=False, + ) + + x_train, x_test = dataset.load_event_dataset(event_id=53, index_column="time_stamp") + info = dataset.get_event_info(53) + +The returned event metadata can then be used for evaluation, filtering, or reporting. + +Formatted loading +----------------- + +If you want a convenience split into values and normal-operation masks, use the formatted loader. + +.. code-block:: python + + x_train, train_normal, x_test, test_normal = dataset.load_and_format_event_dataset(event_id=53) + +In the package, the normal masks are derived from: + +.. code-block:: text + + status_type_id == 0 + +This is mainly useful when training normal-behavior models or when applying pointwise evaluation logic. + +Training-data filtering +----------------------- + +For normal-behavior modeling, it is generally recommended to filter out timestamps that are clearly not +representative of normal operation. + +In practice, this often means using ``status_type_id`` to exclude abnormal operating modes from the +training data. + +However, CARE2Compare should not be understood as fully pre-cleaned data. Depending on the model and +wind farm, additional preprocessing may still be needed, such as: + +- missing-value handling, +- invalid-measurement filtering, +- feature selection, +- angle transformation, +- scaling. + +Example workflow +---------------- + +A typical workflow looks like this: + +#. load an event dataset, +#. separate training and prediction sections, +#. derive or load model predictions for prediction timestamps, +#. evaluate those predictions with :class:`energy_fault_detector.evaluation.care_score.CAREScore`. + +Example: + +.. code-block:: python + + import pandas as pd + from energy_fault_detector.evaluation.care2compare import Care2CompareDataset + from energy_fault_detector.evaluation.care_score import CAREScore + + dataset = Care2CompareDataset(path="./CARE_To_Compare", download_dataset=False) + scorer = CAREScore() + + # Iterate over the events of a chosen wind farm. The data is loaded with + # timestamps as the index so they match the event_start / event_end metadata. + for x_train, train_normal, x_test, test_normal, event_id in dataset.iter_formatted_datasets( + wind_farm="B", index_column="time_stamp", + ): + info = dataset.get_event_info(event_id) + + # Example placeholder prediction: one boolean anomaly prediction per + # timestamp in the prediction section, indexed like x_test. + predicted_anomalies = pd.Series(False, index=x_test.index) + + scorer.evaluate_event( + event_start=info["event_start"], + event_end=info["event_end"], + event_label=info["event_label"], + predicted_anomalies=predicted_anomalies, + normal_index=test_normal, + event_id=event_id, + ) + + final_score = scorer.get_final_score() + +.. note:: + + ``get_final_score`` requires at least one evaluated ``anomaly`` event *and* one + evaluated ``normal`` event. A single ``evaluate_event`` call is fine for a quick + smoke test, but the final CARE score can only be computed once events of both + labels have been added. + +CARE score overview +------------------- + +The CARE score combines four aspects of fault detection quality: + +- **Coverage** +- **Accuracy** +- **Reliability** +- **Earliness** + +At a high level: + +- Coverage and Earliness are computed on anomalous events, +- Accuracy is computed on normal events, +- Reliability is an event-wise score based on event decisions. + +The package implementation follows the CARE logic provided in the publication and subsequent +package support. For detailed interpretation notes, see :doc:`care2compare_faq`. + +Wind-farm-specific interpretation +--------------------------------- + +The meaning and usefulness of ``status_type_id`` differs slightly between wind farms. + +For Wind Farms B and C, status labels reflect anonymized operator-provided status information and are +useful both for training-data filtering and for parts of CARE-style evaluation. + +For Wind Farm A, the status labels should mainly be used for training-data filtering. For prediction-time +CARE evaluation, they should largely be ignored; pass ``ignore_normal_index=True`` to +:meth:`~energy_fault_detector.evaluation.care_score.CAREScore.evaluate_event` so that every timestamp is +evaluated regardless of ``status_type_id``. + +This distinction is important when interpreting evaluation results. + +Known practical limitations +--------------------------- + +Users should be aware of several dataset and benchmark limitations: + +- the dataset is anonymized, +- the data is not fully preprocessed for modeling, +- exact benchmark code from the original paper is not guaranteed to match the current package state, +- overlapping timestamps across different event files and and no temporal order of events from the same asset may occur due to anonymization, +- some version-specific data-quality notes apply. + +These are discussed in more detail in :doc:`care2compare_faq`. + +See also +-------- + +- :doc:`care2compare_faq` +- :class:`energy_fault_detector.evaluation.care2compare.Care2CompareDataset` +- :class:`energy_fault_detector.evaluation.care_score.CAREScore` diff --git a/docs/conf.py b/docs/conf.py index ed747664..ed69f7f7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,6 +53,16 @@ autoclass_content = 'both' +# Exclude sklearn boilerplate methods (metadata routing, set_output) that are +# not re-implemented in our classes. +autodoc_default_options = { + 'exclude-members': ( + 'set_fit_request, set_transform_request, set_inverse_transform_request, ' + 'set_predict_request, set_score_request, set_output, get_metadata_routing, ' + '__sklearn_is_fitted__' + ), +} + # ====================================================================================== # No configurable options below this line @@ -279,8 +289,22 @@ def run_apidoc(app): ]) +def _skip_sklearn_dunders(app, what, name, obj, skip, options): + """Skip sklearn boilerplate dunder methods that have docstrings. + + ``napoleon_include_special_with_doc = True`` forces inclusion of special + methods with docstrings via ``emit_first_result``, overriding + ``exclude-members``. This handler runs with higher priority (lower number) + so it is called before Napoleon's handler and takes precedence. + """ + if name == '__sklearn_is_fitted__': + return True + return None + + def setup(app): app.connect("builder-inited", run_apidoc) + app.connect("autodoc-skip-member", _skip_sklearn_dunders, priority=100) def linkcode_resolve(domain, info): diff --git a/docs/configuration.rst b/docs/configuration.rst index 0abe2f86..de3eec02 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -91,27 +91,25 @@ with the following keys: Allowed step names and aliases: -+-------------------------+-----------------------------------------------+------------------------------------------------+ -| Step name | Purpose | Aliases | -+=========================+===============================================+================================================+ -| column_selector | Drop columns with too many NaNs | \- | -+-------------------------+-----------------------------------------------+------------------------------------------------+ -| low_unique_value_filter | Drop columns with low variance/many zeros | \- | -+-------------------------+-----------------------------------------------+------------------------------------------------+ -| angle_transformer | Convert angles to sin/cos pairs | angle_transform | -+-------------------------+-----------------------------------------------+------------------------------------------------+ -| counter_diff_transformer| Convert counters to differences/rates | counter_diff, counter_diff_transform | -+-------------------------+-----------------------------------------------+------------------------------------------------+ -| timestamp_transformer | Extract time features (hour, day, etc.) | timestamp_transform,timestamp_features | -+-------------------------+-----------------------------------------------+------------------------------------------------+ -| simple_imputer | Impute missing values | imputer | -+-------------------------+-----------------------------------------------+------------------------------------------------+ -| standard_scaler | Standardize features (z-score) | standardize, standardscaler, standard | -+-------------------------+-----------------------------------------------+------------------------------------------------+ -| minmax_scaler | Scale to [0, 1] | minmax | -+-------------------------+-----------------------------------------------+------------------------------------------------+ -| duplicate_to_nan | Replace consecutive duplicate values with NaN | duplicate_value_to_nan, duplicate_values_to_nan| -+-------------------------+-----------------------------------------------+------------------------------------------------+ ++--------------------------+---------------------------------------------------+-------------------------------------------------+ +| Step name | Purpose | Aliases | ++==========================+===================================================+=================================================+ +| column_selector | Drop columns with too many NaNs | \- | ++--------------------------+---------------------------------------------------+-------------------------------------------------+ +| low_unique_value_filter | Drop columns with low variance/many zeros | \- | ++--------------------------+---------------------------------------------------+-------------------------------------------------+ +| angle_transformer | Convert angles to sin/cos pairs | angle_transform | ++--------------------------+---------------------------------------------------+-------------------------------------------------+ +| counter_diff_transformer | Convert counters to differences/rates | counter_diff, counter_diff_transform | ++--------------------------+---------------------------------------------------+-------------------------------------------------+ +| timestamp_transformer | Extract time features (hour, day, etc.) | timestamp_transform, timestamp_features | ++--------------------------+---------------------------------------------------+-------------------------------------------------+ +| simple_imputer | Impute missing values | imputer | ++--------------------------+---------------------------------------------------+-------------------------------------------------+ +| scaler | Standardize (standard) or scale to [0,1] (minmax) | standard_scaler, minmax_scaler (deprecated) | ++--------------------------+---------------------------------------------------+-------------------------------------------------+ +| duplicate_to_nan | Replace consecutive duplicate values with NaN | duplicate_value_to_nan, duplicate_values_to_nan | ++--------------------------+---------------------------------------------------+-------------------------------------------------+ For detailed documentation of the data preprocessor pipeline, refer to the :py:obj:`DataPreprocessor ` docs. diff --git a/docs/examples/advanced_config.yaml b/docs/examples/advanced_config.yaml index ac25ad01..b79496f1 100644 --- a/docs/examples/advanced_config.yaml +++ b/docs/examples/advanced_config.yaml @@ -45,22 +45,39 @@ train: params: min_unique_value_count: 2 max_col_zero_frac: 0.99 - # Transform angles to sin/cos - - name: angle_transformer + # Encode declared categorical features + - name: categorical_encoder params: - angles: - - angle1 - - angle2 + categorical_features: + - categorical_feature1 + - categorical_feature2 # Imputer (explicit; will be auto-inserted if omitted) - name: simple_imputer params: strategy: mean + # Alternatively, you can use a forward-fill imputer: + # - name: ffill_imputer + # params: + # ffill_limit: '1h' # maximum time gap to forward-fill as a pandas Timedelta string (e.g., '15min', '1h') + # categorical_features: + # - categorical_feature1 + # - categorical_feature2 # Scaler (choose one; StandardScaler is auto-added by default if omitted) - - name: standard_scaler + - name: scaler params: + scaler_type: 'standard' # Supported types: 'standard', 'minmax' with_mean: true with_std: true - + scale_categorical_features: True + categorical_features: + - categorical_feature1 + - categorical_feature2 + # Transform angles to sin/cos + - name: angle_transformer + params: + angles: + - angle1 + - angle2 data_splitter: # How to split data in train and validation sets for the autoencoder type: sklearn diff --git a/docs/examples/cnn_config.yaml b/docs/examples/cnn_config.yaml index 5723ca29..a7c6423f 100644 --- a/docs/examples/cnn_config.yaml +++ b/docs/examples/cnn_config.yaml @@ -10,7 +10,7 @@ train: max_nan_frac_per_col: 0.2 - name: low_unique_value_filter - name: simple_imputer - - name: standard_scaler + - name: scaler autoencoder: name: cnn_seq2seq diff --git a/docs/index.rst b/docs/index.rst index b75ddfef..fc1cd0a2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,8 @@ Installation :maxdepth: 1 arcana + care2compare_guide + care2compare_faq models_overview sequence_models diff --git a/docs/quick_fault_detection.rst b/docs/quick_fault_detection.rst index e34a09fd..383e3c59 100644 --- a/docs/quick_fault_detection.rst +++ b/docs/quick_fault_detection.rst @@ -120,12 +120,6 @@ dataclass. An example options file: automatic_optimization: true enable_debug_plots: false -The underlying helper functions are implemented in: - -- :mod:`energy_fault_detector.quick_fault_detection.data_loading` -- :mod:`energy_fault_detector.quick_fault_detection.configuration` -- :mod:`energy_fault_detector.quick_fault_detection.pipeline` - Output ------ The CLI writes: diff --git a/docs/sequence_models.rst b/docs/sequence_models.rst index 91574ef8..bcb9d726 100644 --- a/docs/sequence_models.rst +++ b/docs/sequence_models.rst @@ -133,8 +133,8 @@ Training and prediction are identical to the dense autoencoder case: print(results.reconstruction.shape) -Seq2One models (sequence-to-one) -================================ +Sequence-to-one models +====================== Seq2One models take a window of length ``sequence_length`` and reconstruct the **last timestep** of that window: @@ -210,8 +210,8 @@ The ``merge_mode`` controls how forward and backward encoder outputs are merged. recommended setting is ``"sum"``. -Seq2Seq models (sequence-to-sequence) -===================================== +Sequence-to-sequence models +=========================== Seq2Seq autoencoders reconstruct the **entire input window**: diff --git a/energy_fault_detector/__init__.py b/energy_fault_detector/__init__.py index 68145b36..41c62847 100644 --- a/energy_fault_detector/__init__.py +++ b/energy_fault_detector/__init__.py @@ -1,4 +1,8 @@ -"""The energy-fault-detector package""" +"""Autoencoder-based anomaly and fault detection for renewable energy assets and power grids. + +The package learns a normal-behaviour model from operational data and flags deviations as +anomalies, and includes the ARCANA method for root-cause analysis of detected anomalies. +""" from .__about__ import __version__ from energy_fault_detector.core._logs import setup_logging diff --git a/energy_fault_detector/anomaly_scores/__init__.py b/energy_fault_detector/anomaly_scores/__init__.py index 4148e466..4aea53a1 100644 --- a/energy_fault_detector/anomaly_scores/__init__.py +++ b/energy_fault_detector/anomaly_scores/__init__.py @@ -1,4 +1,12 @@ -"""Anomaly score classes.""" +"""Anomaly scores that reduce per-feature reconstruction errors to one score per sample. + +After an autoencoder reconstructs its input, an :class:`~energy_fault_detector.core.AnomalyScore` +aggregates the per-feature reconstruction errors into a single scalar per sample. +Samples with high scores are candidate anomalies. + +Each scorer subclasses :class:`~energy_fault_detector.core.AnomalyScore` and is selected +via the ``train.anomaly_score`` config key. +""" from energy_fault_detector.anomaly_scores.mahalanobis_score import MahalanobisScore from energy_fault_detector.anomaly_scores.rmse_score import RMSEScore diff --git a/energy_fault_detector/anomaly_scores/mahalanobis_score.py b/energy_fault_detector/anomaly_scores/mahalanobis_score.py index 556e898f..329b9b65 100644 --- a/energy_fault_detector/anomaly_scores/mahalanobis_score.py +++ b/energy_fault_detector/anomaly_scores/mahalanobis_score.py @@ -25,7 +25,7 @@ class MahalanobisScore(AnomalyScore): Configuration example: - .. code-block:: text + .. code-block:: yaml train: anomaly_score: diff --git a/energy_fault_detector/anomaly_scores/rmse_score.py b/energy_fault_detector/anomaly_scores/rmse_score.py index 5c29a0d6..ea3f6850 100644 --- a/energy_fault_detector/anomaly_scores/rmse_score.py +++ b/energy_fault_detector/anomaly_scores/rmse_score.py @@ -19,7 +19,7 @@ class RMSEScore(AnomalyScore): Configuration example: - .. code-block:: text + .. code-block:: yaml train: anomaly_score: diff --git a/energy_fault_detector/autoencoders/__init__.py b/energy_fault_detector/autoencoders/__init__.py index 3aa56ce5..f7e81d98 100644 --- a/energy_fault_detector/autoencoders/__init__.py +++ b/energy_fault_detector/autoencoders/__init__.py @@ -1,4 +1,11 @@ -"""Autoencoder model classes.""" +"""Autoencoder normal-behaviour models trained to reconstruct normal operation. + +Large reconstruction errors at prediction time indicate anomalies. Models range from dense +(point-wise) autoencoders to sequence models (seq2one and seq2seq) for time-series input. + +Every autoencoder model here subclasses :class:`~energy_fault_detector.core.autoencoder.Autoencoder` +and is referenced from the configuration under the ``train.autoencoder`` key. +""" # Dense from .multilayer_autoencoder import MultilayerAutoencoder diff --git a/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py b/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py index 8f31ac93..e00e4f52 100644 --- a/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py +++ b/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py @@ -50,7 +50,7 @@ class BidirectionalLSTMSeq2OneAutoencoder(Seq2OneAutoencoder): Configuration example: - .. code-block:: text + .. code-block:: yaml train: autoencoder: diff --git a/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py b/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py index 0cda7e3b..f525729d 100644 --- a/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py +++ b/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py @@ -64,7 +64,7 @@ class CNNSeq2OneAutoencoder(Seq2OneAutoencoder): Configuration example: - .. code-block:: text + .. code-block:: yaml train: autoencoder: diff --git a/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py b/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py index 0c7b9c1d..bccb9b20 100644 --- a/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py +++ b/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py @@ -38,7 +38,7 @@ class CNNAutoencoder(Seq2SeqAutoencoder): Configuration example: - .. code-block:: text + .. code-block:: yaml train: autoencoder: diff --git a/energy_fault_detector/autoencoders/conditional_autoencoder.py b/energy_fault_detector/autoencoders/conditional_autoencoder.py index e00357a1..4f3119a5 100644 --- a/energy_fault_detector/autoencoders/conditional_autoencoder.py +++ b/energy_fault_detector/autoencoders/conditional_autoencoder.py @@ -35,7 +35,7 @@ class ConditionalAE(Autoencoder): Configuration example: - .. code-block:: text + .. code-block:: yaml train: autoencoder: @@ -44,7 +44,7 @@ class ConditionalAE(Autoencoder): layers: [200] code_size: 40 learning_rate: 0.001 - batch_size: 128, + batch_size: 128 epochs: 15 loss_name: mse conditional_features: diff --git a/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py b/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py index c771f4ee..f02b5cb2 100644 --- a/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py +++ b/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py @@ -47,7 +47,7 @@ class LSTMSeq2OneAutoencoder(Seq2OneAutoencoder): Configuration example: - .. code-block:: text + .. code-block:: yaml train: autoencoder: diff --git a/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py b/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py index 3d1fba66..410b33fe 100644 --- a/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py +++ b/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py @@ -42,7 +42,7 @@ class LSTMSeqAutoencoder(Seq2SeqAutoencoder): Configuration example: - .. code-block:: text + .. code-block:: yaml train: autoencoder: diff --git a/energy_fault_detector/autoencoders/multilayer_autoencoder.py b/energy_fault_detector/autoencoders/multilayer_autoencoder.py index d5cb1b15..18f38536 100644 --- a/energy_fault_detector/autoencoders/multilayer_autoencoder.py +++ b/energy_fault_detector/autoencoders/multilayer_autoencoder.py @@ -35,7 +35,7 @@ class MultilayerAutoencoder(Autoencoder): Configuration example: - .. code-block:: text + .. code-block:: yaml train: autoencoder: @@ -44,7 +44,7 @@ class MultilayerAutoencoder(Autoencoder): layers: [200] code_size: 40 learning_rate: 0.001 - batch_size: 128, + batch_size: 128 epochs: 15 loss_name: mse """ diff --git a/energy_fault_detector/base_config.yaml b/energy_fault_detector/base_config.yaml index aebcc2b0..5267352b 100644 --- a/energy_fault_detector/base_config.yaml +++ b/energy_fault_detector/base_config.yaml @@ -10,7 +10,12 @@ train: max_nan_frac_per_col: 0.2 - name: low_unique_value_filter - name: simple_imputer - - name: standard_scaler + - name: scaler + + data_splitter: + type: sklearn + validation_split: 0.2 + shuffle: true autoencoder: name: default @@ -29,11 +34,6 @@ train: anomaly_score: name: rmse - data_splitter: - type: sklearn - validation_split: 0.2 - shuffle: true - threshold_selector: fit_on_val: true name: quantile diff --git a/energy_fault_detector/config/__init__.py b/energy_fault_detector/config/__init__.py index 887e38f4..a7b6bc61 100644 --- a/energy_fault_detector/config/__init__.py +++ b/energy_fault_detector/config/__init__.py @@ -1,4 +1,4 @@ -"""Configuration classes.""" +"""Configuration of the fault-detection pipeline, loaded and validated from YAML.""" from energy_fault_detector.config.config import Config from energy_fault_detector.config.base_config import InvalidConfigFile diff --git a/energy_fault_detector/config/config.py b/energy_fault_detector/config/config.py index 8fb87e63..998099a4 100644 --- a/energy_fault_detector/config/config.py +++ b/energy_fault_detector/config/config.py @@ -351,8 +351,8 @@ def _data_preprocessor_params_to_steps(params: Dict[str, Any]) -> List[Dict[str, # 6. Scaler scale = p.get("scale", "standardize") if scale in ["standardize", "standard", "standardscaler"]: - steps.append({"name": "standard_scaler", "step_name": "scaler"}) + steps.append({"name": "scaler", "step_name": "scaler", "params": {"scaler_type": "standard"}}) else: - steps.append({"name": "minmax_scaler", "step_name": "scaler"}) + steps.append({"name": "scaler", "step_name": "scaler", "params": {"scaler_type": "minmax"}}) return steps diff --git a/energy_fault_detector/config/quickstart_config.py b/energy_fault_detector/config/quickstart_config.py index 93fe0e70..57506f42 100644 --- a/energy_fault_detector/config/quickstart_config.py +++ b/energy_fault_detector/config/quickstart_config.py @@ -93,9 +93,9 @@ def _build_preprocessor_steps( # Final scaler with aliases supported for convenience. scaler_key = scaler.lower() if scaler_key in ("standard", "standardize", "standard_scaler"): - steps.append({"name": "standard_scaler"}) + steps.append({"name": "scaler", "params": {"scaler_type": "standard"}}) elif scaler_key in ("minmax", "minmax_scaler", "normalize"): - steps.append({"name": "minmax_scaler"}) + steps.append({"name": "scaler", "params": {"scaler_type": "minmax"}}) else: raise ValueError( f"Unknown scaler '{scaler}'. Use 'standard' (aka 'standardize') or 'minmax'." diff --git a/energy_fault_detector/core/__init__.py b/energy_fault_detector/core/__init__.py index e0b6d92a..7fbd9160 100644 --- a/energy_fault_detector/core/__init__.py +++ b/energy_fault_detector/core/__init__.py @@ -1,5 +1,8 @@ -"""This module contains class templates for most of the anomaly detection classes, such as -anomaly scores, threshold selectors and data classes.""" +"""Abstract base classes and shared infrastructure for the fault-detection pipeline. + +Concrete implementations of these contracts live in their own subpackages +(autoencoders, anomaly_scores, threshold_selectors, data_preprocessing). +""" from .anomaly_score import AnomalyScore from .data_transformer import DataTransformer diff --git a/energy_fault_detector/core/autoencoder.py b/energy_fault_detector/core/autoencoder.py index 73321324..4d7a9c22 100644 --- a/energy_fault_detector/core/autoencoder.py +++ b/energy_fault_detector/core/autoencoder.py @@ -496,8 +496,9 @@ def load(self, directory: str, **kwargs) -> "Autoencoder": # pylint: disable=W0 warnings.warn("No fitted autoencoder model was found.") if not encoder_loaded: warnings.warn("No fitted encoder model was found.") - if not decoder_loaded: - warnings.warn("No fitted decoder model was found.") + # Uncommented for now, since many models do not save the decoder. + # if not decoder_loaded: + # warnings.warn("No fitted decoder model was found.") return self diff --git a/energy_fault_detector/core/fault_detection_model.py b/energy_fault_detector/core/fault_detection_model.py index 5ce45798..d4b752b0 100644 --- a/energy_fault_detector/core/fault_detection_model.py +++ b/energy_fault_detector/core/fault_detection_model.py @@ -139,6 +139,52 @@ def predict(self, sensor_data: pd.DataFrame, model_path: Optional[str] = None, a FaultDetectionResult object. """ + @staticmethod + def _handle_duplicate_index(data: Union[pd.DataFrame, pd.Series]) -> Union[pd.DataFrame, pd.Series]: + """Drop true duplicate indices and raise on contradicting ones. + + Rows sharing the same index but with identical values across all columns are + treated as redundant copies and the first occurrence is kept. Rows sharing the + same index but with differing values indicate a data integrity problem and cause + a ``ValueError``. + + Args: + data: A pandas DataFrame or Series with a potentially duplicated index. + + Returns: + The input with true duplicates removed (keep first). + + Raises: + ValueError: If contradicting duplicates (same index, different values) are found. + """ + dup_mask = data.index.duplicated(keep=False) + if not dup_mask.any(): + return data + + dup_indices = data.index[dup_mask].unique() + contradicting = [] + for idx in dup_indices: + rows = data.loc[idx] + if isinstance(rows, pd.DataFrame): + if len(rows.drop_duplicates()) > 1: + contradicting.append(idx) + else: + if rows.nunique() > 1: + contradicting.append(idx) + + if contradicting: + raise ValueError( + f"Found contradicting duplicate indices (same index, different values): " + f"{list(contradicting)[:10]}. Please check your input data for data integrity issues." + ) + + n_before = len(data) + data = data[~data.index.duplicated(keep='first')] + n_dropped = n_before - len(data) + if n_dropped: + logger.info(f"Dropped {n_dropped} duplicate rows (identical index and values).") + return data + def train_val_split(self, x: DataType) -> Tuple[DataType, DataType]: """Split data in train and validation data. diff --git a/energy_fault_detector/data_preprocessing/__init__.py b/energy_fault_detector/data_preprocessing/__init__.py index 29ee04c9..c658be00 100644 --- a/energy_fault_detector/data_preprocessing/__init__.py +++ b/energy_fault_detector/data_preprocessing/__init__.py @@ -1,4 +1,11 @@ -"""Data preprocessing classes and utilities.""" +"""Configurable preprocessing pipeline and outlier clipping for sensor data. + +The :class:`~energy_fault_detector.data_preprocessing.data_preprocessor.DataPreprocessor` is an +sklearn-style ``Pipeline`` assembled from configurable :class:`~energy_fault_detector.core.DataTransformer` +steps (see ``DataPreprocessor.STEP_REGISTRY``). +The pipeline is configured via the ``train.data_preprocessor.steps`` config key. + +""" from energy_fault_detector.data_preprocessing.data_preprocessor import DataPreprocessor from energy_fault_detector.data_preprocessing.data_clipper import DataClipper diff --git a/energy_fault_detector/data_preprocessing/categorical_encoder.py b/energy_fault_detector/data_preprocessing/categorical_encoder.py new file mode 100644 index 00000000..ff446768 --- /dev/null +++ b/energy_fault_detector/data_preprocessing/categorical_encoder.py @@ -0,0 +1,183 @@ +from typing import List +import logging +import numpy as np +import pandas as pd +from sklearn.preprocessing import OneHotEncoder +from sklearn.utils.validation import check_is_fitted +from energy_fault_detector.core.data_transformer import DataTransformer + +logger = logging.getLogger('energy_fault_detector') + + +class CategoricalEncoder(DataTransformer): + """Transformer for encoding categorical features using one-hot encoding. + + Encodes specified categorical features using one-hot encoding while maintaining numerical + features unchanged. Non-declared object-type columns in numerical slots are automatically + detected and excluded. Column matching for ``categorical_features`` is by exact name. + + Attributes: + categorical_features (list): A list of strings representing the categorical feature names to be one-hot encoded (matched by exact name). + If not provided, the encoder will not encode any categorical features. + feature_names_out_ (list): The list of output feature names after transformation. + numerical_columns (list): The list of numerical feature names identified from the input data. + n_features_in_ (int): The total number of input features in the dataset. + feature_names_in_ (list): The list of input feature names identified during the fit process. + one_hot_encoder (OneHotEncoder): An instance of ``OneHotEncoder`` used for transforming categorical features. + categorical_columns (list): The list of categorical columns identified from the input data. + non_declared_categorical_features (list): Object-type columns detected in numerical slots and excluded. + """ + + def __init__(self, categorical_features: list = None): + super().__init__() + self.feature_names_out_ = None + self.numerical_columns = None + self.feature_names_in_ = None + self.categorical_features = categorical_features if categorical_features else [] + self.one_hot_encoder = OneHotEncoder(sparse_output=False, handle_unknown='ignore') + self.categorical_columns = None + self.non_declared_categorical_features = None + + def fit(self, x: pd.DataFrame, y=None) -> "CategoricalEncoder": + """Fits the OneHotEncoder to the categorical features in the provided DataFrame. + + Identifies numerical and categorical columns, detects and excludes non-declared + object-type columns from numerical slots, and fits the internal ``OneHotEncoder``. + + Args: + x (pd.DataFrame): The input data containing all features. + Only the specified categorical features will be fitted. + y: Ignored. Retained for compatibility with scikit-learn API. + + Returns: + CategoricalEncoder: The fitted encoder instance (self). + """ + self.feature_names_in_ = x.columns.tolist() + self.categorical_columns = [col for col in self.feature_names_in_ + if col in self.categorical_features] + self.numerical_columns = [col for col in self.feature_names_in_ if col not in self.categorical_columns] + + # Clean numerical columns from non_declared categorical features + numerical_data = x.loc[:, self.numerical_columns] + self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist() + self.numerical_columns = [col for col in self.numerical_columns + if col not in self.non_declared_categorical_features] + + if self.non_declared_categorical_features: + logger.warning( + "Non-declared categorical features found in data: %s. " + "They will be dropped. Consider adding them to the categorical_features list if they should be treated as categorical.", + self.non_declared_categorical_features, + ) + + self.n_features_in_ = len(self.feature_names_in_) + categorical_data = x[self.categorical_columns] + + self.one_hot_encoder.fit(categorical_data) + return self + + def transform(self, x: pd.DataFrame) -> pd.DataFrame: + """Transforms the input DataFrame by applying one-hot encoding to categorical features. + + Combines one-hot encoded categorical features with unchanged numerical features. + If no categorical features are specified, returns the numerical data unchanged. + + Args: + x (pd.DataFrame): The input data to be transformed. + + Returns: + pd.DataFrame: The transformed DataFrame with one-hot encoded categorical features + and numerical features. + + Raises: + KeyError: If the input DataFrame is missing any of the features fitted during ``fit``. + TypeError: If ``x`` is not a pandas DataFrame. + """ + check_is_fitted(self, "n_features_in_") + + if not isinstance(x, pd.DataFrame): + raise TypeError(f"x must be a pandas DataFrame, got {type(x).__name__}.") + + # Separate data into numerical and categorical features. Only categorical features are transformed + missing = [c for c in self.numerical_columns + self.categorical_columns if c not in x.columns] + if missing: + raise KeyError(f"The following features are not present in the input data: {missing}") + numerical_data = x[self.numerical_columns] + categorical_data = x[self.categorical_columns] + + if not categorical_data.empty: + for i, col in enumerate(self.categorical_columns): + fitted_cats = set(self.one_hot_encoder.categories_[i]) + current_cats = set(categorical_data[col].dropna().unique()) + new_cats = current_cats - fitted_cats + if new_cats: + logger.warning( + "Unseen categories found in feature '%s': %s.", + col, sorted(new_cats), + ) + x_categorical_ = self.one_hot_encoder.transform(categorical_data) + x_numerical_ = numerical_data.values + x_ = np.concatenate((x_numerical_, x_categorical_), axis=1) + self.feature_names_out_ = self.get_feature_names_out(self.feature_names_in_) + transformed_data = pd.DataFrame(x_, columns=self.feature_names_out_, index=x.index) + return transformed_data + else: + return numerical_data # Returns input df if no categorical features are specified in config file + + def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame: + """Reverts transformed data back to original form by decoding one-hot encoded features. + + Maps one-hot encoded categorical columns back to their original categorical values + and concatenates them with the numerical columns. If no categorical features were + encoded, returns the input unchanged. + + Args: + x (pd.DataFrame): The transformed input data with one-hot encoded columns. + + Returns: + pd.DataFrame: The data with categorical features restored to their original values. + + Raises: + KeyError: If the one-hot encoded columns are not present in the input data. + """ + check_is_fitted(self, "n_features_in_") + + # Get the one-hot encoded column names from the encoder + categorical_encoded_columns = list(self.one_hot_encoder.get_feature_names_out(self.categorical_columns)) + + # Separate numerical columns from one-hot-encoded categorical columns + numerical_columns = [col for col in x.columns if col not in categorical_encoded_columns] + + numerical_data = x[numerical_columns] + + if categorical_encoded_columns: + # Check if the categorical_encoded_columns are present in the input DataFrame + try: + categorical_data = x[categorical_encoded_columns] + except KeyError as exc: + raise KeyError(f"The features {categorical_encoded_columns} are not present in the input data.") from exc + + x_categorical_ = self.one_hot_encoder.inverse_transform(categorical_data) + categorical_original = pd.DataFrame(x_categorical_, columns=self.categorical_columns, index=x.index) + return pd.concat([numerical_data, categorical_original], axis=1) + else: + return numerical_data # Returns input df if no categorical features are specified in config file + + def get_feature_names_out(self, input_features=None) -> List[str]: + """Returns the names of features after the transformation. + + The output names are the concatenation of numerical column names and one-hot + encoded categorical column names (e.g. ``category_A``, ``category_B``). + + Args: + input_features: Unused. Retained for scikit-learn API compatibility. + + Returns: + List[str]: List of output feature names including both numerical and + one-hot encoded features. + """ + check_is_fitted(self, "n_features_in_") + self.feature_names_out_ = self.numerical_columns + list( + self.one_hot_encoder.get_feature_names_out(self.categorical_columns) + ) + return self.feature_names_out_ diff --git a/energy_fault_detector/data_preprocessing/data_clipper.py b/energy_fault_detector/data_preprocessing/data_clipper.py index e735d2dc..a0683a32 100644 --- a/energy_fault_detector/data_preprocessing/data_clipper.py +++ b/energy_fault_detector/data_preprocessing/data_clipper.py @@ -25,7 +25,7 @@ class DataClipper(DataTransformer): Configuration example: - .. code-block:: text + .. code-block:: yaml train: data_clipping: diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py index 2387ae75..31399bf7 100644 --- a/energy_fault_detector/data_preprocessing/data_preprocessor.py +++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py @@ -1,12 +1,13 @@ """Generic class for building a preprocessing pipeline.""" +import copy +import warnings from collections import Counter, defaultdict from typing import List, Optional, Dict, Any, Tuple import pandas as pd from sklearn.pipeline import Pipeline -from sklearn.preprocessing import StandardScaler, MinMaxScaler -from sklearn.impute import SimpleImputer +from sklearn.utils.validation import check_is_fitted from ..core.save_load_mixin import SaveLoadMixin from .column_selector import ColumnSelector @@ -15,34 +16,42 @@ from .duplicate_value_to_nan import DuplicateValuesToNan from .counter_diff_transformer import CounterDiffTransformer from .timestamp_transformer import TimestampTransformer +from .categorical_encoder import CategoricalEncoder +from .scaler import Scaler +from .imputer import Imputer +from .ffill_imputer import ForwardFillImputer class DataPreprocessor(Pipeline, SaveLoadMixin): + """ + A configurable data preprocessing pipeline for tabular data. + """ + STEP_REGISTRY = { 'duplicate_to_nan': DuplicateValuesToNan, 'column_selector': ColumnSelector, 'low_unique_value_filter': LowUniqueValueFilter, 'angle_transformer': AngleTransformer, 'counter_diff_transformer': CounterDiffTransformer, - 'simple_imputer': SimpleImputer, - 'standard_scaler': StandardScaler, - 'minmax_scaler': MinMaxScaler, + 'simple_imputer': Imputer, + 'scaler': Scaler, 'timestamp_transformer': TimestampTransformer, + 'categorical_encoder': CategoricalEncoder, + 'ffill_imputer': ForwardFillImputer, } NAME_ALIASES: Dict[str, str] = { "angle_transform": "angle_transformer", "counter_diff": "counter_diff_transformer", "counter_diff_transform": "counter_diff_transformer", - "standardize": "standard_scaler", - "standard": "standard_scaler", - "standardscaler": "standard_scaler", - "minmax": "minmax_scaler", + "scaler": "scaler", "imputer": "simple_imputer", "duplicate_value_to_nan": "duplicate_to_nan", "duplicate_values_to_nan": "duplicate_to_nan", "timestamp_features": "timestamp_transformer", "timestamp_transform": "timestamp_transformer", + "categorical_encoder": "categorical_encoder", + "ffill_imputer": "ffill_imputer", } def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None: @@ -66,13 +75,14 @@ def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None: 1) NaN introducing steps first (DuplicateValuesToNan, CounterDiffTransformer), 2) ColumnSelector (if present), 3) Other steps - 4) SimpleImputer placed before scaler (always present; mean strategy by default), - 5) Scaler always last (StandardScaler by default). - 6) TimestampTransformer (if present). + 4) Imputer placed before scaler (always present; mean strategy by default), + 5) CategoricalEncoder (if present), + 6) Scaler always last (StandardScaler by default). + 7) TimestampTransformer (if present). Configuration example: - .. code-block:: text + .. code-block:: yaml train: data_preprocessor: @@ -93,7 +103,7 @@ def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None: counters: ['energy_total_kwh'] compute_rate: False fill_first: 'zero' - reset_strategy: 'rollover', + reset_strategy: 'rollover' rollover_values: 'energy_total_kwh': 100000.0 """ @@ -110,16 +120,17 @@ def __init__(self, steps: Optional[List[Dict[str, Any]]] = None) -> None: # Ensure pandas output for supported transformers. self.set_output(transform="pandas") - def inverse_transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: - """Inverse-transform scaler and angles (other transforms are not reversed). + def inverse_transform(self, X: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: + """Inverse-transform scaler, timestamp transformer, categorical encoder and angle transformer (other transforms are not reversed). Args: - x: The transformed data. + X: The transformed data. Returns: DataFrame with inverse scaling and angle back-transformation. """ - x_ = x.copy() # avoid modifying the original DataFrame + check_is_fitted(self) + x_ = X.copy() # avoid modifying the original DataFrame # Drop time features timestamp_key, _ = self._find_step_by_type((TimestampTransformer,)) @@ -127,18 +138,23 @@ def inverse_transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: x_ = self.named_steps[timestamp_key].inverse_transform(x_.copy()) # Find scaler by type and reverse scaling - scaler_key, _ = self._find_step_by_type((StandardScaler, MinMaxScaler)) + scaler_key, _ = self._find_step_by_type((Scaler,)) x_ = self.named_steps[scaler_key].inverse_transform(x_) x_ = pd.DataFrame(data=x_, columns=self.named_steps[scaler_key].get_feature_names_out()) + # Try to reverse categorical encoding + encoder_key, _ = self._find_step_by_type((CategoricalEncoder,)) + if encoder_key is not None: + x_ = self.named_steps[encoder_key].inverse_transform(x_) + # Try to reverse angle transformation angle_key, _ = self._find_step_by_type((AngleTransformer,)) if angle_key is not None: x_ = self.named_steps[angle_key].inverse_transform(x_) # Keep original index - if isinstance(x, pd.DataFrame): - x_.index = x.index + if isinstance(X, pd.DataFrame): + x_.index = X.index return x_ @@ -152,8 +168,9 @@ def transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: Returns: DataFrame with the same index as input. """ + check_is_fitted(self) x_ = super().transform(X=x.copy()) - return pd.DataFrame(data=x_, columns=self.get_feature_names_out(), index=x.index) + return x_ # x_ is already a dataframe. Converting to a dataframe here and adding the input index is not working in case rows are dropped during preprocessing (e.g. in ForwardFillImputer). The index of the returned dataframe should be the index of the transformed dataframe, not the input dataframe. # pylint: disable=arguments-renamed def fit(self, X: pd.DataFrame, y=None, **fit_params): @@ -179,18 +196,18 @@ def fit(self, X: pd.DataFrame, y=None, **fit_params): # Call parent fit result = super().fit(X=X, y=y, **fit_params) - # Validate that protected features are in the output + # Validate that protected features survive the pipeline. ColumnSelector and LowUniqueValueFilter + # receive protected_features via fit_params and skip dropping them, but other steps (e.g. imputers + # dropping all-NaN columns) do not. This check enforces that protected features are still present + # in the output and raises if they were dropped (tested in test_data_preprocessor.py). if protected_features: output_features = self.get_feature_names_out() - missing_features = [f for f in protected_features if f not in output_features] + available_protected = [f for f in protected_features if f in X.columns] + missing_features = [f for f in available_protected + if not any(f in col for col in output_features)] if missing_features: raise ValueError( - f"Protected features were dropped by the preprocessing pipeline: {missing_features}. " - f"This may be caused by AngleTransformer or CounterDiffTransformer modifying these features. " - f"Please ensure protected features (e.g., conditional features for autoencoders) are not " - f"transformed by these steps or consider the transformed version of these features (e.g. " - f"_sin, _cos, _rate, _diff) as " - f"conditional features." + f"Protected features were dropped by the preprocessing pipeline: {missing_features}." ) return result @@ -229,8 +246,8 @@ def _validate_singletons(steps_spec: List[Dict[str, Any]]) -> None: "column_selector", "low_unique_value_filter", "simple_imputer", + "ffill_imputer", "timestamp_transformer", - # scaler handled separately (standard_scaler/minmax_scaler) in your code } counts: List[Tuple[str, int]] = [] for name in singleton_names: @@ -249,7 +266,7 @@ def _build_default_pipeline(self) -> List: Steps: - Column selection: A ColumnSelector object filters out columns/features with too many NaN values. - Low unique value filter: Remove columns/features with <= 2 unique values. - - Imputation with sklearn's SimpleImputer + - Simple imputation - Scaling: Apply either sklearn's StandardScaler or MinMaxScaler. Returns: @@ -259,8 +276,8 @@ def _build_default_pipeline(self) -> List: steps = [ ("column_selector", ColumnSelector(max_nan_frac_per_col=0.05)), ("low_unique_value_filter", LowUniqueValueFilter(min_unique_value_count=2, max_col_zero_frac=1.0)), - ("simple_imputer", SimpleImputer(strategy="mean").set_output(transform="pandas")), - ("standard_scaler", StandardScaler(with_mean=True, with_std=True)), + ("simple_imputer", Imputer(strategy="mean").set_output(transform="pandas")), + ("scaler", Scaler(with_mean=True, with_std=True)), ] return steps @@ -281,10 +298,14 @@ def _build_from_steps_spec(self) -> List: ValueError: If a step lacks 'name' or references an unknown step. """ - self._validate_step_spec_keys(self.steps_spec_) + # Work on a copy so the caller's config dict is not mutated by name normalization / step name assignment. + steps_spec = copy.deepcopy(self.steps_spec_) + + self._migrate_legacy_scaler_names(steps_spec) + self._validate_step_spec_keys(steps_spec) # Filter disabled steps first to simplify ordering. - enabled_spec = [s for s in self.steps_spec_ if s.get("enabled", True)] + enabled_spec = [s for s in steps_spec if s.get("enabled", True)] # Order the steps ordered_spec = self._order_steps_spec(enabled_spec) self._validate_singletons(enabled_spec) @@ -314,7 +335,7 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str, - NaN introducing steps first (DuplicateValuesToNan and CounterDiffTransformer) - ColumnSelector (if present). - Other steps - - Any imputer placed at the end, before scaler. If no imputer was defined, the SimpleImputer with imputation + - Any imputer placed at the end, before scaler. If no imputer was defined, the a simple imputer with imputation strategy 'mean' is added. - Scaler last (if present). If no scaler is added, the StandardScaler with default values is added. - TimestampTransformer (if present). @@ -334,26 +355,33 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str, low_unique_value_filter = [s for s in steps_spec if s.get("name") == "low_unique_value_filter"] duplicates = [s for s in steps_spec if s.get("name") == "duplicate_to_nan"] counter = [s for s in steps_spec if s.get("name") == "counter_diff_transformer"] - imputer = [s for s in steps_spec if s.get("name") == "simple_imputer"] - scaler_names = {"standard_scaler", "minmax_scaler"} - scalers = [s for s in steps_spec if s.get("name") in scaler_names] + imputer = [s for s in steps_spec if s.get("name") in {"simple_imputer", "ffill_imputer"}] + scalers = [s for s in steps_spec if s.get("name") == "scaler"] + encoders = [s for s in steps_spec if s.get("name") == "categorical_encoder"] if len(scalers) > 1: raise ValueError(f"Only one scaler can be used, two found in the steps specification: {scalers}") + if len(imputer) > 1: + raise ValueError(f"Only one imputer can be used, two found in the steps specification: {imputer}") timestamp_step = [s for s in steps_spec if s.get("name") == "timestamp_transformer"] others = [ s for s in steps_spec if s.get("name") not in { "column_selector", "duplicate_to_nan", "counter_diff_transformer", "simple_imputer", - "low_unique_value_filter", "timestamp_transformer", - } | scaler_names + "categorical_encoder", "low_unique_value_filter", "timestamp_transformer", "scaler", + "ffill_imputer" + } ] # Add default scaler if empty if not scalers: - scalers = [{'name': 'standard_scaler', + scalers = [{'name': 'scaler', 'step_name': 'scaler', - 'params': {'with_mean': True, 'with_std': True}}] + 'params': { + 'scaler_type': 'standard', + 'scale_categorical_features': True, + 'with_mean': True, + 'with_std': True}}] # Add default imputer if empty if not imputer: imputer = [{'name': 'simple_imputer', @@ -370,8 +398,11 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str, ordered.extend(low_unique_value_filter) # other transformations ordered.extend(others) - # Imputation and scaling + # Imputation ordered.extend(imputer) + # Encoding categorical features before scaling and after imputation + ordered.extend(encoders) + # Scaling ordered.extend(scalers) # No scaling needed for the time features ordered.extend(timestamp_step) @@ -419,6 +450,38 @@ def _assign_unique_step_names(specs: List[Dict[str, Any]]) -> List[Dict[str, Any return specs + @staticmethod + def _migrate_legacy_scaler_names(steps_spec: List[Dict[str, Any]]) -> None: + """Migrate legacy 'standard_scaler'/'minmax_scaler' step names to the canonical 'scaler' name. + + Legacy configs used ``standard_scaler`` or ``minmax_scaler`` as step names. This method rewrites + them in-place to the canonical ``scaler`` name with the appropriate ``scaler_type`` parameter and + emits a ``DeprecationWarning`` showing the correct configuration format. + + Args: + steps_spec: List of step spec dicts (mutated in place). + """ + legacy_map = {"standard_scaler": "standard", "minmax_scaler": "minmax"} + for spec in steps_spec: + name = spec.get("name") + if name in legacy_map: + scaler_type = legacy_map[name] + params = spec.get("params") or {} + params["scaler_type"] = scaler_type + spec["name"] = "scaler" + spec["params"] = params + warnings.warn( + f"Step name '{name}' is deprecated and has been automatically migrated to " + f"'scaler' with params {{'scaler_type': '{scaler_type}'}}. " + f"Please update your configuration to use:\n" + f" - name: scaler\n" + f" params:\n" + f" scaler_type: {scaler_type}\n" + f"This automatic migration will be removed in a future version.", + DeprecationWarning, + stacklevel=3, + ) + @staticmethod def _validate_step_spec_keys(steps_spec: List[Dict[str, Any]]) -> None: """Validate that each step spec uses only allowed keys and includes 'name'. diff --git a/energy_fault_detector/data_preprocessing/ffill_imputer.py b/energy_fault_detector/data_preprocessing/ffill_imputer.py new file mode 100644 index 00000000..c7e291ee --- /dev/null +++ b/energy_fault_detector/data_preprocessing/ffill_imputer.py @@ -0,0 +1,230 @@ +from typing import List, Optional, Union + +import logging +import numpy as np +import pandas as pd +from sklearn.utils.validation import check_is_fitted + +from energy_fault_detector.core.data_transformer import DataTransformer + +logger = logging.getLogger('energy_fault_detector') + + +class ForwardFillImputer(DataTransformer): + + """Impute missing values using time-based forward fill with a maximum gap duration. + + Forward-fills NaN values from the last valid observation, but only when the elapsed + time between the last valid value and the current row does not exceed ``ffill_limit``. + Rows where the gap is too large remain NaN and are subsequently dropped. + + The transformer assumes time-series data with a temporal index (DatetimeIndex or + TimedeltaIndex). During ``fit``, it categorizes features as numerical or categorical, + identifies and logs non-declared categorical features, and converts ``ffill_limit`` + to a :class:`~pandas.Timedelta`. During ``transform``, it forward-fills missing values + (invalidating fills that exceed the time threshold), drops duplicate rows, and removes + any rows still containing NaN values. + + Dropping duplicate rows after forward-filling is necessary because forward-filling + propagates the last valid value into consecutive NaN rows. If the underlying signal did + not change between observations, the filled rows become exact duplicates of the source + row. Removing these redundant rows avoids introducing identical training samples that + carry no additional information and would otherwise bias the reconstruction error. + + Attributes: + feature_names_in_ (List[str]): List of input feature names observed during fitting. + feature_names_out_ (List[str]): List of output feature names (same as input features unless some were dropped). + ffill_limit_ (pd.Timedelta): Fitted ``ffill_limit`` converted to a Timedelta. + categorical_features (List[str]): List of user-specified categorical feature names (matched by exact name). + numerical_columns (List[str]): List of identified numerical column names after filtering out non-declared categoricals. + categorical_columns (List[str]): List of identified categorical column names (from ``categorical_features``). + non_declared_categorical_features (List[str]): List of columns detected as object-type (categorical) but not declared as such. + """ + + def __init__(self, ffill_limit: Union[str, pd.Timedelta] = "15min", + categorical_features: Optional[List[str]] = None): + """Initializes the ForwardFillImputer with a forward-fill time limit and optional categorical features. + + Args: + ffill_limit (str | pd.Timedelta): Maximum time gap to forward-fill, expressed as a + pandas-compatible Timedelta string (e.g. ``"15min"``, ``"1h"``) or a + :class:`~pandas.Timedelta` instance. NaNs whose elapsed time from the last valid + observation exceeds this limit are left unfilled and the corresponding rows are + dropped. Default is ``"15min"``. + categorical_features (Optional[List[str]]): List of column names to treat as categorical features. + Columns matching any of these (by exact name) are treated as categorical; others as numerical. + Non-declared object-type columns in numerical slots are automatically detected and excluded from processing. + Default is None (interpreted as empty list). + """ + super().__init__() + self.ffill_limit = ffill_limit + + # Attributes set during fit. + self.feature_names_in_: List[str] = [] + self.feature_names_out_: List[str] = [] + self.ffill_limit_: pd.Timedelta = pd.Timedelta(0) + self.categorical_features = categorical_features if categorical_features else [] + self.numerical_columns: List[str] = [] + self.categorical_columns: List[str] = [] + self.non_declared_categorical_features: List[str] = [] + + def fit(self, x: pd.DataFrame, y: Optional[pd.Series] = None) -> "ForwardFillImputer": + """Fits the imputer to the input data by identifying feature types and storing metadata. + + Populates ``numerical_columns``, ``categorical_columns``, and ``non_declared_categorical_features`` + based on the input DataFrame's structure and user-provided categorical feature hints. + Converts ``ffill_limit`` to a :class:`~pandas.Timedelta` and stores it as ``ffill_limit_``. + + Args: + x (pd.DataFrame): Input feature DataFrame. Must contain all features used during ``transform``. + y (Optional[pd.Series]): Target variable. Ignored; included for compatibility with the scikit-learn API. + + Returns: + ForwardFillImputer: The fitted transformer instance (self), for method chaining. + + Raises: + TypeError: If `x` is not a pandas DataFrame. + ValueError: If `ffill_limit` cannot be converted to a Timedelta. + """ + if not isinstance(x, pd.DataFrame): + raise TypeError("x must be a pandas DataFrame.") + + self.feature_names_in_ = x.columns.tolist() + self.n_features_in_ = len(self.feature_names_in_) + + try: + self.ffill_limit_ = pd.Timedelta(self.ffill_limit) + except ValueError as e: + raise ValueError( + f"ffill_limit must be a pandas-compatible Timedelta string or Timedelta " + f"(e.g. '15min', '1h'), got: {self.ffill_limit!r}" + ) from e + + self.numerical_columns = [col for col in self.feature_names_in_ + if col not in self.categorical_features] + self.categorical_columns = [col for col in self.feature_names_in_ + if col in self.categorical_features] + # Clean numerical columns from non_declared categorical features + numerical_data = x.loc[:, self.numerical_columns] + self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist() + # Keep numerical and booleans + self.numerical_columns = [col for col in self.numerical_columns if col not in self.non_declared_categorical_features] + + if self.non_declared_categorical_features: + logger.warning(f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. " + f"They will be dropped. Consider adding them to the categorical_features list if they should be" + f" treated as categorical.") + + # Drop columns that are entirely NaN — they cannot be forward-filled and would otherwise + # cause every row to be dropped by dropna(how="any"). + all_nan_cols = [col for col in self.numerical_columns + self.categorical_columns + if x[col].isna().all()] + if all_nan_cols: + logger.warning(f"Columns containing only NaN values found: {all_nan_cols}. " + f"They will be dropped as they cannot be forward-filled.") + self.numerical_columns = [col for col in self.numerical_columns if col not in all_nan_cols] + self.categorical_columns = [col for col in self.categorical_columns if col not in all_nan_cols] + + return self + + def transform(self, x: pd.DataFrame) -> pd.DataFrame: + """Applies time-based forward-fill imputation and cleaning pipeline to input data. + + The steps executed are: + + 1. Forward-fill all NaN values. + 2. Invalidate fills where the elapsed time from the last valid observation exceeds + ``ffill_limit_`` (set those values back to NaN). + 3. Drop duplicate rows (exact duplicates created by forward-filling where the + underlying signal did not change). + 4. Drop any rows still containing NaN values (``dropna(how="any")``). + + Unlike the previous resampling-based approach, the original (possibly irregular) + timestamps are preserved — no synthetic rows are introduced. + + Args: + x (pd.DataFrame): Input feature DataFrame, with the same column names as used in ``fit()``. + + Returns: + pd.DataFrame: Cleaned and imputed DataFrame, containing only the columns from ``get_feature_names_out()``. + + Raises: + TypeError: If `x` is not a pandas DataFrame, or if its index is not temporal (DatetimeIndex/TimedeltaIndex). + ValueError: If `x` is missing columns seen during `fit()`, or if numerical columns cannot be converted to float. + ValueError: If the imputer has not been fitted (via `check_is_fitted`). + """ + check_is_fitted(self, "n_features_in_") + + x = x.copy() # Avoid modifying the original DataFrame + + feature_names_in = self.feature_names_in_ + if feature_names_in is None: + raise ValueError("ForwardFillImputer is not fitted.") + + if not isinstance(x, pd.DataFrame): + raise TypeError("x must be a pandas DataFrame.") + + missing_columns = [col for col in feature_names_in if col not in x.columns] + if missing_columns: + raise ValueError(f"Input is missing columns seen during fit: {missing_columns}") + + if not isinstance(x.index, (pd.DatetimeIndex, pd.TimedeltaIndex)): + raise TypeError( + "x index must be a DatetimeIndex or TimedeltaIndex for time-based forward fill." + ) + # Harmonize data types in numerical columns to avoid issues during concatenation after one-hot encoding + for col in self.numerical_columns: + if col in x.columns: + try: + x[col] = x[col].astype(float, errors='raise') + except ValueError as e: + raise ValueError(f"Column '{col}' cannot be converted to float.") from e + x_selected = x[self.numerical_columns + self.categorical_columns] + + # --- Time-based forward fill --- + idx_series = x.index.to_series() + df_filled = x_selected.copy() + for col in df_filled.columns: + col_series = df_filled[col] + nan_mask = col_series.isna() + if not nan_mask.any(): + continue + filled = col_series.ffill() + # Time elapsed since the last valid observation for each row + last_valid = idx_series.where(col_series.notna()).ffill() + elapsed = idx_series - last_valid + # Invalidate fills where the gap exceeds the threshold + invalid = nan_mask & (elapsed > self.ffill_limit_) + filled[invalid] = np.nan + df_filled[col] = filled + + df_cleaned = df_filled.drop_duplicates(keep="first") + df_final = df_cleaned.dropna(how="any") + + self.feature_names_out_ = self.get_feature_names_out() + return df_final[self.feature_names_out_] + + def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame: + """Returns the input DataFrame with columns reordered to match the original input feature order. + + This method is included for compatibility with the `DataTransformer`/scikit-learn API. + Since forward-fill imputation is not invertible (lost NaNs and duplicates cannot be recovered), + only column reordering is performed—no actual value inversion occurs. + + Args: + x (pd.DataFrame): Transformed DataFrame with columns matching `feature_names_out_`. + + Returns: + pd.DataFrame: DataFrame with columns reordered to match `feature_names_in_` (original order). + + Raises: + TypeError: If `x` is not a pandas DataFrame. + ValueError: If the transformer has not been fitted (via `check_is_fitted`). + """ + check_is_fitted(self, "n_features_in_") + return pd.DataFrame(x, columns=self.feature_names_in_) + + def get_feature_names_out(self, input_features=None) -> List[str]: + """Return output feature names for downstream transformers.""" + check_is_fitted(self, "n_features_in_") + return self.numerical_columns + self.categorical_columns diff --git a/energy_fault_detector/data_preprocessing/imputer.py b/energy_fault_detector/data_preprocessing/imputer.py new file mode 100644 index 00000000..e4490fae --- /dev/null +++ b/energy_fault_detector/data_preprocessing/imputer.py @@ -0,0 +1,235 @@ +from typing import List +import pandas as pd +from sklearn.utils.validation import check_is_fitted +from sklearn.impute import SimpleImputer + +from energy_fault_detector.core.data_transformer import DataTransformer +import logging + +logger = logging.getLogger('energy_fault_detector') + + +class Imputer(DataTransformer): + """Wrapper around scikit-learn's SimpleImputer to handle numerical and categorical features separately. + + Supports imputation via 'mean' (for numerical) and 'most_frequent' (for categorical) strategies. + Automatically detects and excludes non-declared categorical features (object dtype in numerical slots). + Preserves column order and index through fit/transform/inverse_transform. + + Attributes: + strategy (str): Imputation strategy for numerical features ('mean' or 'median'). + categorical_features (List[str]): List of column names to treat as categorical (matched by exact name). + params (dict): Additional keyword arguments passed to `SimpleImputer` for numerical imputation. + numerical_imputer (SimpleImputer): Imputer instance for numerical features. + categorical_imputer (SimpleImputer): Imputer instance for categorical features (always 'most_frequent'). + n_features_in_ (int): Number of features seen during `fit`. + feature_names_in_ (List[str]): Names of features seen during `fit`. + feature_names_out_ (List[str]): Names of features output after `transform`. + input_index_ (pd.Index): Index of input DataFrame during `fit`. + numerical_columns (List[str]): Column names identified as numerical. + categorical_columns (List[str]): Column names identified as categorical. + non_declared_categorical_features (List[str]): Object-type columns detected in numerical slots and excluded. + """ + + def __init__(self, strategy: str = 'mean', categorical_features: list = None, **params): + """Initializes the Imputer with specified strategy and categorical feature handling. + + Args: + strategy (str, optional): Imputation strategy for numerical features. Supports 'mean' (default) and 'median'. + Categorical features are always imputed using 'most_frequent'. + categorical_features (List[str], optional): List of column names to treat as categorical. + Columns matching any of these (by exact name) are treated as categorical; others as numerical. + Non-declared object-type columns in numerical slots are automatically detected and excluded. + Defaults to None (interpreted as empty list). + **params: Additional keyword arguments passed to `SimpleImputer` for numerical imputation. + Example: `{'add_indicator': True}`. + + Raises: + ValueError: If `strategy` is not supported ('mean' or 'median'). + """ + super().__init__() + self.strategy = strategy + self.categorical_features = categorical_features if categorical_features else [] + self.params = params + # Initialize nested estimators + # Separate imputers for numerical and categorical data + if self.strategy == 'mean': + self.numerical_imputer = SimpleImputer(strategy=self.strategy, **self.params) + self.categorical_imputer = SimpleImputer(strategy='most_frequent') # Categorical default to 'most_frequent' + elif self.strategy == 'median': + self.numerical_imputer = SimpleImputer(strategy=self.strategy, **self.params) + self.categorical_imputer = SimpleImputer(strategy='most_frequent') # Categorical default to 'most_frequent' + else: + raise ValueError(f"Unsupported strategy: {self.strategy}. Supported strategies are 'mean' and 'median'.") + + # Attributes to be defined during fitting + self.feature_names_in_: List[str] = [] + self.feature_names_out_ = None + self.input_index_ = None + self.numerical_columns: List[str] = [] + self.categorical_columns: List[str] = [] + self.non_declared_categorical_features: List[str] = [] + + def fit(self, x: pd.DataFrame, y=None) -> "Imputer": + """Fits the imputer separately for numerical and categorical features. + + Identifies and separates features by type, logs warnings for non-declared categorical columns, + and fits `SimpleImputer` instances on their respective subsets. + + Args: + x (pd.DataFrame): Input feature DataFrame. + y (optional): Target variable. Ignored; included for scikit-learn API compatibility. + + Returns: + Imputer: The fitted imputer instance (self), for method chaining. + + Raises: + TypeError: If `x` is not a pandas DataFrame. + ValueError: If numerical columns cannot be safely converted (via downstream usage). + """ + + self.feature_names_in_ = x.columns.tolist() + self.n_features_in_ = len(self.feature_names_in_) + self.input_index_ = x.index + + # Split data into numerical (and boolean) and categorical features + self.numerical_columns = [col for col in self.feature_names_in_ + if col not in self.categorical_features] + self.categorical_columns = [col for col in self.feature_names_in_ + if col in self.categorical_features] + numerical_data = x.loc[:, self.numerical_columns] + categorical_data = x.loc[:, self.categorical_columns] + + # Clean numerical columns from non_declared categorical features + self.non_declared_categorical_features = numerical_data.select_dtypes(include='object').columns.tolist() + self.numerical_columns = [col for col in self.numerical_columns + if col not in self.non_declared_categorical_features] + numerical_data = numerical_data.loc[:, self.numerical_columns] + if self.non_declared_categorical_features: + logger.warning( + f"Non-declared categorical features found in data: {self.non_declared_categorical_features}. " + f"They will be dropped. Consider adding them to the categorical_features list if they should be" + f" treated as categorical.") + + # Drop columns that are entirely NaN — they cannot be imputed and would cause + # SimpleImputer to silently skip them, leading to a column-count mismatch on transform. + all_nan_cols = [col for col in self.numerical_columns + self.categorical_columns + if x[col].isna().all()] + if all_nan_cols: + logger.warning(f"Columns containing only NaN values found: {all_nan_cols}. " + f"They will be dropped as they cannot be imputed.") + self.numerical_columns = [col for col in self.numerical_columns if col not in all_nan_cols] + self.categorical_columns = [col for col in self.categorical_columns if col not in all_nan_cols] + numerical_data = numerical_data.loc[:, self.numerical_columns] + categorical_data = categorical_data.loc[:, self.categorical_columns] + + logger.debug(f"Numerical columns: {self.numerical_columns}") + logger.debug(f"Categorical columns: {self.categorical_columns}") + + # Fit the imputers + if not numerical_data.empty: + self.numerical_imputer.fit(numerical_data) + if not categorical_data.empty: + self.categorical_imputer.fit(categorical_data) + + return self + + def transform(self, x: pd.DataFrame) -> pd.DataFrame: + """Applies imputation to input DataFrame, handling numerical and categorical columns separately. + + Steps: + 1. Validates presence of all training features. + 2. Harmonizes numerical column dtypes to float. + 3. Applies `numerical_imputer` and `categorical_imputer` independently. + 4. Reconcatenates and reorders columns to match original feature order. + + Args: + x (pd.DataFrame): Input feature DataFrame, with same columns as used in `fit()`. + + Returns: + pd.DataFrame: Imputed DataFrame with same column names and original index. + + Raises: + ValueError: If input is missing columns seen during `fit`. + ValueError: If a numerical column cannot be converted to float. + ValueError: If imputer has not been fitted (via `check_is_fitted`). + """ + check_is_fitted(self, "n_features_in_") + missing_columns = [col for col in self.feature_names_in_ if col not in x.columns] + if missing_columns: + raise ValueError(f"Input is missing columns seen during fit: {missing_columns}") + + # Harmonize data types in numerical columns to avoid issues during concatenation after one-hot encoding + x = x.copy() # Avoid modifying the original DataFrame + for col in self.numerical_columns: + if col in x.columns: + try: + x[col] = x[col].astype(float, errors='raise') + except ValueError as e: + raise ValueError(f"Column '{col}' cannot be converted to float.") from e + + # Separate data into numerical and categorical features + numerical_data = x.loc[:, self.numerical_columns] + categorical_data = x.loc[:, self.categorical_columns] + + # Transform the data + if not numerical_data.empty: + numerical_transformed = pd.DataFrame( + self.numerical_imputer.transform(numerical_data), + columns=self.numerical_columns, + index=x.index + ) + else: + numerical_transformed = pd.DataFrame(index=x.index) + if not categorical_data.empty: + categorical_transformed = pd.DataFrame( + self.categorical_imputer.transform(categorical_data), + columns=self.categorical_columns, + index=x.index + ) + else: + categorical_transformed = pd.DataFrame(index=x.index) # Empty DataFrame for consistency + + # Concatenate numerical and categorical data back together + self.feature_names_out_ = self.get_feature_names_out() + transformed_data = pd.concat([numerical_transformed, categorical_transformed], axis=1) + transformed_data = transformed_data[self.feature_names_out_] # Ensure original column order + + return transformed_data + + def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame: + """Returns input DataFrame with columns reordered to match original feature order. + + This method preserves the index of the input `x` and does **not** reverse imputation (since original + NaN positions are not stored). Included for scikit-learn compatibility. + + Args: + x (pd.DataFrame): Transformed DataFrame (output of `transform()`). + + Returns: + pd.DataFrame: DataFrame with columns reordered to match `feature_names_in_`. The index of `x` is preserved. + + Raises: + ValueError: If imputer has not been fitted (via `check_is_fitted`). + """ + check_is_fitted(self, "n_features_in_") + + return pd.DataFrame(x, columns=self.feature_names_in_) + + def get_feature_names_out(self, input_features=None) -> List[str]: + """Returns ordered list of output feature names. + + The output feature names are the concatenation of `numerical_columns` and `categorical_columns`, + in the order they were identified during `fit`. + + Args: + input_features: Ignored; included for scikit-learn API compatibility. + + Returns: + List[str]: Ordered list of output feature names. + + Raises: + ValueError: If imputer has not been fitted (via `check_is_fitted`). + """ + check_is_fitted(self, "n_features_in_") + return self.numerical_columns + self.categorical_columns diff --git a/energy_fault_detector/data_preprocessing/scaler.py b/energy_fault_detector/data_preprocessing/scaler.py new file mode 100644 index 00000000..f23dbff0 --- /dev/null +++ b/energy_fault_detector/data_preprocessing/scaler.py @@ -0,0 +1,208 @@ +import pandas as pd +from sklearn.preprocessing import StandardScaler, MinMaxScaler +from sklearn.utils.validation import check_is_fitted + +from energy_fault_detector.core.data_transformer import DataTransformer +import logging + +logger = logging.getLogger('energy_fault_detector') + + +class Scaler(DataTransformer): + """Preprocessing step for scaling numerical (and optionally categorical) features. + + Supports 'standard' (StandardScaler) and 'minmax' (MinMaxScaler) scalers. + Automatically enforces float conversion for all columns and optionally excludes + encoded categorical features from scaling depending on `scale_categorical_features`. + + Attributes: + scaler_type (str): Type of scaler selected ('standard' or 'minmax'). + scale_categorical_features (bool): Whether to scale features matched in `categorical_features`. + categorical_features (List[str]): List of column names treated as categorical (matched by substring, to cover one-hot encoded columns). + scaler (SklearnScaler): Instantiated sklearn scaler object (e.g., StandardScaler). + feature_names_in_ (List[str]): Names of features seen during `fit()`. + feature_names_out_ (List[str]): Names of features output after `transform()` (same as input). + columns_dropped_ (List[str]): Placeholder (unused in current implementation; preserved for API). + params (dict): Keyword arguments passed to the underlying sklearn scaler constructor. + """ + + SCALER_REGISTRY = { + 'standard': StandardScaler, + 'minmax': MinMaxScaler + } + + def __init__(self, scaler_type: str = 'standard', scale_categorical_features: bool = True, + categorical_features: list = None, **params): + """Initializes the Scaler with specified strategy and feature selection logic. + + Args: + scaler_type (str, optional): Type of scaler to use. Supported: `'standard'` (default), `'minmax'`. + scale_categorical_features (bool, optional): Whether to include categorical (e.g., one-hot encoded) + features in scaling. If `False`, only non-encoded features are scaled. + Defaults to `True`. + categorical_features (List[str], optional): List of column names to treat as categorical (matched by substring, + to cover one-hot encoded columns such as `category_col_A`). + Used only when `scale_categorical_features=False` to determine which features to exclude. + Defaults to `None` (interpreted as empty list). + **params: Additional keyword arguments passed to the underlying sklearn scaler constructor. + Example: `{'with_mean': False, 'with_std': True}`. + + Raises: + ValueError: If `scaler_type` is not supported (must be `'standard'` or `'minmax'`). + """ + super().__init__() + + # Validate scaler_type + if scaler_type not in self.SCALER_REGISTRY: + raise ValueError(f"Unsupported scaler_type '{scaler_type}'. Valid types are: {list(self.SCALER_REGISTRY.keys())}") + + self.scaler_type = scaler_type + self.scale_categorical_features = scale_categorical_features + self.categorical_features = categorical_features if categorical_features else [] + scaler = self.SCALER_REGISTRY.get(scaler_type) + + # Attributes to be defined during fitting + self.feature_names_in_ = None + self.feature_names_out_ = None + self.columns_dropped_ = [] + + # Parameters of nested estimators + self.params = params + # Initialize nested estimators + self.scaler = scaler(**self.params) + + def fit(self, x: pd.DataFrame, y: pd.Series = None) -> 'Scaler': + """Fits the scaler on the input data, optionally excluding categorical features. + + Enforces float dtype for all columns. Scales either all columns or only + non-encoded numerical features, depending on `scale_categorical_features`. + + Args: + x (pd.DataFrame): Input feature DataFrame. + y (pd.Series, optional): Target variable. Ignored; included for scikit-learn API compatibility. + + Returns: + Scaler: The fitted transformer instance (self), for method chaining. + + Raises: + ValueError: If any column cannot be converted to float. + ValueError: If column dtypes are inconsistent across features. + """ + logger.debug("Fitting Scaler transformer...") + x = x.copy() # Avoid modifying the original DataFrame + + # Check that all columns in input are numerical types. + for col in x.columns: + try: + x[col] = x[col].astype(float, errors='raise') + except ValueError as e: + raise ValueError(f"Column '{col}' cannot be converted to float.") from e + + self.feature_names_in_ = list(x.columns) + self.n_features_in_ = len(self.feature_names_in_) + + # Determine features to scale + if self.scale_categorical_features: + subset_to_fit = x + else: + non_encoded_features = [col for col in x.columns if not any(feature in col for feature in self.categorical_features)] + subset_to_fit = x[non_encoded_features] + + # Fit the scaler + self.scaler.fit(subset_to_fit) + + # Define output feature names + self.feature_names_out_ = list(x.columns) + return self + + def transform(self, x: pd.DataFrame) -> pd.DataFrame: + """Applies the learned scaling transformation to input data. + + Enforces float dtypes and validates feature alignment with `fit()`. + Scales either all columns or only numerical (non-encoded) columns, + depending on `scale_categorical_features`. + + Args: + x (pd.DataFrame): Input feature DataFrame, with column order and names matching those seen in `fit()`. + + Returns: + pd.DataFrame: Scaled DataFrame, same shape and index as input. + + Raises: + ValueError: If input feature names/order do not match those seen during `fit()`. + ValueError: If any column cannot be converted to float. + ValueError: If scaler has not been fitted (via `check_is_fitted`). + """ + logger.debug("Transforming data with Scaler transformer...") + check_is_fitted(self, "n_features_in_") + + x_transformed = x.copy() # Avoid modifying the original DataFrame + + # Check that all columns in input are numerical types. + for col in x_transformed.columns: + try: + x_transformed[col] = x_transformed[col].astype(float, errors='raise') + except ValueError as e: + raise ValueError(f"Column '{col}' cannot be converted to float.") from e + + # Check if input features match the features seen during fit + if list(x.columns) != self.feature_names_in_: + raise ValueError(f"Input features {list(x.columns)} do not match the features seen during fit {self.feature_names_in_}.") + # Transform the appropriate features + if self.scale_categorical_features: + x_transformed.iloc[:, :] = self.scaler.transform(x_transformed) + else: + non_encoded_features = [col for col in x.columns if not any(feature in col for feature in self.categorical_features)] + x_transformed[non_encoded_features] = self.scaler.transform(x[non_encoded_features]) + + return x_transformed + + def inverse_transform(self, x: pd.DataFrame) -> pd.DataFrame: + """Applies the inverse scaling transformation to revert scaled data. + + Requires that the scaler was fitted and that input column structure matches `fit()`. + + Args: + x (pd.DataFrame): Scaled input DataFrame, with same columns and order as `fit()` output. + + Returns: + pd.DataFrame: Inversely transformed (descaled) DataFrame, with same shape/index as input. + + Raises: + ValueError: If scaler has not been fitted (via `check_is_fitted`). + ValueError: If input feature names/order do not match those seen during `fit()`. + """ + logger.debug("Applying inverse transformation with Scaler transformer...") + check_is_fitted(self, "n_features_in_") + + x_inverse_transformed = x.copy() + + # Check if input features match the features seen during fit + if list(x.columns) != self.feature_names_in_: + raise ValueError(f"Input features {list(x.columns)} do not match the features seen during fit {self.feature_names_in_}.") + + # Apply inverse transform to the appropriate features + if self.scale_categorical_features: + x_inverse_transformed.iloc[:, :] = self.scaler.inverse_transform(x_inverse_transformed) + else: + non_encoded_features = [col for col in x.columns if not any(feature in col for feature in self.categorical_features)] + x_inverse_transformed[non_encoded_features] = self.scaler.inverse_transform(x[non_encoded_features]) + + return x_inverse_transformed + + def get_feature_names_out(self, input_features=None) -> list: + """Returns the list of output feature names (same as input feature names). + + Preserves original column order and names seen during `fit()`. + + Args: + input_features: Ignored; included for scikit-learn API compatibility. + + Returns: + List[str]: Ordered list of output feature names. + + Raises: + ValueError: If scaler has not been fitted (via `check_is_fitted`). + """ + check_is_fitted(self, "n_features_in_") + return self.feature_names_out_ diff --git a/energy_fault_detector/data_preprocessing/timestamp_transformer.py b/energy_fault_detector/data_preprocessing/timestamp_transformer.py index 39807a04..bd219a97 100644 --- a/energy_fault_detector/data_preprocessing/timestamp_transformer.py +++ b/energy_fault_detector/data_preprocessing/timestamp_transformer.py @@ -107,8 +107,8 @@ class TimestampTransformer(DataTransformer): Args: features (Optional[List[str]]): List of feature names to generate. Supported: - ["second_of_minute", "minute_of_hour", "hour_of_day", "day_of_week", "day_of_month", "month_of_year", - "is_weekend", "year"] + ``second_of_minute``, ``minute_of_hour``, ``hour_of_day``, ``day_of_week``, + ``day_of_month``, ``day_of_year``, ``month_of_year``, ``is_weekend``, ``year``. timestamp_col (Optional[str]): The column name of the DataFrame containing timestamps. If None, the index is assumed to be the timestamp. groupby_level (Optional[str]): Optional index level name or position for grouping (e.g., 'device_id' or 0). @@ -124,7 +124,9 @@ class TimestampTransformer(DataTransformer): train: data_preprocessor: steps: - - name: standard_scaler + - name: scaler + params: + scaler_type: standard - name: timestamp_transformer params: features: diff --git a/energy_fault_detector/data_splitting/__init__.py b/energy_fault_detector/data_splitting/__init__.py index 1033f7b2..e6c334bb 100644 --- a/energy_fault_detector/data_splitting/__init__.py +++ b/energy_fault_detector/data_splitting/__init__.py @@ -1,3 +1,3 @@ -"""Data splitter classes and functions.""" +"""Splitting and windowing of time-series data for training and validation.""" from .data_splitter import BlockDataSplitter diff --git a/energy_fault_detector/data_splitting/data_splitter.py b/energy_fault_detector/data_splitting/data_splitter.py index 3fb9463e..1ad7fb0a 100644 --- a/energy_fault_detector/data_splitting/data_splitter.py +++ b/energy_fault_detector/data_splitting/data_splitter.py @@ -25,7 +25,7 @@ class BlockDataSplitter: Configuration example: - .. code-block:: text + .. code-block:: yaml train: data_splitter: diff --git a/energy_fault_detector/evaluation/__init__.py b/energy_fault_detector/evaluation/__init__.py index 499c600f..97c28717 100644 --- a/energy_fault_detector/evaluation/__init__.py +++ b/energy_fault_detector/evaluation/__init__.py @@ -1,4 +1,4 @@ -"""Evaluation classes and methods, including the CARE-Score and Care2CompareDataset.""" +"""Benchmark metrics and dataset loaders for evaluating fault-detection models.""" from .care_score import CAREScore from .care2compare import Care2CompareDataset diff --git a/energy_fault_detector/evaluation/care2compare.py b/energy_fault_detector/evaluation/care2compare.py index 9e1afb03..37ebe18d 100644 --- a/energy_fault_detector/evaluation/care2compare.py +++ b/energy_fault_detector/evaluation/care2compare.py @@ -15,7 +15,7 @@ class Care2CompareDataset: """Loads the Care to Compare Dataset (accompanying paper https://doi.org/10.3390/data9120138). - The data can be downloaded either manually from https://doi.org/10.5281/zenodo.14958989 (in this case specify + The data can be downloaded either manually from https://doi.org/10.5281/zenodo.10958774 (in this case specify `path`) or it can be downloaded automatically by setting download_dataset to True. By default, only the averages are read. See statistics argument of the data loading methods. @@ -46,7 +46,7 @@ def __init__(self, path: Union[Path, str] = "./CARE_To_Compare", download_datase logger.info("Downloading CARE to Compare dataset (~5 GB) from zenodo. Depending on your internet " "connection this can take some time. Additionally the downloaded zip-file will be unzipped " "(~20 GB) which can also take longer.") - path = download_zenodo_data(identifier="10.5281/zenodo.15846963", dest=path, overwrite=False) + path = download_zenodo_data(identifier="10.5281/zenodo.10958774", dest=path, overwrite=False) self.path: Path = Path(path) self.wind_farms: Dict[str, Path] = { diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py index df4b2fc0..a9ea5c61 100644 --- a/energy_fault_detector/fault_detector.py +++ b/energy_fault_detector/fault_detector.py @@ -39,7 +39,15 @@ def __init__(self, config: Optional[Config] = None, model_directory: str | Path def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Series, fit_preprocessor: bool = True ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.Series]: - """Preprocesses the training data using the configured data_preprocessor.""" + """Preprocesses the training data using the configured data_preprocessor. + If categorical features are declared in config file, data is split into categorical and numerical features. + Boolean features are treated the same as numerical features. Encoded categorical features are scaled or not + depending on the config flag. + + Returns: + Tuple[pd.DataFrame, pd.DataFrame, pd.Series]: Preprocessed training data after filtering only normal index, + original sensor data (clipped if data clipping is enabled), and normal index. + """ x = sensor_data.sort_index() if normal_index is not None: @@ -47,16 +55,18 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri else: y = pd.Series(np.full(len(x), True), index=x.index) - if not x.loc[x.index.duplicated()].empty or not y.loc[y.index.duplicated()].empty: - raise ValueError('There are duplicated indices in the input dataframe `sensor_data` and/or in the ' - '`normal_index`, please check your input data.') + x = self._handle_duplicate_index(x) + if normal_index is not None: + y = self._handle_duplicate_index(y) + # TODO: add list of relevant features to config and protect relevant features (if present) as well. # Determine which features to protect based on config flag protect = self.config.protect_conditional_features if self.config else True protected_features = ( self.autoencoder.conditional_features or [] ) if protect else [] + # Data clipping (outlier clipping) if self.config.data_clipping: logger.debug('Clip data before scaling.') clipper_params = self.config.data_clipping_params.copy() @@ -69,6 +79,7 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri x = data_clipper.transform(x) x_normal = x[y.values] + # Data preprocessing pipeline if fit_preprocessor: logger.info('Fit preprocessor pipeline.') fit_params = {} @@ -82,6 +93,13 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri x_prepped = self.data_preprocessor.transform(x_normal) x_prepped = x_prepped.astype(self.config.dtype) + if x_prepped.empty: + raise ValueError( + "Preprocessed training data is empty after filtering for normal behaviour and applying the " + "preprocessing pipeline. Check that 'normal_index' marks enough rows as normal and that the " + "preprocessor (e.g. ForwardFillImputer, ColumnSelector) is not dropping all rows." + ) + return x_prepped, x, y def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_models: bool = True, @@ -101,34 +119,49 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo "QuantileThresholdSelector or AdaptiveThresholdSelector.") non_numeric = sensor_data.select_dtypes(exclude='number').columns.tolist() - if non_numeric: - raise ValueError(f"`sensor_data` must be numeric. Non-numeric columns: {non_numeric}") - + if non_numeric and fit_preprocessor: + logger.warning("Fitting preprocessor on non-numeric columns. Boolean columns will be treated as " + "floats during imputation. Non-declared categorical features will be dropped.") clear_session() + # --- Resolve conditional features against available data --- + self._resolve_conditional_features(sensor_data) + + # Preprocess data and fit autoencoder model_path = None x_prepped, x, y = self.preprocess_train_data( sensor_data=sensor_data, normal_index=normal_index, fit_preprocessor=fit_preprocessor ) - # --- Resolve conditional features against available data --- + # Capture conditions available in the raw data before the preprocessor may expand + # categorical conditions into one-hot encoded column names (e.g. 'state' -> 'state_A'). + declared_conditions = list(self.autoencoder.conditional_features or []) + + # Resolve declared conditions against preprocessed data (may expand categorical names) self._resolve_conditional_features(x_prepped) - # Post-preprocessing check: conditionals may have been dropped - if not self.config.protect_conditional_features and self.autoencoder.is_conditional: - # Check if conditionals survived preprocessing - surviving = [ - f for f in (self.autoencoder.conditional_features or []) - if f in x_prepped.columns - ] - dropped_by_pipeline = set(self.autoencoder.conditional_features or []) - set(surviving) - if dropped_by_pipeline: - logger.warning(f"Conditional features dropped by preprocessing pipeline: " - f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}") - if surviving: - self.autoencoder.conditional_features = surviving - else: - self._fallback_if_no_conditionals() + # Check conditionals: available conditions in original data and surviving conditions during preprocessing. + if self.autoencoder.is_conditional: + # Uses nested for loop in case categorical cols have been encoded already and contain the original + # conditional feature name as a substring. + available = [declared_condition for declared_condition in declared_conditions + if any(declared_condition in col for col in sensor_data.columns)] + missing = [declared_condition for declared_condition in declared_conditions + if declared_condition not in available] + if missing: + logger.warning(f"Declared conditions not found in sensor_data will be ignored: " + f"{sorted(missing)}. Using: {available or 'none'}") + + if not self.config.protect_conditional_features: + # Uses nested for loop in case categorical cols have been encoded already and contain the original + # conditional feature name as a substring. + surviving = [declared_condition for declared_condition in available + if any(declared_condition in col for col in x_prepped.columns)] + dropped_by_pipeline = set(available or []) - set(surviving) + + if dropped_by_pipeline: + logger.warning(f"Declared conditions dropped by preprocessing pipeline: " + f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}") train_recon_error, val_recon_error = None, None x_train, x_val = self.train_val_split(x_prepped) @@ -198,10 +231,11 @@ def tune(self, sensor_data: pd.DataFrame, normal_index: Optional[pd.Series] = No if tune_method not in ['threshold', 'decoder', 'full']: raise ValueError(f'Unknown tune method {tune_method}.') - if tune_method == 'threshold' and self.config.fit_threshold_on_val: + fit_on_val = self.config.fit_threshold_on_val + if tune_method == 'threshold' and fit_on_val: logger.warning('Fine-tuning using only validation data for threshold does not make sense if only the' ' threshold is tuned! Setting fit_threshold_on_val to False.') - self.config['train']['threshold_selector']['fit_on_val'] = False + fit_on_val = False if pretrained_model_path is not None: self._load_from_path(model_path=pretrained_model_path) @@ -248,7 +282,7 @@ def tune(self, sensor_data: pd.DataFrame, normal_index: Optional[pd.Series] = No ) # tune/fit threshold - self._fit_threshold(x=x, y=y, x_val=x_val, fit_on_validation=self.config.fit_threshold_on_val) + self._fit_threshold(x=x, y=y, x_val=x_val, fit_on_validation=fit_on_val) model_path = None if save_models: @@ -300,8 +334,7 @@ def predict(self, sensor_data: pd.DataFrame, model_path: Optional[str] = None, raise ValueError('No models loaded and no model_path provided!') logger.debug('No model_path provided; using existing model instances.') - if not x.loc[x.index.duplicated()].empty: - raise ValueError('There are duplicated indices in the input dataframe `sensor_data`.') + x = self._handle_duplicate_index(x) x_prepped = self.data_preprocessor.transform(x).sort_index() x_prepped = x_prepped.astype(self.config.dtype) @@ -408,12 +441,19 @@ def run_root_cause_analysis(self, sensor_data: pd.DataFrame, track_losses: bool def _fit_threshold(self, x: pd.DataFrame, y: pd.Series, x_val: pd.DataFrame, fit_on_validation: bool = False ) -> None: - """Fit AnomalyScore and ThresholdSelector objects.""" + """Fit AnomalyScore and ThresholdSelector objects. + + Args: + x: pandas DataFrame with the sensor data. + y: pandas Series with the labels indicating whether each sample is normal (True) or anomalous (False). + x_val: pandas DataFrame with the validation sensor data. + fit_on_validation: bool indicating whether to fit the threshold on the validation data. + """ - # Fit score object only on normal data (all training + validation data) x_prepped_all = self.data_preprocessor.transform(x) deviations = self.autoencoder.get_reconstruction_error(x_prepped_all) y_ = y.loc[deviations.index] + # Fit score object only on normal data (all training + validation data) self.anomaly_score.fit(deviations[y_.values]) # use series values for compatibility with a multi-index scores = self.anomaly_score.transform(deviations) @@ -439,9 +479,9 @@ def _fit_threshold(self, x: pd.DataFrame, y: pd.Series, x_val: pd.DataFrame, fit self.threshold_selector.fit(x=scores, y=y.loc[scores.index]) def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]: - """Resolve which conditional features are actually available in the data. + """Resolve which declared conditions are actually available in the data. - If all conditional features are missing and the autoencoder is a ConditionalAE, + If all conditions are missing and the autoencoder is a ConditionalAE, falls back to MultilayerAutoencoder. For sequence models, simply clears the conditional_features list (they handle None gracefully). @@ -455,12 +495,9 @@ def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]: if not configured: return [] - available = [f for f in configured if f in sensor_data.columns] - missing = set(configured) - set(available) - - if missing: - logger.warning(f"Conditional features not found in sensor_data and will be ignored: " - f"{sorted(missing)}. Using: {available or 'none'}") + # TODO: this can produce undesired results if any column in the original data contains a substring of a declared + # conditional feature name. + available = [col for col in sensor_data.columns if any(declared_condition in col for declared_condition in configured)] if not available: self._fallback_if_no_conditionals() diff --git a/energy_fault_detector/quick_fault_detection/__init__.py b/energy_fault_detector/quick_fault_detection/__init__.py index eef4053e..283df610 100644 --- a/energy_fault_detector/quick_fault_detection/__init__.py +++ b/energy_fault_detector/quick_fault_detection/__init__.py @@ -1,3 +1,10 @@ +"""One-call end-to-end fault detection for testing the +:class:`~energy_fault_detector.fault_detector.FaultDetector` on a single dataset. + +The function :func:`~energy_fault_detector.quick_fault_detector` is also exposed as +the ``quick_fault_detector`` command-line entry point. + +""" from .pipeline import quick_fault_detector diff --git a/energy_fault_detector/registration.py b/energy_fault_detector/registration.py index c4adcb94..3ee874e2 100644 --- a/energy_fault_detector/registration.py +++ b/energy_fault_detector/registration.py @@ -18,7 +18,7 @@ def register(self, module_path: str, class_type: str, class_names: List[str]) -> Args: module_path (str): Path to the module containing the class. - class_type (str): type of the class (anomaly_score, autoencoder, threshold_selector or data_preprocessor) + class_type (str): type of the class (anomaly_score, autoencoder, threshold_selector) class_names (List[str], optional): a list of names which may be used to refer to this class. Default is None """ diff --git a/energy_fault_detector/root_cause_analysis/__init__.py b/energy_fault_detector/root_cause_analysis/__init__.py index 7eb9cfdf..38a11152 100644 --- a/energy_fault_detector/root_cause_analysis/__init__.py +++ b/energy_fault_detector/root_cause_analysis/__init__.py @@ -1 +1 @@ -"""Root cause analysis.""" +"""Root-cause analysis explaining which sensors caused a detected anomaly.""" diff --git a/energy_fault_detector/root_cause_analysis/arcana.py b/energy_fault_detector/root_cause_analysis/arcana.py index 11387af5..3e5d69da 100644 --- a/energy_fault_detector/root_cause_analysis/arcana.py +++ b/energy_fault_detector/root_cause_analysis/arcana.py @@ -65,7 +65,7 @@ class Arcana: Configuration example: - .. code-block:: text + .. code-block:: yaml root_cause_analysis: alpha: 0.8 diff --git a/energy_fault_detector/threshold_selectors/__init__.py b/energy_fault_detector/threshold_selectors/__init__.py index f9dc6e7b..27d1f41c 100644 --- a/energy_fault_detector/threshold_selectors/__init__.py +++ b/energy_fault_detector/threshold_selectors/__init__.py @@ -1,4 +1,10 @@ -"""Threshold selection methods""" +"""Threshold selectors that turn anomaly scores into normal/anomaly labels. + +A :class:`~energy_fault_detector.core.ThresholdSelector` learns a threshold from +(optionally labelled) anomaly scores. Samples above the threshold are flagged as anomalous. +Each selector subclasses :class:`~energy_fault_detector.core.ThresholdSelector` and is selected +via the ``train.threshold_selector`` config key. +""" from energy_fault_detector.threshold_selectors.fdr_threshold import FDRSelector from energy_fault_detector.threshold_selectors.fbeta_threshold import FbetaSelector diff --git a/energy_fault_detector/threshold_selectors/adaptive_threshold.py b/energy_fault_detector/threshold_selectors/adaptive_threshold.py index 837b10bc..bcae66a1 100644 --- a/energy_fault_detector/threshold_selectors/adaptive_threshold.py +++ b/energy_fault_detector/threshold_selectors/adaptive_threshold.py @@ -35,7 +35,7 @@ class AdaptiveThresholdSelector(ThresholdSelector): Configuration example: - .. code-block:: text + .. code-block:: yaml train: threshold_selector: diff --git a/energy_fault_detector/threshold_selectors/fbeta_threshold.py b/energy_fault_detector/threshold_selectors/fbeta_threshold.py index 34bdc676..c7e22651 100644 --- a/energy_fault_detector/threshold_selectors/fbeta_threshold.py +++ b/energy_fault_detector/threshold_selectors/fbeta_threshold.py @@ -31,7 +31,7 @@ class FbetaSelector(ThresholdSelector): Configuration example: - .. code-block:: text + .. code-block:: yaml train: threshold_selector: diff --git a/energy_fault_detector/threshold_selectors/fdr_threshold.py b/energy_fault_detector/threshold_selectors/fdr_threshold.py index beef183f..d4989d89 100644 --- a/energy_fault_detector/threshold_selectors/fdr_threshold.py +++ b/energy_fault_detector/threshold_selectors/fdr_threshold.py @@ -24,7 +24,7 @@ class FDRSelector(ThresholdSelector): Example Configuration: - .. code-block:: text + .. code-block:: yaml train: threshold_selector: diff --git a/energy_fault_detector/threshold_selectors/quantile_threshold.py b/energy_fault_detector/threshold_selectors/quantile_threshold.py index 4a60f3e2..bf7a191f 100644 --- a/energy_fault_detector/threshold_selectors/quantile_threshold.py +++ b/energy_fault_detector/threshold_selectors/quantile_threshold.py @@ -21,7 +21,7 @@ class QuantileThresholdSelector(ThresholdSelector): Example Configuration: - .. code-block:: text + .. code-block:: yaml train: threshold_selector: @@ -64,9 +64,4 @@ def fit(self, x: Array1D, y: pd.Series = None) -> 'QuantileThresholdSelector': x_ = x self.threshold = float(np.quantile(x_, self.quantile)) - - if self.threshold is None: - import warnings - warnings.warn('Could not find suitable threshold, `threshold` is set to max score.', UserWarning) - self.threshold = float(np.sort(x)[-1]) return self diff --git a/energy_fault_detector/utils/__init__.py b/energy_fault_detector/utils/__init__.py index 23eda90a..f1930beb 100644 --- a/energy_fault_detector/utils/__init__.py +++ b/energy_fault_detector/utils/__init__.py @@ -1 +1 @@ -"""Helper utilities for post-processing, data downloading, index handling, and visualization.""" +"""Helper utilities shared across the package.""" diff --git a/energy_fault_detector/utils/data_downloads.py b/energy_fault_detector/utils/data_downloads.py index 0d403535..ce50a697 100644 --- a/energy_fault_detector/utils/data_downloads.py +++ b/energy_fault_detector/utils/data_downloads.py @@ -22,7 +22,7 @@ def parse_record_id(identifier: str) -> str: Accepts: - Numeric ID (e.g., "15846963") - - DOI (e.g., "10.5281/zenodo.15846963") + - DOI (e.g., "10.5281/zenodo.10958774") - Record URL (e.g., "https://zenodo.org/records/15846963") Args: @@ -197,7 +197,7 @@ def prepare_output_dir(out_dir: Path, overwrite: bool) -> None: out_dir.mkdir(parents=True, exist_ok=True) -def download_zenodo_data(identifier: str = "10.5281/zenodo.15846963", dest: Path = "./downloads", +def download_zenodo_data(identifier: str = "10.5281/zenodo.10958774", dest: Path = "./downloads", remove_zip: bool = True, overwrite: bool = False, flatten_file_structure: bool = True, expected_file_types: Union[List[str], str] = "*.csv") -> Path: """ Download a Zenodo record via API and unzip any .zip files. @@ -207,7 +207,7 @@ def download_zenodo_data(identifier: str = "10.5281/zenodo.15846963", dest: Path that result from extracting ZIP archives. Args: - identifier (str): Zenodo record ID, DOI (e.g., 10.5281/zenodo.15846963), or record URL. + identifier (str): Zenodo record ID, DOI (e.g., 10.5281/zenodo.10958774), or record URL. Defaults to the CARE2Compare dataset. dest (Path): Local output directory to save downloaded files. (default: downloads) remove_zip (bool): If True, ZIP archives will be removed after extraction. diff --git a/energy_fault_detector/utils/visualisation.py b/energy_fault_detector/utils/visualisation.py index 7bc5aac6..b605d763 100644 --- a/energy_fault_detector/utils/visualisation.py +++ b/energy_fault_detector/utils/visualisation.py @@ -59,7 +59,7 @@ def plot_reconstruction(data: pd.DataFrame, reconstruction: pd.DataFrame, featur Notes: - Can result in a very large plot, if the dataset contains many columns/features. Use the - `features_to_plot` parameter to specify which columns to plot. + ``features_to_plot`` parameter to specify which columns to plot. - For MultiIndex data (e.g. multiple devices), pass data for a single group/device, i.e. ``df.loc[device_id]`` to select one device before plotting. diff --git a/notebooks/CARE to Compare/CARE to Compare.ipynb b/notebooks/CARE to Compare/CARE to Compare.ipynb index 9303b422..f5fa4b21 100644 --- a/notebooks/CARE to Compare/CARE to Compare.ipynb +++ b/notebooks/CARE to Compare/CARE to Compare.ipynb @@ -50,7 +50,7 @@ "metadata": {}, "outputs": [], "source": [ - "data_dir = Path('..') / '..' / 'Care_To_Compare_v6'" + "data_dir = Path('..') / '..' / 'CARE_To_Compare'" ] }, { @@ -81,7 +81,7 @@ "outputs": [], "source": [ "# select data for a specific event\n", - "x, y = c2c.load_event_dataset(0, statistics=['average', 'std_dev'])\n", + "x, y = c2c.load_event_dataset(0, statistics=['average', 'std_dev'], index_column='time_stamp')\n", "x.head()" ] }, @@ -101,7 +101,7 @@ "outputs": [], "source": [ "c2c = Care2CompareDataset(data_dir)\n", - "index_column = 'id' # us time_stamp as index column if you are using the TimestampTransformer\n", + "index_column = 'time_stamp'\n", "suffix = ''\n", "\n", "configs = {\n", @@ -163,8 +163,8 @@ " \n", " # evaluate event\n", " event_info = c2c.event_info_all[c2c.event_info_all['event_id'] == event_id]\n", - " event_start = event_info['event_start_id'].iloc[0] if index_column == 'id' else event_info['event_start'].iloc[0]\n", - " event_end = event_info['event_end_id'].iloc[0] if index_column == 'id' else event_info['event_end'].iloc[0]\n", + " event_start = event_info['event_start'].iloc[0]\n", + " event_end = event_info['event_end'].iloc[0]\n", " care_score.evaluate_event(\n", " event_id=event_id,\n", " event_start=event_start,\n", @@ -240,13 +240,13 @@ "c2c.update_c2c_config(config, wf)\n", "care_score = CAREScore(coverage_beta=0.5, reliability_beta=0.5, anomaly_detection_method='criticality')\n", "\n", - "for x_train, asset_id, event_ids in c2c.iter_train_datasets_per_asset(wf):\n", + "for x_train, asset_id, event_ids in c2c.iter_train_datasets_per_asset(wf, index_column='time_stamp'):\n", " x_train = x_train.reset_index(drop=True)\n", "\n", " # create normal index\n", " y_train = x_train['status_type_id'] == 0\n", " # drop unnecessary features (keep asset ID as a sort of condition)\n", - " x_train = x_train.drop(['time_stamp', 'status_type_id'], axis=1)\n", + " x_train = x_train.drop(['time_stamp', 'status_type_id'], axis=1, errors='ignore')\n", " \n", " # create model\n", " model = FaultDetector(config)\n", @@ -257,17 +257,17 @@ " for event_id in event_ids:\n", " print(event_id)\n", " # test model\n", - " x_test = c2c.load_event_dataset(event_id=event_id, test_only=True)\n", + " x_test = c2c.load_event_dataset(event_id=event_id, test_only=True, index_column='time_stamp')\n", " y_test = x_test['status_type_id'] == 0\n", - " x_test = x_test.drop(['time_stamp', 'asset_id', 'status_type_id'], axis=1)\n", + " x_test = x_test.drop(['time_stamp', 'asset_id', 'status_type_id'], axis=1, errors='ignore')\n", " prediction = model.predict(x_test)\n", " \n", " # evaluate event\n", " event_info = c2c.event_info_all[c2c.event_info_all['event_id'] == event_id]\n", " care_score.evaluate_event(\n", " event_id=event_id,\n", - " event_start_id=event_info['event_start_id'].iloc[0],\n", - " event_end_id=event_info['event_end_id'].iloc[0],\n", + " event_start=event_info['event_start'].iloc[0],\n", + " event_end=event_info['event_end'].iloc[0],\n", " event_label=event_info['event_label'].iloc[0],\n", " normal_index=y_test,\n", " predicted_anomalies=prediction.predicted_anomalies,\n", diff --git a/notebooks/CARE to Compare/c2c_configs/windfarm_A.yaml b/notebooks/CARE to Compare/c2c_configs/windfarm_A.yaml index 5ef3264a..74c65f54 100644 --- a/notebooks/CARE to Compare/c2c_configs/windfarm_A.yaml +++ b/notebooks/CARE to Compare/c2c_configs/windfarm_A.yaml @@ -12,7 +12,9 @@ train: params: min_unique_value_count: 10 max_col_zero_frac: 0.99 - - name: minmax_scaler + - name: scaler + params: + scaler_type: minmax data_splitter: shuffle: true diff --git a/notebooks/CARE to Compare/c2c_configs/windfarm_B.yaml b/notebooks/CARE to Compare/c2c_configs/windfarm_B.yaml index 68320b93..320cf2b2 100644 --- a/notebooks/CARE to Compare/c2c_configs/windfarm_B.yaml +++ b/notebooks/CARE to Compare/c2c_configs/windfarm_B.yaml @@ -12,7 +12,9 @@ train: params: min_unique_value_count: 10 max_col_zero_frac: 0.8 - - name: minmax_scaler + - name: scaler + params: + scaler_type: minmax data_splitter: shuffle: true diff --git a/notebooks/CARE to Compare/c2c_configs/windfarm_C.yaml b/notebooks/CARE to Compare/c2c_configs/windfarm_C.yaml index 3f7b08a6..0a38f942 100644 --- a/notebooks/CARE to Compare/c2c_configs/windfarm_C.yaml +++ b/notebooks/CARE to Compare/c2c_configs/windfarm_C.yaml @@ -12,7 +12,9 @@ train: params: min_unique_value_count: 10 max_col_zero_frac: 0.99 - - name: minmax_scaler + - name: scaler + params: + scaler_type: minmax data_splitter: shuffle: true diff --git a/notebooks/Example - Create new model classes.ipynb b/notebooks/Example - Create new model classes.ipynb index fe54108a..f8cfa9b3 100644 --- a/notebooks/Example - Create new model classes.ipynb +++ b/notebooks/Example - Create new model classes.ipynb @@ -16,7 +16,7 @@ "- A simple non-symmetric autoencoder.\n", "- A Threshold selection method based on the telemanom model from the paper [Detecting Spacecraft Anomalies Using LSTMs and Nonparametric Dynamic Thresholding](https://doi.org/10.1145/3219819.3219845).\n", "\n", - "For the usage example of the new classes the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.14958989) is used." + "For the usage example of the new classes the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774) is used." ], "id": "7d0933b25f2a9b46" }, @@ -246,7 +246,7 @@ "c2c = Care2CompareDataset(data_path)\n", "\n", "event_id = 44\n", - "train, test = c2c.get_dataset_for_event(event_id, index_column='time_stamp')\n", + "train, test = c2c.load_event_dataset(event_id, index_column='time_stamp')\n", "\n", "y_train = train['status_type_id'] == 0\n", "x_train = train.drop(['asset_id', 'id', 'time_stamp', 'status_type_id'], axis=1, errors='ignore')\n", @@ -540,15 +540,21 @@ " 'upper_percentile': 0.999\n", " },\n", " 'data_preprocessor': {\n", - " 'params': {\n", - " 'include_column_selector': True,\n", - " 'include_low_unique_value_filter': True,\n", - " 'include_duplicate_value_to_nan': False,\n", - " 'max_col_zero_frac': 0.99,\n", - " 'max_nan_frac_per_col': 0.05,\n", - " 'min_unique_value_count': 10,\n", - " 'scale': 'minmax'\n", - " }\n", + " 'steps': [\n", + " {\n", + " 'name': 'column_selector',\n", + " 'params': {\n", + " 'max_nan_frac_per_col': 0.05,\n", + " }\n", + " },\n", + " {\n", + " 'name': 'low_unique_value_filter',\n", + " 'params': {\n", + " 'min_unique_value_count': 10,\n", + " 'max_col_zero_frac': 0.99,\n", + " }\n", + " }\n", + " ]\n", " },\n", " 'data_splitter': {\n", " 'shuffle': True,\n", diff --git a/notebooks/Example - FaultDetector Usage.ipynb b/notebooks/Example - FaultDetector Usage.ipynb index cd8271a1..d25cb45a 100644 --- a/notebooks/Example - FaultDetector Usage.ipynb +++ b/notebooks/Example - FaultDetector Usage.ipynb @@ -13,7 +13,7 @@ "5. Evaluation of results\n", "6. Root cause analysis using ARCANA\n", "\n", - "Get the dataset which is used in this notebook here: [CARE to Compare dataset](https://doi.org/10.5281/zenodo.15846963)" + "Get the dataset which is used in this notebook here: [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774)" ], "id": "687630b4d4423666" }, diff --git a/notebooks/Example - Hyperparameter Optimization.ipynb b/notebooks/Example - Hyperparameter Optimization.ipynb index 344db94e..b1f89be0 100644 --- a/notebooks/Example - Hyperparameter Optimization.ipynb +++ b/notebooks/Example - Hyperparameter Optimization.ipynb @@ -11,7 +11,7 @@ "2. Optimizing the FaultDetector classification performance using the Fbeta score\n", "3. Optimizing the FaultDetector classification performance using the CARE-score\n", "\n", - "The optimization is done using the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.14958989)" + "The optimization is done using the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774)" ], "id": "acc177b6ece47b21" }, @@ -87,7 +87,6 @@ " # sample new parameters\n", " autoencoder_params['batch_size'] = int(trial.suggest_categorical(name='batch_size', choices=[32, 64, 128]))\n", " autoencoder_params['learning_rate'] = trial.suggest_float(name='learning_rate', low=1e-5, high=0.01, log=True)\n", - " autoencoder_params['decay_rate'] = trial.suggest_float(name='decay_rate', low=0.8, high=0.99)\n", "\n", " # architecture\n", " autoencoder_params['layers'][0] = trial.suggest_int(name='layers_0', low=100, high=400)\n", @@ -97,7 +96,7 @@ " # create a new model using our new configuration and train the model\n", " model = FaultDetector(Config(config_dict=cfg))\n", " # For autoencoder optimization, we do not need to fit a threshold\n", - " training_result = model.fit(train_data, normal_index=normal_index, fit_autoencoder_only=True, save_model=False)\n", + " training_result = model.fit(train_data, normal_index=normal_index, fit_autoencoder_only=True, save_models=False)\n", "\n", " # Calculate the MSE of the reconstruction errors of the validation data - this is minimized\n", " deviations = training_result.val_recon_error\n", @@ -199,14 +198,14 @@ " steps = dp.get('steps')\n", "\n", " # Remove any existing scaler step(s)\n", - " scaler_names = {'standard_scaler', 'minmax_scaler'}\n", + " scaler_names = {'scaler'}\n", " steps = [s for s in steps if s.get('name') not in scaler_names]\n", " # Add the chosen scaler step\n", " if choice == 'minmax':\n", - " steps.append({'name': 'minmax_scaler'})\n", + " steps.append({'name': 'scaler', 'params': {'scaler_type': 'minmax'}})\n", " else:\n", " # 'standardize'\n", - " steps.append({'name': 'standard_scaler'})\n", + " steps.append({'name': 'scaler', 'params': {'scaler_type': 'standard'}})\n", "\n", " dp['steps'] = steps\n", " return cfg\n", @@ -232,7 +231,6 @@ " autoencoder_params = cfg['train']['autoencoder']['params']\n", " autoencoder_params['batch_size'] = int(trial.suggest_categorical(name='batch_size', choices=[32, 64, 128]))\n", " autoencoder_params['learning_rate'] = trial.suggest_float(name='learning_rate', low=1e-5, high=0.01, log=True)\n", - " autoencoder_params['decay_rate'] = trial.suggest_float(name='decay_rate', low=0.8, high=0.99)\n", "\n", " # architecture\n", " autoencoder_params['layers'][0] = trial.suggest_int(name='layers_0', low=100, high=400)\n", @@ -330,7 +328,7 @@ " threshold_params['nn_size'] = trial.suggest_int(name='nn_size', low=20, high=50)\n", "\n", " # Create a CAREScore object and train+evaluate each dataset for this wind farm\n", - " care_score = CAREScore(coverage_beta=0.5, eventwise_f_score_beta=0.5, anomaly_detection_method='criticality')\n", + " care_score = CAREScore(coverage_beta=0.5, reliability_beta=0.5, anomaly_detection_method='criticality')\n", " i = 1\n", " for x_train, y_train, x_test, y_test, event_id in c2c.iter_formatted_datasets(wind_farm=wind_farm, index_column='time_stamp'):\n", " print(f\"event {i}/{len(c2c.event_info_all[c2c.event_info_all['wind_farm'] == wind_farm])}\")\n", diff --git a/notebooks/Example - Model Finetuning.ipynb b/notebooks/Example - Model Finetuning.ipynb index 9b6ff27e..4786e5a9 100644 --- a/notebooks/Example - Model Finetuning.ipynb +++ b/notebooks/Example - Model Finetuning.ipynb @@ -5,7 +5,7 @@ "source": [ "# Model Finetuning\n", "\n", - "This notebook explains how to use the fine-tuning function of an FaultDetector on the example use-case of transfer learning, demonstrated on the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.14958989)." + "This notebook explains how to use the fine-tuning function of an FaultDetector on the example use-case of transfer learning, demonstrated on the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774)." ], "metadata": { "collapsed": false @@ -58,11 +58,11 @@ "# select two events from different assets\n", "event_id = 44 # anomaly event of asset_id 44\n", "event_info_1 = c2c.get_event_info(event_id=event_id)\n", - "train_sensor_data_1, train_status_1, test_sensor_data_1, test_status_1 = c2c.get_formatted_event_dataset(event_id=event_id, index_column='time_stamp')\n", + "train_sensor_data_1, train_status_1, test_sensor_data_1, test_status_1 = c2c.load_and_format_event_dataset(event_id=event_id, index_column='time_stamp')\n", "\n", "event_id = 5 # anomaly event of asset_id 32\n", "event_info_2 = c2c.get_event_info(event_id=event_id)\n", - "train_sensor_data_2, train_status_2, test_sensor_data_2, test_status_2 = c2c.get_formatted_event_dataset(event_id=event_id, index_column='time_stamp')" + "train_sensor_data_2, train_status_2, test_sensor_data_2, test_status_2 = c2c.load_and_format_event_dataset(event_id=event_id, index_column='time_stamp')" ], "metadata": { "collapsed": false, diff --git a/notebooks/Example - Quick Fault Detection.ipynb b/notebooks/Example - Quick Fault Detection.ipynb index 6628592f..ff742720 100644 --- a/notebooks/Example - Quick Fault Detection.ipynb +++ b/notebooks/Example - Quick Fault Detection.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "source": [ "# Example: Quick fault detection in one line of code\n", - "Shows how to use quick fault detection module using example data from the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.14958989)\n", + "Shows how to use quick fault detection module using example data from the [CARE to Compare dataset](https://doi.org/10.5281/zenodo.10958774)\n", "Structure:\n", "1. Importing\n", "2. Data format preparation\n", diff --git a/notebooks/HDD Failure/base_model.yaml b/notebooks/HDD Failure/base_model.yaml index cffd17ec..e8d06138 100644 --- a/notebooks/HDD Failure/base_model.yaml +++ b/notebooks/HDD Failure/base_model.yaml @@ -9,7 +9,9 @@ train: params: max_nan_frac_per_col: 0.2 - name: simple_imputer - - name: standard_scaler + - name: scaler + params: + scaler_type: standard autoencoder: name: default diff --git a/tests/config/test_quickstart_config.py b/tests/config/test_quickstart_config.py index 4e93506f..393c98a1 100644 --- a/tests/config/test_quickstart_config.py +++ b/tests/config/test_quickstart_config.py @@ -40,15 +40,7 @@ def test_generate_quickstart_config_valid_dict(self) -> None: step_names = [s["name"] for s in train["data_preprocessor"]["steps"]] self.assertIn("column_selector", step_names) self.assertIn("simple_imputer", step_names) - self.assertTrue( - any(n in ("standard_scaler", "minmax_scaler") for n in step_names), - "Expected a scaler step in the pipeline." - ) - - self.assertTrue( - any(n in ("standard_scaler", "minmax_scaler") for n in step_names), - "Expected a scaler step in the pipeline." - ) + self.assertIn("scaler", step_names) def test_generate_quickstart_config_validation_split_guard(self) -> None: """If validation split not in (0, 1) it should raise ValueError.""" diff --git a/tests/data_preprocessing/test_categorical_encoder.py b/tests/data_preprocessing/test_categorical_encoder.py new file mode 100644 index 00000000..9ca86d53 --- /dev/null +++ b/tests/data_preprocessing/test_categorical_encoder.py @@ -0,0 +1,237 @@ +import unittest +import pandas as pd +import numpy as np +from sklearn.exceptions import NotFittedError + +from energy_fault_detector.data_preprocessing.categorical_encoder import CategoricalEncoder + + +class TestCategoricalEncoder(unittest.TestCase): + def setUp(self): + """Set up common test fixtures.""" + # Create sample data with numerical, categorical, and mixed features + self.data_clean = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'humidity': [60, 62, 58, 65, 67], + 'category': ['A', 'B', 'A', 'C', 'B'], + 'region': ['North', 'South', 'North', 'East', 'West'] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + self.data_clean_encoded_features = ['temperature', 'humidity', 'category_A', 'category_B', 'category_C', 'region_North', 'region_South', 'region_East', 'region_West'] + + # Create data with non-declared categorical features in numerical columns + self.data_with_non_declared = self.data_clean.copy() + self.data_with_non_declared['status'] = ['high', 'medium', 'low', 'high', 'medium'] + + # Create data with only numerical columns + self.data_numerical_only = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'humidity': [60, 62, 58, 65, 67] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + self.data_numerical_only_encoded_features = ['temperature', 'humidity'] + + # Create data with missing values in categorical columns + self.data_with_nan = self.data_clean.copy() + self.data_with_nan.iloc[2, 2] = np.nan + self.data_with_nan.iloc[3, 3] = np.nan + + # Create data with duplicate rows for testing deduplication + self.data_with_duplicates = pd.DataFrame({ + 'temperature': [20.5, 20.5, 21.3, 22.1], + 'category': ['A', 'A', 'B', 'C'] + }, index=pd.date_range('2024-01-01', periods=4, freq='1Min')) + + # Create data for testing inverse transform + self.data_for_inverse = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8], + 'category': ['A', 'B', 'A'], + 'region': ['North', 'South', 'North'] + }, index=pd.date_range('2024-01-01', periods=3, freq='1Min')) + + def tearDown(self): + """Clean up after each test.""" + del (self.data_clean, self.data_with_non_declared, self.data_numerical_only, + self.data_with_nan, self.data_with_duplicates, self.data_for_inverse) + + def test_init_defaults(self): + """Test default initialization.""" + encoder = CategoricalEncoder() + self.assertEqual(encoder.categorical_features, []) + self.assertIsNone(encoder.categorical_columns) + self.assertIsNone(encoder.numerical_columns) + + def test_init_with_categorical_features(self): + """Test initialization with categorical features.""" + encoder = CategoricalEncoder(categorical_features=['category', 'region']) + self.assertListEqual(encoder.categorical_features, ['category', 'region']) + + def test_fit_with_categorical_features(self): + """Test fitting with categorical features.""" + encoder = CategoricalEncoder(categorical_features=['category', 'region']) + encoder.fit(self.data_clean) + + self.assertEqual(encoder.n_features_in_, 4) + self.assertListEqual(encoder.feature_names_in_ or [], ['temperature', 'humidity', 'category', 'region']) + self.assertListEqual(encoder.numerical_columns or [], ['temperature', 'humidity']) + self.assertListEqual(encoder.categorical_columns or [], ['category', 'region']) + + def test_fit_without_declaring_categorical_features(self): + """Test fitting without specifying categorical features.""" + encoder = CategoricalEncoder() + encoder.fit(self.data_clean) + + # Should ignore categorical columns + self.assertEqual(encoder.n_features_in_, 4) + self.assertListEqual(encoder.numerical_columns or [], ['temperature', 'humidity']) + self.assertListEqual(encoder.categorical_columns or [], []) + + def test_fit_with_non_declared_categorical_features(self): + """Test detection of non-declared categorical features in numerical columns.""" + encoder = CategoricalEncoder(categorical_features=['category', 'region']) + encoder.fit(self.data_with_non_declared) + + # Should detect 'status' as non-declared categorical + self.assertIn('status', encoder.non_declared_categorical_features or []) + self.assertNotIn('status', encoder.numerical_columns or []) + self.assertNotIn('status', encoder.categorical_columns or []) + + def test_fit_with_only_numerical_data(self): + """Test fitting on numerical-only data.""" + encoder = CategoricalEncoder() + encoder.fit(self.data_numerical_only) + + self.assertEqual(encoder.n_features_in_, 2) + self.assertListEqual(encoder.numerical_columns or [], ['temperature', 'humidity']) + self.assertEqual(len(encoder.categorical_columns or []), 0) + + def test_transform_with_categorical_features(self): + """Test transformation with categorical features.""" + encoder = CategoricalEncoder(categorical_features=['category', 'region']) + encoder.fit(self.data_clean) + result = encoder.transform(self.data_clean) + + # Should have transformed columns: temperature, humidity, category_A, category_B, category_C, region_* + for col in self.data_clean_encoded_features: + self.assertIn(col, result.columns) + # Should maintain same index + self.assertTrue(result.index.equals(self.data_clean.index)) + # Should not contain NaN + self.assertFalse(result.isna().any().any()) + + def test_transform_with_only_numerical_data(self): + """Test transformation when no categorical features.""" + encoder = CategoricalEncoder() + encoder.fit(self.data_numerical_only) + result = encoder.transform(self.data_numerical_only) + + # Should return same columns and shape + self.assertEqual(result.shape, self.data_numerical_only.shape) + self.assertListEqual(list(result.columns), self.data_numerical_only_encoded_features) + + def test_transform_missing_columns_raises(self): + """Test that missing columns raise KeyError.""" + encoder = CategoricalEncoder(categorical_features=['category']) + encoder.fit(self.data_clean) + + # Create data missing one feature + partial_data = self.data_clean.drop(columns=['temperature']) + + with self.assertRaises(KeyError): + encoder.transform(partial_data) + + def test_transform_invalid_type_raises(self): + """Test that non-DataFrame input raises TypeError.""" + encoder = CategoricalEncoder() + encoder.fit(self.data_clean) + + with self.assertRaises(TypeError): + encoder.transform([1, 2, 3]) + + def test_transform_unfitted_raises(self): + """Test that transform on unfitted model raises error.""" + encoder = CategoricalEncoder() + + with self.assertRaises(NotFittedError): + encoder.transform(self.data_clean) + + def test_inverse_transform(self): + """Test inverse transform returns original categorical values.""" + encoder = CategoricalEncoder(categorical_features=['category', 'region']) + encoder.fit(self.data_for_inverse) + + transformed = encoder.transform(self.data_for_inverse) + inverse = encoder.inverse_transform(transformed) + + # Should have same columns and shape as original + self.assertListEqual(list(inverse.columns), list(self.data_for_inverse.columns)) + self.assertEqual(inverse.shape, self.data_for_inverse.shape) + # Should match original data (except for any NaN handling) + pd.testing.assert_frame_equal(inverse, self.data_for_inverse) + + def test_get_feature_names_out(self): + """Test feature names out method.""" + encoder = CategoricalEncoder(categorical_features=['category', 'region']) + encoder.fit(self.data_clean) + + feature_names = encoder.get_feature_names_out() + + # Should include original numerical features + self.assertIn('temperature', feature_names) + self.assertIn('humidity', feature_names) + # Should include encoded categorical features (exact names depend on OneHotEncoder output) + categorical_feature_names = [name for name in feature_names if 'category' in name or 'region' in name] + self.assertGreater(len(categorical_feature_names), 0) + self.assertEqual(len(feature_names), encoder.n_features_in_ + len(categorical_feature_names) - 2) + + def test_empty_dataframe_handling(self): + """Test behavior with empty DataFrame.""" + empty_df = pd.DataFrame(columns=self.data_clean.columns, + index=pd.DatetimeIndex([])) + + encoder = CategoricalEncoder(categorical_features=['category', 'region']) + encoder.fit(self.data_clean) + + result = encoder.transform(empty_df) + + self.assertEqual(result.shape[0], 0) + + def test_transform_preserves_index_type(self): + """Test that index type is preserved after transformation.""" + encoder = CategoricalEncoder(categorical_features=['category']) + encoder.fit(self.data_clean) + result = encoder.transform(self.data_clean) + + self.assertIsInstance(result.index, pd.DatetimeIndex) + + def test_transform_with_nan_in_categorical(self): + """Test transformation with NaN values in categorical columns.""" + encoder = CategoricalEncoder(categorical_features=['category', 'region']) + encoder.fit(self.data_with_nan) + + # Should ignore NaN values in categorical columns during transformation + result = encoder.transform(self.data_with_nan) + self.assertFalse(result.isna().any().any()) + + def test_transform_with_new_categories(self): + """Test transformation with unseen category values — should not crash, encoded as all-zeros.""" + encoder = CategoricalEncoder(categorical_features=['category']) + encoder.fit(self.data_clean) + + # Create data with unseen category value + new_data = pd.DataFrame({ + 'temperature': [25.0], + 'humidity': [70], + 'category': ['D'], + 'region': ['North'] + }, index=pd.date_range('2024-01-01', periods=1, freq='1Min')) + + # With handle_unknown='ignore', unseen categories are encoded as all-zeros (no error) + transformed = encoder.transform(new_data) + # No new column for 'D' — only fitted categories A, B, C have columns + category_cols = [c for c in transformed.columns if c.startswith('category_')] + self.assertEqual(sorted(category_cols), ['category_A', 'category_B', 'category_C']) + self.assertTrue((transformed[category_cols] == 0).all().all(), + "Unseen category should be encoded as all-zeros.") + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/data_preprocessing/test_data_preprocessor.py b/tests/data_preprocessing/test_data_preprocessor.py index a4a1567d..18375762 100644 --- a/tests/data_preprocessing/test_data_preprocessor.py +++ b/tests/data_preprocessing/test_data_preprocessor.py @@ -1,12 +1,15 @@ +import tempfile from unittest import TestCase import numpy as np import pandas as pd from numpy.testing import assert_array_almost_equal from pandas.testing import assert_frame_equal -from sklearn.utils.validation import check_is_fitted, NotFittedError +from sklearn.utils.validation import NotFittedError +from energy_fault_detector.config.config import Config from energy_fault_detector.data_preprocessing.data_preprocessor import DataPreprocessor +from energy_fault_detector.fault_detector import FaultDetector class TestDataPreprocessorPipeline(TestCase): @@ -43,6 +46,16 @@ def setUp(self) -> None: ] ) + # Includes categorical encoder and ffill imputer + self.preprocessor_with_encoder = DataPreprocessor( + steps=[ + {'name': 'column_selector', 'params': {'max_nan_frac_per_col': 0.8}}, + {'name': 'ffill_imputer', + 'params': {'ffill_limit': '1min', 'categorical_features': ['category', 'region']}}, + {'name': 'categorical_encoder', + 'params': {'categorical_features': ['category', 'region']}}, + ] + ) # generate data for standard and feature consistent preprocessor tests length = 10 # choose an even number for simplicity time_index = pd.date_range(start='1/1/2021', end='10/1/2021', periods=length) @@ -86,6 +99,32 @@ def setUp(self) -> None: } self.test_data3 = pd.DataFrame(index=time_index, data=data) + # generate data for tests with categorical encoder and ffill imputer + self.test_data4 = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'humidity': [60, 62, np.nan, np.nan, 67], + 'category': ['A', 'B', 'A', np.nan, 'B'], + 'region': ['North', 'South', 'North', 'East', 'West'], + 'flowrate': [np.nan, np.nan, np.nan, np.nan, np.nan] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + self.exp_result4 = pd.DataFrame({ + 'temperature': [-0.5, 0.1, -1.1, 1.6], + 'humidity': [-1.1, -0.3, -0.3, 1.6], + 'category_A': [ 1. , -1. , 1. , -1. ], + 'category_B': [-1. , 1. , -1. , 1. ], + 'region_North': [ 1. , -1. , 1. , -1. ], + 'region_South': [-0.6, 1.7, -0.6, -0.6], + 'region_West': [-0.6, -0.6, -0.6, 1.7], + }, index=pd.to_datetime(['2024-01-01 00:00:00', '2024-01-01 00:01:00', '2024-01-01 00:02:00', '2024-01-01 00:04:00'])) + + self.exp_inv4 = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 23.0], + 'humidity': [60.0, 62.0, 62.0, 67.0], + 'category': ['A', 'B', 'A', 'B'], + 'region': ['North', 'South', 'North', 'West'] + }, index=pd.to_datetime(['2024-01-01 00:00:00', '2024-01-01 00:01:00', '2024-01-01 00:02:00', '2024-01-01 00:04:00'])) + def test_transform(self): # expected output exp_result = np.array([[-1.5666989, 0.], @@ -146,6 +185,12 @@ def test_transform_fc(self): assert_array_almost_equal(data, exp_result) + def test_transform_with_encoder_ffill(self): + """Test that the preprocessor with categorical encoder and ffill imputer works as expected.""" + self.preprocessor_with_encoder.fit(self.test_data4) + transformed = self.preprocessor_with_encoder.transform(self.test_data4) + self.assertTrue(transformed.round(1).equals(self.exp_result4)) + def test_not_fitted(self): with self.assertRaises(NotFittedError): self.standard_preprocessor.transform(self.test_data1) @@ -202,22 +247,22 @@ def test_steps_mode_no_duplicate_imputer(self) -> None: steps=[ {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}}, {"name": "simple_imputer", "params": {"strategy": "median"}}, - {"name": "standard_scaler"}, + {"name": "scaler"}, ] ) # Count imputers by estimator type n_imputers = sum( - est.__class__.__name__ == "SimpleImputer" for _, est in dp.steps + est.__class__.__name__ == "Imputer" for _, est in dp.steps ) self.assertEqual(n_imputers, 1, "There should be exactly one SimpleImputer.") # Ensure imputer precedes scaler imputer_idx = next( - i for i, (_, est) in enumerate(dp.steps) if est.__class__.__name__ == "SimpleImputer" + i for i, (_, est) in enumerate(dp.steps) if est.__class__.__name__ == "Imputer" ) scaler_idx = next( i for i, (_, est) in enumerate(dp.steps) - if est.__class__.__name__ in {"StandardScaler", "MinMaxScaler"} + if est.__class__.__name__ in {"Scaler"} ) self.assertLess(imputer_idx, scaler_idx, "Imputer must precede scaler.") @@ -226,22 +271,22 @@ def test_steps_mode_default_imputer_inserted(self) -> None: dp = DataPreprocessor( steps=[ {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}}, - {"name": "standard_scaler"}, + {"name": "scaler"}, ] ) # Exactly one imputer should be present n_imputers = sum( - est.__class__.__name__ == "SimpleImputer" for _, est in dp.steps + est.__class__.__name__ == "Imputer" for _, est in dp.steps ) self.assertEqual(n_imputers, 1, "A single default SimpleImputer should be added.") # Imputer must be before scaler imputer_idx = next( - i for i, (_, est) in enumerate(dp.steps) if est.__class__.__name__ == "SimpleImputer" + i for i, (_, est) in enumerate(dp.steps) if est.__class__.__name__ == "Imputer" ) scaler_idx = next( i for i, (_, est) in enumerate(dp.steps) - if est.__class__.__name__ in {"StandardScaler", "MinMaxScaler"} + if est.__class__.__name__ in {"Scaler"} ) self.assertLess(imputer_idx, scaler_idx, "Default imputer must be inserted before scaler.") @@ -250,7 +295,7 @@ def test_steps_mode_alias_imputer_is_normalized(self) -> None: dp = DataPreprocessor( steps=[ {"name": "imputer", "params": {"strategy": "mean"}}, # alias - {"name": "standard_scaler"}, + {"name": "scaler"}, ] ) # Named steps should include the canonical 'simple_imputer' @@ -263,7 +308,7 @@ def test_singleton_violation_raises(self) -> None: steps=[ {"name": "simple_imputer", "params": {"strategy": "mean"}}, {"name": "simple_imputer", "params": {"strategy": "median"}}, - {"name": "standard_scaler"}, + {"name": "scaler", "params": {"scaler_type": "standard"}}, ] ) @@ -271,13 +316,60 @@ def test_only_one_scaler_allowed(self) -> None: """Defining more than one scaler should raise a ValueError.""" with self.assertRaises(ValueError): _ = DataPreprocessor( + steps=[ + {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}}, + {"name": "scaler", "params": {"scaler_type": "standard"}}, + {"name": "scaler", "params": {"scaler_type": "minmax"}}, + ] + ) + + def test_legacy_scaler_names_migrated(self) -> None: + """Legacy 'standard_scaler'/'minmax_scaler' names should be migrated with a DeprecationWarning.""" + import warnings as _warnings + + with _warnings.catch_warnings(record=True) as caught: + _warnings.simplefilter("always") + dp = DataPreprocessor( steps=[ {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}}, {"name": "standard_scaler"}, + ] + ) + + deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertTrue( + any("standard_scaler" in str(w.message) for w in deprecation_warnings), + "Expected a DeprecationWarning mentioning 'standard_scaler'.", + ) + # The migrated pipeline should have a 'scaler' step with a Scaler instance + self.assertIn("scaler", dp.named_steps) + self.assertEqual(dp.named_steps["scaler"].scaler_type, "standard") + + with _warnings.catch_warnings(record=True) as caught: + _warnings.simplefilter("always") + dp = DataPreprocessor( + steps=[ + {"name": "column_selector", "params": {"max_nan_frac_per_col": 0.2}}, {"name": "minmax_scaler"}, ] ) + deprecation_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertTrue( + any("minmax_scaler" in str(w.message) for w in deprecation_warnings), + "Expected a DeprecationWarning mentioning 'minmax_scaler'.", + ) + self.assertIn("scaler", dp.named_steps) + self.assertEqual(dp.named_steps["scaler"].scaler_type, "minmax") + + def test_inverse_transform_with_encoder_ffill(self): + """Test that the inverse_transform works correctly with the preprocessor that includes categorical encoder and ffill imputer.""" + self.preprocessor_with_encoder.fit(self.test_data4) + transformed = self.preprocessor_with_encoder.transform(self.test_data4) + inversed = self.preprocessor_with_encoder.inverse_transform(transformed) + print("Inversed DataFrame:\n", inversed) + self.assertEqual(inversed.shape, self.exp_inv4.shape, "Inverse transform shape mismatch.") + self.assertTrue(inversed.round(1).equals(self.exp_inv4), "Inverse transform did not return the expected DataFrame.") class TestDataPreprocessorPipelineWithTimestamp(TestCase): def setUp(self) -> None: @@ -416,23 +508,277 @@ def test_protected_features_with_both_filters(self): self.assertIn('high_nan_feature', transformed.columns, "High-NaN protected feature should be kept") - def test_validation_fails_when_protected_feature_missing_from_output(self): - """Test that fit raises ValueError if a protected feature is missing from output.""" - from energy_fault_detector.data_preprocessing.angle_transformer import AngleTransformer - # Create a pipeline that transforms a protected feature - preprocessor = DataPreprocessor( - steps=[ - {'name': 'angle_transformer', - 'params': {'angles': ['conditional_feature']}}, # Transforms conditional_feature - ] - ) +class TestComprehensivePipelineFlow(TestCase): + """Test comprehensive preprocessing pipeline""" + + def setUp(self): + """Set up test fixtures for comprehensive pipeline testing.""" + self.n_samples = 500 + self.time_index = pd.date_range('2025-01-01', periods=self.n_samples, freq='5min') + + # Create realistic test data matching the updated advanced_config.yaml scenario + np.random.seed(42) + self.test_data = pd.DataFrame({ + # Numerical features + 'temp_sensor_1': np.random.normal(100, 5, self.n_samples), + 'temp_sensor_2': np.random.normal(85, 3, self.n_samples), + 'pressure_sensor': np.random.normal(50, 2, self.n_samples), + 'flow_rate': np.random.exponential(10, self.n_samples), + # Categorical features + 'category_col': np.random.choice(['A', 'B', 'C', 'D'], self.n_samples), + 'equipment_type': np.random.choice(['Type1', 'Type2', 'Type3'], self.n_samples), + # Conditional feature (numerical) + 'operating_condition': np.random.normal(0, 1, self.n_samples), + }, index=self.time_index) + + # Add some NaN values to test imputation + self.test_data.iloc[50:100, self.test_data.columns.get_loc('temp_sensor_1')] = np.nan + self.test_data.iloc[200:250, self.test_data.columns.get_loc('pressure_sensor')] = np.nan + self.test_data.iloc[150:170, self.test_data.columns.get_loc('category_col')] = np.nan + + def _create_config_dict(self, protect_conditional_features: bool = True): + """Create configuration matching updated advanced_config.yaml structure.""" + return { + 'train': { + 'data_clipping': { + 'lower_percentile': 0.001, + 'upper_percentile': 0.999, + }, + 'data_preprocessor': { + 'steps': [ + {'name': 'column_selector', 'params': {'max_nan_frac_per_col': 0.9}}, + {'name': 'low_unique_value_filter', 'params': {'min_unique_value_count': 2}}, + {'name': 'ffill_imputer', 'params': {'ffill_limit': '60min', 'categorical_features': ['category_col', 'equipment_type']}}, + {'name': 'categorical_encoder', 'params': {'categorical_features': ['category_col', 'equipment_type']}}, + {'name': 'scaler', 'params': {'scaler_type': 'standard', 'with_mean': True, 'with_std': True, 'scale_categorical_features': False, 'categorical_features': ['category_col', 'equipment_type']}}, + {'name': 'timestamp_transformer', 'params': {'features': ['minute_of_hour', 'hour_of_day', 'day_of_week']}} + ] + }, + 'autoencoder': { + 'name': 'ConditionalAutoencoder', + 'params': { + 'act': 'prelu', + 'batch_size': 256, + 'code_size': 9, + 'epochs': 10, + 'last_act': 'linear', + 'layers': [64, 32], + 'learning_rate': 0.0001, + 'loss_name': 'mean_squared_error', + 'noise': 0.0, + 'conditional_features': ['operating_condition', 'category_col'], + 'verbose': 0 + } + }, + 'protect_conditional_features': protect_conditional_features, + 'anomaly_score': {'name': 'rmse'}, + 'threshold_selector': {'name': 'quantile'}, + } + } + + def test_pipeline_dtype_conversion(self): + """Test that all dtypes are numerical after preprocessing.""" + config_dict = self._create_config_dict() + preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps']) + + preprocessor.fit(self.test_data) + transformed_data = preprocessor.transform(self.test_data) + + # Verify all dtypes are numerical after preprocessing + for dtype in transformed_data.dtypes: + self.assertTrue(pd.api.types.is_numeric_dtype(dtype), + f"Column {dtype.name} should be numeric after preprocessing") + + def test_pipeline_expected_columns(self): + """Test that expected columns exist after preprocessing.""" + config_dict = self._create_config_dict() + preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps']) + + preprocessor.fit(self.test_data) + transformed_data = preprocessor.transform(self.test_data) + + # Check expected numerical columns + expected_numerical = ['temp_sensor_1', 'temp_sensor_2', 'pressure_sensor', 'flow_rate', 'operating_condition'] + for col in expected_numerical: + self.assertIn(col, transformed_data.columns, f"Expected numerical column {col}") + + # Check categorical features were one-hot encoded + expected_categorical = ['category_col_A', 'category_col_B', 'category_col_C', 'category_col_D', + 'equipment_type_Type1', 'equipment_type_Type2', 'equipment_type_Type3'] + for col in expected_categorical: + self.assertIn(col, transformed_data.columns, f"Expected encoded categorical column {col}") + + # Check timestamp features + timestamp_features = ['minute_of_hour_sine', 'minute_of_hour_cosine', + 'hour_of_day_sine', 'hour_of_day_cosine', + 'day_of_week_sine', 'day_of_week_cosine'] + for col in timestamp_features: + self.assertIn(col, transformed_data.columns, f"Expected timestamp feature {col}") + + def test_pipeline_nan_handling(self): + """Test that NaN values are properly handled.""" + config_dict = self._create_config_dict() + preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps']) + + preprocessor.fit(self.test_data) + transformed_data = preprocessor.transform(self.test_data) + + # Verify no NaN values remain after preprocessing + self.assertFalse(transformed_data.isna().any().any(), + "Transformed data should not contain NaN values after preprocessing") + + def test_pipeline_shape_transformation(self): + """Test that data shape is appropriate after preprocessing.""" + config_dict = self._create_config_dict() + preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps']) + + preprocessor.fit(self.test_data) + transformed_data = preprocessor.transform(self.test_data) + + # Verify data shape (should have more columns than original due to encoding) + self.assertGreater(transformed_data.shape[1], self.test_data.shape[1], + "Transformed data should have more columns due to categorical encoding and timestamp features") + + def test_pipeline_inverse_transform(self): + """Test that inverse transform works correctly.""" + config_dict = self._create_config_dict() + preprocessor = DataPreprocessor(steps=config_dict['train']['data_preprocessor']['steps']) + + preprocessor.fit(self.test_data) + transformed_data = preprocessor.transform(self.test_data) + inverse_data = preprocessor.inverse_transform(transformed_data) + + # Verify inverse transform preserves row count of the transformed data + # (the ffill_imputer may drop rows that still contain NaNs after filling) + self.assertEqual(inverse_data.shape[0], transformed_data.shape[0], + "Inverse transform should preserve the transformed data's row count") + + def test_pipeline_fault_detector_integration(self): + """Test integration with FaultDetector.""" + config_dict = self._create_config_dict() + + with tempfile.TemporaryDirectory() as tmpdir: + config = Config(config_dict=config_dict) + fault_detector = FaultDetector(config=config, model_directory=tmpdir) + + # Create normal index (all normal for training) + normal_index = pd.Series([True] * self.n_samples, index=self.time_index) + + # Fit the model + result = fault_detector.fit( + sensor_data=self.test_data, + normal_index=normal_index, + save_models=False + ) + + # Verify model was trained successfully + self.assertIsNotNone(fault_detector.autoencoder, + "Autoencoder should be created and trained") + self.assertEqual(fault_detector.autoencoder.__class__.__name__, 'ConditionalAE', + "Autoencoder should be ConditionalAE as specified in config") + self.assertIsNotNone(result.train_recon_error, + "Training reconstruction error should be available") + + def test_resolution_of_conditions(self): + """Test pipeline handling of missing conditional features.""" + config_dict = self._create_config_dict() + + # Create data without the conditional feature + test_data_no_cond = self.test_data.drop(columns=['category_col']) + + with tempfile.TemporaryDirectory() as tmpdir: + config = Config(config_dict=config_dict) + fault_detector = FaultDetector(config=config, model_directory=tmpdir) + + # Create normal index (all normal for training) + normal_index = pd.Series([True] * self.n_samples, index=self.time_index) + + # Fit the model + result = fault_detector.fit( + sensor_data=test_data_no_cond, + normal_index=normal_index, + save_models=False + ) + + # Verify that autoencoder conditions are updated with available feature names + self.assertIn('operating_condition', fault_detector.autoencoder.conditional_features, + "Autoencoder should have 'operating_condition' as a conditional feature") + self.assertNotIn('category_col', fault_detector.autoencoder.conditional_features, + "Autoencoder should not have 'category_col' as a conditional feature since it's missing") + + def test_declared_category_not_as_condition(self): + """Test pipeline handling of declared categorical features not being declared as conditional features.""" + config_dict = self._create_config_dict() + + with tempfile.TemporaryDirectory() as tmpdir: + config = Config(config_dict=config_dict) + fault_detector = FaultDetector(config=config, model_directory=tmpdir) - # This should raise an error because 'conditional_feature' will be transformed to - # 'conditional_feature_sin' and 'conditional_feature_cos' - with self.assertRaises(ValueError) as context: - preprocessor.fit(self.test_data, protected_features=['conditional_feature']) + # Create normal index (all normal for training) + normal_index = pd.Series([True] * self.n_samples, index=self.time_index) + + # Fit the model + result = fault_detector.fit( + sensor_data=self.test_data, + normal_index=normal_index, + save_models=False + ) + + self.assertIn('equipment_type_Type1', fault_detector.data_preprocessor.get_feature_names_out(), + "Data preprocessor should have 'equipment_type' as a feature") + self.assertNotIn('equipment_type', fault_detector.autoencoder.conditional_features, + "Autoencoder should not have 'equipment_type' as a conditional feature since it's not declared as such.") + + def test_declared_condition_not_as_category(self): + """Test pipeline handling of declared conditional features not being declared as categorical features.""" + config_dict = self._create_config_dict(False) + + # Remove 'category_col' from categorical features in the config to simulate it being declared as conditional but not categorical + config_dict['train']['data_preprocessor']['steps'][2]['params']['categorical_features'] = ['equipment_type'] # Only 'equipment_type' is categorical + config_dict['train']['data_preprocessor']['steps'][3]['params']['categorical_features'] = ['equipment_type'] # Only 'equipment_type' is categorical + config_dict['train']['data_preprocessor']['steps'][4]['params']['categorical_features'] = ['equipment_type'] # Only 'equipment_type' is categorical + + with tempfile.TemporaryDirectory() as tmpdir: + config = Config(config_dict=config_dict) + fault_detector = FaultDetector(config=config, model_directory=tmpdir) + + # Create normal index (all normal for training) + normal_index = pd.Series([True] * self.n_samples, index=self.time_index) + + # Fit the model + result = fault_detector.fit( + sensor_data=self.test_data, + normal_index=normal_index, + save_models=False + ) + + # Verify that autoencoder conditions are updated properly. + self.assertFalse(any('category_col' in col for col in fault_detector.data_preprocessor.get_feature_names_out()), + "Data preprocessor should not have any column containing substring 'category_col' since it's not declared as categorical.") + + def test_protected_features_are_not_dropped_in_pipeline(self): + """Test that protected features are not dropped in the comprehensive pipeline.""" + config_dict = self._create_config_dict() + config = Config(config_dict=config_dict) + + with tempfile.TemporaryDirectory() as tmpdir: + fault_detector = FaultDetector(config=config, model_directory=tmpdir) + + # Create normal index (all normal for training) + normal_index = pd.Series([True] * self.n_samples, index=self.time_index) + + # Fit the model + result = fault_detector.fit( + sensor_data=self.test_data, + normal_index=normal_index, + save_models=False + ) - self.assertIn('Protected features were dropped', str(context.exception)) - self.assertIn('conditional_feature', str(context.exception)) - self.assertIn('AngleTransformer or CounterDiffTransformer', str(context.exception)) + protected_features = config_dict['train']['autoencoder']['params']['conditional_features'] + + output_features = fault_detector.data_preprocessor.get_feature_names_out() + available_protected = [f for f in protected_features if f in self.test_data.columns] + missing_features = [f for f in available_protected + if not any(f in col for col in output_features)] + self.assertEqual(len(missing_features), 0, f"Protected features {available_protected} should not be dropped in the pipeline.") diff --git a/tests/data_preprocessing/test_ffill_imputer.py b/tests/data_preprocessing/test_ffill_imputer.py new file mode 100644 index 00000000..1aa76701 --- /dev/null +++ b/tests/data_preprocessing/test_ffill_imputer.py @@ -0,0 +1,305 @@ +import unittest +import pandas as pd +import numpy as np +from datetime import datetime, timedelta + +from sklearn.exceptions import NotFittedError + +from energy_fault_detector.data_preprocessing.ffill_imputer import ForwardFillImputer + + +class TestForwardFillImputer(unittest.TestCase): + + def setUp(self): + """Set up common test fixtures.""" + # Create datetime index with 1-minute frequency + base_time = datetime(2024, 1, 1, 0, 0, 0) + self.timestamps = pd.date_range(start=base_time, periods=10, freq='1Min') + + # Create numeric data with some NaNs + self.data_numeric = pd.DataFrame({ + 'temp': [20.0, np.nan, 22.0, np.nan, np.nan, 25.0, np.nan, np.nan, np.nan, 30.0], + 'humidity': [60.0, 61.0, np.nan, np.nan, 64.0, 65.0, np.nan, 67.0, 68.0, 69.0], + 'pressure': [1013.0, 1013.5, 1014.0, np.nan, np.nan, np.nan, 1015.5, 1016.0, np.nan, 1016.5] + }, index=self.timestamps) + + # Create categorical data + self.data_categorical = pd.DataFrame({ + 'status': ['A', 'B', 'A', np.nan, 'A', 'B', 'A', 'A', 'B', 'A'], + 'region': ['North', 'South', 'East', 'West', 'North', 'South', 'East', 'West', 'North', 'South'] + }, index=self.timestamps) + + # Combined DataFrame + self.data_full = pd.concat([self.data_numeric, self.data_categorical], axis=1) + + # Create non-declared categorical features in numeric columns (strings) + self.data_mixed_categorical = self.data_numeric.copy() + self.data_mixed_categorical['category_col'] = ['cat1', 'cat2', 'cat1', 'cat3', 'cat2', 'cat1', 'cat3', 'cat1', 'cat2', 'cat1'] + + # Create full dataframe containing non-declared categorical features + self.data_full_with_non_declared = pd.concat([self.data_mixed_categorical, self.data_categorical], axis=1) + + def tearDown(self): + """Clean up after each test.""" + del self.timestamps, self.data_numeric, self.data_categorical, self.data_full, self.data_mixed_categorical, self.data_full_with_non_declared + + def test_init_defaults(self): + """Test default initialization.""" + imputer = ForwardFillImputer() + self.assertEqual(imputer.ffill_limit, "15min") + self.assertEqual(imputer.categorical_features, []) + + def test_init_custom_params(self): + """Test initialization with custom parameters.""" + imputer = ForwardFillImputer(ffill_limit="10min", categorical_features=["status", "region"]) + self.assertEqual(imputer.ffill_limit, "10min") + self.assertEqual(imputer.categorical_features, ["status", "region"]) + + def test_fit(self): + """Test fitting the imputer.""" + imputer = ForwardFillImputer(categorical_features=['status', 'region']) + imputer.fit(self.data_full_with_non_declared) + + self.assertEqual(imputer.n_features_in_, 6) # 3 numeric + 1 non-declared categorical + 2 declared categorical + self.assertListEqual(imputer.feature_names_in_, ['temp', 'humidity', 'pressure', 'category_col', 'status', 'region']) + self.assertListEqual(imputer.numerical_columns, ['temp', 'humidity', 'pressure']) + self.assertListEqual(imputer.categorical_columns, ['status', 'region']) + self.assertEqual(len(imputer.non_declared_categorical_features), 1) + + def test_fit_numerical_only(self): + """Test fitting on numerical-only data.""" + imputer = ForwardFillImputer() + imputer.fit(self.data_numeric) + + self.assertEqual(imputer.n_features_in_, 3) + self.assertListEqual(imputer.feature_names_in_, ['temp', 'humidity', 'pressure']) + self.assertListEqual(imputer.numerical_columns, ['temp', 'humidity', 'pressure']) + self.assertEqual(len(imputer.categorical_columns), 0) + self.assertEqual(len(imputer.non_declared_categorical_features), 0) + + def test_fit_with_categorical(self): + """Test fitting with categorical features declared.""" + imputer = ForwardFillImputer(categorical_features=['status', 'region']) + imputer.fit(self.data_full) + + self.assertEqual(imputer.n_features_in_, 5) + self.assertListEqual(imputer.numerical_columns, ['temp', 'humidity', 'pressure']) + self.assertListEqual(imputer.categorical_columns, ['status', 'region']) + self.assertEqual(len(imputer.non_declared_categorical_features), 0) + + def test_fit_non_declared_categorical_in_numerical_columns(self): + """Test detection of non-declared categorical features.""" + imputer = ForwardFillImputer() + imputer.fit(self.data_mixed_categorical) + + # Should detect 'category_col' as non-declared categorical + self.assertIn('category_col', imputer.non_declared_categorical_features) + self.assertNotIn('category_col', imputer.numerical_columns) + self.assertEqual(imputer.numerical_columns, ['temp', 'humidity', 'pressure']) + + def test_transform_basic(self): + """Test basic forward fill transformation.""" + imputer = ForwardFillImputer(ffill_limit="10min") + imputer.fit(self.data_numeric) + result = imputer.transform(self.data_numeric) + + # Should not have any NaNs + self.assertFalse(result.isna().any().any()) + # Should maintain same columns + self.assertEqual(result.shape[1], 3) + # Should have same column names + self.assertListEqual(list(result.columns), ['temp', 'humidity', 'pressure']) + + def test_transform_with_categorical(self): + """Test transformation with categorical features.""" + imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit="10min") + imputer.fit(self.data_full) + result = imputer.transform(self.data_full) + + # Should not have any NaNs + self.assertFalse(result.isna().any().any()) + # Should maintain same columns + self.assertEqual(result.shape[1], 5) + # Should preserve categorical data + self.assertListEqual(list(result.columns), ['temp', 'humidity', 'pressure', 'status', 'region']) + # Check that status column is preserved (with some imputed) + self.assertEqual(result['status'].iloc[3], 'A') # Should be forward-filled + + def test_transform_limit_exceeded(self): + """Test that values beyond ffill_limit are not filled.""" + # Create data with long gaps (> ffill_limit) + long_gaps_data = pd.DataFrame({ + 'value1': [1.0, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, 2.0], + 'value2': list(range(10, 27)) + }, index=pd.date_range('2024-01-01', periods=17, freq='1Min')) + + imputer = ForwardFillImputer(ffill_limit="5min") + imputer.fit(long_gaps_data) + result = imputer.transform(long_gaps_data) + print(result) + # The 7th row should be 2.0 because elapsed time (6min) exceeds the 5min limit + self.assertTrue(result.iloc[6]['value1']==2.0) + + def test_transform_drops_duplicates(self): + """Test that duplicate rows are removed.""" + # Create data with duplicate rows + dup_data = pd.DataFrame({ + 'value': [1.0, 2.0, 2.0, 3.0] + }, index=pd.date_range('2024-01-01', periods=4, freq='1Min')) + + imputer = ForwardFillImputer() + imputer.fit(dup_data) + result = imputer.transform(dup_data) + + # Should have fewer rows after deduplication + self.assertLess(result.shape[0], dup_data.shape[0]) + + def test_transform_drops_rows_with_any_na(self): + """Test that rows with any NaN are dropped after imputation.""" + # Create data where after ffill, some rows still have NaN + some_na_data = pd.DataFrame({ + 'value1': [1.0, np.nan, np.nan, 3.0, np.nan], + 'value2': [np.nan, 2.0, 3.0, 4.0, 5.0] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + imputer = ForwardFillImputer(ffill_limit="1min") + imputer.fit(some_na_data) + result = imputer.transform(some_na_data) + + # Should have dropped rows with any remaining NaNs + self.assertFalse(result.isna().any().any()) + self.assertEqual(result.shape[0], 3) # Should keep three rows + + def test_transform_preserves_index_type(self): + """Test that index type is preserved after transformation.""" + imputer = ForwardFillImputer() + imputer.fit(self.data_numeric) + result = imputer.transform(self.data_numeric) + + self.assertIsInstance(result.index, pd.DatetimeIndex) + + def test_transform_non_datetime_index_raises(self): + """Test that non-datetime index raises TypeError.""" + imputer = ForwardFillImputer() + imputer.fit(self.data_numeric) + + # Create DataFrame with non-datetime index + bad_index_data = self.data_numeric.copy() + bad_index_data.index = range(len(bad_index_data)) + + with self.assertRaises(TypeError) as context: + imputer.transform(bad_index_data) + self.assertIn("DatetimeIndex", str(context.exception)) + + def test_transform_missing_columns_raises(self): + """Test that missing columns raise ValueError.""" + imputer = ForwardFillImputer() + imputer.fit(self.data_numeric) + + # Drop one column + partial_data = self.data_numeric.drop(columns=['temp']) + + with self.assertRaises(ValueError) as context: + imputer.transform(partial_data) + self.assertIn("missing columns", str(context.exception).lower()) + + def test_transform_invalid_type_raises(self): + """Test that non-DataFrame input raises TypeError.""" + imputer = ForwardFillImputer() + imputer.fit(self.data_numeric) + + with self.assertRaises(TypeError) as context: + imputer.transform([1, 2, 3]) + self.assertIn("pandas DataFrame", str(context.exception)) + + def test_transform_unfitted_raises(self): + """Test that transform on unfitted model raises error.""" + imputer = ForwardFillImputer() + + with self.assertRaises(NotFittedError) as context: + imputer.transform(self.data_numeric) + self.assertIn("not fitted", str(context.exception).lower()) + + def test_inverse_transform(self): + """Test inverse transform maintains column order.""" + imputer = ForwardFillImputer(categorical_features=['status', 'region'], ffill_limit="10min") + imputer.fit(self.data_full) + transformed = imputer.transform(self.data_full) + inverse = imputer.inverse_transform(transformed) + + # Should match original column names + self.assertListEqual(list(inverse.columns), self.data_full.columns.tolist()) + self.assertEqual(inverse.shape[1], 5) + + def test_get_feature_names_out(self): + """Test feature names out method.""" + imputer = ForwardFillImputer(categorical_features=['status', 'region']) + imputer.fit(self.data_full) + + feature_names = imputer.get_feature_names_out() + self.assertEqual(len(feature_names), 5) + self.assertIn('temp', feature_names) + self.assertIn('humidity', feature_names) + self.assertIn('pressure', feature_names) + self.assertIn('status', feature_names) + self.assertIn('region', feature_names) + + def test_empty_dataframe_handling(self): + """Test behavior with empty DataFrame.""" + empty_df = pd.DataFrame(columns=self.data_numeric.columns, + index=pd.DatetimeIndex([])) + + imputer = ForwardFillImputer() + imputer.fit(self.data_numeric) + + result = imputer.transform(empty_df) + + self.assertEqual(result.shape[0], 0) + + def test_all_nan_column_handling(self): + """Test that all-NaN columns are dropped during fit.""" + all_nan_data = pd.DataFrame({ + 'value1': [1.0, 2.0, 3.0], + 'value2': [np.nan, np.nan, np.nan] + }, index=pd.date_range('2024-01-01', periods=3, freq='1Min')) + + imputer = ForwardFillImputer() + imputer.fit(all_nan_data) + + # All-NaN column should be dropped from numerical_columns + self.assertNotIn('value2', imputer.numerical_columns) + self.assertIn('value1', imputer.numerical_columns) + + # Transform should keep rows from the valid column, not drop everything + result = imputer.transform(all_nan_data) + self.assertEqual(result.shape[1], 1) + self.assertListEqual(list(result.columns), ['value1']) + self.assertEqual(result.shape[0], 3) + + def test_numerical_conversion_error(self): + """Test error handling for non-convertible numerical columns.""" + + + imputer = ForwardFillImputer(categorical_features=['status', 'region']) + imputer.fit(self.data_full) + + # Add non-numeric column to numeric section + bad_numeric = self.data_full.copy() + bad_numeric['temp'] = ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B'] + + # Try transform - should raise error because 'bad_numeric' cannot convert to float + with self.assertRaises(ValueError) as context: + imputer.transform(bad_numeric) + self.assertIn("cannot be converted to float", str(context.exception)) + + def test_feature_names_out_after_fit(self): + """Test feature names out are available after fit.""" + imputer = ForwardFillImputer(categorical_features=['status', 'region']) + imputer.fit(self.data_full) + + feature_names = imputer.get_feature_names_out() + self.assertEqual(len(feature_names), imputer.n_features_in_) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/data_preprocessing/test_imputer.py b/tests/data_preprocessing/test_imputer.py new file mode 100644 index 00000000..51a980fd --- /dev/null +++ b/tests/data_preprocessing/test_imputer.py @@ -0,0 +1,329 @@ +import unittest +import pandas as pd +import numpy as np +from sklearn.exceptions import NotFittedError + +from energy_fault_detector.data_preprocessing.imputer import Imputer + + +class TestImputer(unittest.TestCase): + def setUp(self): + """Set up common test fixtures.""" + # Create sample numerical data with missing values + self.data_numerical = pd.DataFrame({ + 'temperature': [20.5, np.nan, 19.8, 22.1, 23.0], + 'humidity': [60, 62, np.nan, 65, 67], + 'pressure': [1013.2, 1014.5, 1012.8, np.nan, 1013.7] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with categorical features (object dtype) + self.data_with_categorical = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'humidity': [60, 62, 58, 65, 67], + 'status': ['A', 'B', 'A', 'C', 'B'], + 'region': ['North', 'South', 'North', 'East', 'West'], + 'category': ['X', 'Y', 'X', 'Z', 'Y'] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with mixed numerical and categorical columns + self.data_mixed = pd.DataFrame({ + 'temperature': [20.5, np.nan, 19.8, 22.1, 23.0], + 'humidity': [60, 62, np.nan, 65, 67], + 'status': ['A', 'B', 'A', 'C', 'B'], + 'region': ['North', 'South', 'North', 'East', 'West'], + 'voltage': [220.5, 221.3, np.nan, 219.8, 220.1] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with NaN values in categorical columns + self.data_with_nan_categorical = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, np.nan, 23.0], + 'humidity': [60, 62, 58, 65, np.nan], + 'status': ['A', 'B', np.nan, 'C', 'B'], + 'region': ['North', 'South', 'North', 'East', 'West'] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with all NaN in a column + self.data_all_nan = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'humidity': [60, 62, 58, 65, 67], + 'broken_sensor': [np.nan, np.nan, np.nan, np.nan, np.nan] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data for inverse transform testing + self.data_for_inverse = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8], + 'humidity': [60, 62, 58], + 'status': ['A', 'B', 'A'] + }, index=pd.date_range('2024-01-01', periods=3, freq='1Min')) + + # Create transformed data (with NaN values filled) + self.transformed_data = pd.DataFrame({ + 'temperature': [20.5, 21.1, 19.8, 22.1, 23.0], + 'humidity': [60, 62, 63.5, 65, 67], + 'status': ['A', 'B', 'A', 'C', 'B'], + 'region': ['North', 'South', 'North', 'East', 'West'] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with missing columns (for error testing) + self.data_missing_columns = pd.DataFrame({ + 'temperature': [21.0, 22.0], + 'humidity': [61, 63] + }, index=pd.date_range('2024-01-01', periods=2, freq='1Min')) + + # Create data with non-declared categorical features (object dtype in numerical columns) + self.data_with_object_in_num = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'humidity': [60, 62, 58, 65, 67], + 'sensor_id': ['S1', 'S2', 'S3', 'S4', 'S5'] # This should be detected as categorical + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with only categorical features + self.data_only_categorical = pd.DataFrame({ + 'status': ['A', 'B', 'A', 'C', 'B'], + 'region': ['North', 'South', 'North', 'East', 'West'] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with boolean columns + self.data_with_boolean = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'is_active': [True, False, True, False, True], + 'status': ['A', 'B', 'A', 'C', 'B'] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + def tearDown(self): + """Clean up after each test.""" + del (self.data_numerical, self.data_with_categorical, + self.data_mixed, self.data_with_nan_categorical, + self.data_all_nan, self.data_for_inverse, + self.transformed_data, self.data_missing_columns, + self.data_with_object_in_num, self.data_only_categorical, + self.data_with_boolean) + + def test_init_defaults(self): + """Test default initialization.""" + imputer = Imputer() + self.assertEqual(imputer.strategy, 'mean') + self.assertEqual(imputer.categorical_features, []) + self.assertIsNotNone(imputer.numerical_imputer) + self.assertIsNotNone(imputer.categorical_imputer) + + def test_init_median_strategy(self): + """Test initialization with median strategy.""" + imputer = Imputer(strategy='median') + self.assertEqual(imputer.strategy, 'median') + self.assertEqual(imputer.numerical_imputer.strategy, 'median') + self.assertEqual(imputer.categorical_imputer.strategy, 'most_frequent') + + def test_init_invalid_strategy(self): + """Test initialization with invalid strategy raises ValueError.""" + with self.assertRaises(ValueError): + Imputer(strategy='invalid_strategy') + + def test_init_with_categorical_features(self): + """Test initialization with categorical features.""" + imputer = Imputer(categorical_features=['status', 'region']) + self.assertEqual(imputer.categorical_features, ['status', 'region']) + + def test_fit_numerical_data(self): + """Test fitting with numerical data containing NaN.""" + imputer = Imputer() + imputer.fit(self.data_numerical) + + self.assertEqual(imputer.n_features_in_, 3) + self.assertEqual(len(imputer.feature_names_in_), 3) + self.assertEqual(len(imputer.numerical_columns), 3) + self.assertEqual(len(imputer.categorical_columns), 0) + self.assertIsNotNone(imputer.numerical_imputer.statistics_) + + def test_fit_with_categorical_features(self): + """Test fitting with categorical features.""" + imputer = Imputer(categorical_features=['status']) + imputer.fit(self.data_with_categorical) + + self.assertEqual(imputer.n_features_in_, 5) + self.assertIn('status', imputer.categorical_columns) + self.assertIn('temperature', imputer.numerical_columns) + self.assertEqual(len(imputer.numerical_columns), 2) # temperature and humidity + self.assertEqual(len(imputer.categorical_columns), 1) # status + + def test_fit_mixed_data(self): + """Test fitting with mixed numerical and categorical data.""" + imputer = Imputer(categorical_features=['status', 'region']) + imputer.fit(self.data_mixed) + + self.assertEqual(imputer.n_features_in_, 5) + self.assertEqual(len(imputer.numerical_columns), 3) # temperature, humidity, voltage + self.assertEqual(len(imputer.categorical_columns), 2) # status, region + + def test_fit_detects_non_declared_categorical(self): + """Test that non-declared categorical features are detected.""" + imputer = Imputer() + imputer.fit(self.data_with_object_in_num) + + # sensor_id should be detected as non-declared categorical + self.assertIn('sensor_id', imputer.non_declared_categorical_features) + # sensor_id should be excluded from numerical columns + self.assertNotIn('sensor_id', imputer.numerical_columns) + # sensor_id should be dropped entirely (not in categorical either) + self.assertEqual(len(imputer.numerical_columns), 2) # temperature, humidity only + + def test_fit_with_all_nan_column(self): + """Test fitting with a column that has all NaN values.""" + imputer = Imputer() + imputer.fit(self.data_all_nan) + + # All-NaN column should be dropped from numerical_columns + self.assertEqual(imputer.n_features_in_, 3) + self.assertNotIn('broken_sensor', imputer.numerical_columns) + self.assertListEqual(imputer.numerical_columns, ['temperature', 'humidity']) + + # Transform should work and return only the valid columns + result = imputer.transform(self.data_all_nan) + self.assertEqual(result.shape[1], 2) + self.assertListEqual(list(result.columns), ['temperature', 'humidity']) + self.assertFalse(result.isna().any().any()) + + def test_fit_empty_categorical(self): + """Test fitting when categorical columns DataFrame is empty.""" + imputer = Imputer(categorical_features=['nonexistent']) + imputer.fit(self.data_numerical) + + # Should handle empty categorical data gracefully + self.assertEqual(len(imputer.categorical_columns), 0) + self.assertIsNotNone(imputer.categorical_imputer) + + def test_transform_with_numerical_data(self): + """Test transformation with numerical data.""" + imputer = Imputer() + imputer.fit(self.data_numerical) + result = imputer.transform(self.data_numerical) + + # Should return DataFrame with same shape and columns + self.assertEqual(result.shape, self.data_numerical.shape) + self.assertListEqual(list(result.columns), self.data_numerical.columns.tolist()) + # No NaN values should remain + self.assertFalse(result.isna().any().any()) + # Index should be preserved + self.assertTrue(result.index.equals(self.data_numerical.index)) + + def test_transform_with_categorical_data(self): + """Test transformation with categorical data.""" + imputer = Imputer(categorical_features=['status']) + imputer.fit(self.data_with_categorical) + result = imputer.transform(self.data_with_categorical) + + # Should return DataFrame without non-declared categorical features + self.assertEqual(result.shape[1], 3) # temperature, humidity, status + # Categorical values should be imputed (most frequent) + self.assertFalse(result['status'].isna().any()) + # Index should be preserved + self.assertTrue(result.index.equals(self.data_with_categorical.index)) + + def test_transform_mixed_data(self): + """Test transformation with mixed data.""" + imputer = Imputer(categorical_features=['status', 'region']) + imputer.fit(self.data_mixed) + result = imputer.transform(self.data_mixed) + + # Should handle mixed data correctly + self.assertEqual(result.shape, self.data_mixed.shape) + # No NaN values should remain + self.assertFalse(result.isna().any().any()) + + def test_transform_unfitted_raises(self): + """Test that transform on unfitted model raises error.""" + imputer = Imputer() + with self.assertRaises(NotFittedError): + imputer.transform(self.data_numerical) + + def test_transform_with_nan_categorical(self): + """Test transformation with NaN values in categorical columns.""" + imputer = Imputer(categorical_features=['status', 'region']) + imputer.fit(self.data_with_nan_categorical) + result = imputer.transform(self.data_with_nan_categorical) + + # Categorical NaN values should be imputed + self.assertFalse(result[['status', 'region']].isna().any().any()) + + def test_inverse_transform(self): + """Test inverse transform method.""" + imputer = Imputer(categorical_features=['status']) + imputer.fit(self.data_for_inverse) + transformed = imputer.transform(self.data_for_inverse) + inverse = imputer.inverse_transform(transformed) + + # Should return DataFrame with same shape + self.assertEqual(inverse.shape, self.data_for_inverse.shape) + # Index should be restored to original + self.assertTrue(inverse.index.equals(self.data_for_inverse.index)) + # Column names should match original + self.assertListEqual(list(inverse.columns), self.data_for_inverse.columns.tolist()) + + def test_get_feature_names_out(self): + """Test get_feature_names_out method.""" + imputer = Imputer(categorical_features=['status', 'region']) + imputer.fit(self.data_mixed) + + feature_names = imputer.get_feature_names_out() + + # Should return numerical columns first, then categorical + expected_order = ['temperature', 'humidity', 'voltage', 'status', 'region'] + self.assertListEqual(feature_names, expected_order) + + def test_transform_with_missing_columns(self): + """Test that transform raises error for missing columns.""" + imputer = Imputer() + imputer.fit(self.data_numerical) + + with self.assertRaises(ValueError): + imputer.transform(self.data_missing_columns) + + def test_transform_with_boolean_columns(self): + """Test transformation with boolean columns.""" + imputer = Imputer(categorical_features=['status']) + imputer.fit(self.data_with_boolean) + result = imputer.transform(self.data_with_boolean) + + # Boolean column should be handled as numerical + self.assertFalse(result.isna().any().any()) + + def test_transform_preserves_index_type(self): + """Test that index type is preserved after transformation.""" + imputer = Imputer() + imputer.fit(self.data_numerical) + result = imputer.transform(self.data_numerical) + + self.assertIsInstance(result.index, pd.DatetimeIndex) + + def test_transform_with_median_strategy(self): + """Test transformation with median strategy.""" + imputer = Imputer(strategy='median') + imputer.fit(self.data_numerical) + result = imputer.transform(self.data_numerical) + + # Should handle NaN values correctly + self.assertFalse(result.isna().any().any()) + # Should return DataFrame with same shape + self.assertEqual(result.shape, self.data_numerical.shape) + + def test_transform_with_empty_categorical_data(self): + """Test transformation when no categorical columns match.""" + imputer = Imputer(categorical_features=['nonexistent']) + imputer.fit(self.data_numerical) + result = imputer.transform(self.data_numerical) + + # Should handle empty categorical data gracefully + self.assertEqual(result.shape, self.data_numerical.shape) + + def test_transform_with_only_categorical(self): + """Test transformation with only categorical features.""" + imputer = Imputer(categorical_features=['status', 'region']) + imputer.fit(self.data_only_categorical) + result = imputer.transform(self.data_only_categorical) + + # Should handle categorical-only data + self.assertEqual(result.shape, self.data_only_categorical.shape) + self.assertFalse(result.isna().any().any()) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/data_preprocessing/test_scaler.py b/tests/data_preprocessing/test_scaler.py new file mode 100644 index 00000000..06466d16 --- /dev/null +++ b/tests/data_preprocessing/test_scaler.py @@ -0,0 +1,336 @@ +import unittest +import pandas as pd +import numpy as np +from sklearn.exceptions import NotFittedError + +from energy_fault_detector.data_preprocessing.scaler import Scaler + + +class TestScaler(unittest.TestCase): + def setUp(self): + """Set up common test fixtures.""" + # Create sample numerical data + self.data_numerical = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'humidity': [60, 62, 58, 65, 67], + 'pressure': [1013.2, 1014.5, 1012.8, 1015.0, 1013.7] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with categorical features (already encoded for this test) + self.data_with_categorical = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'humidity': [60, 62, 58, 65, 67], + 'category_A': [1, 0, 1, 0, 0], + 'category_B': [0, 1, 0, 0, 1], + 'category_C': [0, 0, 0, 1, 0], + 'region_East': [0, 0, 1, 0, 0], + 'region_North': [1, 0, 1, 0, 0], + 'region_South': [0, 1, 0, 0, 0], + 'region_West': [0, 0, 0, 1, 1] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with negative values + self.data_with_negatives = pd.DataFrame({ + 'temperature': [-5.2, -3.1, 0.0, 2.5, 4.8], + 'humidity': [-10, -5, 0, 15, 20] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with outliers + self.data_with_outliers = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 150.0], + 'humidity': [60, 62, 58, 65, 500] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with missing values + self.data_with_nan = pd.DataFrame({ + 'temperature': [20.5, np.nan, 19.8, 22.1, 23.0], + 'humidity': [60, 62, np.nan, 65, 67] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data for inverse transform testing + self.data_for_inverse = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8], + 'humidity': [60, 62, 58] + }, index=pd.date_range('2024-01-01', periods=3, freq='1Min')) + + # Create transformed data for testing + self.transformed_data = pd.DataFrame({ + 'temperature': [0.612, 0.918, -0.408, 1.428, 1.734], + 'humidity': [-0.408, -0.102, -0.714, 0.816, 1.122] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + # Create data with different columns than during fit + self.data_with_different_columns = pd.DataFrame({ + 'temperature': [21.0, 22.0], + 'humidity': [61, 63] + }, index=pd.date_range('2024-01-01', periods=2, freq='1Min')) + + def tearDown(self): + """Clean up after each test.""" + del (self.data_numerical, self.data_with_categorical, + self.data_with_negatives, self.data_with_outliers, + self.data_with_nan, self.data_for_inverse, + self.transformed_data, self.data_with_different_columns) + + def test_init_defaults(self): + """Test default initialization.""" + scaler = Scaler() + self.assertEqual(scaler.scaler_type, 'standard') + self.assertTrue(scaler.scale_categorical_features) + self.assertEqual(scaler.categorical_features, []) + self.assertIsNotNone(scaler.scaler) + + def test_init_invalid_scaler_type(self): + """Test initialization with invalid scaler type raises ValueError.""" + with self.assertRaises(ValueError): + Scaler(scaler_type='invalid_type') + + def test_init_with_minmax_scaler(self): + """Test initialization with MinMax scaler.""" + scaler = Scaler(scaler_type='minmax') + self.assertEqual(scaler.scaler_type, 'minmax') + self.assertIsNotNone(scaler.scaler) + self.assertTrue(hasattr(scaler.scaler, 'feature_range')) + + def test_init_with_custom_params(self): + """Test initialization with custom parameters.""" + scaler = Scaler(scaler_type='minmax', feature_range=(0, 1)) + self.assertEqual(scaler.scaler.feature_range, (0, 1)) + + def test_init_with_categorical_features(self): + """Test initialization with categorical features.""" + scaler = Scaler(scale_categorical_features=False, categorical_features=['category_A', 'category_B']) + self.assertEqual(scaler.categorical_features, ['category_A', 'category_B']) + self.assertFalse(scaler.scale_categorical_features) + + def test_fit_standard_scaler(self): + """Test fitting with standard scaler.""" + scaler = Scaler() + scaler.fit(self.data_numerical) + + self.assertEqual(scaler.n_features_in_, 3) + self.assertListEqual(scaler.feature_names_in_, ['temperature', 'humidity', 'pressure']) + self.assertIsNotNone(scaler.scaler.mean_) + self.assertIsNotNone(scaler.scaler.var_) + + def test_fit_minmax_scaler(self): + """Test fitting with minmax scaler.""" + scaler = Scaler(scaler_type='minmax') + scaler.fit(self.data_numerical) + + self.assertEqual(scaler.n_features_in_, 3) + self.assertIsNotNone(scaler.scaler.data_min_) + self.assertIsNotNone(scaler.scaler.data_max_) + + def test_fit_with_categorical_features_scaled(self): + """Test fitting with categorical features and scale_categorical_features=True.""" + scaler = Scaler(scale_categorical_features=True) + scaler.fit(self.data_with_categorical) + + self.assertEqual(scaler.n_features_in_, 9) + self.assertListEqual(scaler.feature_names_out_, self.data_with_categorical.columns.tolist()) + + def test_fit_with_categorical_features_not_scaled(self): + """Test fitting with categorical features and scale_categorical_features=False.""" + scaler = Scaler(scale_categorical_features=False, categorical_features=['category', 'region']) + scaler.fit(self.data_with_categorical) + + # Should only fit on non-categorical features + self.assertEqual(scaler.n_features_in_, 9) + # Check that scaling was applied only to numerical features + numerical_cols = ['temperature', 'humidity'] + # Verify the scaler was fitted on the numerical columns only + subset_to_fit = self.data_with_categorical[numerical_cols] + self.assertEqual(scaler.scaler.mean_.shape[0], len(numerical_cols)) + + def test_transform_standard_scaler(self): + """Test transformation with standard scaler.""" + scaler = Scaler() + scaler.fit(self.data_numerical) + result = scaler.transform(self.data_numerical) + + # Should return DataFrame with same shape and columns + self.assertEqual(result.shape, self.data_numerical.shape) + self.assertListEqual(list(result.columns), self.data_numerical.columns.tolist()) + + # For standard scaler, the transformed data should have mean ~0 and std ~1 + np.testing.assert_array_almost_equal(result.mean(), [0.0, 0.0, 0.0], decimal=5) + np.testing.assert_array_almost_equal(result.std(ddof=0), [1.0, 1.0, 1.0], decimal=5) + + def test_transform_minmax_scaler(self): + """Test transformation with minmax scaler.""" + scaler = Scaler(scaler_type='minmax') + scaler.fit(self.data_numerical) + result = scaler.transform(self.data_numerical) + + # For minmax scaler, values should be scaled to [0,1] + self.assertGreaterEqual(result.min().min(), 0.0) + self.assertLessEqual(result.max().max(), 1.0) + + # Should preserve index + self.assertTrue(result.index.equals(self.data_numerical.index)) + + def test_transform_with_categorical_features_scaled(self): + """Test transformation with categorical features and scale_categorical_features=True.""" + scaler = Scaler(scale_categorical_features=True) + scaler.fit(self.data_with_categorical) + result = scaler.transform(self.data_with_categorical) + + # Should have same shape and columns + self.assertEqual(result.shape, self.data_with_categorical.shape) + self.assertListEqual(list(result.columns), self.data_with_categorical.columns.tolist()) + + def test_transform_with_categorical_features_not_scaled(self): + """Test transformation with categorical features and scale_categorical_features=False.""" + scaler = Scaler(scale_categorical_features=False, categorical_features=['category_A', 'category_B']) + scaler.fit(self.data_with_categorical) + result = scaler.transform(self.data_with_categorical) + + # Only numerical columns should be scaled + numerical_cols = ['temperature', 'humidity'] + self.assertFalse(result[numerical_cols].equals(self.data_with_categorical[numerical_cols])) + # Categorical columns should remain unchanged + pd.testing.assert_frame_equal(result[['category_A', 'category_B']], self.data_with_categorical[['category_A', 'category_B']].astype(float)) # Ensure numeric types for comparison + + def test_transform_unfitted_raises(self): + """Test that transform on unfitted model raises error.""" + scaler = Scaler() + with self.assertRaises(NotFittedError): + scaler.transform(self.data_numerical) + + def test_inverse_transform_standard_scaler(self): + """Test inverse transform with standard scaler.""" + scaler = Scaler() + scaler.fit(self.data_for_inverse) + transformed = scaler.transform(self.data_for_inverse) + inverse = scaler.inverse_transform(transformed) + + # Should return to original values + pd.testing.assert_frame_equal(inverse, self.data_for_inverse.apply(lambda x: x.astype(float))) + + def test_inverse_transform_minmax_scaler(self): + """Test inverse transform with minmax scaler.""" + scaler = Scaler(scaler_type='minmax') + scaler.fit(self.data_for_inverse) + transformed = scaler.transform(self.data_for_inverse) + inverse = scaler.inverse_transform(transformed) + + # Should return to original values + pd.testing.assert_frame_equal(inverse, self.data_for_inverse.apply(lambda x: x.astype(float))) # Ensure numeric types for comparison + + def test_inverse_transform_with_categorical_features_not_scaled(self): + """Test inverse transform with categorical features and scale_categorical_features=False.""" + scaler = Scaler(scale_categorical_features=False, categorical_features=['category_A']) + scaler.fit(self.data_with_categorical) + transformed = scaler.transform(self.data_with_categorical) + inverse = scaler.inverse_transform(transformed) + + # Categorical columns should be unchanged + pd.testing.assert_frame_equal(inverse[['category_A']], self.data_with_categorical[['category_A']].astype(float)) + # Numerical columns should return to original values + pd.testing.assert_frame_equal(inverse[['temperature', 'humidity']], self.data_with_categorical[['temperature', 'humidity']].astype(float)) + + def test_get_feature_names_out(self): + """Test get_feature_names_out method.""" + scaler = Scaler() + scaler.fit(self.data_numerical) + + feature_names = scaler.get_feature_names_out() + + self.assertListEqual(feature_names, self.data_numerical.columns.tolist()) + + def test_transform_preserves_index_type(self): + """Test that index type is preserved after transformation.""" + scaler = Scaler() + scaler.fit(self.data_numerical) + result = scaler.transform(self.data_numerical) + + self.assertIsInstance(result.index, pd.DatetimeIndex) + + def test_transform_with_different_columns(self): + """Test that transform works with different number of columns than fitted.""" + scaler = Scaler() + scaler.fit(self.data_numerical) + + # This should raise an error as per sklearn behavior + with self.assertRaises(ValueError): + scaler.transform(self.data_with_different_columns) + + def test_transform_with_negative_values(self): + """Test transformation with negative values.""" + scaler = Scaler() + scaler.fit(self.data_with_negatives) + result = scaler.transform(self.data_with_negatives) + + # Should handle negative values correctly + self.assertFalse(result.isna().any().any()) + + def test_transform_with_outliers(self): + """Test transformation with outliers.""" + scaler = Scaler() + scaler.fit(self.data_with_outliers) + result = scaler.transform(self.data_with_outliers) + + # Should handle outliers, though they may affect the distribution + self.assertFalse(result.isna().any().any()) + # Standard scaled values should be reasonable (not NaN or infinite) + self.assertTrue(np.all(np.isfinite(result.values))) + + def test_transform_with_nan_values(self): + """Test transformation with NaN values.""" + scaler = Scaler() + scaler.fit(self.data_with_nan) + + result = scaler.transform(self.data_with_nan) + # Should handle NaN values, resulting in NaN in the same positions + self.assertTrue(result.isna().any().any()) + + def test_transform_with_all_zeros(self): + """Test transformation with all zero variance columns.""" + data_all_zeros = pd.DataFrame({ + 'temperature': [0.0, 0.0, 0.0, 0.0, 0.0], + 'humidity': [60, 62, 58, 65, 67] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + scaler = Scaler() + scaler.fit(data_all_zeros) + result = scaler.transform(data_all_zeros) + + # Zero variance column should be scaled to zeros + self.assertTrue((result['temperature'] == 0.0).all()) + # Should not raise any errors + + def test_transform_with_constant_column(self): + """Test transformation with constant value columns.""" + data_constant = pd.DataFrame({ + 'temperature': [5.0, 5.0, 5.0, 5.0, 5.0], + 'humidity': [60, 62, 58, 65, 67] + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + scaler = Scaler() + scaler.fit(data_constant) + result = scaler.transform(data_constant) + + # Constant column should have zero variance after scaling + self.assertTrue((result['temperature'] == 0.0).all()) + # Should not raise any errors + + def test_transform_with_mixed_types(self): + """Test transformation with mixed data types after preprocessing.""" + # Simulate data after categorical encoding + data_mixed = pd.DataFrame({ + 'temperature': [20.5, 21.3, 19.8, 22.1, 23.0], + 'category_A': [1, 0, 1, 0, 0], + 'category_B': [0, 1, 0, 0, 1], + }, index=pd.date_range('2024-01-01', periods=5, freq='1Min')) + + scaler = Scaler() + scaler.fit(data_mixed) + result = scaler.transform(data_mixed) + + # Should scale all columns when scale_categorical_features=True + self.assertEqual(result.shape, data_mixed.shape) + self.assertFalse(result.isna().any().any()) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/test_fault_detector.py b/tests/test_fault_detector.py index 60c1d32c..7cd8d3b5 100644 --- a/tests/test_fault_detector.py +++ b/tests/test_fault_detector.py @@ -740,3 +740,41 @@ def test_explicit_true(self): } }) self.assertTrue(config.protect_conditional_features) + + +class TestHandleDuplicateIndex(unittest.TestCase): + """Test _handle_duplicate_index on the FaultDetectionModel base class.""" + + def test_no_duplicates_unchanged(self): + """Data without duplicate indices should be returned unchanged.""" + df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]}, index=[0, 1, 2]) + result = FaultDetector._handle_duplicate_index(df) + self.assertEqual(len(result), 3) + pd.testing.assert_frame_equal(result, df) + + def test_true_duplicates_dropped(self): + """Rows with same index and same values should be collapsed to one.""" + df = pd.DataFrame({'a': [1, 2, 2, 3], 'b': [4, 5, 5, 6]}, index=[0, 1, 1, 2]) + result = FaultDetector._handle_duplicate_index(df) + self.assertEqual(len(result), 3) + self.assertEqual(list(result.index), [0, 1, 2]) + + def test_contradicting_duplicates_raise(self): + """Rows with same index but different values should raise ValueError.""" + df = pd.DataFrame({'a': [1, 2, 3, 3], 'b': [4, 5, 6, 7]}, index=[0, 1, 2, 2]) + with self.assertRaises(ValueError) as ctx: + FaultDetector._handle_duplicate_index(df) + self.assertIn('contradicting', str(ctx.exception)) + + def test_series_true_duplicates_dropped(self): + """Series with same index and same values should be collapsed.""" + s = pd.Series([True, True, True], index=[0, 1, 1]) + result = FaultDetector._handle_duplicate_index(s) + self.assertEqual(len(result), 2) + + def test_series_contradicting_raise(self): + """Series with same index but different values should raise ValueError.""" + s = pd.Series([True, True, False], index=[0, 1, 1]) + with self.assertRaises(ValueError) as ctx: + FaultDetector._handle_duplicate_index(s) + self.assertIn('contradicting', str(ctx.exception))