create ValueRangeDetector

config/pipeline_config_default.yaml

@@ -2,7 +2,7 @@ parsers:
     MatcherParser:
         method_type: matcher_parser
         auto_config: False
-        log_format: "type=<Type> msg=audit(<Time>:*): <Content>"
+        log_format: "type=<Type> msg=audit(<Time>:<Line>): <Content>"
         time_format: null
         params:
             remove_spaces: True
@@ -83,6 +83,20 @@ detectors:
         auto_config: False
         params: {}
 
+    ValueRangeDetector:
+        method_type: value_range_detector
+        auto_config: False
+        params:
+            ignore_non_numerical_val: True
+        events:
+            1:
+                test:
+                    params: {}
+                    variables:
+                        - pos: 1
+                          name: test
+                          params: {}
+
     CharsetDetector:
         method_type: charset_detector
         auto_config: False

docs/detectors.md

@@ -88,6 +88,7 @@ List of detectors:
 * [New Value](detectors/new_value.md): Detect new values in the variables in the logs.
 * [Combo Detector](detectors/combo.md): Detect new combination of variables in the logs.
 * [New Event](detectors/new_event.md): Detect new events in the variables in the logs.
+* [Value Range](...): Detect numeric value ranges in variables in the logs.
 * [Rule Based](detectors/rule_based.md): Detect anomalies based in a set of rules.
 * [Charset](detectors/charset.md): Detect new characters in the variables in the logs.
 

docs/detectors/combo.md

@@ -33,9 +33,11 @@ detectors:
 ## Example usage
 
 ```python
-from detectmatelibrary.detectors.combo_detector import ComboDetector, ComboConfig
+from detectmatelibrary.detectors.new_value_combo_detector import NewValueComboDetector, NewValueComboDetectorConfig
 import detectmatelibrary.schemas as schemas
 
+detector = NewValueComboDetector(name="NewValueTest", config=cfg)
+
 test_data = schemas.ParserSchema({
     "parserType": "test",
     "EventID": 12,

docs/detectors/new_event.md

@@ -26,7 +26,7 @@ detectors:
 ## Example usage
 
 ```python
-from detectmatelibrary.detectors.new_event_detector import NewEventDetector, BufferMode
+from detectmatelibrary.detectors.new_event_detector import NewEventDetector, NewEventDetectorConfig
 import detectmatelibrary.schemas as schemas
 
 detector = NewEventDetector(name="NewEventTest", config=cfg)
@@ -44,7 +44,7 @@ parser_data = schemas.ParserSchema({
 })
 
 
-alert = detector.process(parsed_data)
+alert = detector.process(parser_data)
 ```
 
 Go back [Index](../index.md)

docs/detectors/new_value.md

@@ -60,7 +60,7 @@ parser_data = schemas.ParserSchema({
 })
 
 
-alert = detector.process(parsed_data)
+alert = detector.process(parser_data)
 ```
 
 Go back [Index](../index.md)

docs/detectors/value_range.md

@@ -0,0 +1,62 @@
+# Value Range Detector
+
+The Value Range Detector raises alerts when numerical values outside of known ranges appear in configured fields. It is useful to detect unexpected changes, configuration drift, or the appearance of new actors in the environment.
+
+|            | Schema                     | Description        |
+|------------|----------------------------|--------------------|
+| **Input**  | [ParserSchema](../schemas.md) | Structured log  |
+| **Output** | [DetectorSchema](../schemas.md) | Alert / finding |
+
+## Description
+
+This detector maintains a lightweight set of observed values per monitored field and emits an alert when a value outside the learned range is seen (subject to configuration).
+
+
+## Configuration example
+
+```yaml
+detectors:
+    ValueRangeDetector:
+        method_type: value_range_detector
+        auto_config: False
+        params: {"ignore_non_numerical_val": True}
+        events:
+            1:
+                test:
+                    params: {}
+                    variables:
+                        - pos: 0
+                          name: var1
+                          params:
+                              threshold: 0.
+                    header_variables:
+                        - pos: level
+                          params: {}
+```
+
+
+## Example usage
+
+```python
+from detectmatelibrary.detectors.value_range_detector import ValueRangeDetector, BufferMode
+import detectmatelibrary.schemas as schemas
+
+detector = ValueRangeDetector(name="NewValueTest", config=cfg)
+
+parser_data = schemas.ParserSchema({
+    "parserType": "test",
+    "EventID": 1,
+    "template": "test template",
+    "variables": ["1", "2", "17"],
+    "logID": "1",
+    "parsedLogID": "1",
+    "parserID": "test_parser",
+    "log": "test log message",
+    "logFormatVariables": {"timestamp": "123456"}
+})
+
+
+alert = detector.process(parser_data)
+```
+
+Go back [Index](../index.md)

src/detectmatelibrary/detectors/__init__.py

@@ -1,6 +1,7 @@
 from .random_detector import RandomDetector, RandomDetectorConfig
 from .new_value_detector import NewValueDetector, NewValueDetectorConfig
 from .new_event_detector import NewEventDetector, NewEventDetectorConfig
+from .value_range_detector import ValueRangeDetector, ValueRangeDetectorConfig
 from .charset_detector import CharsetDetector, CharsetDetectorConfig
 
 __all__ = [
@@ -11,6 +12,8 @@
     "RandomDetector",
     "NewEventDetector",
     "NewEventDetectorConfig",
+    "ValueRangeDetector",
+    "ValueRangeDetectorConfig",
     "CharsetDetector",
     "CharsetDetectorConfig"
 ]

src/detectmatelibrary/detectors/value_range_detector.py

@@ -0,0 +1,184 @@
+from detectmatelibrary.common._config._compile import generate_detector_config
+from detectmatelibrary.common._config._formats import EventsConfig
+from detectmatelibrary.common.detector import (
+    CoreDetectorConfig,
+    CoreDetector,
+    get_configured_variables,
+    get_global_variables,
+    validate_config_coverage,
+)
+from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability.stability_tracker import (
+    EventStabilityTracker
+)
+from detectmatelibrary.utils.persistency.event_persistency import EventPersistency
+from detectmatelibrary.utils.data_buffer import BufferMode
+from detectmatelibrary.schemas import ParserSchema, DetectorSchema
+from detectmatelibrary.constants import GLOBAL_EVENT_ID
+from typing_extensions import override
+from tools.logging import logger
+from typing import Dict, List, Any
+
+
+class ValueRangeDetectorConfig(CoreDetectorConfig):
+    method_type: str = "value_range_detector"
+
+    ignore_non_numerical_val: bool = True
+    use_stable_vars: bool = True
+    use_static_vars: bool = True
+
+
+class ValueRangeDetector(CoreDetector):
+    """Detect new value ranges in logs as anomalies based on learned values."""
+
+    def __init__(
+        self,
+        name: str = "ValueRangeDetector",
+        config: ValueRangeDetectorConfig = ValueRangeDetectorConfig()
+    ) -> None:
+
+        if isinstance(config, dict):
+            config = ValueRangeDetectorConfig.from_dict(config, name)
+
+        super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config)
+        self.config: ValueRangeDetectorConfig  # type narrowing for IDE
+        self.persistency = EventPersistency(
+            event_data_class=EventStabilityTracker,
+        )
+        # auto config checks if individual variables are stable to select combos from
+        self.auto_conf_persistency = EventPersistency(
+            event_data_class=EventStabilityTracker
+        )
+
+    def cast_val_to_numeric(self, configured_variables: Dict[str, Any], k: str, remove: List[str],
+                            stage: str) -> bool:
+        v = configured_variables[k]
+        if not isinstance(v, (int, float)):
+            try:
+                configured_variables[k] = int(v)
+            except ValueError:
+                try:
+                    configured_variables[k] = float(v)
+                except ValueError:
+                    logger.error(f"Non-numeric value '{v}' appeared in {stage} of {self.__class__.__name__}"
+                                 f" with the name {self.name}.")
+                    if not self.config.ignore_non_numerical_val:
+                        exit(1)
+                    remove.append(k)
+                    return False
+        return True
+
+    def train(self, input_: ParserSchema) -> None:  # type: ignore
+        """Train the detector by learning values from the input data."""
+        configured_variables = get_configured_variables(input_, self.config.events)
+        remove: List[str] = []
+        for k in configured_variables.keys():
+            self.cast_val_to_numeric(configured_variables, k, remove, "training")
+        for k in remove:
+            del configured_variables[k]
+        self.persistency.ingest_event(
+            event_id=input_["EventID"],
+            event_template=input_["template"],
+            named_variables=configured_variables
+        )
+        if self.config.global_instances:
+            global_vars = get_global_variables(input_, self.config.global_instances)
+            if global_vars:
+                self.persistency.ingest_event(
+                    event_id=GLOBAL_EVENT_ID,
+                    event_template=input_["template"],
+                    named_variables=global_vars
+                )
+
+    def detect(
+        self, input_:  ParserSchema, output_: DetectorSchema  # type: ignore
+    ) -> bool:
+        """Detect new value ranges in the input data."""
+        alerts: dict[str, str] = {}
+        configured_variables = get_configured_variables(input_, self.config.events)
+        overall_score = 0.0
+
+        current_event_id = input_["EventID"]
+        known_events = self.persistency.get_events_data()
+
+        if current_event_id in known_events:
+            event_tracker = known_events[current_event_id]
+            for var_name, multi_tracker in event_tracker.get_data().items():
+                cast = self.cast_val_to_numeric(configured_variables, var_name, [], "detection")
+                value = configured_variables.get(var_name)
+                if value is None or not cast:
+                    continue
+                min_ = min(multi_tracker.unique_set)
+                max_ = max(multi_tracker.unique_set)
+                if value < min_ or value > max_:
+                    alerts[f"EventID {current_event_id} - {var_name}"] = (
+                        f"Out of range value: '{value}' ({min_} - {max_})"
+                    )
+                    overall_score += 1.0
+
+        if self.config.global_instances and GLOBAL_EVENT_ID in known_events:
+            global_vars = get_global_variables(input_, self.config.global_instances)
+            global_tracker = known_events[GLOBAL_EVENT_ID]
+            for var_name, multi_tracker in global_tracker.get_data().items():
+                cast = self.cast_val_to_numeric(global_vars, var_name, [], "detection")
+                value = global_vars.get(var_name)
+                if value is None or not cast:
+                    continue
+                min_ = min(multi_tracker.unique_set)
+                max_ = max(multi_tracker.unique_set)
+                if isinstance(value, float):
+                    min_ = float(min_)
+                    max_ = float(max_)
+                else:
+                    min_ = int(min_)
+                    max_ = int(max_)
+                if value < min_ or value > max_:
+                    alerts[f"Global - {var_name}"] = f"Out of range value: '{value}' ({min_} - {max_})"
+                    overall_score += 1.0
+
+        if overall_score > 0:
+            output_["score"] = overall_score
+            output_["description"] = f"{self.name} detects values not encountered in training as anomalies."
+            output_["alertsObtain"].update(alerts)
+            return True
+
+        return False
+
+    def configure(self, input_: ParserSchema) -> None:  # type: ignore
+        self.auto_conf_persistency.ingest_event(
+            event_id=input_["EventID"],
+            event_template=input_["template"],
+            variables=input_["variables"],
+            named_variables=input_["logFormatVariables"],
+        )
+
+    @override
+    def post_train(self) -> None:
+        if not self.config.auto_config:
+            validate_config_coverage(self.name, self.config.events, self.persistency)
+
+    def set_configuration(self) -> None:
+        variables = {}
+        for event_id, tracker in self.auto_conf_persistency.get_events_data().items():
+            stable = []
+            if self.config.use_stable_vars:
+                stable = tracker.get_features_by_classification("STABLE")  # type: ignore
+            static = []
+            if self.config.use_static_vars:
+                static = tracker.get_features_by_classification("STATIC")  # type: ignore
+            vars_ = stable + static
+            if len(vars_) > 0:
+                variables[event_id] = vars_
+        config_dict = generate_detector_config(
+            variable_selection=variables,
+            detector_name=self.name,
+            method_type=self.config.method_type,
+        )
+        # Update the config object from the dictionary instead of replacing it
+        self.config = ValueRangeDetectorConfig.from_dict(config_dict, self.name)
+        events = self.config.events
+        if isinstance(events, EventsConfig) and not events.events:
+            logger.warning(
+                f"[{self.name}] auto_config=True generated an empty configuration. "
+                "No stable variables were found in configure-phase data. "
+                "The detector will produce no alerts."
+            )