From 1e1e5d32bd52c94277617b6559399fd86dcc5625 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Thu, 16 Jul 2026 11:21:33 +0200 Subject: [PATCH 01/23] Implement CalculatedChannel aggregation and enhance QueryBuilder for calculated channels - Introduced the CalculatedChannel class for computing derived time-series channels, including validation and identity handling. - Enhanced the QueryBuilder class with methods to solve calculated channels, including a new filter pipeline and validation for selections. - Updated the DefaultSolver to support solving calculated channels, emitting narrow output DataFrames. - Added comprehensive unit and integration tests for CalculatedChannel and its interactions within the query engine. - Improved documentation for new methods and classes, ensuring clarity on usage and expected behavior. --- .../query/aggregations/calculated_channel.py | 117 +++++++++ .../analyze/query/query_builder.py | 109 +++++++- .../analyze/query/solvers/default_solver.py | 158 ++++++++++++ .../analyze/query/solvers/query_solver.py | 74 ++++++ .../aggregations/calculated_channel_test.py | 128 +++++++++ ...default_solver_calculated_channels_test.py | 243 ++++++++++++++++++ 6 files changed, 828 insertions(+), 1 deletion(-) create mode 100644 src/impulse_query_engine/analyze/query/aggregations/calculated_channel.py create mode 100644 tests/impulse_query_engine/unit/analyze/query/aggregations/calculated_channel_test.py create mode 100644 tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py diff --git a/src/impulse_query_engine/analyze/query/aggregations/calculated_channel.py b/src/impulse_query_engine/analyze/query/aggregations/calculated_channel.py new file mode 100644 index 00000000..036a5a6d --- /dev/null +++ b/src/impulse_query_engine/analyze/query/aggregations/calculated_channel.py @@ -0,0 +1,117 @@ +"""CalculatedChannel aggregation for computing derived time-series channels.""" + +import pyspark.sql.types as T + +from impulse_query_engine.analyze.metadata.tag_expression import TagExpression +from impulse_query_engine.analyze.metadata.time_series_expression import ( + TimeSeriesExpression, + TimeSeriesSelector, +) +from impulse_query_engine.analyze.query.solvers.series_cache import SeriesCache + +from .aggregation import Aggregation + +# Sentinel distinguishing "derive a deterministic channel_id" (the default) from +# an explicit ``channel_id=None`` (emit a SQL null). A plain default of ``None`` +# could not tell these two intents apart. +_AUTO = object() + + +class CalculatedChannel(Aggregation): + """A derived channel: a wrapped time-series expression plus output identity. + + A ``CalculatedChannel`` wraps an arbitrary :class:`TimeSeriesExpression` + built from the operator DSL (e.g. ``q.channel(channel_name="raw_speed") * 3.6`` + or ``rpm + speed``). When evaluated per container the wrapped expression must + yield a :class:`SampleSeries`; :meth:`QueryBuilder.solve_calculated_channels` + explodes that series into narrow rows matching the silver ``channel_data`` + shape (``container_id, channel_id, tstart, tend, value``) plus one string + column per identity entry. + + Parameters + ---------- + expr : TimeSeriesExpression + The wrapped expression; must ``build()`` to a ``SampleSeries``. + identity : dict of str + Identity columns for the output rows, e.g. + ``{"channel_name": "Eng_RPM", "data_key": "TM"}``. Each key becomes a + ``StringType`` output column and each value is emitted as a literal on + every row of this channel. Must be non-empty; the identity also seeds + the deterministic ``channel_id`` hash. + channel_id : int or None, optional + Output ``channel_id`` for every emitted row. When omitted (the + ``_AUTO`` sentinel) a deterministic id is derived from the identity by + the solver, typed to match the source ``channel_id`` column. Pass an + ``int`` to use it verbatim, or ``None`` to emit a SQL null. + + Notes + ----- + ``_alias`` is set explicitly in ``__init__`` — as the identity values joined + by ``"::"`` (e.g. ``"Eng_RPM::TM"``) — rather than left for the caller to + chain ``.alias(...)``: reading a missing ``_alias`` would trigger + ``TimeSeriesExpression.__getattr__`` and silently return a callable instead + of raising. The alias is not consumed by the calculated-channels solve path + (that emits the identity columns directly); it exists so the object stays a + well-formed ``TimeSeriesExpression``. A later ``.alias(...)`` still overrides + it. + + Examples + -------- + :: + + CalculatedChannel(rpm * 3.6, {"channel_name": "speed_kmh", "data_key": "CALC"}) + """ + + _ALIAS_SEPARATOR = "::" + + def __init__( + self, + expr: TimeSeriesExpression, + identity: dict[str, str], + *, + channel_id=_AUTO, + ): + if not identity: + raise ValueError( + "CalculatedChannel requires a non-empty identity dict " + "(e.g. {'channel_name': 'Eng_RPM', 'data_key': 'TM'}); identity " + "defines the output identifier columns and seeds the " + "deterministic channel_id." + ) + self.expr = expr + self.identity = dict(identity) + self._explicit_channel_id = channel_id + self._alias = self._ALIAS_SEPARATOR.join(str(v) for v in identity.values()) + self.is_single_signal = getattr(expr, "is_single_signal", True) + self.requires_udf = getattr(expr, "requires_udf", False) + + def __str__(self) -> str: + return f"" + + def canonical_identity(self) -> str: + """Return a stable string encoding of the identity, used for the id hash. + + Keys are sorted so the encoding (and the derived ``channel_id``) is + independent of kwarg order. + """ + return "&".join(f"{k}={self.identity[k]}" for k in sorted(self.identity)) + + def build(self, cache: SeriesCache): + """Evaluate the wrapped expression against the cache (yields a SampleSeries).""" + return self.expr.build(cache) + + def dtype(self) -> T.DataType: + """Spark type of the emitted ``value`` column (matches CHANNELS_SCHEMA).""" + return T.DoubleType() + + def get_selectors(self) -> list[TimeSeriesSelector]: + return self.expr.get_selectors() + + def get_selector_expr(self): + return self.expr.get_selector_expr() + + def get_required_tag_exprs(self) -> set[TagExpression]: + return self.expr.get_required_tag_exprs() + + def required_tags(self) -> set[str]: + return self.expr.required_tags() diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 20939fdb..514d0f1e 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -11,7 +11,11 @@ TimeSeriesExpression, TimeSeriesSelector, ) +from impulse_query_engine.analyze.query.aggregations.calculated_channel import ( + CalculatedChannel, +) from impulse_query_engine.analyze.query.solvers.empty_cache import EmptyTimeSeriesCache +from impulse_query_engine.model.series.sample_series import SampleSeries from .solvers.blob_solver import BlobSolver from .solvers.query_solver import QuerySolver @@ -232,6 +236,20 @@ def solve( self.result_dtypes, ) = self._determine_result_objects_dtypes() + channel_metrics_df = self._run_filter_pipeline(spark, solver, pre_filtered_containers_df) + + return solver.solve(self, channel_metrics_df, self.selections, self.result_dtypes) + + def _run_filter_pipeline(self, spark, solver, pre_filtered_containers_df) -> DataFrame: + """Run the shared metadata filter pipeline and return the channel-match frame. + + Extracts the selector split, the four filter stages + (container tags → container metrics → channel tags → channel metrics) and + the optional channel-alias resolution that both :meth:`solve` and + :meth:`solve_calculated_channels` drive before their differing final + ``solver`` call. Returns the ``(container_id, channel_id, selector_ids …)`` + DataFrame identifying the channels selected by the current selections. + """ # extract selectors upfront direct_selectors = TimeSeriesExpression.collect_selectors( self.selections, uses_alias=False @@ -260,7 +278,96 @@ def solve( spark, channel_metrics_df, aliased_channel_metrics_df ) - return solver.solve(self, channel_metrics_df, self.selections, self.result_dtypes) + return channel_metrics_df + + @telemetry_logger("query", "solve_calculated_channels") + def solve_calculated_channels( + self, + spark, + solver: QuerySolver = BlobSolver(), + pre_filtered_containers_df: DataFrame = None, + ) -> DataFrame: + """ + Compute calculated channels and return a narrow silver-shaped DataFrame. + + Every selection must be a :class:`CalculatedChannel`. This runs the same + metadata filter pipeline as :meth:`solve` (resolving the input channels + each calculated channel depends on), then evaluates each calculated + channel per container and emits rows in the silver ``channel_data`` shape + — ``container_id, channel_id, tstart, tend, value`` — plus one string + column per identity kwarg shared by the selections. + + Parameters + ---------- + spark : SparkSession + Spark session used for query execution. + solver : QuerySolver, optional + Query solver to use. Must implement ``solve_calculated_channels`` + (``DefaultSolver`` does); the default ``BlobSolver`` does not. + pre_filtered_containers_df : DataFrame, optional + Pre-filtered container metrics for incremental processing. When + provided, only these containers are processed; when None, all + containers matching the query filters are processed. + + Returns + ------- + pyspark.sql.DataFrame + Narrow DataFrame ``[container_id, channel_id, tstart, tend, value, + ]``. + + Raises + ------ + ValueError + If any selection is not a ``CalculatedChannel``, if the selections + declare inconsistent identity-key sets, or if a wrapped expression + does not evaluate to a ``SampleSeries``. + """ + self._validate_calculated_channels() + + ( + self.result_objects, + self.result_dtypes, + ) = self._determine_result_objects_dtypes() + + channel_metrics_df = self._run_filter_pipeline(spark, solver, pre_filtered_containers_df) + + return solver.solve_calculated_channels( + self, channel_metrics_df, self.selections, self.result_dtypes + ) + + def _validate_calculated_channels(self) -> None: + """Validate the selections for :meth:`solve_calculated_channels`. + + Every selection must be a ``CalculatedChannel``, all must declare the + same set of identity keys (so the narrow output has a single, stable + schema), and each wrapped expression must evaluate to a ``SampleSeries``. + """ + if not self.selections: + raise ValueError( + "solve_calculated_channels() requires at least one CalculatedChannel." + ) + + for i, s in enumerate(self.selections): + if not isinstance(s, CalculatedChannel): + raise ValueError( + "solve_calculated_channels() requires all selections to be " + f"CalculatedChannel; got {type(s).__name__} at index {i}." + ) + + key_sets = {frozenset(s.identity) for s in self.selections} + if len(key_sets) > 1: + rendered = ", ".join(sorted(str(sorted(ks)) for ks in key_sets)) + raise ValueError( + "All CalculatedChannels in one solve_calculated_channels() call must " + f"declare the same identity keys; got differing key sets: {rendered}." + ) + + for s in self.selections: + s.expr.require_evaluation_type( + SampleSeries, + owner="CalculatedChannel", + example="q.channel(channel_name='raw_speed') * 3.6", + ) @telemetry_logger("query", "to_pandas") def toPandas(self, spark, solver: QuerySolver = BlobSolver()) -> pd.DataFrame: diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 92c8c432..91bb7f7d 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib from collections.abc import Iterable from functools import partial from typing import TYPE_CHECKING @@ -953,3 +954,160 @@ def solve(self, query, channels_df, selections, dtypes) -> DataFrame: .apply(solve_udf) ) return res + + # ------------------------------------------------------------------ + # Calculated channels (narrow output) + # ------------------------------------------------------------------ + + @staticmethod + def _calculated_channel_id(cc, channel_id_dtype: T.DataType): + """Resolve the output ``channel_id`` for a single ``CalculatedChannel``. + + An explicit ``channel_id`` on the channel is used verbatim (an ``int``, + or ``None`` to emit a SQL null). Otherwise a deterministic id is derived + from the channel's identity via a stable BLAKE2b digest (never the + process-randomized ``hash()``), sized to the source ``channel_id`` type: + signed int32 for ``IntegerType``, signed int64 otherwise. Determinism + makes writes idempotent and joins predictable across runs. + """ + from impulse_query_engine.analyze.query.aggregations.calculated_channel import _AUTO + + if cc._explicit_channel_id is not _AUTO: + return cc._explicit_channel_id + digest = hashlib.blake2b(cc.canonical_identity().encode(), digest_size=8).digest() + if isinstance(channel_id_dtype, T.IntegerType): + return int.from_bytes(digest[:4], "big", signed=True) + return int.from_bytes(digest[:8], "big", signed=True) + + @staticmethod + def _solve_calculated_channels_udf( + pdf, selections: Iterable, col_map: dict[str, str], identity_keys, channel_ids + ) -> pd.DataFrame: + """ + UDF to solve calculated channels for a single container. + + Unlike :meth:`_solve_udf` (one wide row per container), this emits many + narrow rows: each ``CalculatedChannel`` builds to a ``SampleSeries`` that + is exploded into ``(container_id, channel_id, tstart, tend, value)`` rows, + with the channel's identity values attached as string columns. + + Parameters + ---------- + pdf : pd.DataFrame + Rows for a single container (all input channels joined in). + selections : Iterable + The ``CalculatedChannel`` selections to evaluate. + col_map : dict[str, str] + Column-name mapping for the cache (cid/ch/ts/te/val/conv). + identity_keys : Iterable[str] + Identity column names shared by the selections (sorted upstream). + channel_ids : list + Output ``channel_id`` per selection, positionally paired with + *selections* (precomputed on the driver). + + Returns + ------- + pd.DataFrame + Narrow rows for this container; empty (but full-width) when no + calculated channel produced any samples. + """ + cache = TimeSeriesCache(pdf, col_map=col_map) + cid_col = col_map["cid"] + ch_col = col_map["ch"] + container_id = pdf[cid_col].iloc[0] + + cols = {cid_col: [], ch_col: [], "tstart": [], "tend": [], "value": []} + for key in identity_keys: + cols[key] = [] + + for cc, ch_id in zip(selections, channel_ids, strict=False): + series = cc.build(cache) + for tstart, tend, value in series.get_data(): + cols[cid_col].append(container_id) + cols[ch_col].append(ch_id) + cols["tstart"].append(int(tstart)) + cols["tend"].append(int(tend)) + cols["value"].append(float(value)) + for key in identity_keys: + cols[key].append(cc.identity.get(key)) + + return pd.DataFrame(cols) + + def solve_calculated_channels(self, query, channels_df, selections, dtypes=None) -> DataFrame: + """ + Solve calculated channels by grouping channels and exploding each result. + + Structurally parallels :meth:`solve` — same unit-conversion prelude, + same channel-data read and ``[container_id, channel_id]`` join — but the + grouped-map UDF emits narrow silver-shaped rows (many per container) + instead of one wide row. Output columns are + ``[container_id, channel_id, tstart, tend, value, ]``. + + Parameters + ---------- + query : QueryBuilder + Query object containing database and filter information. + channels_df : pyspark.sql.DataFrame + Channel-match DataFrame from the filter pipeline. + selections : list + List of ``CalculatedChannel`` selections to evaluate. + dtypes : list, optional + Unused (``value`` is always ``DoubleType``); accepted for signature + parity with :meth:`solve`. + + Returns + ------- + pyspark.sql.DataFrame + Narrow DataFrame of calculated-channel samples. + """ + col_map = self.config.col_map + source_unit_col = self.config.source_unit_col + target_unit_col = self.config.target_unit_col + + has_conversion_table = getattr(query.db.config, "unit_conversion_table", None) is not None + has_unit_cols = ( + source_unit_col in channels_df.columns and target_unit_col in channels_df.columns + ) + + if has_conversion_table and has_unit_cols: + channels_df = self._compute_conversion_factors(self.spark, query, channels_df) + + for col_name in (source_unit_col, target_unit_col): + if col_name in channels_df.columns: + channels_df = channels_df.drop(col_name) + + q = query.db.channels(self.spark) + q = self._apply_column_mapping(q, self.config.channels.column_name_mapping) + + if self.is_raw_data: + q = self.interval_encoder.prepare_channels_df(q) + + identity_keys = sorted(set().union(*[set(cc.identity) for cc in selections])) + channel_id_dtype = q.schema[self.config.channel_id_col].dataType + channel_ids = [self._calculated_channel_id(cc, channel_id_dtype) for cc in selections] + + schema = self._build_calculated_channels_output_schema(q, identity_keys) + solve_udf = F.pandas_udf( + partial( + DefaultSolver._solve_calculated_channels_udf, + selections=selections, + col_map=col_map, + identity_keys=identity_keys, + channel_ids=channel_ids, + ), + schema, + F.PandasUDFType.GROUPED_MAP, + ) + df = q.join( + F.broadcast(channels_df), on=[self.config.container_id_col, self.config.channel_id_col] + ) + + container_count = channels_df.select(self.config.container_id_col).distinct().count() + if container_count == 0: + return self.spark.createDataFrame([], schema=schema) + res = ( + df.repartition(container_count, self.config.container_id_col) + .groupBy(self.config.container_id_col) + .apply(solve_udf) + ) + return res diff --git a/src/impulse_query_engine/analyze/query/solvers/query_solver.py b/src/impulse_query_engine/analyze/query/solvers/query_solver.py index ee208f14..86765f77 100644 --- a/src/impulse_query_engine/analyze/query/solvers/query_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/query_solver.py @@ -99,6 +99,49 @@ def _build_solve_output_schema(self, channels: DataFrame, selections, dtypes) -> entries.append(T.StructField(s._alias, dtype)) return T.StructType(entries) + def _build_calculated_channels_output_schema( + self, channels: DataFrame, identity_keys + ) -> T.StructType: + """Build the narrow grouped-map output schema for calculated channels. + + The output mirrors the silver ``channel_data`` table + (``container_id, channel_id, tstart, tend, value``) plus one + ``StringType`` identity column per key in *identity_keys* (emitted in + sorted order for a stable schema). The ``container_id`` and + ``channel_id`` field types are derived from *channels* (the + column-mapped channels DataFrame) so they match the physical source + types instead of being hardcoded. + + Parameters + ---------- + channels : pyspark.sql.DataFrame + The column-mapped channels DataFrame whose ``container_id`` / + ``channel_id`` column types are authoritative. + identity_keys : Iterable[str] + Identity column names shared by the calculated channels. + + Returns + ------- + pyspark.sql.types.StructType + ``[container_id, channel_id, tstart, tend, value, ]``. + """ + entries = [ + T.StructField( + self.config.container_id_col, + channels.schema[self.config.container_id_col].dataType, + ), + T.StructField( + self.config.channel_id_col, + channels.schema[self.config.channel_id_col].dataType, + ), + T.StructField(self.config.tstart_col, T.LongType()), + T.StructField(self.config.tend_col, T.LongType()), + T.StructField(self.config.value_col, T.DoubleType()), + ] + for key in sorted(identity_keys): + entries.append(T.StructField(key, T.StringType())) + return T.StructType(entries) + def _empty_channel_match_df(self, spark, db: MeasurementDB) -> DataFrame: """Return an empty ``(container_id, channel_id, selector_ids)`` DataFrame. @@ -367,3 +410,34 @@ def solve(self, query, channels_df, selections, dtypes): DataFrame containing results for each container. """ pass + + def solve_calculated_channels(self, query, channels_df, selections, dtypes=None) -> DataFrame: + """ + Solve calculated channels into a narrow, silver-shaped DataFrame. + + Optional stage, parallel to :meth:`solve` but emitting many rows per + container (the exploded ``SampleSeries`` of each ``CalculatedChannel``) + instead of one wide row. Solvers that support calculated channels + (e.g. ``DefaultSolver``) override this; the base implementation raises. + + Parameters + ---------- + query : QueryBuilder + Query object containing database and filter information. + channels_df : pyspark.sql.DataFrame + Channel-match DataFrame from the filter pipeline. + selections : list + List of ``CalculatedChannel`` selections to evaluate. + dtypes : list, optional + Result data types (unused by the narrow path; ``value`` is always + ``DoubleType``). + + Returns + ------- + pyspark.sql.DataFrame + Narrow ``[container_id, channel_id, tstart, tend, value, + ]`` DataFrame. + """ + raise NotImplementedError( + f"{self.__class__.__name__} does not support calculated channels" + ) diff --git a/tests/impulse_query_engine/unit/analyze/query/aggregations/calculated_channel_test.py b/tests/impulse_query_engine/unit/analyze/query/aggregations/calculated_channel_test.py new file mode 100644 index 00000000..fe9c72fd --- /dev/null +++ b/tests/impulse_query_engine/unit/analyze/query/aggregations/calculated_channel_test.py @@ -0,0 +1,128 @@ +# pylint: disable=missing-function-docstring +"""Unit tests for the CalculatedChannel aggregation class. + +Covers construction (identity storage, the `_alias` rule, the channel_id +sentinel), the `canonical_identity` encoding, delegation of the expression +interface to the wrapped expression, and validation. +""" + +import pyspark.sql.types as T +import pytest + +from impulse_query_engine.analyze.metadata.time_series_expression import ( + TimeSeriesExpression, +) +from impulse_query_engine.analyze.query.aggregations.aggregation import Aggregation +from impulse_query_engine.analyze.query.aggregations.calculated_channel import ( + _AUTO, + CalculatedChannel, +) +from impulse_query_engine.model.series.sample_series import SampleSeries + + +class _StubExpr(TimeSeriesExpression): + """Minimal TimeSeriesExpression recording delegation and returning a series.""" + + def __init__(self, series=None): + super().__init__() + self._series = series if series is not None else SampleSeries([0], [100], [1.0]) + self._selectors = ["sel"] + + def build(self, cache): + return self._series + + def get_selectors(self): + return self._selectors + + def get_selector_expr(self): + return "selector_expr" + + def get_required_tag_exprs(self): + return {"tag_expr"} + + def required_tags(self): + return {"tag"} + + def __str__(self): + return "" + + +class TestConstruction: + def test_is_aggregation(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "speed_kmh"}) + assert isinstance(cc, Aggregation) + assert isinstance(cc, TimeSeriesExpression) + + def test_identity_stored(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "speed_kmh", "data_key": "CALC"}) + assert cc.identity == {"channel_name": "speed_kmh", "data_key": "CALC"} + + def test_alias_concatenates_identity_values(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "Eng_RPM", "data_key": "TM"}) + assert cc._alias == "Eng_RPM::TM" + + def test_alias_single_identity(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "speed_kmh"}) + assert cc._alias == "speed_kmh" + + def test_explicit_alias_overrides(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "speed_kmh"}) + cc.alias("renamed") + assert cc._alias == "renamed" + + def test_channel_id_defaults_to_auto_sentinel(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) + assert cc._explicit_channel_id is _AUTO + + def test_explicit_channel_id_stored(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}, channel_id=999) + assert cc._explicit_channel_id == 999 + + def test_explicit_none_channel_id_stored(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}, channel_id=None) + assert cc._explicit_channel_id is None + assert cc._explicit_channel_id is not _AUTO + + def test_empty_identity_raises(self): + with pytest.raises(ValueError, match="non-empty identity"): + CalculatedChannel(_StubExpr(), {}) + + +class TestCanonicalIdentity: + def test_sorted_and_stable(self): + cc1 = CalculatedChannel(_StubExpr(), {"channel_name": "s", "data_key": "CALC"}) + cc2 = CalculatedChannel(_StubExpr(), {"data_key": "CALC", "channel_name": "s"}) + assert cc1.canonical_identity() == "channel_name=s&data_key=CALC" + # Order-independent: same identity, different key order → same encoding. + assert cc1.canonical_identity() == cc2.canonical_identity() + + def test_distinct_identities_differ(self): + a = CalculatedChannel(_StubExpr(), {"channel_name": "a"}).canonical_identity() + b = CalculatedChannel(_StubExpr(), {"channel_name": "b"}).canonical_identity() + assert a != b + + +class TestDelegation: + def test_build_returns_wrapped_series(self): + series = SampleSeries([0, 100], [100, 200], [3.0, 4.0]) + cc = CalculatedChannel(_StubExpr(series), {"channel_name": "x"}) + assert cc.build(cache=None) is series + + def test_get_selectors_delegates(self): + expr = _StubExpr() + cc = CalculatedChannel(expr, {"channel_name": "x"}) + assert cc.get_selectors() == expr._selectors + + def test_selector_interface_delegates(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) + assert cc.get_selector_expr() == "selector_expr" + assert cc.get_required_tag_exprs() == {"tag_expr"} + assert cc.required_tags() == {"tag"} + + def test_dtype_is_double(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) + assert cc.dtype() == T.DoubleType() + + def test_evaluation_type_is_sample_series(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) + assert cc.evaluation_type() is SampleSeries diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py new file mode 100644 index 00000000..56386781 --- /dev/null +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py @@ -0,0 +1,243 @@ +# pylint: disable=missing-function-docstring +"""End-to-end tests for QueryBuilder.solve_calculated_channels + DefaultSolver. + +Exercises the narrow calculated-channel output against the wide-only +`basic_narrow_db` fixture: real computed values, output schema, deterministic +channel_id, dynamic container_id typing, validation, and empty results. +""" + +import pyspark.sql.functions as F +import pyspark.sql.types as T +import pytest + +from impulse_query_engine.analyze.query.aggregations.calculated_channel import ( + CalculatedChannel, +) +from impulse_query_engine.analyze.query.aggregations.stats_aggregator import ( + StatsAggregator, +) +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver +from impulse_query_engine.measurement_db import MeasurementDB, MeasurementDBConfig +from impulse_query_engine.model.series.sample_series import SampleSeries +from tests.conftest import basic_narrow_db, spark # noqa: F401 (pytest fixtures) + +# Known datum from tests/unit/data/basic_narrow_csv/channel_data.csv: +# container 1, channel 5 (Engine RPM), second RLE row. +_C1_RPM_TSTART = 1499929245761999 +_C1_RPM_VALUE = 1081.0 + + +def _recast_container_id(db: MeasurementDB, cid_type: T.DataType) -> MeasurementDB: + """Clone a ``for_debug`` db, casting ``container_id`` on every table.""" + tables = { + name: ( + df.withColumn("container_id", F.col("container_id").cast(cid_type)) + if "container_id" in df.columns + else df + ) + for name, df in db.config.debug_tables.items() + } + return MeasurementDB(MeasurementDBConfig.for_debug(tables), ws=db.ws) + + +class TestCalculatedChannelValues: + def test_scaling_produces_real_values(self, spark, basic_narrow_db): + q = basic_narrow_db.query + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, + {"channel_name": "rpm_x2", "data_key": "CALC"}, + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + + row = result.filter( + (F.col("container_id") == 1) & (F.col("tstart") == _C1_RPM_TSTART) + ).collect() + assert len(row) == 1 + assert row[0]["value"] == pytest.approx(_C1_RPM_VALUE * 2) + assert row[0]["channel_name"] == "rpm_x2" + assert row[0]["data_key"] == "CALC" + + def test_all_rows_carry_identity(self, spark, basic_narrow_db): + q = basic_narrow_db.query + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") + 1.0, + {"channel_name": "rpm_plus_1", "data_key": "CALC"}, + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + distinct_names = {r["channel_name"] for r in result.select("channel_name").collect()} + distinct_keys = {r["data_key"] for r in result.select("data_key").collect()} + assert distinct_names == {"rpm_plus_1"} + assert distinct_keys == {"CALC"} + assert result.count() > 0 + + def test_multi_channel_sync_matches_core_model(self, spark, basic_narrow_db): + """A two-channel sum matches an in-process SampleSeries synchronization.""" + q = basic_narrow_db.query + rpm = q.channel(channel_name="Engine RPM") + speed = q.channel(channel_name="Vehicle Speed Sensor") + cc = CalculatedChannel(rpm + speed, {"channel_name": "rpm_plus_speed", "data_key": "CALC"}) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + + # Container 1 has both channel 5 (RPM) and channel 7 (Speed). + got = { + r["tstart"]: r["value"] for r in result.filter(F.col("container_id") == 1).collect() + } + assert got, "expected calculated rows for container 1" + + # Expected: build the two source series and add them via the same core model. + channels = basic_narrow_db.channels(spark).filter(F.col("container_id") == 1) + + def _series(channel_id): + rows = sorted( + ( + (r["tstart"], r["tend"], r["value"]) + for r in channels.filter(F.col("channel_id") == channel_id).collect() + ), + key=lambda x: x[0], + ) + return SampleSeries([r[0] for r in rows], [r[1] for r in rows], [r[2] for r in rows]) + + expected_series = _series(5) + _series(7) + expected = {int(ts): val for ts, _, val in expected_series.get_data()} + + assert set(got) == set(expected) + for ts, val in expected.items(): + assert got[ts] == pytest.approx(val) + + +class TestOutputSchema: + def test_output_columns_and_order(self, spark, basic_narrow_db): + q = basic_narrow_db.query + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, + {"channel_name": "rpm_x2", "data_key": "CALC"}, + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + # Identity columns appear in sorted order after the fixed silver columns. + assert result.columns == [ + "container_id", + "channel_id", + "tstart", + "tend", + "value", + "channel_name", + "data_key", + ] + assert result.schema["tstart"].dataType == T.LongType() + assert result.schema["value"].dataType == T.DoubleType() + assert result.schema["channel_name"].dataType == T.StringType() + + @pytest.mark.parametrize( + "cid_type", [T.StringType(), T.IntegerType(), T.LongType()], ids=lambda t: t.simpleString() + ) + def test_dynamic_container_id_type(self, spark, basic_narrow_db, cid_type): + db = _recast_container_id(basic_narrow_db, cid_type) + q = db.query + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, {"channel_name": "rpm_x2"} + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + assert result.schema["container_id"].dataType == cid_type + # channel_id type follows the source channels table (IntegerType here). + assert result.schema["channel_id"].dataType == T.IntegerType() + assert result.count() > 0 + + +class TestChannelId: + def test_deterministic_across_runs(self, spark, basic_narrow_db): + q = basic_narrow_db.query + + def _ids(): + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, + {"channel_name": "rpm_x2", "data_key": "CALC"}, + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + return {r["channel_id"] for r in result.select("channel_id").distinct().collect()} + + assert _ids() == _ids() + + def test_distinct_identities_distinct_ids(self, spark, basic_narrow_db): + q = basic_narrow_db.query + cc_a = CalculatedChannel(q.channel(channel_name="Engine RPM") * 2, {"channel_name": "a"}) + cc_b = CalculatedChannel(q.channel(channel_name="Engine RPM") * 3, {"channel_name": "b"}) + result = q.select(cc_a, cc_b).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + by_name = { + r["channel_name"]: r["channel_id"] + for r in result.select("channel_name", "channel_id").distinct().collect() + } + assert by_name["a"] != by_name["b"] + + def test_explicit_channel_id_honored(self, spark, basic_narrow_db): + q = basic_narrow_db.query + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, {"channel_name": "rpm_x2"}, channel_id=999 + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + ids = {r["channel_id"] for r in result.select("channel_id").distinct().collect()} + assert ids == {999} + + def test_explicit_none_channel_id_emits_null(self, spark, basic_narrow_db): + q = basic_narrow_db.query + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, {"channel_name": "rpm_x2"}, channel_id=None + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + ids = {r["channel_id"] for r in result.select("channel_id").distinct().collect()} + assert ids == {None} + + +class TestValidation: + def test_plain_selector_rejected(self, spark, basic_narrow_db): + q = basic_narrow_db.query + q.select(q.channel(channel_name="Engine RPM")) + with pytest.raises(ValueError, match="requires all selections to be"): + q.solve_calculated_channels(spark, solver=DefaultSolver(spark)) + + def test_aggregation_rejected(self, spark, basic_narrow_db): + q = basic_narrow_db.query + agg = StatsAggregator( + input_expressions=[q.channel(channel_name="Engine RPM")], statistics=["mean"] + ) + q.select(agg) + with pytest.raises(ValueError, match="requires all selections to be"): + q.solve_calculated_channels(spark, solver=DefaultSolver(spark)) + + def test_mismatched_identity_keys_rejected(self, spark, basic_narrow_db): + q = basic_narrow_db.query + cc_a = CalculatedChannel(q.channel(channel_name="Engine RPM") * 2, {"channel_name": "a"}) + cc_b = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 3, {"channel_name": "b", "data_key": "CALC"} + ) + q.select(cc_a, cc_b) + with pytest.raises(ValueError, match="same identity keys"): + q.solve_calculated_channels(spark, solver=DefaultSolver(spark)) + + +class TestEmptyResults: + def test_no_matching_channels_empty_frame(self, spark, basic_narrow_db): + q = basic_narrow_db.query + cc = CalculatedChannel( + q.channel(channel_name="Nonexistent Channel") * 2, + {"channel_name": "nope", "data_key": "CALC"}, + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + assert result.count() == 0 + assert result.columns == [ + "container_id", + "channel_id", + "tstart", + "tend", + "value", + "channel_name", + "data_key", + ] + + def test_base_solver_not_supported(self, spark, basic_narrow_db): + from impulse_query_engine.analyze.query.solvers.blob_solver import BlobSolver + + q = basic_narrow_db.query + cc = CalculatedChannel(q.channel(channel_name="Engine RPM") * 2, {"channel_name": "x"}) + q.select(cc) + with pytest.raises(NotImplementedError, match="calculated channels"): + q.solve_calculated_channels(spark, solver=BlobSolver()) From 46e24f7de83afde780d31aa80e30da492467255d Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Thu, 16 Jul 2026 12:00:44 +0200 Subject: [PATCH 02/23] Refactor query structure and introduce CalculatedChannel class - Updated import paths to reflect the new structure for calculated channels. - Introduced the CalculatedChannel class, encapsulating derived time-series channel logic. - Simplified the QueryBuilder's solve_calculated_channels method by removing unused parameters. - Enhanced the DefaultSolver to accommodate the new CalculatedChannel class. - Added unit tests for CalculatedChannel and updated existing tests to ensure compatibility with the new structure. --- .../analyze/query/channels/__init__.py | 0 .../calculated_channel.py | 30 ++++++++++++------- .../analyze/query/query_builder.py | 11 ++----- .../analyze/query/solvers/default_solver.py | 7 ++--- .../analyze/query/solvers/query_solver.py | 5 +--- .../unit/analyze/query/channels/__init__.py | 0 .../calculated_channel_test.py | 13 ++++---- ...default_solver_calculated_channels_test.py | 2 +- 8 files changed, 33 insertions(+), 35 deletions(-) create mode 100644 src/impulse_query_engine/analyze/query/channels/__init__.py rename src/impulse_query_engine/analyze/query/{aggregations => channels}/calculated_channel.py (78%) create mode 100644 tests/impulse_query_engine/unit/analyze/query/channels/__init__.py rename tests/impulse_query_engine/unit/analyze/query/{aggregations => channels}/calculated_channel_test.py (90%) diff --git a/src/impulse_query_engine/analyze/query/channels/__init__.py b/src/impulse_query_engine/analyze/query/channels/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/impulse_query_engine/analyze/query/aggregations/calculated_channel.py b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py similarity index 78% rename from src/impulse_query_engine/analyze/query/aggregations/calculated_channel.py rename to src/impulse_query_engine/analyze/query/channels/calculated_channel.py index 036a5a6d..0af43ce3 100644 --- a/src/impulse_query_engine/analyze/query/aggregations/calculated_channel.py +++ b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py @@ -1,4 +1,4 @@ -"""CalculatedChannel aggregation for computing derived time-series channels.""" +"""CalculatedChannel: a labeled derived time-series channel.""" import pyspark.sql.types as T @@ -9,24 +9,25 @@ ) from impulse_query_engine.analyze.query.solvers.series_cache import SeriesCache -from .aggregation import Aggregation - # Sentinel distinguishing "derive a deterministic channel_id" (the default) from # an explicit ``channel_id=None`` (emit a SQL null). A plain default of ``None`` # could not tell these two intents apart. _AUTO = object() -class CalculatedChannel(Aggregation): +class CalculatedChannel(TimeSeriesExpression): """A derived channel: a wrapped time-series expression plus output identity. A ``CalculatedChannel`` wraps an arbitrary :class:`TimeSeriesExpression` built from the operator DSL (e.g. ``q.channel(channel_name="raw_speed") * 3.6`` - or ``rpm + speed``). When evaluated per container the wrapped expression must - yield a :class:`SampleSeries`; :meth:`QueryBuilder.solve_calculated_channels` - explodes that series into narrow rows matching the silver ``channel_data`` - shape (``container_id, channel_id, tstart, tend, value``) plus one string - column per identity entry. + or ``rpm + speed``). Like the other ``TimeSeriesExpression`` leaves/nodes + (``TimeSeriesSelector``, ``TimeSeriesOp``) it ``build()``s to a + :class:`SampleSeries` — it is a *labeled derived signal*, not a reduction, so + it is a plain ``TimeSeriesExpression`` rather than an ``Aggregation``. + :meth:`QueryBuilder.solve_calculated_channels` explodes that series into + narrow rows matching the silver ``channel_data`` shape + (``container_id, channel_id, tstart, tend, value``) plus one string column + per identity entry. Parameters ---------- @@ -101,8 +102,15 @@ def build(self, cache: SeriesCache): return self.expr.build(cache) def dtype(self) -> T.DataType: - """Spark type of the emitted ``value`` column (matches CHANNELS_SCHEMA).""" - return T.DoubleType() + """Spark type of ``build()``'s result: a serialized ``SampleSeries``. + + Matches :meth:`TimeSeriesSelector.dtype` (``BinaryType``), since the + wrapped expression evaluates to a ``SampleSeries``. The narrow + calculated-channels solve path does not consume this — it emits its own + ``container_id, channel_id, tstart, tend, value`` schema — but keeping it + honest makes the expression safe if ever routed through ``solve()``. + """ + return T.BinaryType() def get_selectors(self) -> list[TimeSeriesSelector]: return self.expr.get_selectors() diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 514d0f1e..7287387f 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -11,7 +11,7 @@ TimeSeriesExpression, TimeSeriesSelector, ) -from impulse_query_engine.analyze.query.aggregations.calculated_channel import ( +from impulse_query_engine.analyze.query.channels.calculated_channel import ( CalculatedChannel, ) from impulse_query_engine.analyze.query.solvers.empty_cache import EmptyTimeSeriesCache @@ -324,16 +324,9 @@ def solve_calculated_channels( """ self._validate_calculated_channels() - ( - self.result_objects, - self.result_dtypes, - ) = self._determine_result_objects_dtypes() - channel_metrics_df = self._run_filter_pipeline(spark, solver, pre_filtered_containers_df) - return solver.solve_calculated_channels( - self, channel_metrics_df, self.selections, self.result_dtypes - ) + return solver.solve_calculated_channels(self, channel_metrics_df, self.selections) def _validate_calculated_channels(self) -> None: """Validate the selections for :meth:`solve_calculated_channels`. diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 91bb7f7d..39476c79 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -970,7 +970,7 @@ def _calculated_channel_id(cc, channel_id_dtype: T.DataType): signed int32 for ``IntegerType``, signed int64 otherwise. Determinism makes writes idempotent and joins predictable across runs. """ - from impulse_query_engine.analyze.query.aggregations.calculated_channel import _AUTO + from impulse_query_engine.analyze.query.channels.calculated_channel import _AUTO if cc._explicit_channel_id is not _AUTO: return cc._explicit_channel_id @@ -1033,7 +1033,7 @@ def _solve_calculated_channels_udf( return pd.DataFrame(cols) - def solve_calculated_channels(self, query, channels_df, selections, dtypes=None) -> DataFrame: + def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame: """ Solve calculated channels by grouping channels and exploding each result. @@ -1051,9 +1051,6 @@ def solve_calculated_channels(self, query, channels_df, selections, dtypes=None) Channel-match DataFrame from the filter pipeline. selections : list List of ``CalculatedChannel`` selections to evaluate. - dtypes : list, optional - Unused (``value`` is always ``DoubleType``); accepted for signature - parity with :meth:`solve`. Returns ------- diff --git a/src/impulse_query_engine/analyze/query/solvers/query_solver.py b/src/impulse_query_engine/analyze/query/solvers/query_solver.py index 86765f77..66cac6fa 100644 --- a/src/impulse_query_engine/analyze/query/solvers/query_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/query_solver.py @@ -411,7 +411,7 @@ def solve(self, query, channels_df, selections, dtypes): """ pass - def solve_calculated_channels(self, query, channels_df, selections, dtypes=None) -> DataFrame: + def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame: """ Solve calculated channels into a narrow, silver-shaped DataFrame. @@ -428,9 +428,6 @@ def solve_calculated_channels(self, query, channels_df, selections, dtypes=None) Channel-match DataFrame from the filter pipeline. selections : list List of ``CalculatedChannel`` selections to evaluate. - dtypes : list, optional - Result data types (unused by the narrow path; ``value`` is always - ``DoubleType``). Returns ------- diff --git a/tests/impulse_query_engine/unit/analyze/query/channels/__init__.py b/tests/impulse_query_engine/unit/analyze/query/channels/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/impulse_query_engine/unit/analyze/query/aggregations/calculated_channel_test.py b/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py similarity index 90% rename from tests/impulse_query_engine/unit/analyze/query/aggregations/calculated_channel_test.py rename to tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py index fe9c72fd..501e2e58 100644 --- a/tests/impulse_query_engine/unit/analyze/query/aggregations/calculated_channel_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py @@ -13,7 +13,7 @@ TimeSeriesExpression, ) from impulse_query_engine.analyze.query.aggregations.aggregation import Aggregation -from impulse_query_engine.analyze.query.aggregations.calculated_channel import ( +from impulse_query_engine.analyze.query.channels.calculated_channel import ( _AUTO, CalculatedChannel, ) @@ -48,10 +48,12 @@ def __str__(self): class TestConstruction: - def test_is_aggregation(self): + def test_is_time_series_expression_not_aggregation(self): + # It builds to a SampleSeries (a labeled derived signal), so it is a + # plain TimeSeriesExpression, not an Aggregation (a reduction). cc = CalculatedChannel(_StubExpr(), {"channel_name": "speed_kmh"}) - assert isinstance(cc, Aggregation) assert isinstance(cc, TimeSeriesExpression) + assert not isinstance(cc, Aggregation) def test_identity_stored(self): cc = CalculatedChannel(_StubExpr(), {"channel_name": "speed_kmh", "data_key": "CALC"}) @@ -119,9 +121,10 @@ def test_selector_interface_delegates(self): assert cc.get_required_tag_exprs() == {"tag_expr"} assert cc.required_tags() == {"tag"} - def test_dtype_is_double(self): + def test_dtype_is_binary(self): + # Builds to a SampleSeries → BinaryType, matching TimeSeriesSelector. cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) - assert cc.dtype() == T.DoubleType() + assert cc.dtype() == T.BinaryType() def test_evaluation_type_is_sample_series(self): cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py index 56386781..311c1554 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py @@ -10,7 +10,7 @@ import pyspark.sql.types as T import pytest -from impulse_query_engine.analyze.query.aggregations.calculated_channel import ( +from impulse_query_engine.analyze.query.channels.calculated_channel import ( CalculatedChannel, ) from impulse_query_engine.analyze.query.aggregations.stats_aggregator import ( From 215d172d908da8a3bc31ce412142fb2cc31c4e29 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Thu, 16 Jul 2026 13:49:28 +0200 Subject: [PATCH 03/23] Add CalculatedChannel and ChannelType classes with associated schemas and utilities - Introduced the CalculatedChannel class for managing derived time-series channels, including validation and identity handling. - Added ChannelType enum to categorize channel types and provide methods for retrieving associated fact and dimension table names and schemas. - Implemented CALCULATED_CHANNEL_FACT_SCHEMA and CALCULATED_CHANNEL_DIMENSION_SCHEMA for structured data persistence. - Enhanced report utilities to support calculated channels, including methods for dispatching and persisting calculated channel data. - Added integration and unit tests to validate the functionality of the new classes and ensure correct behavior in various scenarios. --- src/impulse_reporting/channels/__init__.py | 0 .../channels/calculated_channel.py | 209 ++++++++++++++++++ .../channels/channel_types.py | 150 +++++++++++++ src/impulse_reporting/core/report.py | 194 +++++++++++++++- src/impulse_reporting/core/report_utils.py | 72 +++++- .../incremental/definition_hash_comparator.py | 53 +++++ .../persist/dimension_schema.py | 18 ++ src/impulse_reporting/persist/fact_schema.py | 16 ++ .../persist/report_storage.py | 52 +++-- .../integration/calculated_channel_test.py | 171 ++++++++++++++ .../unit/channels/__init__.py | 0 .../unit/channels/calculated_channel_test.py | 125 +++++++++++ .../unit/channels/channel_types_test.py | 44 ++++ .../unit/core/report_incremental_test.py | 10 +- .../unit/core/report_utils_test.py | 14 +- 15 files changed, 1083 insertions(+), 45 deletions(-) create mode 100644 src/impulse_reporting/channels/__init__.py create mode 100644 src/impulse_reporting/channels/calculated_channel.py create mode 100644 src/impulse_reporting/channels/channel_types.py create mode 100644 tests/impulse_reporting/integration/calculated_channel_test.py create mode 100644 tests/impulse_reporting/unit/channels/__init__.py create mode 100644 tests/impulse_reporting/unit/channels/calculated_channel_test.py create mode 100644 tests/impulse_reporting/unit/channels/channel_types_test.py diff --git a/src/impulse_reporting/channels/__init__.py b/src/impulse_reporting/channels/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py new file mode 100644 index 00000000..23ed17f1 --- /dev/null +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import hashlib +import zlib +from collections.abc import Mapping + +import pyspark.sql.functions as f +from pyspark.sql import DataFrame, Row, SparkSession + +from impulse_query_engine.analyze.metadata.time_series_expression import ( + TimeSeriesExpression, +) +from impulse_query_engine.analyze.query.channels.calculated_channel import ( + CalculatedChannel as QeCalculatedChannel, +) +from impulse_query_engine.analyze.query.query_builder import QueryBuilder +from impulse_query_engine.analyze.query.solvers.query_solver import QuerySolver +from impulse_query_engine.model.series.sample_series import SampleSeries +from impulse_reporting.persist.dimension_schema import CALCULATED_CHANNEL_DIMENSION_SCHEMA +from impulse_reporting.persist.fact_schema import CALCULATED_CHANNEL_FACT_SCHEMA + +# Identity keys required on every reporting-layer calculated channel. Fixed so the +# gold fact table has a single, stable schema whose identity columns are named +# ``channel_name`` and ``data_key`` (see CALCULATED_CHANNEL_FACT_SCHEMA). +_REQUIRED_IDENTITY_KEYS = {"channel_name", "data_key"} + + +class CalculatedChannel: + """A reporting-layer calculated (derived) channel. + + Orchestration counterpart to a query-engine ``CalculatedChannel``: it wraps a + :class:`TimeSeriesExpression` built from the operator DSL (e.g. + ``q.channel(channel_name="raw_speed") * 3.6``) plus an ``identity`` dict, and + is driven by :class:`Report` to compute the channel across containers, persist + the narrow result to a gold fact table, and update it incrementally. + + Structurally parallels :class:`BasicEvent` (holds an aliased expression, + name-derived id, SHA-256 definition hash) but — like ``ContainerEvent`` — it + drives its own solve via ``QueryBuilder.solve_calculated_channels`` rather than + riding the centralized wide ``solved_df``. Accordingly :meth:`get_expression` + returns ``None`` so it is excluded from the batch solve. + + Parameters + ---------- + name : str + Name of the calculated channel (used as the entity id seed's fallback and + stored on the dimension row). + expr : TimeSeriesExpression + The wrapped expression; must evaluate to a ``SampleSeries``. + identity : Mapping[str, str] + Output identity columns. Must contain exactly the keys + ``{"channel_name", "data_key"}``; the values are emitted as literals on + every fact row and seed the deterministic ``channel_id``. + desc : str, optional + Human-readable description (stored on the dimension row, excluded from the + definition hash). + attributes : Mapping[str, str], optional + Key-value metadata stored on the dimension row. + """ + + def __init__( + self, + name: str, + expr: TimeSeriesExpression, + identity: Mapping[str, str], + desc: str = None, + attributes: Mapping[str, str] = None, + ): + self.name = name + self.report_id = -1 + self.description = desc + + expr.require_evaluation_type( + SampleSeries, + owner="CalculatedChannel", + example="q.channel(channel_name='raw_speed') * 3.6", + ) + + self.identity = {str(k): str(v) for k, v in identity.items()} + if set(self.identity) != _REQUIRED_IDENTITY_KEYS: + raise ValueError( + "CalculatedChannel identity must contain exactly the keys " + f"{sorted(_REQUIRED_IDENTITY_KEYS)}; got {sorted(self.identity)}. " + "These become the fixed identity columns of the gold fact table." + ) + + normalized_attributes: dict[str, str] = {} + if attributes is not None: + normalized_attributes = {str(k): str(v) for k, v in attributes.items()} + self.attributes = normalized_attributes + + # Deterministic entity id from the identity (same canonical encoding the + # query-engine layer uses). Passed explicitly to the query-engine channel + # so fact.channel_id == get_id() == dimension.channel_id. + self._entity_id = zlib.crc32(self._canonical_identity().encode()) & 0x7FFFFFFF + self.expression = QeCalculatedChannel(expr, self.identity, channel_id=self._entity_id) + + def _canonical_identity(self) -> str: + """Stable identity encoding (sorted keys), matching the query-engine layer.""" + return "&".join(f"{k}={self.identity[k]}" for k in sorted(self.identity)) + + def get_name(self) -> str: + """Return the channel name.""" + return self.name + + def set_report_id(self, report_id: int): + """Set the owning report id.""" + self.report_id = report_id + + def get_id(self) -> int: + """Return the deterministic entity id (also the fact/dimension ``channel_id``).""" + return self._entity_id + + def get_expression(self) -> TimeSeriesExpression | None: + """Return ``None`` — calculated channels drive their own narrow solve. + + Returning ``None`` keeps this channel out of the centralized wide batch + solve (``collect_solvable_expressions``), mirroring ``ContainerEvent``. + """ + return None + + def get_expression_str(self) -> str: + """String form of the wrapped expression (identity + expr, no name/desc).""" + if isinstance(self.expression, TimeSeriesExpression): + return self.expression.__str__() + return "NA" + + def get_channel_type_str(self) -> str: + """Channel type string, matching the ``ChannelType`` enum member name.""" + return "CALCULATED_CHANNEL" + + def determine_definition_hash(self) -> int: + """Hash of the computation-affecting definition (expression + identity). + + Uses the wrapped-expression string, which encodes identity and the + expression but not name/description/report_id/attributes. SHA-256, first + 8 bytes as a signed int (fits ``LongType``) — same technique as events. + """ + hash_bytes = hashlib.sha256(self.get_expression_str().encode()).digest() + return int.from_bytes(hash_bytes[:8], byteorder="big", signed=True) + + def as_dict(self) -> dict: + """Dictionary representation matching ``CALCULATED_CHANNEL_DIMENSION_SCHEMA``.""" + return { + "channel_id": self.get_id(), + "report_id": self.report_id, + "channel_type": self.get_channel_type_str(), + "channel_name": self.identity.get("channel_name"), + "channel_description": self.description, + "channel_expression": self.get_expression_str(), + "identity": self.identity, + "definition_hash": self.determine_definition_hash(), + "attributes": self.attributes, + } + + def as_spark_row(self) -> Row: + """Spark Row representation of the dimension metadata.""" + return Row(**self.as_dict()) + + @classmethod + def determine_calculated_channels( + cls, + spark: SparkSession, + channels: list[CalculatedChannel], + *, + query: QueryBuilder = None, + solver: QuerySolver = None, + pre_filtered_containers_df: DataFrame = None, + ) -> DataFrame | None: + """Solve the given channels and shape the result into fact rows. + + Drives ``QueryBuilder.solve_calculated_channels`` (the narrow, many-rows- + per-container endpoint) with the report's ``query`` + ``solver``, then + projects to :data:`CALCULATED_CHANNEL_FACT_SCHEMA`. Because each channel's + ``channel_id`` was fixed to its entity id at construction, no id-join is + needed. + + Parameters + ---------- + spark : SparkSession + Spark session (unused directly; kept for interface parity). + channels : list of CalculatedChannel + Channels to solve; all share the same identity key set. + query : QueryBuilder + Query builder used to select and solve the channels. + solver : QuerySolver + Solver implementing ``solve_calculated_channels`` (a ``DefaultSolver``). + pre_filtered_containers_df : DataFrame, optional + Incremental container subset; ``None`` processes all containers. + + Returns + ------- + DataFrame or None + Narrow fact DataFrame, or ``None`` when there are no channels. + """ + if not channels: + return None + + qe_channels = [channel.expression for channel in channels] + df = query.select(*qe_channels).solve_calculated_channels( + spark, solver, pre_filtered_containers_df + ) + return df.select(*CALCULATED_CHANNEL_FACT_SCHEMA.fieldNames()) + + @classmethod + def determine_metadata_df(cls, spark: SparkSession, channels: list[CalculatedChannel]): + """Create the dimension DataFrame for the given channels.""" + rows = [channel.as_spark_row() for channel in channels] + return spark.createDataFrame(rows, schema=CALCULATED_CHANNEL_DIMENSION_SCHEMA) diff --git a/src/impulse_reporting/channels/channel_types.py b/src/impulse_reporting/channels/channel_types.py new file mode 100644 index 00000000..bcb63330 --- /dev/null +++ b/src/impulse_reporting/channels/channel_types.py @@ -0,0 +1,150 @@ +from enum import Enum + +from pyspark.sql.types import StructType + +from impulse_reporting.channels.calculated_channel import CalculatedChannel +from impulse_reporting.persist.dimension_schema import CALCULATED_CHANNEL_DIMENSION_SCHEMA +from impulse_reporting.persist.fact_schema import CALCULATED_CHANNEL_FACT_SCHEMA + + +class ChannelType(Enum): + """ + Enumeration of available calculated-channel types. + + Mirrors :class:`EventType` / :class:`AggregationType`: maps each enum member + to its class and resolves the associated gold fact/dimension table names and + schemas. + + Attributes + ---------- + CALCULATED_CHANNEL : CalculatedChannel + Derived-channel type computed via ``solve_calculated_channels``. + """ + + CALCULATED_CHANNEL = CalculatedChannel + + def get_fact_table_name(self) -> str: + """ + Get the fact table name for the channel type. + + Returns + ------- + str + The name of the fact table associated with this channel type. + + Raises + ------ + ValueError + If the channel type is not supported. + """ + match self: + case ChannelType.CALCULATED_CHANNEL: + return "calculated_channel_fact" + case _: + raise ValueError(f"Unsupported channel type: {self}") + + def get_fact_schema(self) -> StructType: + """ + Get the fact schema for the channel type. + + Returns + ------- + StructType + The PySpark schema structure for this channel type. + + Raises + ------ + ValueError + If the channel type is not supported. + """ + match self: + case ChannelType.CALCULATED_CHANNEL: + return CALCULATED_CHANNEL_FACT_SCHEMA + case _: + raise ValueError(f"Unsupported channel type: {self}") + + def get_dimension_table_name(self) -> str: + """ + Get the dimension table name for the channel type. + + Returns + ------- + str + The name of the dimension table associated with this channel type. + + Raises + ------ + ValueError + If the channel type is not supported. + """ + match self: + case ChannelType.CALCULATED_CHANNEL: + return "calculated_channel_dimension" + case _: + raise ValueError(f"Unsupported channel type: {self}") + + def get_dimension_schema(self) -> StructType: + """ + Get the dimension schema for the channel type. + + Returns + ------- + StructType + The PySpark schema structure for this channel type. + + Raises + ------ + ValueError + If the channel type is not supported. + """ + match self: + case ChannelType.CALCULATED_CHANNEL: + return CALCULATED_CHANNEL_DIMENSION_SCHEMA + case _: + raise ValueError(f"Unsupported channel type: {self}") + + @classmethod + def get_any_for_fact_table(cls, table_name: str) -> "ChannelType": + """Return the first ChannelType whose fact table name matches. + + Parameters + ---------- + table_name : str + Fact table name to look up. + + Returns + ------- + ChannelType + + Raises + ------ + ValueError + If no ChannelType matches the given table name. + """ + for ct in cls: + if ct.get_fact_table_name() == table_name: + return ct + raise ValueError(f"No ChannelType found for fact table: {table_name}") + + @classmethod + def get_any_for_dimension_table(cls, table_name: str) -> "ChannelType": + """Return the first ChannelType whose dimension table name matches. + + Parameters + ---------- + table_name : str + Dimension table name to look up. + + Returns + ------- + ChannelType + + Raises + ------ + ValueError + If no ChannelType matches the given table name. + """ + for ct in cls: + if ct.get_dimension_table_name() == table_name: + return ct + raise ValueError(f"No ChannelType found for dimension table: {table_name}") diff --git a/src/impulse_reporting/core/report.py b/src/impulse_reporting/core/report.py index d5d50bd1..0cb9bb8a 100644 --- a/src/impulse_reporting/core/report.py +++ b/src/impulse_reporting/core/report.py @@ -14,6 +14,7 @@ from impulse_query_engine.analyze.query.solvers.query_solver import QuerySolver from impulse_query_engine.measurement_db import MeasurementDB, MeasurementDBConfig from impulse_reporting.aggregations.aggregation_types import AggregationType +from impulse_reporting.channels.channel_types import ChannelType from impulse_reporting.config.config_parser import ( ImpulseConfig, Solvers, @@ -24,6 +25,7 @@ cleanup_temp_tables, collect_solvable_expressions, dispatch_aggregations, + dispatch_calculated_channels, dispatch_events, solve_expressions_batched, split_by_hash_change, @@ -90,14 +92,18 @@ def __init__( self.pages = [] self.events = [] + self.calculated_channels = [] self.event_dfs = {} self.event_metadata_dfs = {} self.aggregation_dfs = {} self.aggregation_metadata_dfs = {} + self.calculated_channel_dfs = {} + self.calculated_channel_metadata_dfs = {} self.container_dimension_df = None self.channel_mapping_resolution_dimension_df = None self._is_incremental = None + self._changed_channel_ids = {} if config: self.config = Report.load_config_from_dict(config) @@ -431,6 +437,50 @@ def _group_events_by_type(self): break return event_types + def add_calculated_channel(self, channel): + """ + Add a calculated channel to the report. + + Parameters + ---------- + channel : CalculatedChannel + The calculated channel to add. + + Returns + ------- + None + """ + self.calculated_channels.append(channel) + channel.set_report_id(self.report_id) + + def get_calculated_channels(self) -> list: + """ + Get the list of calculated channels associated with the report. + + Returns + ------- + list of CalculatedChannel + List of calculated channels. + """ + return self.calculated_channels + + def _group_calculated_channels_by_type(self): + """ + Group calculated channels by their type. + + Returns + ------- + dict + Dictionary mapping channel type names to lists of channels. + """ + channel_types = {channel_type.name: [] for channel_type in ChannelType} + for channel in self.calculated_channels: + for channel_type in channel_types.keys(): + if isinstance(channel, ChannelType[channel_type].value): + channel_types[channel_type].append(channel) + break + return channel_types + def _group_aggregations_by_type(self): """ Group aggregations by their type. @@ -501,9 +551,12 @@ def persist_results(self): # Use tracked state from determine_report changed_aggregation_ids = getattr(self, "_changed_aggregation_ids", {}) changed_event_ids = getattr(self, "_changed_event_ids", {}) + changed_channel_ids = getattr(self, "_changed_channel_ids", {}) if self._is_incremental: - self._persist_incremental(changed_aggregation_ids, changed_event_ids) + self._persist_incremental( + changed_aggregation_ids, changed_event_ids, changed_channel_ids + ) else: self._persist_full() @@ -596,6 +649,42 @@ def _persist_full(self): schema, uri = writer.extract_metadata_schema_and_output_uri(event_type) writer.write(event_meta_dfs_list, schema=schema, uri=uri) + # calculated channel fact tables + channel_fact_by_table = {} + for channel_type_str, channel_dfs in self.calculated_channel_dfs.items(): + table_name = ChannelType[channel_type_str].get_fact_table_name() + channel_fact_by_table.setdefault(table_name, []) + if isinstance(channel_dfs, dict): + if channel_dfs.get("changed") is not None: + channel_fact_by_table[table_name].append(channel_dfs["changed"]) + if channel_dfs.get("unchanged") is not None: + channel_fact_by_table[table_name].append(channel_dfs["unchanged"]) + elif channel_dfs is not None: + channel_fact_by_table[table_name].append(channel_dfs) + + for table_name, channel_dfs_list in channel_fact_by_table.items(): + if not channel_dfs_list: + continue + channel_type = ChannelType.get_any_for_fact_table(table_name) + writer = storage_factory.create_writer(channel_type) + schema, uri = writer.extract_fact_schema_and_output_uri(channel_type) + writer.write(channel_dfs_list, schema=schema, uri=uri) + + # calculated channel dimension tables + channel_dim_by_table = {} + for channel_type_str, channel_metadata_df in self.calculated_channel_metadata_dfs.items(): + table_name = ChannelType[channel_type_str].get_dimension_table_name() + channel_dim_by_table.setdefault(table_name, []) + channel_dim_by_table[table_name].append(channel_metadata_df) + + for table_name, channel_meta_dfs_list in channel_dim_by_table.items(): + if not channel_meta_dfs_list: + continue + channel_type = ChannelType.get_any_for_dimension_table(table_name) + writer = storage_factory.create_writer(channel_type) + schema, uri = writer.extract_metadata_schema_and_output_uri(channel_type) + writer.write(channel_meta_dfs_list, schema=schema, uri=uri) + # persist measurement dimensions if self.container_dimension_df: writer = storage_factory.create_container_dimension_writer() @@ -613,6 +702,7 @@ def _persist_incremental( self, changed_aggregation_ids: dict[str, list[int]], changed_event_ids: dict[str, list[int]], + changed_channel_ids: dict[str, list[int]] = None, ): """ Persist results using incremental strategy. @@ -625,11 +715,14 @@ def _persist_incremental( Mapping of aggregation type to list of visual_ids with changed definitions. changed_event_ids : dict[str, list[int]] Mapping of event type to list of event_ids with changed definitions. + changed_channel_ids : dict[str, list[int]], optional + Mapping of channel type to list of channel_ids with changed definitions. Returns ------- None """ + changed_channel_ids = changed_channel_ids or {} storage_factory = WriterFactory(self.sink) transformer = ReportEntityTransformer() @@ -774,6 +867,49 @@ def _persist_incremental( ], ) + # Persist calculated channel facts + for channel_type_str, channel_data in self.calculated_channel_dfs.items(): + channel_type = ChannelType[channel_type_str] + writer = storage_factory.create_writer(channel_type) + schema, uri = writer.extract_fact_schema_and_output_uri(channel_type) + merge_keys = ["container_id", "channel_id", "tstart"] + + if isinstance(channel_data, dict): + changed_df = channel_data.get("changed") + unchanged_df = channel_data.get("unchanged") + + # Changed definitions: replaceWhere (atomic) over all containers + if changed_df is not None and channel_type_str in changed_channel_ids: + changed_ids = changed_channel_ids[channel_type_str] + df_enriched = self._transform_for_persistence(changed_df, schema, transformer) + self.sink.replace_by_ids( + df=df_enriched, + uri=uri, + id_column="channel_id", + ids_to_replace=changed_ids, + ) + + # Unchanged definitions: MERGE over the incremental subset + if unchanged_df is not None: + df_enriched = self._transform_for_persistence( + unchanged_df, schema, transformer + ) + self.sink.upsert(df_enriched, uri, merge_keys) + elif channel_data is not None: + df_enriched = self._transform_for_persistence(channel_data, schema, transformer) + self.sink.upsert(df_enriched, uri, merge_keys) + + # Persist calculated channel dimensions (always upsert by channel_id) + for ( + channel_type_str, + channel_metadata_df, + ) in self.calculated_channel_metadata_dfs.items(): + channel_type = ChannelType[channel_type_str] + writer = storage_factory.create_writer(channel_type) + schema, uri = writer.extract_metadata_schema_and_output_uri(channel_type) + df_enriched = self._transform_for_persistence(channel_metadata_df, schema, transformer) + self.sink.upsert(df_enriched, uri, ["channel_id"]) + def _transform_for_persistence( self, df: DataFrame, @@ -929,7 +1065,7 @@ def determine_report(self, is_incremental: bool = None): # Split changed/unchanged definitions changed_events_by_type, unchanged_events_by_type, self._changed_event_ids = ( split_by_hash_change( - events_by_type, EventType, self.sink, self.spark, hash_comparator, is_event=True + events_by_type, EventType, self.sink, self.spark, hash_comparator, kind="event" ) ) changed_aggs_by_type, unchanged_aggs_by_type, self._changed_aggregation_ids = ( @@ -939,7 +1075,7 @@ def determine_report(self, is_incremental: bool = None): self.sink, self.spark, hash_comparator, - is_event=False, + kind="aggregation", ) ) @@ -1035,6 +1171,58 @@ def determine_report(self, is_incremental: bool = None): self.aggregation_dfs = aggregation_dfs self.aggregation_metadata_dfs = aggregation_metadata_dfs + # Calculated channels: own narrow solve (not the wide solved_df). + # Changed definitions recompute over all containers; unchanged ones over + # the incrementally-detected subset. + channels_by_type = self._group_calculated_channels_by_type() + changed_channels_by_type, unchanged_channels_by_type, self._changed_channel_ids = ( + split_by_hash_change( + channels_by_type, + ChannelType, + self.sink, + self.spark, + hash_comparator, + kind="channel", + ) + ) + changed_channel_dfs = dispatch_calculated_channels( + self.spark, + changed_channels_by_type, + ChannelType, + self.query, + self.solver, + None, + ) + unchanged_channel_dfs = dispatch_calculated_channels( + self.spark, + unchanged_channels_by_type, + ChannelType, + self.query, + self.solver, + pre_filtered_containers_df, + ) + calculated_channel_dfs = {} + all_channel_types = set( + list(changed_channel_dfs.keys()) + list(unchanged_channel_dfs.keys()) + ) + for t in all_channel_types: + calculated_channel_dfs[t] = { + "changed": changed_channel_dfs.get(t), + "unchanged": unchanged_channel_dfs.get(t), + } + + calculated_channel_metadata_dfs = {} + for channel_name, channel_list in channels_by_type.items(): + if not channel_list: + continue + cls = ChannelType[channel_name].value + calculated_channel_metadata_dfs[channel_name] = cls.determine_metadata_df( + self.spark, channel_list + ) + + self.calculated_channel_dfs = calculated_channel_dfs + self.calculated_channel_metadata_dfs = calculated_channel_metadata_dfs + # Determine container dimension self.container_dimension_df = ContainerDimension.get_dimension( spark=self.spark, diff --git a/src/impulse_reporting/core/report_utils.py b/src/impulse_reporting/core/report_utils.py index bd6091f3..9851cd71 100644 --- a/src/impulse_reporting/core/report_utils.py +++ b/src/impulse_reporting/core/report_utils.py @@ -111,7 +111,7 @@ def split_by_hash_change( sink: Sink | None, spark: SparkSession, hash_comparator: DefinitionHashComparator, - is_event: bool = True, + kind: str = "event", ) -> tuple[dict[str, list], dict[str, list], dict[str, list[int]]]: """Split items into changed/unchanged using definition-hash comparison. @@ -120,21 +120,31 @@ def split_by_hash_change( items_by_type : dict[str, list] ``{type_name: [items]}`` as returned by ``_group_*_by_type()``. type_enum : type - ``EventType`` or ``AggregationType`` enum class. + ``EventType``, ``AggregationType``, or ``ChannelType`` enum class. sink : Sink | None Report sink (``None`` in sinkless mode). spark : SparkSession Active Spark session. hash_comparator : DefinitionHashComparator Comparator instance. - is_event : bool - ``True`` for events, ``False`` for aggregations. + kind : str + One of ``"event"``, ``"aggregation"``, or ``"channel"`` — selects which + comparator method to use. Returns ------- tuple[dict, dict, dict] ``(changed_by_type, unchanged_by_type, changed_ids)`` """ + comparators = { + "event": hash_comparator.group_events_by_hash_change, + "aggregation": hash_comparator.group_aggregations_by_hash_change, + "channel": hash_comparator.group_calculated_channels_by_hash_change, + } + if kind not in comparators: + raise ValueError(f"Unsupported kind '{kind}'; expected one of {sorted(comparators)}.") + compare = comparators[kind] + changed_by_type: dict[str, list] = {} unchanged_by_type: dict[str, list] = {} changed_ids: dict[str, list[int]] = {} @@ -150,13 +160,7 @@ def split_by_hash_change( continue dim_table = sink.config.get_output_uri_dimension_table(type_enum[type_name]) - - if is_event: - changed, unchanged = hash_comparator.group_events_by_hash_change(items, dim_table) - else: - changed, unchanged = hash_comparator.group_aggregations_by_hash_change( - items, dim_table - ) + changed, unchanged = compare(items, dim_table) if changed: changed_by_type[type_name] = changed @@ -294,6 +298,52 @@ def dispatch_aggregations( return aggregation_dfs +def dispatch_calculated_channels( + spark: SparkSession, + channels_by_type: dict[str, list], + type_enum, + query: QueryBuilder, + solver: QuerySolver, + pre_filtered_containers_df: DataFrame | None, +) -> dict: + """Dispatch ``determine_calculated_channels`` calls per type. + + Unlike events/aggregations, calculated channels never ride the wide + ``solved_df`` — each type drives its own narrow solve via + ``query.solve_calculated_channels`` (the ``ContainerEvent`` pattern), so this + always passes ``query``/``solver``/``pre_filtered_containers_df``. + + Parameters + ---------- + spark : SparkSession + channels_by_type : dict[str, list] + type_enum : ChannelType enum + query : QueryBuilder + solver : QuerySolver + pre_filtered_containers_df : DataFrame | None + + Returns + ------- + dict + ``channel_dfs`` + """ + channel_dfs: dict = {} + + for type_name, channels in channels_by_type.items(): + if not channels: + continue + cls = type_enum[type_name].value + channel_dfs[type_name] = cls.determine_calculated_channels( + spark, + channels, + query=query, + solver=solver, + pre_filtered_containers_df=pre_filtered_containers_df, + ) + + return channel_dfs + + def solve_expressions_batched( spark: SparkSession, expressions: list[TimeSeriesExpression], diff --git a/src/impulse_reporting/incremental/definition_hash_comparator.py b/src/impulse_reporting/incremental/definition_hash_comparator.py index 105ec8d4..ec0b66c9 100644 --- a/src/impulse_reporting/incremental/definition_hash_comparator.py +++ b/src/impulse_reporting/incremental/definition_hash_comparator.py @@ -3,6 +3,7 @@ from pyspark.sql import SparkSession from impulse_reporting.aggregations.aggregation import Aggregation +from impulse_reporting.channels.calculated_channel import CalculatedChannel from impulse_reporting.events.event import Event @@ -160,6 +161,58 @@ def group_aggregations_by_hash_change( return (changed, unchanged) + def group_calculated_channels_by_hash_change( + self, + channels: list[CalculatedChannel], + dimension_table: str, + ) -> tuple[list[CalculatedChannel], list[CalculatedChannel]]: + """ + Group calculated channels into changed and unchanged based on definition hash. + + Compares the current definition hash of each channel against the stored + hash in the gold layer dimension table (keyed on ``channel_id``). Channels + with different hashes (or new channels not in gold) are "changed" and need + full reprocessing of all containers. + + Parameters + ---------- + channels : List[CalculatedChannel] + Current calculated-channel definitions to check. + dimension_table : str + URI of the gold layer calculated-channel dimension table. + + Returns + ------- + Tuple[List[CalculatedChannel], List[CalculatedChannel]] + A tuple of (changed_channels, unchanged_channels). + """ + + if not self._table_exists(dimension_table): + # No gold table exists - all channels are "changed" (need full processing) + return (channels, []) + + stored_hashes = ( + self.spark.read.table(dimension_table) + .select("channel_id", "definition_hash") + .collect() + ) + stored_hash_map = {row.channel_id: row.definition_hash for row in stored_hashes} + + changed: list[CalculatedChannel] = [] + unchanged: list[CalculatedChannel] = [] + + for channel in channels: + channel_id = channel.get_id() + current_hash = channel.determine_definition_hash() + stored_hash = stored_hash_map.get(channel_id) + + if stored_hash is None or stored_hash != current_hash: + changed.append(channel) + else: + unchanged.append(channel) + + return (changed, unchanged) + def _table_exists(self, table_uri: str) -> bool: """ Check if a table exists in the catalog. diff --git a/src/impulse_reporting/persist/dimension_schema.py b/src/impulse_reporting/persist/dimension_schema.py index ee9ccae5..ecf89f8c 100644 --- a/src/impulse_reporting/persist/dimension_schema.py +++ b/src/impulse_reporting/persist/dimension_schema.py @@ -80,3 +80,21 @@ StructField("definition_hash", LongType(), True), ] ) + +# Metadata for calculated channels. ``channel_id`` is the deterministic entity id +# (identical to the fact's channel_id), ``definition_hash`` drives incremental +# reprocessing, and ``identity`` retains the full identity dict (the fact keeps +# only the fixed identity columns). +CALCULATED_CHANNEL_DIMENSION_SCHEMA = StructType( + [ + StructField("channel_id", LongType(), False), + StructField("report_id", IntegerType(), False), + StructField("channel_type", StringType(), True), + StructField("channel_name", StringType(), True), + StructField("channel_description", StringType(), True), + StructField("channel_expression", StringType(), True), + StructField("identity", MapType(StringType(), StringType()), True), + StructField("definition_hash", LongType(), True), + StructField("attributes", MapType(StringType(), StringType()), True), + ] +) diff --git a/src/impulse_reporting/persist/fact_schema.py b/src/impulse_reporting/persist/fact_schema.py index 58f14a6c..cdd21b18 100644 --- a/src/impulse_reporting/persist/fact_schema.py +++ b/src/impulse_reporting/persist/fact_schema.py @@ -58,3 +58,19 @@ StructField("statistic_value", DoubleType(), False), ] ) + +# Narrow, silver-shaped facts for calculated channels. Mirrors the query engine's +# solve_calculated_channels() output: one row per RLE sample interval, plus the +# identity columns. Field *types* are cosmetic — persistence projects by name only +# and the real container_id/channel_id types flow from the solved DataFrame. +CALCULATED_CHANNEL_FACT_SCHEMA = StructType( + [ + StructField("container_id", IntegerType(), False), + StructField("channel_id", LongType(), False), + StructField("tstart", LongType(), False), + StructField("tend", LongType(), False), + StructField("value", DoubleType(), False), + StructField("channel_name", StringType(), False), + StructField("data_key", StringType(), False), + ] +) diff --git a/src/impulse_reporting/persist/report_storage.py b/src/impulse_reporting/persist/report_storage.py index 0174a7c2..d9a1b759 100644 --- a/src/impulse_reporting/persist/report_storage.py +++ b/src/impulse_reporting/persist/report_storage.py @@ -8,6 +8,7 @@ from pyspark.sql.types import StructType from impulse_reporting.aggregations.aggregation_types import AggregationType +from impulse_reporting.channels.channel_types import ChannelType from impulse_reporting.events.event_types import EventType @@ -26,13 +27,13 @@ class SinkConfig(ABC): table_prefix: str @abstractmethod - def get_output_uri_fact_table(self, element: AggregationType | EventType) -> str: + def get_output_uri_fact_table(self, element: AggregationType | EventType | ChannelType) -> str: """ Get the corresponding output URI for the fact table. Parameters ---------- - element : AggregationType | EventType + element : AggregationType | EventType | ChannelType The aggregation or event type to get the URI for. Returns @@ -43,13 +44,15 @@ def get_output_uri_fact_table(self, element: AggregationType | EventType) -> str pass @abstractmethod - def get_output_uri_dimension_table(self, element: AggregationType | EventType) -> str: + def get_output_uri_dimension_table( + self, element: AggregationType | EventType | ChannelType + ) -> str: """ Get the corresponding output URI for a dimension table. Parameters ---------- - element : AggregationType | EventType + element : AggregationType | EventType | ChannelType The aggregation or event type to get the URI for. Returns @@ -102,13 +105,13 @@ class UnitySinkConfig(SinkConfig): catalog_name: str schema_name: str - def get_output_uri_fact_table(self, element: AggregationType | EventType) -> str: + def get_output_uri_fact_table(self, element: AggregationType | EventType | ChannelType) -> str: """ Get the output URI for a fact table in Unity Catalog format. Parameters ---------- - element : AggregationType | EventType + element : AggregationType | EventType | ChannelType The aggregation or event type to get the URI for. Returns @@ -123,13 +126,15 @@ def get_output_uri_fact_table(self, element: AggregationType | EventType) -> str uri = f"{self.catalog_name}.{self.schema_name}.{table_name}" return uri - def get_output_uri_dimension_table(self, element: AggregationType | EventType) -> str: + def get_output_uri_dimension_table( + self, element: AggregationType | EventType | ChannelType + ) -> str: """ Get the output URI for the dimension table in Unity Catalog format. Parameters ---------- - element : AggregationType | EventType + element : AggregationType | EventType | ChannelType The aggregation or event type to get the URI for. Returns @@ -492,14 +497,14 @@ def write(self, df: DataFrame | list[DataFrame], schema: StructType, uri: str): @abstractmethod def extract_fact_schema_and_output_uri( - self, aggregation_type: AggregationType | EventType + self, aggregation_type: AggregationType | EventType | ChannelType ) -> tuple[StructType, str]: """ Extract fact schema and output URI for the given aggregation or event type. Parameters ---------- - aggregation_type : AggregationType | EventType + aggregation_type : AggregationType | EventType | ChannelType The aggregation or event type to extract information for. Returns @@ -561,14 +566,14 @@ def write(self, df: DataFrame | list[DataFrame], schema: StructType, uri: str): self.sink.store(df_enriched, uri) def extract_fact_schema_and_output_uri( - self, entity_type: AggregationType | EventType + self, entity_type: AggregationType | EventType | ChannelType ) -> tuple[StructType, str]: """ Extract fact schema and output URI for the given aggregation type. Parameters ---------- - entity_type : AggregationType | EventType + entity_type : AggregationType | EventType | ChannelType The aggregation or event type to extract information for. Returns @@ -581,14 +586,14 @@ def extract_fact_schema_and_output_uri( return schema, uri def extract_metadata_schema_and_output_uri( - self, entity_type: AggregationType | EventType + self, entity_type: AggregationType | EventType | ChannelType ) -> tuple[StructType, str]: """ Extract metadata schema and output URI for the given aggregation type. Parameters ---------- - entity_type : AggregationType | EventType + entity_type : AggregationType | EventType | ChannelType The aggregation type to extract information for. Returns @@ -682,14 +687,16 @@ def __init__(self, sink: Sink): self.sink_config: SinkConfig = sink.config self._default_transformer = ReportEntityTransformer() - def create_writer(self, element: AggregationType | EventType) -> DefaultReportEntityWriter: + def create_writer( + self, element: AggregationType | EventType | ChannelType + ) -> DefaultReportEntityWriter: """ - Get the appropriate writer for the given aggregation or event type. + Get the appropriate writer for the given aggregation, event, or channel type. Parameters ---------- - element : AggregationType | EventType - The aggregation or event type to create a writer for. + element : AggregationType | EventType | ChannelType + The aggregation, event, or channel type to create a writer for. Returns ------- @@ -702,12 +709,13 @@ def create_writer(self, element: AggregationType | EventType) -> DefaultReportEn If the element type is not supported. """ - if isinstance(element, AggregationType): - return DefaultReportEntityWriter(self.sink, self._default_transformer) - elif isinstance(element, EventType): + if isinstance(element, (AggregationType, EventType, ChannelType)): return DefaultReportEntityWriter(self.sink, self._default_transformer) else: - error_msg = f"No writer found for element: {element}. Supported types are: AggregationType and EventType" + error_msg = ( + f"No writer found for element: {element}. Supported types are: " + "AggregationType, EventType and ChannelType" + ) raise ValueError(error_msg) def create_container_dimension_writer(self) -> ContainerDimensionWriter: diff --git a/tests/impulse_reporting/integration/calculated_channel_test.py b/tests/impulse_reporting/integration/calculated_channel_test.py new file mode 100644 index 00000000..722cd178 --- /dev/null +++ b/tests/impulse_reporting/integration/calculated_channel_test.py @@ -0,0 +1,171 @@ +# pylint: disable=missing-function-docstring +"""Integration tests: Report orchestration of calculated channels. + +Builds a Report against the autouse ``spark_catalog.silver`` fixtures, adds a +calculated channel, runs determine_report + persist_results, and asserts real +values land in the gold fact/dimension tables — then verifies incremental +re-run behavior (idempotent unchanged upsert; changed-definition replace). +""" + +from unittest.mock import create_autospec + +import pyspark.sql.functions as F +import pytest +from databricks.sdk import WorkspaceClient + +from impulse_reporting.channels.calculated_channel import CalculatedChannel +from impulse_reporting.config.config_parser import ( + ImpulseConfig, + IncrementalConfig, + Source, + UnitySink, +) +from impulse_reporting.core.report import Report +from tests.conftest import spark # noqa: F401 (pytest fixture) + +_FACT = "spark_catalog.gold.evaluation_calculated_channel_fact" +_DIM = "spark_catalog.gold.evaluation_calculated_channel_dimension" + + +def _config(silver_table="container_metrics", is_enabled=False): + return ImpulseConfig( + source=Source( + container_metrics_table=f"spark_catalog.silver.{silver_table}", + channel_metrics_table="spark_catalog.silver.channel_metrics", + channels_uri="spark_catalog.silver.channels", + ), + unity_sink=UnitySink( + catalog="spark_catalog", + schema="gold", + table_prefix="evaluation", + ), + incremental=IncrementalConfig( + enabled=is_enabled, + silver_last_modified_column="timestamp", + gold_last_modified_column="_created_at", + ), + ) + + +def _add_channel(report, factor=3.6, name="speed_kmh"): + q = report.get_db().query + ch = CalculatedChannel( + name=name, + expr=q.channel(channel_name="Vehicle Speed Sensor") * factor, + identity={"channel_name": name, "data_key": "CALC"}, + ) + report.add_calculated_channel(ch) + return ch + + +def test_persist_calculated_channel_full(spark): + report = Report( + name="calc_channel_report", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(_config(is_enabled=False)), + ) + ch = _add_channel(report, factor=3.6) + + report.determine_report() + report.persist_results() + + assert spark.catalog.tableExists(_FACT) + assert spark.catalog.tableExists(_DIM) + + fact = spark.read.table(_FACT) + assert fact.count() > 0 + # channel_id in the fact matches the reporting entity id. + ids = {r["channel_id"] for r in fact.select("channel_id").distinct().collect()} + assert ids == {ch.get_id()} + # identity columns carried through. + assert {r["channel_name"] for r in fact.select("channel_name").distinct().collect()} == { + "speed_kmh" + } + assert {r["data_key"] for r in fact.select("data_key").distinct().collect()} == {"CALC"} + + # Values are the derived signal: compare against the raw source scaled by 3.6. + raw = ( + report.get_db() + .channels(spark) + .join( + report.get_db() + .channel_metrics(spark) + .filter(F.col("channel_name") == "Vehicle Speed Sensor") + .select("container_id", "channel_id"), + on=["container_id", "channel_id"], + ) + ) + raw_sum = raw.select(F.sum("value")).first()[0] + calc_sum = fact.select(F.sum("value")).first()[0] + assert calc_sum == pytest.approx(raw_sum * 3.6) + + dim = spark.read.table(_DIM) + assert dim.filter(F.col("channel_name") == "speed_kmh").count() == 1 + assert dim.filter(F.col("definition_hash").isNotNull()).count() == 1 + + +def test_incremental_unchanged_is_idempotent(spark): + # Seed gold (full), then re-run incrementally with the same definition/data. + r1 = Report( + name="calc_channel_report", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(_config(is_enabled=False)), + ) + _add_channel(r1, factor=3.6) + r1.determine_report() + r1.persist_results() + count_before = spark.read.table(_FACT).count() + + r2 = Report( + name="calc_channel_report", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(_config(is_enabled=True)), + ) + _add_channel(r2, factor=3.6) + r2.determine_report() + r2.persist_results() + + # Idempotent: unchanged definition + unchanged silver → row count stable. + assert spark.read.table(_FACT).count() == count_before + + +def test_incremental_changed_definition_replaces(spark): + # Seed gold with factor 3.6. + r1 = Report( + name="calc_channel_report", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(_config(is_enabled=False)), + ) + ch1 = _add_channel(r1, factor=3.6) + r1.determine_report() + r1.persist_results() + hash_before = ( + spark.read.table(_DIM) + .filter(F.col("channel_id") == ch1.get_id()) + .select("definition_hash") + .first()[0] + ) + + # Re-run incrementally with a changed factor → replace_by_ids rewrites the rows. + r2 = Report( + name="calc_channel_report", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(_config(is_enabled=True)), + ) + ch2 = _add_channel(r2, factor=3.7) + assert ch2.get_id() == ch1.get_id() # same identity → same entity id + r2.determine_report() + r2.persist_results() + + dim = spark.read.table(_DIM) + hash_after = ( + dim.filter(F.col("channel_id") == ch2.get_id()).select("definition_hash").first()[0] + ) + assert hash_after != hash_before + # A single dimension row per channel_id (upsert, not append). + assert dim.filter(F.col("channel_id") == ch2.get_id()).count() == 1 diff --git a/tests/impulse_reporting/unit/channels/__init__.py b/tests/impulse_reporting/unit/channels/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py new file mode 100644 index 00000000..c3e4b449 --- /dev/null +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -0,0 +1,125 @@ +# pylint: disable=missing-function-docstring +"""Unit tests for the reporting-layer CalculatedChannel class.""" + +import pytest + +from impulse_query_engine.analyze.metadata.time_series_expression import TimeSeriesSelector +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver +from impulse_reporting.channels.calculated_channel import CalculatedChannel +from tests.conftest import basic_narrow_db, spark # noqa: F401 (pytest fixtures) + +_IDENTITY = {"channel_name": "speed_kmh", "data_key": "CALC"} + + +def _channel(name="speed_kmh", identity=None): + # TimeSeriesSelector builds to a SampleSeries; * scalar keeps it a SampleSeries. + expr = TimeSeriesSelector(None) * 3.6 + return CalculatedChannel(name=name, expr=expr, identity=identity or dict(_IDENTITY)) + + +class TestConstruction: + def test_stores_identity(self): + ch = _channel() + assert ch.identity == _IDENTITY + + def test_get_id_deterministic_and_identity_derived(self): + # Same identity → same id regardless of name; different identity → different id. + a = _channel(name="a") + b = _channel(name="b") + assert a.get_id() == b.get_id() + c = _channel(identity={"channel_name": "other", "data_key": "CALC"}) + assert c.get_id() != a.get_id() + + def test_get_id_positive_int32(self): + ch = _channel() + assert 0 <= ch.get_id() <= 0x7FFFFFFF + + def test_get_expression_is_none(self): + # Channels drive their own solve, so they are excluded from the batch solve. + assert _channel().get_expression() is None + + def test_channel_type_str(self): + assert _channel().get_channel_type_str() == "CALCULATED_CHANNEL" + + def test_rejects_non_sample_series_expr(self): + # `> 0` yields an Intervals-producing op, not a SampleSeries. + with pytest.raises(ValueError, match="SampleSeries"): + CalculatedChannel( + name="bad", expr=(TimeSeriesSelector(None) > 0), identity=dict(_IDENTITY) + ) + + def test_rejects_wrong_identity_keys(self): + with pytest.raises(ValueError, match="identity must contain exactly"): + CalculatedChannel( + name="bad", expr=(TimeSeriesSelector(None) * 2), identity={"channel_name": "x"} + ) + + def test_rejects_empty_identity(self): + with pytest.raises(ValueError, match="identity must contain exactly"): + CalculatedChannel(name="bad", expr=(TimeSeriesSelector(None) * 2), identity={}) + + +class TestMetadata: + def test_as_dict_keys_and_values(self): + ch = _channel() + d = ch.as_dict() + assert set(d) == { + "channel_id", + "report_id", + "channel_type", + "channel_name", + "channel_description", + "channel_expression", + "identity", + "definition_hash", + "attributes", + } + assert d["channel_id"] == ch.get_id() + assert d["report_id"] == -1 + assert d["channel_type"] == "CALCULATED_CHANNEL" + assert d["channel_name"] == "speed_kmh" + assert d["identity"] == _IDENTITY + assert isinstance(d["definition_hash"], int) + + def test_as_spark_row_field_count(self): + assert len(_channel().as_spark_row()) == 9 + + def test_definition_hash_ignores_name_and_desc(self): + a = CalculatedChannel("a", TimeSeriesSelector(None) * 3.6, dict(_IDENTITY), desc="one") + b = CalculatedChannel("b", TimeSeriesSelector(None) * 3.6, dict(_IDENTITY), desc="two") + assert a.determine_definition_hash() == b.determine_definition_hash() + + def test_definition_hash_changes_with_expression(self): + a = CalculatedChannel("a", TimeSeriesSelector(None) * 3.6, dict(_IDENTITY)) + b = CalculatedChannel("a", TimeSeriesSelector(None) * 2.0, dict(_IDENTITY)) + assert a.determine_definition_hash() != b.determine_definition_hash() + + +class TestDetermineCalculatedChannels: + def test_returns_none_when_empty(self, spark): + assert ( + CalculatedChannel.determine_calculated_channels(spark, [], query=None, solver=None) + is None + ) + + def test_returns_fact_columns_with_matching_channel_id(self, spark, basic_narrow_db): + q = basic_narrow_db.query + ch = CalculatedChannel( + name="rpm_x2", + expr=q.channel(channel_name="Engine RPM") * 2, + identity={"channel_name": "rpm_x2", "data_key": "CALC"}, + ) + df = CalculatedChannel.determine_calculated_channels( + spark, [ch], query=q, solver=DefaultSolver(spark) + ) + assert df.columns == [ + "container_id", + "channel_id", + "tstart", + "tend", + "value", + "channel_name", + "data_key", + ] + ids = {r["channel_id"] for r in df.select("channel_id").distinct().collect()} + assert ids == {ch.get_id()} diff --git a/tests/impulse_reporting/unit/channels/channel_types_test.py b/tests/impulse_reporting/unit/channels/channel_types_test.py new file mode 100644 index 00000000..54c02859 --- /dev/null +++ b/tests/impulse_reporting/unit/channels/channel_types_test.py @@ -0,0 +1,44 @@ +# pylint: disable=missing-function-docstring +"""Unit tests for the ChannelType registry.""" + +import pytest +from pyspark.sql.types import StructType + +from impulse_reporting.channels.calculated_channel import CalculatedChannel +from impulse_reporting.channels.channel_types import ChannelType + + +def test_member_maps_to_class(): + assert ChannelType.CALCULATED_CHANNEL.value is CalculatedChannel + + +def test_table_names(): + assert ChannelType.CALCULATED_CHANNEL.get_fact_table_name() == "calculated_channel_fact" + assert ( + ChannelType.CALCULATED_CHANNEL.get_dimension_table_name() == "calculated_channel_dimension" + ) + + +def test_schemas_non_empty(): + assert isinstance(ChannelType.CALCULATED_CHANNEL.get_fact_schema(), StructType) + assert isinstance(ChannelType.CALCULATED_CHANNEL.get_dimension_schema(), StructType) + assert len(ChannelType.CALCULATED_CHANNEL.get_fact_schema().fields) > 0 + assert len(ChannelType.CALCULATED_CHANNEL.get_dimension_schema().fields) > 0 + + +def test_reverse_lookup_round_trips(): + assert ( + ChannelType.get_any_for_fact_table("calculated_channel_fact") + is ChannelType.CALCULATED_CHANNEL + ) + assert ( + ChannelType.get_any_for_dimension_table("calculated_channel_dimension") + is ChannelType.CALCULATED_CHANNEL + ) + + +def test_reverse_lookup_raises_on_unknown(): + with pytest.raises(ValueError, match="No ChannelType found"): + ChannelType.get_any_for_fact_table("nonexistent_table") + with pytest.raises(ValueError, match="No ChannelType found"): + ChannelType.get_any_for_dimension_table("nonexistent_table") diff --git a/tests/impulse_reporting/unit/core/report_incremental_test.py b/tests/impulse_reporting/unit/core/report_incremental_test.py index b8413250..a31b19b2 100644 --- a/tests/impulse_reporting/unit/core/report_incremental_test.py +++ b/tests/impulse_reporting/unit/core/report_incremental_test.py @@ -301,6 +301,7 @@ def test_persist_incremental_calls_persist_incremental_method(self, spark): report._is_incremental = True report._changed_aggregation_ids = {"HISTOGRAM": [1]} report._changed_event_ids = {"BASIC_EVENT": [2]} + report._changed_channel_ids = {"CALCULATED_CHANNEL": [3]} with ( patch.object(report, "_persist_full") as mock_full, @@ -308,7 +309,9 @@ def test_persist_incremental_calls_persist_incremental_method(self, spark): ): report.persist_results() - mock_incr.assert_called_once_with({"HISTOGRAM": [1]}, {"BASIC_EVENT": [2]}) + mock_incr.assert_called_once_with( + {"HISTOGRAM": [1]}, {"BASIC_EVENT": [2]}, {"CALCULATED_CHANNEL": [3]} + ) mock_full.assert_not_called() def test_persist_uses_tracked_state_from_determine_report(self, spark): @@ -319,6 +322,7 @@ def test_persist_uses_tracked_state_from_determine_report(self, spark): report._is_incremental = True report._changed_aggregation_ids = {"HISTOGRAM": [100]} report._changed_event_ids = {"BASIC_EVENT": [200]} + report._changed_channel_ids = {"CALCULATED_CHANNEL": [300]} with ( patch.object(report, "_persist_full") as mock_full, @@ -327,7 +331,9 @@ def test_persist_uses_tracked_state_from_determine_report(self, spark): # Call with defaults - should use tracked state report.persist_results() - mock_incr.assert_called_once_with({"HISTOGRAM": [100]}, {"BASIC_EVENT": [200]}) + mock_incr.assert_called_once_with( + {"HISTOGRAM": [100]}, {"BASIC_EVENT": [200]}, {"CALCULATED_CHANNEL": [300]} + ) mock_full.assert_not_called() diff --git a/tests/impulse_reporting/unit/core/report_utils_test.py b/tests/impulse_reporting/unit/core/report_utils_test.py index 80051fed..4977e7d7 100644 --- a/tests/impulse_reporting/unit/core/report_utils_test.py +++ b/tests/impulse_reporting/unit/core/report_utils_test.py @@ -193,7 +193,7 @@ def test_sinkless_all_items_go_to_changed_events(self): sink=None, spark=MagicMock(), hash_comparator=MagicMock(), - is_event=True, + kind="event", ) assert changed == {"BASIC_EVENT": [item1, item2]} @@ -211,7 +211,7 @@ def test_sinkless_all_items_go_to_changed_aggregations(self): sink=None, spark=MagicMock(), hash_comparator=MagicMock(), - is_event=False, + kind="aggregation", ) assert changed == {"HISTOGRAM": [item]} @@ -253,7 +253,7 @@ def test_with_sink_delegates_to_group_events_by_hash_change(self): sink=mock_sink, spark=MagicMock(), hash_comparator=mock_comparator, - is_event=True, + kind="event", ) mock_comparator.group_events_by_hash_change.assert_called_once() @@ -262,7 +262,7 @@ def test_with_sink_delegates_to_group_events_by_hash_change(self): assert changed_ids == {"BASIC_EVENT": [10]} def test_with_sink_delegates_to_group_aggregations_by_hash_change(self): - """When is_event=False, group_aggregations_by_hash_change is called.""" + """When kind="aggregation", group_aggregations_by_hash_change is called.""" item = MagicMock() item.get_id.return_value = 42 @@ -278,7 +278,7 @@ def test_with_sink_delegates_to_group_aggregations_by_hash_change(self): sink=mock_sink, spark=MagicMock(), hash_comparator=mock_comparator, - is_event=False, + kind="aggregation", ) mock_comparator.group_aggregations_by_hash_change.assert_called_once() @@ -301,7 +301,7 @@ def test_with_sink_all_unchanged_produces_no_changed_entry(self): sink=mock_sink, spark=MagicMock(), hash_comparator=mock_comparator, - is_event=True, + kind="event", ) assert changed == {} @@ -335,7 +335,7 @@ def side_effect(items, _table): sink=mock_sink, spark=MagicMock(), hash_comparator=mock_comparator, - is_event=True, + kind="event", ) assert call_count[0] == 2 From d00962d95c5364a11b4e692688175b58a787a31e Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Thu, 16 Jul 2026 16:06:43 +0200 Subject: [PATCH 04/23] Enhance documentation and structure for Calculated Channels - Updated pydoc-markdown.yml to include new loaders for calculated channels. - Added detailed definitions for calculated_channel_dimension and calculated_channel_fact in the gold layer event documentation. - Enhanced the API reference to include methods for adding and retrieving calculated channels in reports. - Improved tutorial documentation to demonstrate the materialization of calculated channels. - Updated the skills documentation to reflect the new calculated channels feature and its usage in the Impulse framework. --- .../data_model/gold_layer_event_normalized.md | 28 +++ docs/impulse/docs/data_model/index.md | 6 +- .../query/channels/calculated_channel.md | 76 ++++++++ .../analyze/query/query_builder.md | 38 ++++ .../analyze/query/solvers/default_solver.md | 27 ++- .../analyze/query/solvers/query_solver.md | 24 +++ .../analyze/query/solvers/solver_config.md | 21 ++- .../channels/calculated_channel.md | 168 ++++++++++++++++++ .../channels/channel_types.md | 125 +++++++++++++ .../impulse_reporting/config/config_parser.md | 3 +- .../api/impulse_reporting/core/report.md | 28 +++ docs/impulse/docs/references/api/sidebar.json | 15 ++ .../references/query_engine/query_solvers.md | 27 +++ .../query_engine/tsal/defining_expressions.md | 6 + .../impulse/docs/references/report/channel.md | 131 ++++++++++++++ docs/impulse/docs/references/report/index.md | 23 +-- docs/impulse/docs/tutorial/demo.md | 31 +++- docs/impulse/pydoc-markdown.yml | 3 + skills/README.md | 1 + skills/impulse-analyze/SKILL.md | 19 ++ skills/impulse-channels/SKILL.md | 96 ++++++++++ skills/impulse-data-model/SKILL.md | 5 +- skills/impulse-reporting/SKILL.md | 31 ++-- skills/impulse-tsal/SKILL.md | 2 + skills/impulse/SKILL.md | 3 + 25 files changed, 908 insertions(+), 29 deletions(-) create mode 100644 docs/impulse/docs/references/api/impulse_query_engine/analyze/query/channels/calculated_channel.md create mode 100644 docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md create mode 100644 docs/impulse/docs/references/api/impulse_reporting/channels/channel_types.md create mode 100644 docs/impulse/docs/references/report/channel.md create mode 100644 skills/impulse-channels/SKILL.md diff --git a/docs/impulse/docs/data_model/gold_layer_event_normalized.md b/docs/impulse/docs/data_model/gold_layer_event_normalized.md index be06e88a..d7a8cbd1 100644 --- a/docs/impulse/docs/data_model/gold_layer_event_normalized.md +++ b/docs/impulse/docs/data_model/gold_layer_event_normalized.md @@ -165,6 +165,30 @@ channel_mapping_resolution_dimension { timestamp _created_at } +calculated_channel_dimension { + long channel_id + int report_id + string channel_type + string channel_name + string channel_description + string channel_expression + map identity + long definition_hash + map attributes + timestamp _created_at +} + +calculated_channel_fact { + int container_id FK + long channel_id FK + long tstart + long tend + double value + string channel_name + string data_key + timestamp _created_at +} + histogram_fact }o--|| event_dimension: event_id histogram2d_fact }o--|| event_dimension: event_id stats_aggregator_fact }o--|| event_instance_fact: event_instance_id @@ -172,10 +196,12 @@ stats_aggregator_fact }o--|| event_instance_fact: event_instance_id histogram_dimension ||--o{ histogram_fact : by_visual_id histogram2d_dimension ||--o{ histogram2d_fact : by_visual_id stats_aggregator_dimension ||--o{ stats_aggregator_fact: by_visual_id +calculated_channel_dimension ||--o{ calculated_channel_fact : by_channel_id measurement_dimension ||--o{ histogram_fact : container_id measurement_dimension ||--o{ histogram2d_fact : container_id measurement_dimension ||--o{ stats_aggregator_fact : container_id +measurement_dimension ||--o{ calculated_channel_fact : container_id measurement_dimension ||--o{ channel_mapping_resolution_dimension : container_id @@ -198,6 +224,7 @@ guaranteed. | `{prefix}_histogram2d_fact` | `container_id`, `visual_id`, `event_id`, `x_bin_id`, `y_bin_id` | 2D histogram bin values per container. | | `{prefix}_stats_aggregator_fact` | `container_id`, `visual_id`, `event_instance_id`, `channel_name`, `aggregation_label` | Statistics values per signal, event instance, and container. | | `{prefix}_event_instance_fact` | `container_id`, `event_id`, `event_instance_id` | Materialized event occurrences with start/end timestamps. | +| `{prefix}_calculated_channel_fact` | `container_id`, `channel_id`, `tstart` | Materialized derived signal — one row per sample interval, in the silver `channels` shape (`tstart`, `tend`, `value`) plus identity columns. | --- @@ -211,6 +238,7 @@ guaranteed. | `{prefix}_event_dimension` | `event_id`, `report_id` | Event definitions (name, expression, required channels). | | `{prefix}_measurement_dimension` | `container_id` | Container metadata. Always carries `container_id`, `config_hash`, `_created_at`; additional columns are populated from [`config.measurement_dimensions`](../config/configuration.md#measurement_dimensions-optional). | | `{prefix}_channel_mapping_resolution_dimension` | `container_id`, `channel_id`, `channel_alias` | Resolves each channel alias to its physical channel per container (physical join keys, alias `priority`). Written only when the report uses aliased selectors. The `source_unit` / `target_unit` columns are present only when a [`config.unit_conversion_table`](../config/configuration.md) is configured. | +| `{prefix}_calculated_channel_dimension` | `channel_id`, `report_id` | Calculated-channel definitions (name, TSAL expression, `identity`, `definition_hash`). Written only when the report defines calculated channels. See [Channels](../references/report/channel.md). | The `channel_mapping_resolution_dimension` table lets BI consumers join a fact back to the physical channel that an alias resolved to: join on diff --git a/docs/impulse/docs/data_model/index.md b/docs/impulse/docs/data_model/index.md index e1a78135..e18b0c24 100644 --- a/docs/impulse/docs/data_model/index.md +++ b/docs/impulse/docs/data_model/index.md @@ -61,6 +61,7 @@ The Gold layer uses a **star schema** with fact and dimension tables. All table | `histogram_fact` | One row per bin per container | 1D histogram bin values, duration-weighted. | | `histogram2d_fact` | One row per (x, y) bin per container | 2D histogram bin values, duration-weighted. | | `stats_aggregator_fact` | One row per signal per event instance | Descriptive statistics (min, max, mean, median). | +| `calculated_channel_fact` | One row per sample interval per container | Materialized derived signal (a *channel*, not a summary), in the silver `channels` shape. | ### Dimension tables @@ -71,14 +72,16 @@ The Gold layer uses a **star schema** with fact and dimension tables. All table | `histogram_dimension` | Histogram metadata (bins, signal info, units). | | `histogram2d_dimension` | 2D histogram metadata (axes, bins, signal info, units). | | `stats_aggregator_dimension` | Statistics metadata (channel names, aggregation labels). | +| `calculated_channel_dimension` | Calculated-channel definitions (name, expression, identity). | ### Join pattern -Fact and dimension tables are connected through three key columns: +Fact and dimension tables are connected through these key columns: - **`container_id`** -- links all fact tables to `measurement_dimension` - **`event_id`** -- links `event_instance_fact`, `histogram_fact`, and `histogram2d_fact` to `event_dimension` - **`visual_id`** -- links each aggregation fact table to its corresponding dimension table +- **`channel_id`** -- links `calculated_channel_fact` to `calculated_channel_dimension` `stats_aggregator_fact` additionally joins to `event_instance_fact` via `event_instance_id`, enabling per-interval breakdowns. @@ -93,3 +96,4 @@ Fact and dimension tables are connected through three key columns: | **Channel** | A time-series signal within a container (e.g. "Engine RPM"). Identified by `(container_id, channel_id)`. | `channels`, `channel_metrics`, `channel_tags` | | **Event** | A time window of interest, defined by a condition or spanning the full recording. | `event_dimension`, `event_instance_fact` | | **Aggregation** | A computation over channel data within event windows (histogram, 2D histogram, or statistics). | `*_fact`, `*_dimension` | +| **Calculated channel** | A derived time-series signal computed from existing channels and materialized at the same per-sample grain. | `calculated_channel_dimension`, `calculated_channel_fact` | diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/channels/calculated_channel.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/channels/calculated_channel.md new file mode 100644 index 00000000..ff60d241 --- /dev/null +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/channels/calculated_channel.md @@ -0,0 +1,76 @@ +--- +sidebar_label: calculated_channel +title: impulse_query_engine.analyze.query.channels.calculated_channel +--- + +CalculatedChannel: a labeled derived time-series channel. + + +## CalculatedChannel + +```python +class CalculatedChannel(TimeSeriesExpression) +``` + +A derived channel: a wrapped time-series expression plus output identity. + +A ``CalculatedChannel`` wraps an arbitrary :class:`TimeSeriesExpression` +built from the operator DSL (e.g. ``q.channel(channel_name="raw_speed") * 3.6`` +or ``rpm + speed``). Like the other ``TimeSeriesExpression`` leaves/nodes +(``TimeSeriesSelector``, ``TimeSeriesOp``) it ``build()``s to a +:class:`SampleSeries` — it is a *labeled derived signal*, not a reduction, so +it is a plain ``TimeSeriesExpression`` rather than an ``Aggregation``. +:meth:`QueryBuilder.solve_calculated_channels` explodes that series into +narrow rows matching the silver ``channel_data`` shape +(``container_id, channel_id, tstart, tend, value``) plus one string column +per identity entry. + +**Arguments**: + +- `expr` (`TimeSeriesExpression`): The wrapped expression; must ``build()`` to a ``SampleSeries``. +- `identity` (`dict of str`): Identity columns for the output rows, e.g. +``{"channel_name": "Eng_RPM", "data_key": "TM"}``. Each key becomes a +``StringType`` output column and each value is emitted as a literal on +every row of this channel. Must be non-empty; the identity also seeds +the deterministic ``channel_id`` hash. +- `channel_id` (`int or None`): Output ``channel_id`` for every emitted row. When omitted (the +``_AUTO`` sentinel) a deterministic id is derived from the identity by +the solver, typed to match the source ``channel_id`` column. Pass an +``int`` to use it verbatim, or ``None`` to emit a SQL null. + +#### canonical\_identity + +```python +def canonical_identity() -> str +``` + +Return a stable string encoding of the identity, used for the id hash. + +Keys are sorted so the encoding (and the derived ``channel_id``) is +independent of kwarg order. + + +#### build + +```python +def build(cache: SeriesCache) +``` + +Evaluate the wrapped expression against the cache (yields a SampleSeries). + + +#### dtype + +```python +def dtype() -> T.DataType +``` + +Spark type of ``build()``'s result: a serialized ``SampleSeries``. + +Matches :meth:`TimeSeriesSelector.dtype` (``BinaryType``), since the +wrapped expression evaluates to a ``SampleSeries``. The narrow +calculated-channels solve path does not consume this — it emits its own +``container_id, channel_id, tstart, tend, value`` schema — but keeping it +honest makes the expression safe if ever routed through ``solve()``. + + diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md index 9b800938..ca3f8950 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md @@ -157,6 +157,44 @@ When None, all containers matching query filters are processed (full mode). `pyspark.sql.DataFrame`: DataFrame containing query results. +#### solve\_calculated\_channels + +```python +def solve_calculated_channels( + spark, + solver: QuerySolver = BlobSolver(), + pre_filtered_containers_df: DataFrame = None) -> DataFrame +``` + +Compute calculated channels and return a narrow silver-shaped DataFrame. + +Every selection must be a :class:`CalculatedChannel`. This runs the same +metadata filter pipeline as :meth:`solve` (resolving the input channels +each calculated channel depends on), then evaluates each calculated +channel per container and emits rows in the silver ``channel_data`` shape +— ``container_id, channel_id, tstart, tend, value`` — plus one string +column per identity kwarg shared by the selections. + +**Arguments**: + +- `spark` (`SparkSession`): Spark session used for query execution. +- `solver` (`QuerySolver`): Query solver to use. Must implement ``solve_calculated_channels`` +(``DefaultSolver`` does); the default ``BlobSolver`` does not. +- `pre_filtered_containers_df` (`DataFrame`): Pre-filtered container metrics for incremental processing. When +provided, only these containers are processed; when None, all +containers matching the query filters are processed. + +**Raises**: + +- `ValueError`: If any selection is not a ``CalculatedChannel``, if the selections +declare inconsistent identity-key sets, or if a wrapped expression +does not evaluate to a ``SampleSeries``. + +**Returns**: + +`pyspark.sql.DataFrame`: Narrow DataFrame ``[container_id, channel_id, tstart, tend, value, +]``. + #### toPandas ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/default_solver.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/default_solver.md index 2ac09763..4f29b807 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/default_solver.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/default_solver.md @@ -46,7 +46,8 @@ processing. Requires an ``is_plausible`` column in the silver layer. - `raw_encoder` (`RawEncoder`): Which encoder converts RAW point data into intervals for solving. ``RawEncoder.RLE`` (default) run-length encodes equal-valued runs; -``RawEncoder.INTERVAL`` only derives ``tend``. Only consulted when ``is_raw_data`` is ``True``. +``RawEncoder.INTERVAL`` only derives ``tend`` and drops exact +duplicates. Only consulted when ``is_raw_data`` is ``True``. #### filter\_container\_tags @@ -288,3 +289,27 @@ the source to the target unit on the fly. `pyspark.sql.DataFrame`: DataFrame containing results for each container. +#### solve\_calculated\_channels + +```python +def solve_calculated_channels(query, channels_df, selections) -> DataFrame +``` + +Solve calculated channels by grouping channels and exploding each result. + +Structurally parallels :meth:`solve` — same unit-conversion prelude, +same channel-data read and ``[container_id, channel_id]`` join — but the +grouped-map UDF emits narrow silver-shaped rows (many per container) +instead of one wide row. Output columns are +``[container_id, channel_id, tstart, tend, value, ]``. + +**Arguments**: + +- `query` (`QueryBuilder`): Query object containing database and filter information. +- `channels_df` (`pyspark.sql.DataFrame`): Channel-match DataFrame from the filter pipeline. +- `selections` (`list`): List of ``CalculatedChannel`` selections to evaluate. + +**Returns**: + +`pyspark.sql.DataFrame`: Narrow DataFrame of calculated-channel samples. + diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/query_solver.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/query_solver.md index e14377ac..6c651fe6 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/query_solver.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/query_solver.md @@ -202,3 +202,27 @@ Stage 6: Solve query. `pyspark.sql.DataFrame`: DataFrame containing results for each container. +#### solve\_calculated\_channels + +```python +def solve_calculated_channels(query, channels_df, selections) -> DataFrame +``` + +Solve calculated channels into a narrow, silver-shaped DataFrame. + +Optional stage, parallel to :meth:`solve` but emitting many rows per +container (the exploded ``SampleSeries`` of each ``CalculatedChannel``) +instead of one wide row. Solvers that support calculated channels +(e.g. ``DefaultSolver``) override this; the base implementation raises. + +**Arguments**: + +- `query` (`QueryBuilder`): Query object containing database and filter information. +- `channels_df` (`pyspark.sql.DataFrame`): Channel-match DataFrame from the filter pipeline. +- `selections` (`list`): List of ``CalculatedChannel`` selections to evaluate. + +**Returns**: + +`pyspark.sql.DataFrame`: Narrow ``[container_id, channel_id, tstart, tend, value, +]`` DataFrame. + diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md index 5f34b502..8d65a48d 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md @@ -37,7 +37,8 @@ RLE ``value`` within a container/channel into a single interval (see :class:`~impulse_query_engine.analyze.query.solvers.utils.rle_encoder.RleEncoder`). INTERVAL - Derive ``tend`` from the following sample's timestamp, *without* merging equal-valued runs (see + Derive ``tend`` from the following sample's timestamp and drop exact + duplicate points, *without* merging equal-valued runs (see :class:`~impulse_query_engine.analyze.query.solvers.utils.interval_encoder.IntervalEncoder`). @@ -385,6 +386,24 @@ def group_id_col() -> str Internal column name for the unit group id on the unit_conversion table. +#### timestamp\_col + +```python +def timestamp_col() -> str +``` + +Internal column name for the timestamp on the channels table for raw encoded channel data. + + +#### is\_plausible\_col + +```python +def is_plausible_col() -> str +``` + +Internal column name for the plausibility flag on the channels table for raw encoded channel data. + + #### effective\_alias\_join\_keys ```python diff --git a/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md b/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md new file mode 100644 index 00000000..53de6c14 --- /dev/null +++ b/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md @@ -0,0 +1,168 @@ +--- +sidebar_label: calculated_channel +title: impulse_reporting.channels.calculated_channel +--- + +## CalculatedChannel + +```python +class CalculatedChannel() +``` + +A reporting-layer calculated (derived) channel. + +Orchestration counterpart to a query-engine ``CalculatedChannel``: it wraps a +:class:`TimeSeriesExpression` built from the operator DSL (e.g. +``q.channel(channel_name="raw_speed") * 3.6``) plus an ``identity`` dict, and +is driven by :class:`Report` to compute the channel across containers, persist +the narrow result to a gold fact table, and update it incrementally. + +Structurally parallels :class:`BasicEvent` (holds an aliased expression, +name-derived id, SHA-256 definition hash) but — like ``ContainerEvent`` — it +drives its own solve via ``QueryBuilder.solve_calculated_channels`` rather than +riding the centralized wide ``solved_df``. Accordingly :meth:`get_expression` +returns ``None`` so it is excluded from the batch solve. + +**Arguments**: + +- `name` (`str`): Name of the calculated channel (used as the entity id seed's fallback and +stored on the dimension row). +- `expr` (`TimeSeriesExpression`): The wrapped expression; must evaluate to a ``SampleSeries``. +- `identity` (`Mapping[str, str]`): Output identity columns. Must contain exactly the keys +``{"channel_name", "data_key"}``; the values are emitted as literals on +every fact row and seed the deterministic ``channel_id``. +- `desc` (`str`): Human-readable description (stored on the dimension row, excluded from the +definition hash). +- `attributes` (`Mapping[str, str]`): Key-value metadata stored on the dimension row. + +#### get\_name + +```python +def get_name() -> str +``` + +Return the channel name. + + +#### set\_report\_id + +```python +def set_report_id(report_id: int) +``` + +Set the owning report id. + + +#### get\_id + +```python +def get_id() -> int +``` + +Return the deterministic entity id (also the fact/dimension ``channel_id``). + + +#### get\_expression + +```python +def get_expression() -> TimeSeriesExpression | None +``` + +Return ``None`` — calculated channels drive their own narrow solve. + +Returning ``None`` keeps this channel out of the centralized wide batch +solve (``collect_solvable_expressions``), mirroring ``ContainerEvent``. + + +#### get\_expression\_str + +```python +def get_expression_str() -> str +``` + +String form of the wrapped expression (identity + expr, no name/desc). + + +#### get\_channel\_type\_str + +```python +def get_channel_type_str() -> str +``` + +Channel type string, matching the ``ChannelType`` enum member name. + + +#### determine\_definition\_hash + +```python +def determine_definition_hash() -> int +``` + +Hash of the computation-affecting definition (expression + identity). + +Uses the wrapped-expression string, which encodes identity and the +expression but not name/description/report_id/attributes. SHA-256, first +8 bytes as a signed int (fits ``LongType``) — same technique as events. + + +#### as\_dict + +```python +def as_dict() -> dict +``` + +Dictionary representation matching ``CALCULATED_CHANNEL_DIMENSION_SCHEMA``. + + +#### as\_spark\_row + +```python +def as_spark_row() -> Row +``` + +Spark Row representation of the dimension metadata. + + +#### determine\_calculated\_channels + +```python +def determine_calculated_channels( + cls, + spark: SparkSession, + channels: list[CalculatedChannel], + *, + query: QueryBuilder = None, + solver: QuerySolver = None, + pre_filtered_containers_df: DataFrame = None) -> DataFrame | None +``` + +Solve the given channels and shape the result into fact rows. + +Drives ``QueryBuilder.solve_calculated_channels`` (the narrow, many-rows- +per-container endpoint) with the report's ``query`` + ``solver``, then +projects to :data:`CALCULATED_CHANNEL_FACT_SCHEMA`. Because each channel's +``channel_id`` was fixed to its entity id at construction, no id-join is +needed. + +**Arguments**: + +- `spark` (`SparkSession`): Spark session (unused directly; kept for interface parity). +- `channels` (`list of CalculatedChannel`): Channels to solve; all share the same identity key set. +- `query` (`QueryBuilder`): Query builder used to select and solve the channels. +- `solver` (`QuerySolver`): Solver implementing ``solve_calculated_channels`` (a ``DefaultSolver``). +- `pre_filtered_containers_df` (`DataFrame`): Incremental container subset; ``None`` processes all containers. + +**Returns**: + +`DataFrame or None`: Narrow fact DataFrame, or ``None`` when there are no channels. + +#### determine\_metadata\_df + +```python +def determine_metadata_df(cls, spark: SparkSession, + channels: list[CalculatedChannel]) +``` + +Create the dimension DataFrame for the given channels. + + diff --git a/docs/impulse/docs/references/api/impulse_reporting/channels/channel_types.md b/docs/impulse/docs/references/api/impulse_reporting/channels/channel_types.md new file mode 100644 index 00000000..46e7e05d --- /dev/null +++ b/docs/impulse/docs/references/api/impulse_reporting/channels/channel_types.md @@ -0,0 +1,125 @@ +--- +sidebar_label: channel_types +title: impulse_reporting.channels.channel_types +--- + +## ChannelType + +```python +class ChannelType(Enum) +``` + +Enumeration of available calculated-channel types. + +Mirrors :class:`EventType` / :class:`AggregationType`: maps each enum member +to its class and resolves the associated gold fact/dimension table names and +schemas. + +**Arguments**: + +- `CALCULATED_CHANNEL` (`CalculatedChannel`): Derived-channel type computed via ``solve_calculated_channels``. + +#### get\_fact\_table\_name + +```python +def get_fact_table_name() -> str +``` + +Get the fact table name for the channel type. + +**Raises**: + +- `ValueError`: If the channel type is not supported. + +**Returns**: + +`str`: The name of the fact table associated with this channel type. + +#### get\_fact\_schema + +```python +def get_fact_schema() -> StructType +``` + +Get the fact schema for the channel type. + +**Raises**: + +- `ValueError`: If the channel type is not supported. + +**Returns**: + +`StructType`: The PySpark schema structure for this channel type. + +#### get\_dimension\_table\_name + +```python +def get_dimension_table_name() -> str +``` + +Get the dimension table name for the channel type. + +**Raises**: + +- `ValueError`: If the channel type is not supported. + +**Returns**: + +`str`: The name of the dimension table associated with this channel type. + +#### get\_dimension\_schema + +```python +def get_dimension_schema() -> StructType +``` + +Get the dimension schema for the channel type. + +**Raises**: + +- `ValueError`: If the channel type is not supported. + +**Returns**: + +`StructType`: The PySpark schema structure for this channel type. + +#### get\_any\_for\_fact\_table + +```python +def get_any_for_fact_table(cls, table_name: str) -> "ChannelType" +``` + +Return the first ChannelType whose fact table name matches. + +**Arguments**: + +- `table_name` (`str`): Fact table name to look up. + +**Raises**: + +- `ValueError`: If no ChannelType matches the given table name. + +**Returns**: + +`ChannelType`: + +#### get\_any\_for\_dimension\_table + +```python +def get_any_for_dimension_table(cls, table_name: str) -> "ChannelType" +``` + +Return the first ChannelType whose dimension table name matches. + +**Arguments**: + +- `table_name` (`str`): Dimension table name to look up. + +**Raises**: + +- `ValueError`: If no ChannelType matches the given table name. + +**Returns**: + +`ChannelType`: + diff --git a/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md b/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md index f7ff81ed..3658b3a5 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md +++ b/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md @@ -211,8 +211,7 @@ def validate_drop_implausible_data_requires_raw() `drop_implausible_data=True` currently only takes effect with RAW data. The filter is applied inside the RAW -> interval conversion path by the -selected ``raw_encoder`` (``RleEncoder`` / ``IntervalEncoder``). -This only happens when ``data_type=RAW``; for RLE input the field is ignored. +selected ``raw_encoder`` (``RleEncoder`` / ``IntervalEncoder``). #### default\_raw\_encoder\_for\_raw\_data diff --git a/docs/impulse/docs/references/api/impulse_reporting/core/report.md b/docs/impulse/docs/references/api/impulse_reporting/core/report.md index 9eae8dea..fd9a5bf1 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/core/report.md +++ b/docs/impulse/docs/references/api/impulse_reporting/core/report.md @@ -268,6 +268,34 @@ Get a dictionary of events part of the report keyed by event name. `dict`: Dictionary mapping event names to Event objects. +#### add\_calculated\_channel + +```python +def add_calculated_channel(channel) +``` + +Add a calculated channel to the report. + +**Arguments**: + +- `channel` (`CalculatedChannel`): The calculated channel to add. + +**Returns**: + +`None`: + +#### get\_calculated\_channels + +```python +def get_calculated_channels() -> list +``` + +Get the list of calculated channels associated with the report. + +**Returns**: + +`list of CalculatedChannel`: List of calculated channels. + #### persist\_results ```python diff --git a/docs/impulse/docs/references/api/sidebar.json b/docs/impulse/docs/references/api/sidebar.json index 7547c8a2..0c79f03c 100644 --- a/docs/impulse/docs/references/api/sidebar.json +++ b/docs/impulse/docs/references/api/sidebar.json @@ -22,6 +22,13 @@ "label": "impulse_query_engine.analyze.query.aggregations", "type": "category" }, + { + "items": [ + "references/api/impulse_query_engine/analyze/query/channels/calculated_channel" + ], + "label": "impulse_query_engine.analyze.query.channels", + "type": "category" + }, { "items": [ "references/api/impulse_query_engine/analyze/query/solvers/default_solver", @@ -71,6 +78,14 @@ "label": "impulse_reporting.aggregations", "type": "category" }, + { + "items": [ + "references/api/impulse_reporting/channels/calculated_channel", + "references/api/impulse_reporting/channels/channel_types" + ], + "label": "impulse_reporting.channels", + "type": "category" + }, { "items": [ "references/api/impulse_reporting/config/config_parser" diff --git a/docs/impulse/docs/references/query_engine/query_solvers.md b/docs/impulse/docs/references/query_engine/query_solvers.md index a3112c2b..17170f6a 100644 --- a/docs/impulse/docs/references/query_engine/query_solvers.md +++ b/docs/impulse/docs/references/query_engine/query_solvers.md @@ -108,6 +108,33 @@ Solver tuning lives under the `query_engine` section of your report config: If `query_engine` is omitted from your config entirely, the engine runs with `DefaultSolver` and `data_type = "RLE"`. +## Calculated channels + +The standard `solve()` produces a **wide** result — one row per container, one column per selection. +`DefaultSolver.solve_calculated_channels` is a parallel entry point that produces a **narrow**, +silver-shaped result instead: it explodes each selection's `SampleSeries` into one row per sample +interval, emitting `container_id, channel_id, tstart, tend, value` plus the identity columns. + +Every selection must be a [`CalculatedChannel`](../report/channel.md) (all sharing the same identity +key set), and each wrapped expression must evaluate to a `SampleSeries`. The stage reuses the same +metadata filter pipeline as `solve()` to resolve the input channels, then runs a grouped-map UDF per +container. Only `DefaultSolver` implements it — the base `QuerySolver` raises `NotImplementedError`. + +```python +from impulse_query_engine.analyze.query.channels.calculated_channel import CalculatedChannel +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver + +cc = CalculatedChannel( + db.query.channel(channel_name="Vehicle Speed Sensor") * 3.6, + {"channel_name": "speed_kmh", "data_key": "CALC"}, +) +df = db.query.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) +# df: [container_id, channel_id, tstart, tend, value, channel_name, data_key] +``` + +The reporting layer builds on this stage to persist derived channels to the gold layer — see +[Channels](../report/channel.md). + ## API reference Auto-generated symbol-level docs: diff --git a/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md b/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md index d8d7deab..70f609ae 100644 --- a/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md +++ b/docs/impulse/docs/references/query_engine/tsal/defining_expressions.md @@ -222,6 +222,12 @@ squared_rpm = custom_transform(eng_rpm) Virtual signals are TSAL expressions that derive new channels from physical ones. They are not stored in the Silver layer but computed on-the-fly by the solver. +:::note Persisting a virtual signal +A virtual signal is ephemeral — it exists only for the query that defines it. To *materialize* a derived +signal into a queryable gold table (with incremental updates), wrap the same expression in a +[`CalculatedChannel`](../../report/channel.md) and register it on a report. +::: + ### Derived signals Combine physical channels with arithmetic: diff --git a/docs/impulse/docs/references/report/channel.md b/docs/impulse/docs/references/report/channel.md new file mode 100644 index 00000000..56b513c5 --- /dev/null +++ b/docs/impulse/docs/references/report/channel.md @@ -0,0 +1,131 @@ +--- +sidebar_position: 3 +title: Channels +--- + +# Channels + +A calculated channel is a **derived time-series signal** computed from existing channels and persisted +alongside them — for example, `speed_kmh = raw_speed * 3.6`, or the sum of two sensors. Unlike an +aggregation (which summarizes a signal into bins or statistics), a calculated channel materializes a new +signal at the same per-sample grain as the silver `channels` table, so it can be queried and charted like +any physical channel. + +A calculated channel wraps a TSAL expression (see [TSAL](../query_engine/tsal/index.md)) that **must +evaluate to a `SampleSeries`**. This is the *persisted, labeled* form of the ephemeral +[virtual signals](../query_engine/tsal/defining_expressions.md#virtual-signals) you can build ad-hoc in a +query. + +Every calculated channel used in a report **must** be registered with the report via +`add_calculated_channel()` before `determine_report()` is called. + +```python +my_report.add_calculated_channel(my_channel) +``` + +--- + +## CalculatedChannel + +A `CalculatedChannel` couples a TSAL expression with an **identity** — the set of identifier columns +(`channel_name`, `data_key`) emitted on every output row. When the report is solved, the wrapped +expression is evaluated per measurement container and exploded into narrow rows matching the silver +`channels` shape. + +```python +from impulse_reporting.channels.calculated_channel import CalculatedChannel + +speed_kmh = CalculatedChannel( + name="speed_kmh", + expr=veh_spd * 3.6, + identity={"channel_name": "speed_kmh", "data_key": "CALC"}, + desc="Vehicle speed converted to km/h", +) +``` + +### Parameters + +| Parameter | Type | Required | Description | +|--------------|------------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `name` | `str` | Yes | Human-readable channel name, stored in the channel dimension table. | +| `expr` | `TimeSeriesExpression` | Yes | TSAL expression defining the derived signal. Must evaluate to `SampleSeries`; validated at construction (raises `ValueError` otherwise). | +| `identity` | `dict[str, str]` | Yes | Output identifier columns. Must contain **exactly** the keys `{"channel_name", "data_key"}`. Values are emitted as literals on every fact row and seed the `channel_id`. | +| `desc` | `str` | No | Human-readable description stored in the channel dimension table. | +| `attributes` | `Mapping[str, str]` | No | Free-form key-value metadata stored in the channel dimension table. Values are coerced to strings. | + +### How it works + +1. The `expr` is evaluated per measurement container by the solver, yielding a `SampleSeries` for each container. +2. Calculated channels take their **own narrow solve path** (`QueryBuilder.solve_calculated_channels`), distinct from the wide batch solve used by events and aggregations. Multi-channel expressions (e.g. `rpm + speed`) are time-base synchronized automatically; emitted intervals are the intersection. +3. Each series is exploded into one row per sample interval: `container_id, channel_id, tstart, tend, value`, with the `identity` values attached as columns. +4. The `channel_id` is a deterministic hash of the sorted `identity` — the same value appears in both the fact and dimension tables, so they join directly. +5. The sample rows are written to the `calculated_channel_fact` table; the channel definition (name, description, expression, identity) is written to the `calculated_channel_dimension` table. + +:::note Requires a DefaultSolver +The calculated-channels solve path is implemented by `DefaultSolver`. A `Report` uses `DefaultSolver` +automatically; if you call `solve_calculated_channels` directly, pass a `DefaultSolver` — the default +`BlobSolver` raises `NotImplementedError`. +::: + +### Examples + +**Unit conversion (single channel):** + +```python +speed_kmh = CalculatedChannel( + name="speed_kmh", + expr=veh_spd * 3.6, + identity={"channel_name": "speed_kmh", "data_key": "CALC"}, + desc="Vehicle speed converted to km/h", +) +``` + +**Derived signal from multiple channels:** + +```python +power = CalculatedChannel( + name="power_w", + expr=eng_rpm * torque, + identity={"channel_name": "power_w", "data_key": "CALC"}, + desc="Mechanical power (RPM × torque), time-synchronized across both inputs", +) +``` + +--- + +## Calculated channel output schema + +### calculated_channel_dimension + +Stores channel definitions (one row per calculated channel per report). + +| Column | Type | Description | +|-----------------------|-------------------|----------------------------------------------------------------------------------------------| +| `channel_id` | `long` | Unique channel identifier (deterministic hash of the sorted `identity`). | +| `report_id` | `int` | Report identifier. | +| `channel_type` | `str` | `"CALCULATED_CHANNEL"`. | +| `channel_name` | `str` | Channel name (from `identity["channel_name"]`). | +| `channel_description` | `str` | Channel description. | +| `channel_expression` | `str` | String representation of the wrapped TSAL expression (encodes identity + expression). | +| `identity` | `map[str, str]` | Full identity dict. | +| `definition_hash` | `long` | Hash of the channel definition; used by incremental processing to detect definition changes. | +| `attributes` | `map[str, str]` | Free-form key-value metadata supplied via the `attributes` constructor argument. | + +### calculated_channel_fact + +Stores the materialized derived signal (one row per sample interval per container) — the same narrow, +run-length-encoded shape as the silver `channels` table. + +| Column | Type | Description | +|----------------|----------|-----------------------------------------------------------------| +| `container_id` | `int` | Container identifier. Type is inherited from the silver source. | +| `channel_id` | `long` | Foreign key to `calculated_channel_dimension`. | +| `tstart` | `long` | Sample interval start timestamp. | +| `tend` | `long` | Sample interval end timestamp. | +| `value` | `double` | Computed value over the interval. | +| `channel_name` | `str` | Identity column (from `identity`). | +| `data_key` | `str` | Identity column (from `identity`). | + +Both tables carry the configurable `table_prefix` (e.g. `{prefix}_calculated_channel_fact`). See the +[gold layer schema](../../data_model/gold_layer_event_normalized.md) for how they fit the star schema, and +[incremental processing](./index.md#incremental-processing) for how definition changes are reprocessed. diff --git a/docs/impulse/docs/references/report/index.md b/docs/impulse/docs/references/report/index.md index d4bad140..0c95b08f 100644 --- a/docs/impulse/docs/references/report/index.md +++ b/docs/impulse/docs/references/report/index.md @@ -41,7 +41,9 @@ Either `config` or `config_path` must be provided. | `get_sink_config()` | Returns the active `SinkConfig` (e.g. `UnitySinkConfig` with `catalog_name`, `schema_name`, `table_prefix`). | -- | | `add_page(page)` | Adds a `Page` to the report. | `page`: `Page` instance. | | `add_event(event)` | Registers an `Event` with the report. All events used by aggregations must be registered. | `event`: `Event` instance. | -| `determine_report(is_incremental)` | Computes all events, aggregations, and container dimensions. Results are stored on the report object. | `is_incremental`: `bool` or `None`. Mode hint; overridden by `config.incremental` when present. See [Incremental processing](#incremental-processing). | +| `add_calculated_channel(channel)` | Registers a [`CalculatedChannel`](./channel.md) with the report. | `channel`: `CalculatedChannel` instance. | +| `get_calculated_channels()` | Returns the list of registered calculated channels. | -- | +| `determine_report(is_incremental)` | Computes all events, aggregations, calculated channels, and container dimensions. Results are stored on the report object. | `is_incremental`: `bool` or `None`. Mode hint; overridden by `config.incremental` when present. See [Incremental processing](#incremental-processing). | | `persist_results()` | Writes all computed results (fact and dimension tables) to the configured Gold layer sink. | -- | ### Execution workflow @@ -89,7 +91,7 @@ or pass `is_incremental=True` at call time. 1. Compare every event and aggregation against its stored `definition_hash` in the gold dimension table. Classify each as **changed** (hash differs, or it's brand new) or **unchanged** (hash matches). 2. For unchanged definitions, process only the containers that are new or have newer silver data than gold. Skip the rest. 3. For changed definitions, reprocess all containers that match the report's filters. -4. Persist via Delta `MERGE` on natural keys for unchanged definitions; replace atomically via `replaceWhere` on `visual_id` or `event_id` for changed ones. +4. Persist via Delta `MERGE` on natural keys for unchanged definitions; replace atomically via `replaceWhere` on `visual_id`, `event_id`, or `channel_id` for changed ones. #### Mode resolution @@ -106,13 +108,14 @@ The first run of a new report is always full. Subsequent runs pick up where the Only the hashed attributes matter. Anything else is cosmetic and won't trigger reprocessing. -| Type | Hashed | -|-------------------|-------------------------------------------------------------| -| `BasicEvent` | `expr` string | -| `ContainerEvent` | `name` | -| `Histogram` | `base_expr`, `bins`, `event` | -| `Histogram2D` | `x_expr`, `y_expr`, `x_bins`, `y_bins`, `event` | -| `StatsAggregator` | `input_expressions`, `statistics`, `event` | +| Type | Hashed | +|---------------------|-------------------------------------------------------------| +| `BasicEvent` | `expr` string | +| `ContainerEvent` | `name` | +| `Histogram` | `base_expr`, `bins`, `event` | +| `Histogram2D` | `x_expr`, `y_expr`, `x_bins`, `y_bins`, `event` | +| `StatsAggregator` | `input_expressions`, `statistics`, `event` | +| `CalculatedChannel` | `expr` string + `identity` | Renaming an aggregation, tweaking the description, or swapping `channel_name` or units keeps the hash stable. No reprocessing. @@ -126,5 +129,5 @@ Renaming an aggregation, tweaking the description, or swapping `channel_name` or #### Operational notes - A single run can be partly incremental: one event is changed (full reprocess), another is unchanged (upserted containers only), a newly added aggregation is brand new (also full reprocess). Each entity walks its own path. -- `replaceWhere` is atomic per fact table. When a definition changes, all rows for that `visual_id` or `event_id` get deleted and rewritten in one transaction. No intermediate inconsistent state, but there is a brief rewrite window. +- `replaceWhere` is atomic per fact table. When a definition changes, all rows for that `visual_id`, `event_id`, or `channel_id` get deleted and rewritten in one transaction. No intermediate inconsistent state, but there is a brief rewrite window. - `MERGE` keeps existing rows that don't conflict, so unchanged definitions accumulate rows for new containers without rewriting the old ones. diff --git a/docs/impulse/docs/tutorial/demo.md b/docs/impulse/docs/tutorial/demo.md index b48ac915..ba4953c8 100644 --- a/docs/impulse/docs/tutorial/demo.md +++ b/docs/impulse/docs/tutorial/demo.md @@ -258,6 +258,31 @@ distance bins. --- +## Step 4b: Materialize calculated channels + +Aggregations *summarize* signals. A [calculated channel](../references/report/channel.md) instead +*materializes a derived signal* — persisting one of the Step 2 virtual channels as a first-class, queryable +channel at the same per-sample grain as the silver `channels` table. + +```python +from impulse_reporting.channels.calculated_channel import CalculatedChannel + +avg_temp_channel = CalculatedChannel( + name="avg_temp", + expr=avg_temp, # the virtual signal defined in Step 2 + identity={"channel_name": "avg_temp", "data_key": "CALC"}, + desc="Average of ambient and intake air temperature", +) +my_report.add_calculated_channel(avg_temp_channel) +``` + +- The wrapped expression must evaluate to a `SampleSeries` (a signal), not `Intervals`. +- `identity` must contain exactly `channel_name` and `data_key`; these become columns on every output row + and seed the deterministic `channel_id`. +- Registration mirrors events: `add_calculated_channel()` before `determine_report()`. + +--- + ## Step 5: Compute and persist Two calls execute the full pipeline: @@ -267,8 +292,8 @@ my_report.determine_report() my_report.persist_results() ``` -- `determine_report()` resolves all TSAL expressions, computes event instances, and runs all aggregations across all - containers matched by the configuration. +- `determine_report()` resolves all TSAL expressions, computes event instances, runs all aggregations, and solves + calculated channels across all containers matched by the configuration. - `persist_results()` writes fact and dimension tables to the Gold layer in Unity Catalog. --- @@ -365,5 +390,7 @@ After running the notebook, the following tables are created in Unity Catalog un | `t_histogram2d_fact` | 2D histogram bin values per container. | | `t_stats_aggregator_dimension` | Statistics metadata (3 stats aggregations). | | `t_stats_aggregator_fact` | Statistics values per signal, event instance, and container. | +| `t_calculated_channel_dimension` | Calculated-channel definitions (1 channel: `avg_temp`). | +| `t_calculated_channel_fact` | Materialized derived signal, one row per sample interval. | See the [Data Model](../data_model) for complete schema details. diff --git a/docs/impulse/pydoc-markdown.yml b/docs/impulse/pydoc-markdown.yml index 277b6096..3a07927c 100644 --- a/docs/impulse/pydoc-markdown.yml +++ b/docs/impulse/pydoc-markdown.yml @@ -10,8 +10,11 @@ loaders: - impulse_reporting.aggregations.histogram - impulse_reporting.aggregations.histogram2d - impulse_reporting.aggregations.stats_aggregator + - impulse_reporting.channels.calculated_channel + - impulse_reporting.channels.channel_types - impulse_reporting.config.config_parser - impulse_query_engine.analyze.query.query_builder + - impulse_query_engine.analyze.query.channels.calculated_channel - impulse_query_engine.analyze.metadata.time_series_expression - impulse_query_engine.analyze.metadata.tag_expression - impulse_query_engine.analyze.metadata.metric_expression diff --git a/skills/README.md b/skills/README.md index dcda6761..cb98eccf 100644 --- a/skills/README.md +++ b/skills/README.md @@ -14,6 +14,7 @@ Each skill is a folder with a `SKILL.md` file that documents usage patterns. Sta | [`impulse-config`](./impulse-config/SKILL.md) | The `ImpulseConfig` schema — source tables, sink, container filters, solver options, incremental processing, and sinkless mode. | | [`impulse-events`](./impulse-events/SKILL.md) | Defining event windows: `BasicEvent`, `ContainerEvent`, `SequenceOfEvents`, `PointsInTimeEvent`. | | [`impulse-aggregations`](./impulse-aggregations/SKILL.md)| Computing results over channels: 1D/2D histograms (duration/distance/custom-weight), `StatsAggregator`, `PointValueAggregator`, and pages. | +| [`impulse-channels`](./impulse-channels/SKILL.md) | Calculated (derived) channels — materializing a new signal from existing channels via `CalculatedChannel` and `solve_calculated_channels`. | | [`impulse-reporting`](./impulse-reporting/SKILL.md) | The batch pipeline that persists events and aggregations to the gold-layer star schema with `Report` / `Page`, plus incremental runs. | | [`impulse-analyze`](./impulse-analyze/SKILL.md) | Ad-hoc analysis — evaluating TSAL directly through the query engine and returning Spark or pandas DataFrames, no gold-layer write. | | [`impulse-ml`](./impulse-ml/SKILL.md) | Extracting event-scoped statistics as a flat feature matrix for MLflow / AutoML. | diff --git a/skills/impulse-analyze/SKILL.md b/skills/impulse-analyze/SKILL.md index c7698a7a..48d2bdd3 100644 --- a/skills/impulse-analyze/SKILL.md +++ b/skills/impulse-analyze/SKILL.md @@ -124,3 +124,22 @@ Each selected expression is typed by what it evaluates to (see `impulse-tsal` fo For scalar-per-container summaries (means, maxima, counts), `select()` the reducer expressions as above. When you want the results persisted as a star schema instead of returned inline, use `impulse-reporting`. + +## Solving calculated channels + +`solve()` returns one **wide** row per container. To get a **narrow**, silver-shaped result instead — +one row per sample interval — use `solve_calculated_channels()` with `CalculatedChannel` selections (see +`impulse-channels`). It requires a `DefaultSolver` (the default `BlobSolver` raises). + +```python +from impulse_query_engine.analyze.query.channels.calculated_channel import CalculatedChannel + +cc = CalculatedChannel( + db.query.channel(channel_name="Vehicle Speed Sensor") * 3.6, + {"channel_name": "speed_kmh", "data_key": "CALC"}, +) +df = db.query.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) +# df: [container_id, channel_id, tstart, tend, value, channel_name, data_key] +``` + +All selections must be `CalculatedChannel`s sharing one identity key set. diff --git a/skills/impulse-channels/SKILL.md b/skills/impulse-channels/SKILL.md new file mode 100644 index 00000000..16380019 --- /dev/null +++ b/skills/impulse-channels/SKILL.md @@ -0,0 +1,96 @@ +--- +name: impulse-channels +description: > + Compute calculated (derived) channels in Impulse — new time-series signals built from existing + channels and materialized at the same per-sample grain. Use when the user wants to "add a calculated + channel", derive/persist a signal (e.g. "speed in km/h", "power = rpm × torque"), materialize a virtual + signal into a queryable table, or run `solve_calculated_channels`. Covers the reporting-layer + CalculatedChannel, the ad-hoc `QueryBuilder.solve_calculated_channels` endpoint, and the + calculated_channel_fact/dimension gold output. +--- + +# Impulse — calculated channels + +A calculated channel is a **derived signal** computed from existing channels (see `impulse-tsal`) and +persisted at the same per-sample grain as the silver `channels` table — unlike an aggregation (see +`impulse-aggregations`), which summarizes a signal into bins or statistics. It is the *persisted, labeled* +form of a virtual signal. + +**Every calculated channel must be registered with the report before computing:** + +```python +report.add_calculated_channel(my_channel) +``` + +The wrapped TSAL expression **must evaluate to a `SampleSeries`** (a signal), not `Intervals` — validated +at construction (raises `ValueError`). + +## CalculatedChannel + +Couples a TSAL expression with an `identity` — the identifier columns emitted on every output row. The +expression is evaluated per container and exploded into narrow rows (one per sample interval). + +```python +from impulse_reporting.channels.calculated_channel import CalculatedChannel + +veh_spd = report.get_db().query.channel(channel_name="Vehicle Speed Sensor") + +speed_kmh = CalculatedChannel( + name="speed_kmh", + expr=veh_spd * 3.6, + identity={"channel_name": "speed_kmh", "data_key": "CALC"}, + desc="Vehicle speed converted to km/h", +) +report.add_calculated_channel(speed_kmh) +``` + +| Parameter | Type | Required | Description | +|--------------|------------------------|----------|---------------------------------------------------------------------------------------------------| +| `name` | `str` | Yes | Human-readable channel name; stored in `calculated_channel_dimension`. | +| `expr` | `TimeSeriesExpression` | Yes | Derived signal. **Must evaluate to `SampleSeries`.** | +| `identity` | `dict[str, str]` | Yes | Output identifier columns. **Must contain exactly `{"channel_name", "data_key"}`.** Seeds `channel_id`. | +| `desc` | `str` | No | Description stored in `calculated_channel_dimension`. | +| `attributes` | `Mapping[str, str]` | No | Free-form metadata; values coerced to strings. | + +Multi-channel expressions (e.g. `eng_rpm * torque`) are time-base synchronized automatically; emitted +intervals are the intersection. The `channel_id` is a deterministic hash of the sorted `identity` — the +same value in both the fact and dimension tables, so they join on it. + +## Ad-hoc: solve_calculated_channels + +To compute channels without a report (see `impulse-analyze` for the ad-hoc query pattern), use the query +engine's narrow-solve endpoint directly. It returns a Spark DataFrame; it requires a `DefaultSolver` (the +default `BlobSolver` raises `NotImplementedError`). + +```python +from impulse_query_engine.analyze.query.channels.calculated_channel import CalculatedChannel +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver + +cc = CalculatedChannel( + db.query.channel(channel_name="Vehicle Speed Sensor") * 3.6, + {"channel_name": "speed_kmh", "data_key": "CALC"}, + # channel_id defaults to a deterministic hash; pass an int to override, or None to emit null. +) +df = db.query.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) +df.show() # [container_id, channel_id, tstart, tend, value, channel_name, data_key] +``` + +All selections in one call must share the same identity key set (validated up front). + +## Output schema + +**calculated_channel_dimension** (one row per channel per report) — key columns: `channel_id`, +`report_id`, `channel_type` (`"CALCULATED_CHANNEL"`), `channel_name`, `channel_description`, +`channel_expression` (TSAL string), `identity` (map), `definition_hash`, `attributes` (map). + +**calculated_channel_fact** (one row per sample interval per container) — `container_id`, `channel_id`, +`tstart`, `tend`, `value`, `channel_name`, `data_key`. Same narrow, run-length-encoded shape as the silver +`channels` table. Join to the dimension on `channel_id`, and to `measurement_dimension` on `container_id` +(see `impulse-data-model`). + +## Incremental + +Calculated channels reuse the report's incremental engine (see `impulse-reporting`). A definition change — +the `expr` string or the `identity` — reprocesses that channel across all containers (`replaceWhere` on +`channel_id`); unchanged definitions only recompute new/updated containers (`MERGE`). Renaming or editing +`desc`/`attributes` does not change the hash. diff --git a/skills/impulse-data-model/SKILL.md b/skills/impulse-data-model/SKILL.md index 1fe3feb0..1a13b622 100644 --- a/skills/impulse-data-model/SKILL.md +++ b/skills/impulse-data-model/SKILL.md @@ -93,6 +93,7 @@ A star schema. Every table is prefixed with your configured `table_prefix` (e.g. | `histogram_fact` | One row per bin per container | | `histogram2d_fact` | One row per (x, y) bin per container | | `stats_aggregator_fact` | One row per signal per event instance | +| `calculated_channel_fact` | One row per sample interval per container (a derived signal, silver `channels` shape) | **Dimension tables** @@ -103,12 +104,14 @@ A star schema. Every table is prefixed with your configured `table_prefix` (e.g. | `histogram_dimension` | Histogram metadata (bins, signal info, units). | | `histogram2d_dimension` | 2D histogram metadata. | | `stats_aggregator_dimension` | Statistics metadata (channel names, labels). | +| `calculated_channel_dimension` | Calculated-channel definitions (name, expression, identity). See `impulse-channels`. | -**Join pattern** — three key columns connect facts to dimensions: +**Join pattern** — key columns connect facts to dimensions: - `container_id` → links every fact to `measurement_dimension`. - `event_id` → links `event_instance_fact`, `histogram_fact`, `histogram2d_fact` to `event_dimension`. - `visual_id` → links each aggregation fact to its own dimension table. +- `channel_id` → links `calculated_channel_fact` to `calculated_channel_dimension`. `stats_aggregator_fact` additionally joins to `event_instance_fact` via `event_instance_id` for per-interval breakdowns. diff --git a/skills/impulse-reporting/SKILL.md b/skills/impulse-reporting/SKILL.md index c414b9ed..1894ef73 100644 --- a/skills/impulse-reporting/SKILL.md +++ b/skills/impulse-reporting/SKILL.md @@ -46,7 +46,8 @@ Exactly one of `config` / `config_path` must be provided. Useful methods: `get_db()` → the `MeasurementDB` for signal selection; `get_solver()` → the configured solver; `get_sink_config()` → the resolved sink; `add_event(event)`; `add_page(page)`; -`determine_report(is_incremental=None)`; `persist_results()`. +`add_calculated_channel(channel)` (see `impulse-channels`); `determine_report(is_incremental=None)`; +`persist_results()`. ## The lifecycle @@ -73,10 +74,17 @@ page.add_aggregation(HistogramDuration( channel_name="Engine RPM", bins_unit="rpm", values_unit="s", )) -# 4. Compute everything (in parallel across containers) +# 4. (optional) Materialize derived channels (impulse-channels) +from impulse_reporting.channels.calculated_channel import CalculatedChannel +report.add_calculated_channel(CalculatedChannel( + name="speed_kmh", expr=veh_spd * 3.6, + identity={"channel_name": "speed_kmh", "data_key": "CALC"}, +)) + +# 5. Compute everything (in parallel across containers) report.determine_report() -# 5. Write the gold star schema +# 6. Write the gold star schema report.persist_results() ``` @@ -110,18 +118,19 @@ in silver. Turn it on via `incremental.enabled` in config (see `impulse-config`) **On each run:** each event/aggregation is compared against its stored `definition_hash`. Unchanged definitions reprocess only new/updated containers (persisted via Delta `MERGE` on natural keys); changed or brand-new definitions reprocess all matching containers (replaced atomically via -`replaceWhere` on `visual_id`/`event_id`). A single run can mix both per entity. +`replaceWhere` on `visual_id`/`event_id`/`channel_id`). A single run can mix both per entity. **What counts as a definition change** — only the hashed attributes; renames, descriptions, and units are cosmetic and do not trigger reprocessing: -| Type | Hashed | -|-------------------|----------------------------------------------------| -| `BasicEvent` | `expr` string | -| `ContainerEvent` | `name` | -| `Histogram` | `base_expr`, `bins`, `event` | -| `Histogram2D` | `x_expr`, `y_expr`, `x_bins`, `y_bins`, `event` | -| `StatsAggregator` | `input_expressions`, `statistics`, `event` | +| Type | Hashed | +|---------------------|----------------------------------------------------| +| `BasicEvent` | `expr` string | +| `ContainerEvent` | `name` | +| `Histogram` | `base_expr`, `bins`, `event` | +| `Histogram2D` | `x_expr`, `y_expr`, `x_bins`, `y_bins`, `event` | +| `StatsAggregator` | `input_expressions`, `statistics`, `event` | +| `CalculatedChannel` | `expr` string + `identity` | **Container-update detection** unions two sets: new containers (silver rows absent from gold) and updated containers (silver `silver_last_modified_column` newer than the gold `gold_last_modified_column`). diff --git a/skills/impulse-tsal/SKILL.md b/skills/impulse-tsal/SKILL.md index 7695c8d8..3d88a179 100644 --- a/skills/impulse-tsal/SKILL.md +++ b/skills/impulse-tsal/SKILL.md @@ -179,6 +179,8 @@ that falls in a gap where the signal is not valid, so the result may be shorter - **Events** (`impulse-events`): `BasicEvent` and `SequenceOfEvents` need **`Intervals`**; `PointsInTimeEvent` needs **`PointsInTime`**. Validated at construction — a wrong type raises `ValueError`. - **Aggregations** (`impulse-aggregations`): histogram/stats signal inputs must be **`SampleSeries`**. +- **Calculated channels** (`impulse-channels`): the wrapped expression must evaluate to **`SampleSeries`** + (it materializes a derived signal). Validated at construction. ## Common virtual-signal recipes diff --git a/skills/impulse/SKILL.md b/skills/impulse/SKILL.md index ff3eec31..9b41ad56 100644 --- a/skills/impulse/SKILL.md +++ b/skills/impulse/SKILL.md @@ -29,6 +29,7 @@ notebook or job. | **Channel** | One sensor signal within a container — e.g. "Engine RPM". Selected by metadata tags, not columns. | | **Event** | A time window of interest, defined by a TSAL condition or spanning the whole recording. | | **Aggregation** | A computation over channel data within event windows — histogram, 2D histogram, or statistics. | +| **Calculated channel** | A derived signal computed from existing channels and materialized as a new channel (same per-sample grain), not a summary. | | **Silver layer**| The input Delta tables Impulse reads (`container_metrics`, `channel_metrics`, `channels`, …). | | **Gold layer** | The output star schema Impulse writes (fact + dimension tables) for dashboards and apps. | @@ -48,6 +49,7 @@ All three build on the same foundation. Whichever mode, you will almost always a - **`impulse-config`** — the config dict that points Impulse at your tables. - **`impulse-data-model`** — the shape of the input tables and output star schema. - **`impulse-events`** and **`impulse-aggregations`** — the building blocks of reporting and ML. +- **`impulse-channels`** — when the goal is to *materialize a derived signal* (a new channel) rather than summarize one. ## Setup (every mode) @@ -136,6 +138,7 @@ report.persist_results() # write the gold star schema - Building signal expressions → **`impulse-tsal`** - Defining event windows → **`impulse-events`** - Histograms / statistics → **`impulse-aggregations`** +- Materializing a derived signal as a new channel → **`impulse-channels`** - The full reporting lifecycle and incremental runs → **`impulse-reporting`** - Notebook exploration without writes → **`impulse-analyze`** - ML feature matrices → **`impulse-ml`** From e70c58dc4de2485711f8ab93d4ff569213dbc85b Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Thu, 16 Jul 2026 17:19:13 +0200 Subject: [PATCH 05/23] Implement canonical identity handling and validation for calculated channels - Added `canonical_identity` method to `CalculatedChannel` for generating order-independent identity keys. - Introduced `_validate_unique_calculated_channels` method in `Report` to ensure calculated channels have unique identities, raising a `ValueError` for duplicates. - Updated `determine_definition_hash` to use the canonical identity for hash computation. - Enhanced unit and integration tests to cover identity reordering and duplicate identity scenarios, ensuring correct behavior in incremental report processing. --- .../channels/calculated_channel.py | 16 ++-- src/impulse_reporting/core/report.py | 36 +++++++++ .../integration/calculated_channel_test.py | 47 +++++++++++- .../unit/channels/calculated_channel_test.py | 20 +++++ .../unit/core/report_incremental_test.py | 46 +++++++++++ .../definition_hash_comparator_test.py | 76 +++++++++++++++++++ 6 files changed, 234 insertions(+), 7 deletions(-) diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py index 23ed17f1..3afb4d1b 100644 --- a/src/impulse_reporting/channels/calculated_channel.py +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -99,6 +99,15 @@ def _canonical_identity(self) -> str: """Stable identity encoding (sorted keys), matching the query-engine layer.""" return "&".join(f"{k}={self.identity[k]}" for k in sorted(self.identity)) + def canonical_identity(self) -> str: + """Public, order-independent identity key. + + Two channels with the same ``identity`` (regardless of key insertion + order) share this value and therefore the same ``channel_id``. Used by + :class:`Report` to reject duplicate channel identities. + """ + return self._canonical_identity() + def get_name(self) -> str: """Return the channel name.""" return self.name @@ -131,12 +140,9 @@ def get_channel_type_str(self) -> str: def determine_definition_hash(self) -> int: """Hash of the computation-affecting definition (expression + identity). - - Uses the wrapped-expression string, which encodes identity and the - expression but not name/description/report_id/attributes. SHA-256, first - 8 bytes as a signed int (fits ``LongType``) — same technique as events. """ - hash_bytes = hashlib.sha256(self.get_expression_str().encode()).digest() + payload = f"{self._canonical_identity()}|{self.expression.expr}" + hash_bytes = hashlib.sha256(payload.encode()).digest() return int.from_bytes(hash_bytes[:8], byteorder="big", signed=True) def as_dict(self) -> dict: diff --git a/src/impulse_reporting/core/report.py b/src/impulse_reporting/core/report.py index b6835718..5fdb35e8 100644 --- a/src/impulse_reporting/core/report.py +++ b/src/impulse_reporting/core/report.py @@ -482,6 +482,41 @@ def _group_calculated_channels_by_type(self): break return channel_types + def _validate_unique_calculated_channels(self): + """ + Reject calculated channels that share a canonical identity. + + ``channel_id`` is derived from the identity, so two channels with the + same identity would collide on the fact-table merge key + ``(container_id, channel_id, tstart)`` and overwrite each other + non-deterministically. Fail fast, naming the colliding identity and the + offending channel names. + + Raises + ------ + ValueError + If any canonical identity appears on more than one registered + calculated channel. + """ + names_by_identity: dict[str, list[str]] = {} + for channel in self.calculated_channels: + names_by_identity.setdefault(channel.canonical_identity(), []).append( + channel.get_name() + ) + + duplicates = { + identity: names for identity, names in names_by_identity.items() if len(names) > 1 + } + if duplicates: + details = "; ".join( + f"identity [{identity}] used by channels {names}" + for identity, names in duplicates.items() + ) + raise ValueError( + "Calculated channels must have unique identities — each identity maps " + f"to one channel_id and one set of fact rows. Duplicates found: {details}." + ) + def _group_aggregations_by_type(self): """ Group aggregations by their type. @@ -1175,6 +1210,7 @@ def determine_report(self, is_incremental: bool = None): # Calculated channels: own narrow solve (not the wide solved_df). # Changed definitions recompute over all containers; unchanged ones over # the incrementally-detected subset. + self._validate_unique_calculated_channels() channels_by_type = self._group_calculated_channels_by_type() changed_channels_by_type, unchanged_channels_by_type, self._changed_channel_ids = ( split_by_hash_change( diff --git a/tests/impulse_reporting/integration/calculated_channel_test.py b/tests/impulse_reporting/integration/calculated_channel_test.py index 722cd178..84742cbf 100644 --- a/tests/impulse_reporting/integration/calculated_channel_test.py +++ b/tests/impulse_reporting/integration/calculated_channel_test.py @@ -47,12 +47,12 @@ def _config(silver_table="container_metrics", is_enabled=False): ) -def _add_channel(report, factor=3.6, name="speed_kmh"): +def _add_channel(report, factor=3.6, name="speed_kmh", identity=None): q = report.get_db().query ch = CalculatedChannel( name=name, expr=q.channel(channel_name="Vehicle Speed Sensor") * factor, - identity={"channel_name": name, "data_key": "CALC"}, + identity=identity or {"channel_name": name, "data_key": "CALC"}, ) report.add_calculated_channel(ch) return ch @@ -169,3 +169,46 @@ def test_incremental_changed_definition_replaces(spark): assert hash_after != hash_before # A single dimension row per channel_id (upsert, not append). assert dim.filter(F.col("channel_id") == ch2.get_id()).count() == 1 + + +def test_incremental_identity_reorder_does_not_reprocess(spark): + # Seed gold with identity keys in one order. + r1 = Report( + name="calc_channel_report", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(_config(is_enabled=False)), + ) + ch1 = _add_channel(r1, factor=3.6, identity={"channel_name": "speed_kmh", "data_key": "CALC"}) + r1.determine_report() + r1.persist_results() + count_before = spark.read.table(_FACT).count() + hash_before = ( + spark.read.table(_DIM) + .filter(F.col("channel_id") == ch1.get_id()) + .select("definition_hash") + .first()[0] + ) + + # Re-run incrementally with the SAME identity but reversed key insertion order + # and the same expression. This must NOT be seen as a definition change. + r2 = Report( + name="calc_channel_report", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(_config(is_enabled=True)), + ) + ch2 = _add_channel(r2, factor=3.6, identity={"data_key": "CALC", "channel_name": "speed_kmh"}) + assert ch2.get_id() == ch1.get_id() + r2.determine_report() + r2.persist_results() + + # Stable hash → classified unchanged → idempotent upsert, no row growth. + hash_after = ( + spark.read.table(_DIM) + .filter(F.col("channel_id") == ch2.get_id()) + .select("definition_hash") + .first()[0] + ) + assert hash_after == hash_before + assert spark.read.table(_FACT).count() == count_before diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py index c3e4b449..52bf0c83 100644 --- a/tests/impulse_reporting/unit/channels/calculated_channel_test.py +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -94,6 +94,26 @@ def test_definition_hash_changes_with_expression(self): b = CalculatedChannel("a", TimeSeriesSelector(None) * 2.0, dict(_IDENTITY)) assert a.determine_definition_hash() != b.determine_definition_hash() + def test_definition_hash_independent_of_identity_key_order(self): + # Same identity, different key insertion order → same hash (no spurious + # "changed" classification in incremental runs). + a = CalculatedChannel( + "a", TimeSeriesSelector(None) * 3.6, {"channel_name": "s", "data_key": "CALC"} + ) + b = CalculatedChannel( + "a", TimeSeriesSelector(None) * 3.6, {"data_key": "CALC", "channel_name": "s"} + ) + assert a.determine_definition_hash() == b.determine_definition_hash() + + def test_definition_hash_changes_with_identity_value(self): + a = CalculatedChannel( + "a", TimeSeriesSelector(None) * 3.6, {"channel_name": "s", "data_key": "CALC"} + ) + b = CalculatedChannel( + "a", TimeSeriesSelector(None) * 3.6, {"channel_name": "other", "data_key": "CALC"} + ) + assert a.determine_definition_hash() != b.determine_definition_hash() + class TestDetermineCalculatedChannels: def test_returns_none_when_empty(self, spark): diff --git a/tests/impulse_reporting/unit/core/report_incremental_test.py b/tests/impulse_reporting/unit/core/report_incremental_test.py index a31b19b2..8b8bf9e8 100644 --- a/tests/impulse_reporting/unit/core/report_incremental_test.py +++ b/tests/impulse_reporting/unit/core/report_incremental_test.py @@ -920,3 +920,49 @@ def test_persist_full_skips_when_all_dfs_none(self, spark): report._persist_full() mock_writer.write.assert_not_called() + + +# ============================================================================ +# Tests: duplicate calculated-channel identity rejection +# ============================================================================ +class TestDuplicateCalculatedChannelIdentities: + """determine_report must reject two calculated channels sharing an identity.""" + + @staticmethod + def _channel(name, identity): + from impulse_query_engine.analyze.metadata.time_series_expression import ( + TimeSeriesSelector, + ) + from impulse_reporting.channels.calculated_channel import CalculatedChannel + + return CalculatedChannel(name=name, expr=TimeSeriesSelector(None) * 3.6, identity=identity) + + def test_duplicate_identity_raises_from_determine_report(self, spark): + report = _build_report(spark) + identity = {"channel_name": "speed_kmh", "data_key": "CALC"} + # Same identity, different key order — must still be detected as duplicate. + report.calculated_channels = [ + self._channel("a", dict(identity)), + self._channel("b", {"data_key": "CALC", "channel_name": "speed_kmh"}), + ] + + with ( + patch.object(report, "_gold_layer_exists", return_value=True), + patch.object(report, "_group_events_by_type", return_value={}), + patch.object(report, "_group_aggregations_by_type", return_value={}), + patch( + "impulse_reporting.core.report.ContainerDimension.get_dimension", + return_value=None, + ), + pytest.raises(ValueError, match="unique identities"), + ): + report.determine_report(is_incremental=False) + + def test_distinct_identities_do_not_raise(self, spark): + report = _build_report(spark) + report.calculated_channels = [ + self._channel("a", {"channel_name": "speed_kmh", "data_key": "CALC"}), + self._channel("b", {"channel_name": "rpm_x2", "data_key": "CALC"}), + ] + # Should not raise (validation passes); call the guard directly. + report._validate_unique_calculated_channels() diff --git a/tests/impulse_reporting/unit/incremental/definition_hash_comparator_test.py b/tests/impulse_reporting/unit/incremental/definition_hash_comparator_test.py index 483307ea..0d20e999 100644 --- a/tests/impulse_reporting/unit/incremental/definition_hash_comparator_test.py +++ b/tests/impulse_reporting/unit/incremental/definition_hash_comparator_test.py @@ -320,3 +320,79 @@ def test_table_exists_returns_false_for_nonexistent_table(spark): result = comparator._table_exists("nonexistent_catalog.nonexistent_schema.nonexistent_table") assert result is False + + +# --------------------------------------------------------------------------- +# Calculated channel variant (group_calculated_channels_by_hash_change) +# --------------------------------------------------------------------------- +def _calc_channel(name, factor=3.6, identity=None): + from impulse_reporting.channels.calculated_channel import CalculatedChannel + + return CalculatedChannel( + name=name, + expr=TimeSeriesSelector(None) * factor, + identity=identity or {"channel_name": name, "data_key": "CALC"}, + ) + + +def test_all_calculated_channels_changed_when_table_does_not_exist(spark): + """All channels are changed when the gold dimension table doesn't exist.""" + comparator = DefinitionHashComparator(spark) + channels = [_calc_channel("speed_kmh"), _calc_channel("rpm_x2")] + + changed, unchanged = comparator.group_calculated_channels_by_hash_change( + channels=channels, + dimension_table="nonexistent_catalog.nonexistent_schema.nonexistent_table", + ) + + assert len(changed) == 2 + assert len(unchanged) == 0 + + +def test_unchanged_calculated_channel_with_matching_hash(spark): + """A channel whose stored hash matches is marked unchanged.""" + comparator = DefinitionHashComparator(spark) + ch = _calc_channel("speed_kmh") + + dimension_data = [Row(channel_id=ch.get_id(), definition_hash=ch.determine_definition_hash())] + spark.createDataFrame(dimension_data).write.mode("overwrite").saveAsTable( + "spark_catalog.default.test_calc_channel_dim_match" + ) + + try: + changed, unchanged = comparator.group_calculated_channels_by_hash_change( + channels=[ch], + dimension_table="spark_catalog.default.test_calc_channel_dim_match", + ) + assert changed == [] + assert unchanged == [ch] + finally: + spark.sql("DROP TABLE IF EXISTS spark_catalog.default.test_calc_channel_dim_match") + + +def test_changed_calculated_channel_with_different_hash(spark): + """A channel whose expression changed (different hash) is marked changed.""" + comparator = DefinitionHashComparator(spark) + ch = _calc_channel("speed_kmh", factor=3.7) + + # Store a stale hash (as if the channel was previously defined with * 3.6). + stale = _calc_channel("speed_kmh", factor=3.6) + dimension_data = [ + Row(channel_id=stale.get_id(), definition_hash=stale.determine_definition_hash()) + ] + spark.createDataFrame(dimension_data).write.mode("overwrite").saveAsTable( + "spark_catalog.default.test_calc_channel_dim_changed" + ) + + try: + changed, unchanged = comparator.group_calculated_channels_by_hash_change( + channels=[ch], + dimension_table="spark_catalog.default.test_calc_channel_dim_changed", + ) + # Same identity → same channel_id as the stored row, but the expression + # hash differs, so it must be classified as changed. + assert ch.get_id() == stale.get_id() + assert changed == [ch] + assert unchanged == [] + finally: + spark.sql("DROP TABLE IF EXISTS spark_catalog.default.test_calc_channel_dim_changed") From 0d0ef710163cd9423518fc3d63299b21c4eccbe5 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Fri, 17 Jul 2026 10:40:25 +0200 Subject: [PATCH 06/23] Refactor identity handling in CalculatedChannel and QueryBuilder - Updated the identity handling in CalculatedChannel to allow arbitrary keys, emitting the entire identity as a single VARIANT column. - Modified QueryBuilder and DefaultSolver to accommodate the new identity structure, ensuring compatibility with heterogeneous identity keys across calculated channels. - Enhanced output schema definitions to reflect the changes in identity representation. - Updated unit and integration tests to validate the new identity handling and ensure correct behavior across various scenarios. --- .../query/channels/calculated_channel.py | 16 ++--- .../analyze/query/query_builder.py | 26 +++---- .../analyze/query/solvers/default_solver.py | 42 +++++------ .../analyze/query/solvers/query_solver.py | 57 +++++++-------- .../channels/calculated_channel.py | 54 +++++++++----- .../persist/dimension_schema.py | 9 +-- src/impulse_reporting/persist/fact_schema.py | 11 +-- ...default_solver_calculated_channels_test.py | 71 ++++++++++++------ .../integration/calculated_channel_test.py | 72 +++++++++++++++++-- .../unit/channels/calculated_channel_test.py | 24 ++++--- 10 files changed, 242 insertions(+), 140 deletions(-) diff --git a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py index 0af43ce3..4df4dd15 100644 --- a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py +++ b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py @@ -26,19 +26,19 @@ class CalculatedChannel(TimeSeriesExpression): it is a plain ``TimeSeriesExpression`` rather than an ``Aggregation``. :meth:`QueryBuilder.solve_calculated_channels` explodes that series into narrow rows matching the silver ``channel_data`` shape - (``container_id, channel_id, tstart, tend, value``) plus one string column - per identity entry. + (``container_id, channel_id, tstart, tend, value``) plus a single + ``identity`` ``VARIANT`` column holding the whole identity dict. Parameters ---------- expr : TimeSeriesExpression The wrapped expression; must ``build()`` to a ``SampleSeries``. identity : dict of str - Identity columns for the output rows, e.g. - ``{"channel_name": "Eng_RPM", "data_key": "TM"}``. Each key becomes a - ``StringType`` output column and each value is emitted as a literal on - every row of this channel. Must be non-empty; the identity also seeds - the deterministic ``channel_id`` hash. + Identity for the output rows, e.g. + ``{"channel_name": "Eng_RPM", "data_key": "TM"}``. The whole dict is + emitted per row in a single ``identity`` ``VARIANT`` column (keys are + arbitrary). Must be non-empty; the identity also seeds the + deterministic ``channel_id`` hash. channel_id : int or None, optional Output ``channel_id`` for every emitted row. When omitted (the ``_AUTO`` sentinel) a deterministic id is derived from the identity by @@ -52,7 +52,7 @@ class CalculatedChannel(TimeSeriesExpression): chain ``.alias(...)``: reading a missing ``_alias`` would trigger ``TimeSeriesExpression.__getattr__`` and silently return a callable instead of raising. The alias is not consumed by the calculated-channels solve path - (that emits the identity columns directly); it exists so the object stays a + (that emits the identity column directly); it exists so the object stays a well-formed ``TimeSeriesExpression``. A later ``.alias(...)`` still overrides it. diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 7287387f..05e78418 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -294,8 +294,8 @@ def solve_calculated_channels( metadata filter pipeline as :meth:`solve` (resolving the input channels each calculated channel depends on), then evaluates each calculated channel per container and emits rows in the silver ``channel_data`` shape - — ``container_id, channel_id, tstart, tend, value`` — plus one string - column per identity kwarg shared by the selections. + — ``container_id, channel_id, tstart, tend, value`` — plus a single + ``identity`` ``VARIANT`` column holding each channel's identity dict. Parameters ---------- @@ -313,14 +313,13 @@ def solve_calculated_channels( ------- pyspark.sql.DataFrame Narrow DataFrame ``[container_id, channel_id, tstart, tend, value, - ]``. + identity]``. Raises ------ ValueError - If any selection is not a ``CalculatedChannel``, if the selections - declare inconsistent identity-key sets, or if a wrapped expression - does not evaluate to a ``SampleSeries``. + If any selection is not a ``CalculatedChannel``, or if a wrapped + expression does not evaluate to a ``SampleSeries``. """ self._validate_calculated_channels() @@ -331,9 +330,10 @@ def solve_calculated_channels( def _validate_calculated_channels(self) -> None: """Validate the selections for :meth:`solve_calculated_channels`. - Every selection must be a ``CalculatedChannel``, all must declare the - same set of identity keys (so the narrow output has a single, stable - schema), and each wrapped expression must evaluate to a ``SampleSeries``. + Every selection must be a ``CalculatedChannel`` and each wrapped + expression must evaluate to a ``SampleSeries``. Identity key sets need + not match across selections — the identity is emitted as a single + self-describing ``VARIANT`` column, so heterogeneous keys are fine. """ if not self.selections: raise ValueError( @@ -347,14 +347,6 @@ def _validate_calculated_channels(self) -> None: f"CalculatedChannel; got {type(s).__name__} at index {i}." ) - key_sets = {frozenset(s.identity) for s in self.selections} - if len(key_sets) > 1: - rendered = ", ".join(sorted(str(sorted(ks)) for ks in key_sets)) - raise ValueError( - "All CalculatedChannels in one solve_calculated_channels() call must " - f"declare the same identity keys; got differing key sets: {rendered}." - ) - for s in self.selections: s.expr.require_evaluation_type( SampleSeries, diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index afd9fa0f..dac6edac 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -1004,7 +1004,7 @@ def _calculated_channel_id(cc, channel_id_dtype: T.DataType): @staticmethod def _solve_calculated_channels_udf( - pdf, selections: Iterable, col_map: dict[str, str], identity_keys, channel_ids + pdf, selections: Iterable, col_map: dict[str, str], channel_ids ) -> pd.DataFrame: """ UDF to solve calculated channels for a single container. @@ -1012,7 +1012,9 @@ def _solve_calculated_channels_udf( Unlike :meth:`_solve_udf` (one wide row per container), this emits many narrow rows: each ``CalculatedChannel`` builds to a ``SampleSeries`` that is exploded into ``(container_id, channel_id, tstart, tend, value)`` rows, - with the channel's identity values attached as string columns. + with the channel's whole identity dict attached in a single ``identity`` + map column. The map is converted to a ``VARIANT`` by the caller (a pandas + UDF cannot emit ``VARIANT`` directly, since it is not Arrow-representable). Parameters ---------- @@ -1022,8 +1024,6 @@ def _solve_calculated_channels_udf( The ``CalculatedChannel`` selections to evaluate. col_map : dict[str, str] Column-name mapping for the cache (cid/ch/ts/te/val/conv). - identity_keys : Iterable[str] - Identity column names shared by the selections (sorted upstream). channel_ids : list Output ``channel_id`` per selection, positionally paired with *selections* (precomputed on the driver). @@ -1039,9 +1039,7 @@ def _solve_calculated_channels_udf( ch_col = col_map["ch"] container_id = pdf[cid_col].iloc[0] - cols = {cid_col: [], ch_col: [], "tstart": [], "tend": [], "value": []} - for key in identity_keys: - cols[key] = [] + cols = {cid_col: [], ch_col: [], "tstart": [], "tend": [], "value": [], "identity": []} for cc, ch_id in zip(selections, channel_ids, strict=False): series = cc.build(cache) @@ -1051,8 +1049,7 @@ def _solve_calculated_channels_udf( cols["tstart"].append(int(tstart)) cols["tend"].append(int(tend)) cols["value"].append(float(value)) - for key in identity_keys: - cols[key].append(cc.identity.get(key)) + cols["identity"].append(dict(cc.identity)) return pd.DataFrame(cols) @@ -1064,7 +1061,8 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame same channel-data read and ``[container_id, channel_id]`` join — but the grouped-map UDF emits narrow silver-shaped rows (many per container) instead of one wide row. Output columns are - ``[container_id, channel_id, tstart, tend, value, ]``. + ``[container_id, channel_id, tstart, tend, value, identity]`` where + ``identity`` is a ``VARIANT`` holding the channel's identity dict. Parameters ---------- @@ -1100,19 +1098,17 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame q = self._apply_column_mapping(q, self.config.channels.column_name_mapping) if self.is_raw_data: - q = self.interval_encoder.prepare_channels_df(q) + q = self.channel_encoder.prepare_channels_df(q) - identity_keys = sorted(set().union(*[set(cc.identity) for cc in selections])) channel_id_dtype = q.schema[self.config.channel_id_col].dataType channel_ids = [self._calculated_channel_id(cc, channel_id_dtype) for cc in selections] - schema = self._build_calculated_channels_output_schema(q, identity_keys) + schema = self._build_calculated_channels_output_schema(q) solve_udf = F.pandas_udf( partial( DefaultSolver._solve_calculated_channels_udf, selections=selections, col_map=col_map, - identity_keys=identity_keys, channel_ids=channel_ids, ), schema, @@ -1124,10 +1120,14 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame container_count = channels_df.select(self.config.container_id_col).distinct().count() if container_count == 0: - return self.spark.createDataFrame([], schema=schema) - res = ( - df.repartition(container_count, self.config.container_id_col) - .groupBy(self.config.container_id_col) - .apply(solve_udf) - ) - return res + res = self.spark.createDataFrame([], schema=schema) + else: + res = ( + df.repartition(container_count, self.config.container_id_col) + .groupBy(self.config.container_id_col) + .apply(solve_udf) + ) + # The grouped-map UDF emits ``identity`` as a MapType column (Arrow-safe); + # convert it to a self-describing VARIANT here so both the populated and + # empty branches return an identical schema. + return res.withColumn("identity", F.to_variant_object(F.col("identity"))) diff --git a/src/impulse_query_engine/analyze/query/solvers/query_solver.py b/src/impulse_query_engine/analyze/query/solvers/query_solver.py index 66cac6fa..08189ec4 100644 --- a/src/impulse_query_engine/analyze/query/solvers/query_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/query_solver.py @@ -99,48 +99,49 @@ def _build_solve_output_schema(self, channels: DataFrame, selections, dtypes) -> entries.append(T.StructField(s._alias, dtype)) return T.StructType(entries) - def _build_calculated_channels_output_schema( - self, channels: DataFrame, identity_keys - ) -> T.StructType: + def _build_calculated_channels_output_schema(self, channels: DataFrame) -> T.StructType: """Build the narrow grouped-map output schema for calculated channels. The output mirrors the silver ``channel_data`` table - (``container_id, channel_id, tstart, tend, value``) plus one - ``StringType`` identity column per key in *identity_keys* (emitted in - sorted order for a stable schema). The ``container_id`` and - ``channel_id`` field types are derived from *channels* (the - column-mapped channels DataFrame) so they match the physical source - types instead of being hardcoded. + (``container_id, channel_id, tstart, tend, value``) plus a single + ``identity`` column holding each channel's identity dict as a + ``MapType(string, string)``. This is the **grouped-map UDF** schema; the + map is Arrow-representable so the pandas UDF can emit it directly. The + solver converts the map to a ``VARIANT`` on the returned DataFrame (via + :func:`pyspark.sql.functions.to_variant_object`), so the identity is + self-describing and no gold column depends on the identity key set. + + The ``container_id`` and ``channel_id`` field types are derived from + *channels* (the column-mapped channels DataFrame) so they match the + physical source types instead of being hardcoded. Parameters ---------- channels : pyspark.sql.DataFrame The column-mapped channels DataFrame whose ``container_id`` / ``channel_id`` column types are authoritative. - identity_keys : Iterable[str] - Identity column names shared by the calculated channels. Returns ------- pyspark.sql.types.StructType - ``[container_id, channel_id, tstart, tend, value, ]``. + ``[container_id, channel_id, tstart, tend, value, identity]``. """ - entries = [ - T.StructField( - self.config.container_id_col, - channels.schema[self.config.container_id_col].dataType, - ), - T.StructField( - self.config.channel_id_col, - channels.schema[self.config.channel_id_col].dataType, - ), - T.StructField(self.config.tstart_col, T.LongType()), - T.StructField(self.config.tend_col, T.LongType()), - T.StructField(self.config.value_col, T.DoubleType()), - ] - for key in sorted(identity_keys): - entries.append(T.StructField(key, T.StringType())) - return T.StructType(entries) + return T.StructType( + [ + T.StructField( + self.config.container_id_col, + channels.schema[self.config.container_id_col].dataType, + ), + T.StructField( + self.config.channel_id_col, + channels.schema[self.config.channel_id_col].dataType, + ), + T.StructField(self.config.tstart_col, T.LongType()), + T.StructField(self.config.tend_col, T.LongType()), + T.StructField(self.config.value_col, T.DoubleType()), + T.StructField("identity", T.MapType(T.StringType(), T.StringType())), + ] + ) def _empty_channel_match_df(self, spark, db: MeasurementDB) -> DataFrame: """Return an empty ``(container_id, channel_id, selector_ids)`` DataFrame. diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py index 3afb4d1b..7329fe87 100644 --- a/src/impulse_reporting/channels/calculated_channel.py +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -5,6 +5,7 @@ from collections.abc import Mapping import pyspark.sql.functions as f +import pyspark.sql.types as T from pyspark.sql import DataFrame, Row, SparkSession from impulse_query_engine.analyze.metadata.time_series_expression import ( @@ -19,11 +20,6 @@ from impulse_reporting.persist.dimension_schema import CALCULATED_CHANNEL_DIMENSION_SCHEMA from impulse_reporting.persist.fact_schema import CALCULATED_CHANNEL_FACT_SCHEMA -# Identity keys required on every reporting-layer calculated channel. Fixed so the -# gold fact table has a single, stable schema whose identity columns are named -# ``channel_name`` and ``data_key`` (see CALCULATED_CHANNEL_FACT_SCHEMA). -_REQUIRED_IDENTITY_KEYS = {"channel_name", "data_key"} - class CalculatedChannel: """A reporting-layer calculated (derived) channel. @@ -48,9 +44,9 @@ class CalculatedChannel: expr : TimeSeriesExpression The wrapped expression; must evaluate to a ``SampleSeries``. identity : Mapping[str, str] - Output identity columns. Must contain exactly the keys - ``{"channel_name", "data_key"}``; the values are emitted as literals on - every fact row and seed the deterministic ``channel_id``. + Output identity. Any non-empty set of keys; the whole dict is emitted + per fact row in a single ``identity`` ``VARIANT`` column and seeds the + deterministic ``channel_id``. desc : str, optional Human-readable description (stored on the dimension row, excluded from the definition hash). @@ -77,11 +73,11 @@ def __init__( ) self.identity = {str(k): str(v) for k, v in identity.items()} - if set(self.identity) != _REQUIRED_IDENTITY_KEYS: + if not self.identity: raise ValueError( - "CalculatedChannel identity must contain exactly the keys " - f"{sorted(_REQUIRED_IDENTITY_KEYS)}; got {sorted(self.identity)}. " - "These become the fixed identity columns of the gold fact table." + "CalculatedChannel requires a non-empty identity dict " + "(e.g. {'channel_name': 'speed_kmh', 'data_key': 'CALC'}); it defines " + "the output identity and seeds the deterministic channel_id." ) normalized_attributes: dict[str, str] = {} @@ -139,19 +135,22 @@ def get_channel_type_str(self) -> str: return "CALCULATED_CHANNEL" def determine_definition_hash(self) -> int: - """Hash of the computation-affecting definition (expression + identity). - """ + """Hash of the computation-affecting definition (expression + identity).""" payload = f"{self._canonical_identity()}|{self.expression.expr}" hash_bytes = hashlib.sha256(payload.encode()).digest() return int.from_bytes(hash_bytes[:8], byteorder="big", signed=True) def as_dict(self) -> dict: - """Dictionary representation matching ``CALCULATED_CHANNEL_DIMENSION_SCHEMA``.""" + """Dictionary representation of the dimension metadata. + + ``identity`` is emitted as a plain dict here; ``determine_metadata_df`` + converts it to a ``VARIANT`` column so the dimension identity mirrors the + fact identity (no fixed per-key columns). + """ return { "channel_id": self.get_id(), "report_id": self.report_id, "channel_type": self.get_channel_type_str(), - "channel_name": self.identity.get("channel_name"), "channel_description": self.description, "channel_expression": self.get_expression_str(), "identity": self.identity, @@ -186,7 +185,7 @@ def determine_calculated_channels( spark : SparkSession Spark session (unused directly; kept for interface parity). channels : list of CalculatedChannel - Channels to solve; all share the same identity key set. + Channels to solve; identity keys may differ across channels. query : QueryBuilder Query builder used to select and solve the channels. solver : QuerySolver @@ -210,6 +209,23 @@ def determine_calculated_channels( @classmethod def determine_metadata_df(cls, spark: SparkSession, channels: list[CalculatedChannel]): - """Create the dimension DataFrame for the given channels.""" + """Create the dimension DataFrame for the given channels. + + ``identity`` in ``CALCULATED_CHANNEL_DIMENSION_SCHEMA`` is a ``VARIANT``, + which ``createDataFrame`` cannot build from a Python dict directly. So + the frame is staged with ``identity`` as a ``MapType`` (matching the plain + dict from :meth:`as_dict`) and then converted to ``VARIANT`` via + :func:`pyspark.sql.functions.to_variant_object` — mirroring the fact-side + conversion in the solver. + """ rows = [channel.as_spark_row() for channel in channels] - return spark.createDataFrame(rows, schema=CALCULATED_CHANNEL_DIMENSION_SCHEMA) + staging_fields = [ + ( + T.StructField("identity", T.MapType(T.StringType(), T.StringType())) + if field.name == "identity" + else field + ) + for field in CALCULATED_CHANNEL_DIMENSION_SCHEMA.fields + ] + df = spark.createDataFrame(rows, schema=T.StructType(staging_fields)) + return df.withColumn("identity", f.to_variant_object(f.col("identity"))) diff --git a/src/impulse_reporting/persist/dimension_schema.py b/src/impulse_reporting/persist/dimension_schema.py index ecf89f8c..04030eab 100644 --- a/src/impulse_reporting/persist/dimension_schema.py +++ b/src/impulse_reporting/persist/dimension_schema.py @@ -7,6 +7,7 @@ StringType, StructField, StructType, + VariantType, ) HISTOGRAM_DIMENSION_SCHEMA = StructType( @@ -83,17 +84,17 @@ # Metadata for calculated channels. ``channel_id`` is the deterministic entity id # (identical to the fact's channel_id), ``definition_hash`` drives incremental -# reprocessing, and ``identity`` retains the full identity dict (the fact keeps -# only the fixed identity columns). +# reprocessing, and ``identity`` is a VARIANT holding the full identity dict — +# the same self-describing representation as the fact table (no fixed per-key +# columns). CALCULATED_CHANNEL_DIMENSION_SCHEMA = StructType( [ StructField("channel_id", LongType(), False), StructField("report_id", IntegerType(), False), StructField("channel_type", StringType(), True), - StructField("channel_name", StringType(), True), StructField("channel_description", StringType(), True), StructField("channel_expression", StringType(), True), - StructField("identity", MapType(StringType(), StringType()), True), + StructField("identity", VariantType(), True), StructField("definition_hash", LongType(), True), StructField("attributes", MapType(StringType(), StringType()), True), ] diff --git a/src/impulse_reporting/persist/fact_schema.py b/src/impulse_reporting/persist/fact_schema.py index cdd21b18..6c80718f 100644 --- a/src/impulse_reporting/persist/fact_schema.py +++ b/src/impulse_reporting/persist/fact_schema.py @@ -5,6 +5,7 @@ StringType, StructField, StructType, + VariantType, ) HISTOGRAM_FACT_SCHEMA = StructType( @@ -60,9 +61,10 @@ ) # Narrow, silver-shaped facts for calculated channels. Mirrors the query engine's -# solve_calculated_channels() output: one row per RLE sample interval, plus the -# identity columns. Field *types* are cosmetic — persistence projects by name only -# and the real container_id/channel_id types flow from the solved DataFrame. +# solve_calculated_channels() output: one row per RLE sample interval, plus a single +# ``identity`` VARIANT column holding the channel's identity dict (no fixed per-key +# columns). Field *types* are cosmetic — persistence projects by name only and the +# real container_id/channel_id/identity types flow from the solved DataFrame. CALCULATED_CHANNEL_FACT_SCHEMA = StructType( [ StructField("container_id", IntegerType(), False), @@ -70,7 +72,6 @@ StructField("tstart", LongType(), False), StructField("tend", LongType(), False), StructField("value", DoubleType(), False), - StructField("channel_name", StringType(), False), - StructField("data_key", StringType(), False), + StructField("identity", VariantType(), False), ] ) diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py index 311c1554..b648be46 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py @@ -27,6 +27,13 @@ _C1_RPM_VALUE = 1081.0 +def _id_str(col="identity", key=None): + """Extract identity as a JSON string, or a single key's string value, from VARIANT.""" + if key is None: + return F.to_json(F.col(col)) + return F.variant_get(F.col(col), f"$.{key}", "string") + + def _recast_container_id(db: MeasurementDB, cid_type: T.DataType) -> MeasurementDB: """Clone a ``for_debug`` db, casting ``container_id`` on every table.""" tables = { @@ -49,13 +56,19 @@ def test_scaling_produces_real_values(self, spark, basic_narrow_db): ) result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - row = result.filter( - (F.col("container_id") == 1) & (F.col("tstart") == _C1_RPM_TSTART) - ).collect() + row = ( + result.filter((F.col("container_id") == 1) & (F.col("tstart") == _C1_RPM_TSTART)) + .select( + "value", + _id_str(key="channel_name").alias("cn"), + _id_str(key="data_key").alias("dk"), + ) + .collect() + ) assert len(row) == 1 assert row[0]["value"] == pytest.approx(_C1_RPM_VALUE * 2) - assert row[0]["channel_name"] == "rpm_x2" - assert row[0]["data_key"] == "CALC" + assert row[0]["cn"] == "rpm_x2" + assert row[0]["dk"] == "CALC" def test_all_rows_carry_identity(self, spark, basic_narrow_db): q = basic_narrow_db.query @@ -64,12 +77,25 @@ def test_all_rows_carry_identity(self, spark, basic_narrow_db): {"channel_name": "rpm_plus_1", "data_key": "CALC"}, ) result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - distinct_names = {r["channel_name"] for r in result.select("channel_name").collect()} - distinct_keys = {r["data_key"] for r in result.select("data_key").collect()} - assert distinct_names == {"rpm_plus_1"} - assert distinct_keys == {"CALC"} + distinct = {r["j"] for r in result.select(_id_str().alias("j")).collect()} + assert distinct == {'{"channel_name":"rpm_plus_1","data_key":"CALC"}'} assert result.count() > 0 + def test_arbitrary_identity_keys_round_trip(self, spark, basic_narrow_db): + # Identity is a self-describing VARIANT, so non-{channel_name,data_key} keys work. + q = basic_narrow_db.query + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, + {"sensor_id": "s1", "unit": "rpm"}, + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + assert result.schema["identity"].dataType == T.VariantType() + row = result.select( + _id_str(key="sensor_id").alias("sid"), _id_str(key="unit").alias("unit") + ).first() + assert row["sid"] == "s1" + assert row["unit"] == "rpm" + def test_multi_channel_sync_matches_core_model(self, spark, basic_narrow_db): """A two-channel sum matches an in-process SampleSeries synchronization.""" q = basic_narrow_db.query @@ -113,19 +139,18 @@ def test_output_columns_and_order(self, spark, basic_narrow_db): {"channel_name": "rpm_x2", "data_key": "CALC"}, ) result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - # Identity columns appear in sorted order after the fixed silver columns. + # Identity is a single self-describing VARIANT column after the silver columns. assert result.columns == [ "container_id", "channel_id", "tstart", "tend", "value", - "channel_name", - "data_key", + "identity", ] assert result.schema["tstart"].dataType == T.LongType() assert result.schema["value"].dataType == T.DoubleType() - assert result.schema["channel_name"].dataType == T.StringType() + assert result.schema["identity"].dataType == T.VariantType() @pytest.mark.parametrize( "cid_type", [T.StringType(), T.IntegerType(), T.LongType()], ids=lambda t: t.simpleString() @@ -163,8 +188,10 @@ def test_distinct_identities_distinct_ids(self, spark, basic_narrow_db): cc_b = CalculatedChannel(q.channel(channel_name="Engine RPM") * 3, {"channel_name": "b"}) result = q.select(cc_a, cc_b).solve_calculated_channels(spark, solver=DefaultSolver(spark)) by_name = { - r["channel_name"]: r["channel_id"] - for r in result.select("channel_name", "channel_id").distinct().collect() + r["cn"]: r["channel_id"] + for r in result.select(_id_str(key="channel_name").alias("cn"), "channel_id") + .distinct() + .collect() } assert by_name["a"] != by_name["b"] @@ -203,15 +230,16 @@ def test_aggregation_rejected(self, spark, basic_narrow_db): with pytest.raises(ValueError, match="requires all selections to be"): q.solve_calculated_channels(spark, solver=DefaultSolver(spark)) - def test_mismatched_identity_keys_rejected(self, spark, basic_narrow_db): + def test_mismatched_identity_keys_allowed(self, spark, basic_narrow_db): + # Identity is a self-describing VARIANT, so heterogeneous key sets are fine. q = basic_narrow_db.query cc_a = CalculatedChannel(q.channel(channel_name="Engine RPM") * 2, {"channel_name": "a"}) cc_b = CalculatedChannel( q.channel(channel_name="Engine RPM") * 3, {"channel_name": "b", "data_key": "CALC"} ) - q.select(cc_a, cc_b) - with pytest.raises(ValueError, match="same identity keys"): - q.solve_calculated_channels(spark, solver=DefaultSolver(spark)) + result = q.select(cc_a, cc_b).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + assert result.schema["identity"].dataType == T.VariantType() + assert result.count() > 0 class TestEmptyResults: @@ -229,9 +257,10 @@ def test_no_matching_channels_empty_frame(self, spark, basic_narrow_db): "tstart", "tend", "value", - "channel_name", - "data_key", + "identity", ] + # Empty branch funnels through the same parse → identity is VARIANT too. + assert result.schema["identity"].dataType == T.VariantType() def test_base_solver_not_supported(self, spark, basic_narrow_db): from impulse_query_engine.analyze.query.solvers.blob_solver import BlobSolver diff --git a/tests/impulse_reporting/integration/calculated_channel_test.py b/tests/impulse_reporting/integration/calculated_channel_test.py index 84742cbf..374b3ed5 100644 --- a/tests/impulse_reporting/integration/calculated_channel_test.py +++ b/tests/impulse_reporting/integration/calculated_channel_test.py @@ -10,18 +10,21 @@ from unittest.mock import create_autospec import pyspark.sql.functions as F +import pyspark.sql.types as T import pytest from databricks.sdk import WorkspaceClient from impulse_reporting.channels.calculated_channel import CalculatedChannel from impulse_reporting.config.config_parser import ( + DataType, ImpulseConfig, IncrementalConfig, + QueryEngine, Source, UnitySink, ) from impulse_reporting.core.report import Report -from tests.conftest import spark # noqa: F401 (pytest fixture) +from tests.conftest import setup_raw_channels_db, spark # noqa: F401 (pytest fixtures) _FACT = "spark_catalog.gold.evaluation_calculated_channel_fact" _DIM = "spark_catalog.gold.evaluation_calculated_channel_dimension" @@ -78,11 +81,24 @@ def test_persist_calculated_channel_full(spark): # channel_id in the fact matches the reporting entity id. ids = {r["channel_id"] for r in fact.select("channel_id").distinct().collect()} assert ids == {ch.get_id()} - # identity columns carried through. - assert {r["channel_name"] for r in fact.select("channel_name").distinct().collect()} == { - "speed_kmh" + # identity is a single VARIANT column carrying the whole identity dict. + assert fact.schema["identity"].dataType == T.VariantType() + id_names = { + r["cn"] + for r in fact.select( + F.variant_get(F.col("identity"), "$.channel_name", "string").alias("cn") + ) + .distinct() + .collect() + } + id_keys = { + r["dk"] + for r in fact.select(F.variant_get(F.col("identity"), "$.data_key", "string").alias("dk")) + .distinct() + .collect() } - assert {r["data_key"] for r in fact.select("data_key").distinct().collect()} == {"CALC"} + assert id_names == {"speed_kmh"} + assert id_keys == {"CALC"} # Values are the derived signal: compare against the raw source scaled by 3.6. raw = ( @@ -101,7 +117,12 @@ def test_persist_calculated_channel_full(spark): assert calc_sum == pytest.approx(raw_sum * 3.6) dim = spark.read.table(_DIM) - assert dim.filter(F.col("channel_name") == "speed_kmh").count() == 1 + assert ( + dim.filter( + F.variant_get(F.col("identity"), "$.channel_name", "string") == "speed_kmh" + ).count() + == 1 + ) assert dim.filter(F.col("definition_hash").isNotNull()).count() == 1 @@ -212,3 +233,42 @@ def test_incremental_identity_reorder_does_not_reprocess(spark): ) assert hash_after == hash_before assert spark.read.table(_FACT).count() == count_before + + +def test_calculated_channel_raw_mode(spark, setup_raw_channels_db): + """A calculated channel solves over RAW (point-sample) silver data. + + In RAW mode the solver run-length/interval-encodes the point samples before + the calc-channel grouped-map UDF runs — exercising the ``is_raw_data`` branch + of ``DefaultSolver.solve_calculated_channels``. + """ + config = ImpulseConfig( + source=Source( + container_metrics_table="spark_catalog.silver_raw.container_metrics", + channel_metrics_table="spark_catalog.silver_raw.channel_metrics", + channels_uri="spark_catalog.silver_raw.channels", + ), + query_engine=QueryEngine(data_type=DataType.RAW), + ) + report = Report( + name="calc_channel_raw_report", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(config), + ) + q = report.get_db().query + ch = CalculatedChannel( + name="rpm_x2", + expr=q.channel(channel_name="Engine RPM") * 2, + identity={"channel_name": "rpm_x2", "data_key": "CALC"}, + ) + report.add_calculated_channel(ch) + + # This is the branch that regressed: RAW-mode solve must not raise. + df = CalculatedChannel.determine_calculated_channels( + spark, [ch], query=q, solver=report.get_solver() + ) + assert df.count() > 0 + assert df.schema["identity"].dataType == T.VariantType() + ids = {r["channel_id"] for r in df.select("channel_id").distinct().collect()} + assert ids == {ch.get_id()} diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py index 52bf0c83..39795728 100644 --- a/tests/impulse_reporting/unit/channels/calculated_channel_test.py +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -1,6 +1,7 @@ # pylint: disable=missing-function-docstring """Unit tests for the reporting-layer CalculatedChannel class.""" +import pyspark.sql.types as T import pytest from impulse_query_engine.analyze.metadata.time_series_expression import TimeSeriesSelector @@ -48,14 +49,17 @@ def test_rejects_non_sample_series_expr(self): name="bad", expr=(TimeSeriesSelector(None) > 0), identity=dict(_IDENTITY) ) - def test_rejects_wrong_identity_keys(self): - with pytest.raises(ValueError, match="identity must contain exactly"): - CalculatedChannel( - name="bad", expr=(TimeSeriesSelector(None) * 2), identity={"channel_name": "x"} - ) + def test_accepts_arbitrary_identity_keys(self): + # Identity persists as a VARIANT, so any non-empty key set is valid. + ch = CalculatedChannel( + name="ok", + expr=(TimeSeriesSelector(None) * 2), + identity={"sensor_id": "s1", "unit": "rpm"}, + ) + assert ch.identity == {"sensor_id": "s1", "unit": "rpm"} def test_rejects_empty_identity(self): - with pytest.raises(ValueError, match="identity must contain exactly"): + with pytest.raises(ValueError, match="non-empty identity"): CalculatedChannel(name="bad", expr=(TimeSeriesSelector(None) * 2), identity={}) @@ -67,7 +71,6 @@ def test_as_dict_keys_and_values(self): "channel_id", "report_id", "channel_type", - "channel_name", "channel_description", "channel_expression", "identity", @@ -77,12 +80,11 @@ def test_as_dict_keys_and_values(self): assert d["channel_id"] == ch.get_id() assert d["report_id"] == -1 assert d["channel_type"] == "CALCULATED_CHANNEL" - assert d["channel_name"] == "speed_kmh" assert d["identity"] == _IDENTITY assert isinstance(d["definition_hash"], int) def test_as_spark_row_field_count(self): - assert len(_channel().as_spark_row()) == 9 + assert len(_channel().as_spark_row()) == 8 def test_definition_hash_ignores_name_and_desc(self): a = CalculatedChannel("a", TimeSeriesSelector(None) * 3.6, dict(_IDENTITY), desc="one") @@ -138,8 +140,8 @@ def test_returns_fact_columns_with_matching_channel_id(self, spark, basic_narrow "tstart", "tend", "value", - "channel_name", - "data_key", + "identity", ] + assert df.schema["identity"].dataType == T.VariantType() ids = {r["channel_id"] for r in df.select("channel_id").distinct().collect()} assert ids == {ch.get_id()} From 967cdcab1210f02c7e3f681039447c502f74d1a9 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Fri, 17 Jul 2026 11:00:33 +0200 Subject: [PATCH 07/23] Refactor identity handling in calculated channel schemas and tests - Removed the VARIANT identity column from the CALCULATED_CHANNEL_FACT_SCHEMA, reflecting that identity is now managed in the dimension. - Updated related tests to assert the absence of the identity column in the fact schema and confirm its presence in the dimension schema. - Enhanced test assertions to validate the correct mapping of channel identities through joins, ensuring accurate data representation. --- src/impulse_reporting/persist/fact_schema.py | 12 +++-- .../integration/calculated_channel_test.py | 45 +++++++++---------- .../unit/channels/calculated_channel_test.py | 4 +- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/src/impulse_reporting/persist/fact_schema.py b/src/impulse_reporting/persist/fact_schema.py index 6c80718f..59c8d2d0 100644 --- a/src/impulse_reporting/persist/fact_schema.py +++ b/src/impulse_reporting/persist/fact_schema.py @@ -5,7 +5,6 @@ StringType, StructField, StructType, - VariantType, ) HISTOGRAM_FACT_SCHEMA = StructType( @@ -60,11 +59,11 @@ ] ) -# Narrow, silver-shaped facts for calculated channels. Mirrors the query engine's -# solve_calculated_channels() output: one row per RLE sample interval, plus a single -# ``identity`` VARIANT column holding the channel's identity dict (no fixed per-key -# columns). Field *types* are cosmetic — persistence projects by name only and the -# real container_id/channel_id/identity types flow from the solved DataFrame. +# Narrow, silver-shaped facts for calculated channels: one row per RLE sample +# interval. The channel's identity lives on ``calculated_channel_dimension`` (joined +# via ``channel_id``), so it is intentionally absent here. Field *types* are cosmetic +# — persistence projects by name only and the real container_id/channel_id types flow +# from the solved DataFrame. CALCULATED_CHANNEL_FACT_SCHEMA = StructType( [ StructField("container_id", IntegerType(), False), @@ -72,6 +71,5 @@ StructField("tstart", LongType(), False), StructField("tend", LongType(), False), StructField("value", DoubleType(), False), - StructField("identity", VariantType(), False), ] ) diff --git a/tests/impulse_reporting/integration/calculated_channel_test.py b/tests/impulse_reporting/integration/calculated_channel_test.py index 374b3ed5..4c4db95c 100644 --- a/tests/impulse_reporting/integration/calculated_channel_test.py +++ b/tests/impulse_reporting/integration/calculated_channel_test.py @@ -81,24 +81,9 @@ def test_persist_calculated_channel_full(spark): # channel_id in the fact matches the reporting entity id. ids = {r["channel_id"] for r in fact.select("channel_id").distinct().collect()} assert ids == {ch.get_id()} - # identity is a single VARIANT column carrying the whole identity dict. - assert fact.schema["identity"].dataType == T.VariantType() - id_names = { - r["cn"] - for r in fact.select( - F.variant_get(F.col("identity"), "$.channel_name", "string").alias("cn") - ) - .distinct() - .collect() - } - id_keys = { - r["dk"] - for r in fact.select(F.variant_get(F.col("identity"), "$.data_key", "string").alias("dk")) - .distinct() - .collect() - } - assert id_names == {"speed_kmh"} - assert id_keys == {"CALC"} + # Identity is NOT on the fact — it lives on the dimension, joined via channel_id. + assert "identity" not in fact.columns + assert fact.columns == ["container_id", "channel_id", "tstart", "tend", "value", "_created_at"] # Values are the derived signal: compare against the raw source scaled by 3.6. raw = ( @@ -116,14 +101,23 @@ def test_persist_calculated_channel_full(spark): calc_sum = fact.select(F.sum("value")).first()[0] assert calc_sum == pytest.approx(raw_sum * 3.6) + # The dimension carries the identity (VARIANT), keyed by channel_id — the join + # target for the fact. Confirm the fact's channel_id resolves to the identity. dim = spark.read.table(_DIM) - assert ( - dim.filter( - F.variant_get(F.col("identity"), "$.channel_name", "string") == "speed_kmh" - ).count() - == 1 + assert dim.schema["identity"].dataType == T.VariantType() + dim_row = ( + dim.filter(F.col("channel_id") == ch.get_id()) + .select( + F.variant_get(F.col("identity"), "$.channel_name", "string").alias("cn"), + F.variant_get(F.col("identity"), "$.data_key", "string").alias("dk"), + "definition_hash", + ) + .collect() ) - assert dim.filter(F.col("definition_hash").isNotNull()).count() == 1 + assert len(dim_row) == 1 + assert dim_row[0]["cn"] == "speed_kmh" + assert dim_row[0]["dk"] == "CALC" + assert dim_row[0]["definition_hash"] is not None def test_incremental_unchanged_is_idempotent(spark): @@ -269,6 +263,7 @@ def test_calculated_channel_raw_mode(spark, setup_raw_channels_db): spark, [ch], query=q, solver=report.get_solver() ) assert df.count() > 0 - assert df.schema["identity"].dataType == T.VariantType() + # Identity is dimension-only; the fact projection carries just the silver columns. + assert "identity" not in df.columns ids = {r["channel_id"] for r in df.select("channel_id").distinct().collect()} assert ids == {ch.get_id()} diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py index 39795728..6ed1f5c6 100644 --- a/tests/impulse_reporting/unit/channels/calculated_channel_test.py +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -1,7 +1,6 @@ # pylint: disable=missing-function-docstring """Unit tests for the reporting-layer CalculatedChannel class.""" -import pyspark.sql.types as T import pytest from impulse_query_engine.analyze.metadata.time_series_expression import TimeSeriesSelector @@ -134,14 +133,13 @@ def test_returns_fact_columns_with_matching_channel_id(self, spark, basic_narrow df = CalculatedChannel.determine_calculated_channels( spark, [ch], query=q, solver=DefaultSolver(spark) ) + # Identity lives on the dimension (joined via channel_id), not the fact. assert df.columns == [ "container_id", "channel_id", "tstart", "tend", "value", - "identity", ] - assert df.schema["identity"].dataType == T.VariantType() ids = {r["channel_id"] for r in df.select("channel_id").distinct().collect()} assert ids == {ch.get_id()} From a69295644e90d2d8023b16813cef0bf6a17a2fee Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Fri, 17 Jul 2026 14:40:34 +0200 Subject: [PATCH 08/23] Refactor CalculatedChannel to use deterministic channel_id and improve identity handling - Introduced a deterministic `channel_id` derived from the identity using CRC32, ensuring consistent identification across runs. - Removed the `_AUTO` sentinel for channel_id, simplifying the constructor and enhancing clarity. - Updated the `canonical_identity` method to use a consistent separator for identity keys. - Refactored related tests to validate the new channel_id behavior and identity encoding, ensuring correct functionality and stability. --- .../query/channels/calculated_channel.py | 50 ++++--- .../analyze/query/solvers/default_solver.py | 137 +++++++----------- .../channels/calculated_channel.py | 23 ++- .../query/channels/calculated_channel_test.py | 31 ++-- ...default_solver_calculated_channels_test.py | 17 +-- 5 files changed, 115 insertions(+), 143 deletions(-) diff --git a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py index 4df4dd15..1056490d 100644 --- a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py +++ b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py @@ -1,5 +1,7 @@ """CalculatedChannel: a labeled derived time-series channel.""" +import zlib + import pyspark.sql.types as T from impulse_query_engine.analyze.metadata.tag_expression import TagExpression @@ -9,11 +11,6 @@ ) from impulse_query_engine.analyze.query.solvers.series_cache import SeriesCache -# Sentinel distinguishing "derive a deterministic channel_id" (the default) from -# an explicit ``channel_id=None`` (emit a SQL null). A plain default of ``None`` -# could not tell these two intents apart. -_AUTO = object() - class CalculatedChannel(TimeSeriesExpression): """A derived channel: a wrapped time-series expression plus output identity. @@ -38,18 +35,14 @@ class CalculatedChannel(TimeSeriesExpression): ``{"channel_name": "Eng_RPM", "data_key": "TM"}``. The whole dict is emitted per row in a single ``identity`` ``VARIANT`` column (keys are arbitrary). Must be non-empty; the identity also seeds the - deterministic ``channel_id`` hash. - channel_id : int or None, optional - Output ``channel_id`` for every emitted row. When omitted (the - ``_AUTO`` sentinel) a deterministic id is derived from the identity by - the solver, typed to match the source ``channel_id`` column. Pass an - ``int`` to use it verbatim, or ``None`` to emit a SQL null. + deterministic :attr:`channel_id`. Notes ----- - ``_alias`` is set explicitly in ``__init__`` — as the identity values joined - by ``"::"`` (e.g. ``"Eng_RPM::TM"``) — rather than left for the caller to - chain ``.alias(...)``: reading a missing ``_alias`` would trigger + ``_alias`` is set in ``__init__`` (via ``super().__init__(alias=...)``) — as + the identity values joined by ``"::"`` (e.g. ``"Eng_RPM::TM"``) — rather than + left for the caller to chain ``.alias(...)``: reading a missing ``_alias`` + would trigger ``TimeSeriesExpression.__getattr__`` and silently return a callable instead of raising. The alias is not consumed by the calculated-channels solve path (that emits the identity column directly); it exists so the object stays a @@ -63,14 +56,12 @@ class CalculatedChannel(TimeSeriesExpression): CalculatedChannel(rpm * 3.6, {"channel_name": "speed_kmh", "data_key": "CALC"}) """ - _ALIAS_SEPARATOR = "::" + _IDENTITY_SEPARATOR = "::" def __init__( self, expr: TimeSeriesExpression, identity: dict[str, str], - *, - channel_id=_AUTO, ): if not identity: raise ValueError( @@ -79,12 +70,13 @@ def __init__( "defines the output identifier columns and seeds the " "deterministic channel_id." ) + super().__init__( + alias=self._IDENTITY_SEPARATOR.join(str(v) for v in identity.values()), + is_single_signal=getattr(expr, "is_single_signal", True), + requires_udf=getattr(expr, "requires_udf", False), + ) self.expr = expr self.identity = dict(identity) - self._explicit_channel_id = channel_id - self._alias = self._ALIAS_SEPARATOR.join(str(v) for v in identity.values()) - self.is_single_signal = getattr(expr, "is_single_signal", True) - self.requires_udf = getattr(expr, "requires_udf", False) def __str__(self) -> str: return f"" @@ -95,7 +87,21 @@ def canonical_identity(self) -> str: Keys are sorted so the encoding (and the derived ``channel_id``) is independent of kwarg order. """ - return "&".join(f"{k}={self.identity[k]}" for k in sorted(self.identity)) + return self._IDENTITY_SEPARATOR.join( + f"{k}={self.identity[k]}" for k in sorted(self.identity) + ) + + @property + def channel_id(self) -> int: + """Deterministic output ``channel_id`` derived from the identity. + + A CRC32 of :meth:`canonical_identity` masked to a positive int32, so the + value is stable across runs/processes and fits both ``IntegerType`` and + ``LongType`` source ``channel_id`` columns. Determinism makes writes + idempotent and joins predictable; sharing this one derivation across the + query-engine and reporting layers keeps their ids in lockstep. + """ + return zlib.crc32(self.canonical_identity().encode()) & 0x7FFFFFFF def build(self, cache: SeriesCache): """Evaluate the wrapped expression against the cache (yields a SampleSeries).""" diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index dac6edac..8277fafb 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib from collections.abc import Iterable from functools import partial from typing import TYPE_CHECKING @@ -936,6 +935,39 @@ def solve(self, query, channels_df, selections, dtypes) -> DataFrame: DataFrame containing results for each container. """ col_map = self.config.col_map + q, joined_df, container_count = self._prepare_channels_join(query, channels_df) + + schema = self._build_solve_output_schema(q, selections, dtypes) + solve_udf = F.pandas_udf( + partial(DefaultSolver._solve_udf, selections=selections, col_map=col_map), + schema, + F.PandasUDFType.GROUPED_MAP, + ) + return self._apply_grouped_map(joined_df, container_count, schema, solve_udf) + + def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFrame, int]: + """Shared prelude for :meth:`solve` and :meth:`solve_calculated_channels`. + + Applies optional per-channel unit conversion, reads and column-maps the + channel-data table (raw-encoding it when in raw mode), broadcast-joins it + to the channel-match frame on ``[container_id, channel_id]``, and counts + the distinct containers. + + Parameters + ---------- + query : QueryBuilder + Query object containing database and filter information. + channels_df : pyspark.sql.DataFrame + Channel-match DataFrame from the filter pipeline. + + Returns + ------- + tuple[DataFrame, DataFrame, int] + ``(channels_q, joined_df, container_count)`` — the column-mapped + channel-data DataFrame (authoritative for output ``container_id`` / + ``channel_id`` types), the join fed to the grouped-map UDF, and the + number of distinct containers. + """ source_unit_col = self.config.source_unit_col target_unit_col = self.config.target_unit_col @@ -958,53 +990,30 @@ def solve(self, query, channels_df, selections, dtypes) -> DataFrame: # Encode the raw samples into intervals (RLE or interval) for the solving step. q = self.channel_encoder.prepare_channels_df(q) - schema = self._build_solve_output_schema(q, selections, dtypes) - solve_udf = F.pandas_udf( - partial(DefaultSolver._solve_udf, selections=selections, col_map=col_map), - schema, - F.PandasUDFType.GROUPED_MAP, + joined_df = q.join( + F.broadcast(channels_df), + on=[self.config.container_id_col, self.config.channel_id_col], ) - df = q.join( - F.broadcast(channels_df), on=[self.config.container_id_col, self.config.channel_id_col] - ) - container_count = channels_df.select(self.config.container_id_col).distinct().count() + return q, joined_df, container_count + + def _apply_grouped_map(self, joined_df, container_count, schema, solve_udf) -> DataFrame: + """Run *solve_udf* per container, or return an empty frame when none match.""" if container_count == 0: return self.spark.createDataFrame([], schema=schema) - res = ( - df.repartition(container_count, self.config.container_id_col) + return ( + joined_df.repartition(container_count, self.config.container_id_col) .groupBy(self.config.container_id_col) .apply(solve_udf) ) - return res # ------------------------------------------------------------------ # Calculated channels (narrow output) # ------------------------------------------------------------------ - @staticmethod - def _calculated_channel_id(cc, channel_id_dtype: T.DataType): - """Resolve the output ``channel_id`` for a single ``CalculatedChannel``. - - An explicit ``channel_id`` on the channel is used verbatim (an ``int``, - or ``None`` to emit a SQL null). Otherwise a deterministic id is derived - from the channel's identity via a stable BLAKE2b digest (never the - process-randomized ``hash()``), sized to the source ``channel_id`` type: - signed int32 for ``IntegerType``, signed int64 otherwise. Determinism - makes writes idempotent and joins predictable across runs. - """ - from impulse_query_engine.analyze.query.channels.calculated_channel import _AUTO - - if cc._explicit_channel_id is not _AUTO: - return cc._explicit_channel_id - digest = hashlib.blake2b(cc.canonical_identity().encode(), digest_size=8).digest() - if isinstance(channel_id_dtype, T.IntegerType): - return int.from_bytes(digest[:4], "big", signed=True) - return int.from_bytes(digest[:8], "big", signed=True) - @staticmethod def _solve_calculated_channels_udf( - pdf, selections: Iterable, col_map: dict[str, str], channel_ids + pdf, selections: Iterable, col_map: dict[str, str] ) -> pd.DataFrame: """ UDF to solve calculated channels for a single container. @@ -1013,8 +1022,10 @@ def _solve_calculated_channels_udf( narrow rows: each ``CalculatedChannel`` builds to a ``SampleSeries`` that is exploded into ``(container_id, channel_id, tstart, tend, value)`` rows, with the channel's whole identity dict attached in a single ``identity`` - map column. The map is converted to a ``VARIANT`` by the caller (a pandas - UDF cannot emit ``VARIANT`` directly, since it is not Arrow-representable). + map column. Each channel supplies its own deterministic ``channel_id`` + (:attr:`CalculatedChannel.channel_id`). The map is converted to a + ``VARIANT`` by the caller (a pandas UDF cannot emit ``VARIANT`` directly, + since it is not Arrow-representable). Parameters ---------- @@ -1024,9 +1035,6 @@ def _solve_calculated_channels_udf( The ``CalculatedChannel`` selections to evaluate. col_map : dict[str, str] Column-name mapping for the cache (cid/ch/ts/te/val/conv). - channel_ids : list - Output ``channel_id`` per selection, positionally paired with - *selections* (precomputed on the driver). Returns ------- @@ -1041,11 +1049,11 @@ def _solve_calculated_channels_udf( cols = {cid_col: [], ch_col: [], "tstart": [], "tend": [], "value": [], "identity": []} - for cc, ch_id in zip(selections, channel_ids, strict=False): + for cc in selections: series = cc.build(cache) for tstart, tend, value in series.get_data(): cols[cid_col].append(container_id) - cols[ch_col].append(ch_id) + cols[ch_col].append(cc.channel_id) cols["tstart"].append(int(tstart)) cols["tend"].append(int(tend)) cols["value"].append(float(value)) @@ -1057,10 +1065,10 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame """ Solve calculated channels by grouping channels and exploding each result. - Structurally parallels :meth:`solve` — same unit-conversion prelude, - same channel-data read and ``[container_id, channel_id]`` join — but the - grouped-map UDF emits narrow silver-shaped rows (many per container) - instead of one wide row. Output columns are + Structurally parallels :meth:`solve` — sharing the + :meth:`_prepare_channels_join` prelude and :meth:`_apply_grouped_map` + tail — but the grouped-map UDF emits narrow silver-shaped rows (many per + container) instead of one wide row. Output columns are ``[container_id, channel_id, tstart, tend, value, identity]`` where ``identity`` is a ``VARIANT`` holding the channel's identity dict. @@ -1079,29 +1087,7 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame Narrow DataFrame of calculated-channel samples. """ col_map = self.config.col_map - source_unit_col = self.config.source_unit_col - target_unit_col = self.config.target_unit_col - - has_conversion_table = getattr(query.db.config, "unit_conversion_table", None) is not None - has_unit_cols = ( - source_unit_col in channels_df.columns and target_unit_col in channels_df.columns - ) - - if has_conversion_table and has_unit_cols: - channels_df = self._compute_conversion_factors(self.spark, query, channels_df) - - for col_name in (source_unit_col, target_unit_col): - if col_name in channels_df.columns: - channels_df = channels_df.drop(col_name) - - q = query.db.channels(self.spark) - q = self._apply_column_mapping(q, self.config.channels.column_name_mapping) - - if self.is_raw_data: - q = self.channel_encoder.prepare_channels_df(q) - - channel_id_dtype = q.schema[self.config.channel_id_col].dataType - channel_ids = [self._calculated_channel_id(cc, channel_id_dtype) for cc in selections] + q, joined_df, container_count = self._prepare_channels_join(query, channels_df) schema = self._build_calculated_channels_output_schema(q) solve_udf = F.pandas_udf( @@ -1109,24 +1095,11 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame DefaultSolver._solve_calculated_channels_udf, selections=selections, col_map=col_map, - channel_ids=channel_ids, ), schema, F.PandasUDFType.GROUPED_MAP, ) - df = q.join( - F.broadcast(channels_df), on=[self.config.container_id_col, self.config.channel_id_col] - ) - - container_count = channels_df.select(self.config.container_id_col).distinct().count() - if container_count == 0: - res = self.spark.createDataFrame([], schema=schema) - else: - res = ( - df.repartition(container_count, self.config.container_id_col) - .groupBy(self.config.container_id_col) - .apply(solve_udf) - ) + res = self._apply_grouped_map(joined_df, container_count, schema, solve_udf) # The grouped-map UDF emits ``identity`` as a MapType column (Arrow-safe); # convert it to a self-describing VARIANT here so both the populated and # empty branches return an identical schema. diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py index 7329fe87..c356caf3 100644 --- a/src/impulse_reporting/channels/calculated_channel.py +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -1,7 +1,6 @@ from __future__ import annotations import hashlib -import zlib from collections.abc import Mapping import pyspark.sql.functions as f @@ -85,24 +84,20 @@ def __init__( normalized_attributes = {str(k): str(v) for k, v in attributes.items()} self.attributes = normalized_attributes - # Deterministic entity id from the identity (same canonical encoding the - # query-engine layer uses). Passed explicitly to the query-engine channel - # so fact.channel_id == get_id() == dimension.channel_id. - self._entity_id = zlib.crc32(self._canonical_identity().encode()) & 0x7FFFFFFF - self.expression = QeCalculatedChannel(expr, self.identity, channel_id=self._entity_id) - - def _canonical_identity(self) -> str: - """Stable identity encoding (sorted keys), matching the query-engine layer.""" - return "&".join(f"{k}={self.identity[k]}" for k in sorted(self.identity)) + # The wrapped query-engine channel owns the deterministic id and the + # canonical identity encoding, so fact.channel_id == get_id() == + # dimension.channel_id with a single source of truth. + self.expression = QeCalculatedChannel(expr, self.identity) def canonical_identity(self) -> str: """Public, order-independent identity key. Two channels with the same ``identity`` (regardless of key insertion order) share this value and therefore the same ``channel_id``. Used by - :class:`Report` to reject duplicate channel identities. + :class:`Report` to reject duplicate channel identities. Delegates to the + wrapped query-engine channel so both layers encode identity identically. """ - return self._canonical_identity() + return self.expression.canonical_identity() def get_name(self) -> str: """Return the channel name.""" @@ -114,7 +109,7 @@ def set_report_id(self, report_id: int): def get_id(self) -> int: """Return the deterministic entity id (also the fact/dimension ``channel_id``).""" - return self._entity_id + return self.expression.channel_id def get_expression(self) -> TimeSeriesExpression | None: """Return ``None`` — calculated channels drive their own narrow solve. @@ -136,7 +131,7 @@ def get_channel_type_str(self) -> str: def determine_definition_hash(self) -> int: """Hash of the computation-affecting definition (expression + identity).""" - payload = f"{self._canonical_identity()}|{self.expression.expr}" + payload = f"{self.canonical_identity()}|{self.expression.expr}" hash_bytes = hashlib.sha256(payload.encode()).digest() return int.from_bytes(hash_bytes[:8], byteorder="big", signed=True) diff --git a/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py b/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py index 501e2e58..f7324002 100644 --- a/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py @@ -1,11 +1,13 @@ # pylint: disable=missing-function-docstring """Unit tests for the CalculatedChannel aggregation class. -Covers construction (identity storage, the `_alias` rule, the channel_id -sentinel), the `canonical_identity` encoding, delegation of the expression +Covers construction (identity storage, the `_alias` rule, the deterministic +`channel_id`), the `canonical_identity` encoding, delegation of the expression interface to the wrapped expression, and validation. """ +import zlib + import pyspark.sql.types as T import pytest @@ -14,7 +16,6 @@ ) from impulse_query_engine.analyze.query.aggregations.aggregation import Aggregation from impulse_query_engine.analyze.query.channels.calculated_channel import ( - _AUTO, CalculatedChannel, ) from impulse_query_engine.model.series.sample_series import SampleSeries @@ -72,18 +73,22 @@ def test_explicit_alias_overrides(self): cc.alias("renamed") assert cc._alias == "renamed" - def test_channel_id_defaults_to_auto_sentinel(self): + def test_channel_id_deterministic_from_identity(self): cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) - assert cc._explicit_channel_id is _AUTO + expected = zlib.crc32(cc.canonical_identity().encode()) & 0x7FFFFFFF + assert cc.channel_id == expected - def test_explicit_channel_id_stored(self): - cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}, channel_id=999) - assert cc._explicit_channel_id == 999 + def test_channel_id_positive_int32(self): + cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) + assert 0 <= cc.channel_id <= 0x7FFFFFFF - def test_explicit_none_channel_id_stored(self): - cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}, channel_id=None) - assert cc._explicit_channel_id is None - assert cc._explicit_channel_id is not _AUTO + def test_channel_id_identity_derived_not_name(self): + # Same identity (any key order) → same id; different identity → different id. + a = CalculatedChannel(_StubExpr(), {"channel_name": "s", "data_key": "CALC"}) + b = CalculatedChannel(_StubExpr(), {"data_key": "CALC", "channel_name": "s"}) + c = CalculatedChannel(_StubExpr(), {"channel_name": "other", "data_key": "CALC"}) + assert a.channel_id == b.channel_id + assert a.channel_id != c.channel_id def test_empty_identity_raises(self): with pytest.raises(ValueError, match="non-empty identity"): @@ -94,7 +99,7 @@ class TestCanonicalIdentity: def test_sorted_and_stable(self): cc1 = CalculatedChannel(_StubExpr(), {"channel_name": "s", "data_key": "CALC"}) cc2 = CalculatedChannel(_StubExpr(), {"data_key": "CALC", "channel_name": "s"}) - assert cc1.canonical_identity() == "channel_name=s&data_key=CALC" + assert cc1.canonical_identity() == "channel_name=s::data_key=CALC" # Order-independent: same identity, different key order → same encoding. assert cc1.canonical_identity() == cc2.canonical_identity() diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py index b648be46..3ac384b2 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py @@ -195,23 +195,16 @@ def test_distinct_identities_distinct_ids(self, spark, basic_narrow_db): } assert by_name["a"] != by_name["b"] - def test_explicit_channel_id_honored(self, spark, basic_narrow_db): + def test_emitted_id_matches_channel_property(self, spark, basic_narrow_db): + # The solver emits the channel's own deterministic channel_id — no + # separate id derivation in the solver. q = basic_narrow_db.query cc = CalculatedChannel( - q.channel(channel_name="Engine RPM") * 2, {"channel_name": "rpm_x2"}, channel_id=999 - ) - result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - ids = {r["channel_id"] for r in result.select("channel_id").distinct().collect()} - assert ids == {999} - - def test_explicit_none_channel_id_emits_null(self, spark, basic_narrow_db): - q = basic_narrow_db.query - cc = CalculatedChannel( - q.channel(channel_name="Engine RPM") * 2, {"channel_name": "rpm_x2"}, channel_id=None + q.channel(channel_name="Engine RPM") * 2, {"channel_name": "rpm_x2"} ) result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) ids = {r["channel_id"] for r in result.select("channel_id").distinct().collect()} - assert ids == {None} + assert ids == {cc.channel_id} class TestValidation: From f1c70155c32ab12ee344c22731e2116af3d912ed Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Fri, 17 Jul 2026 14:52:08 +0200 Subject: [PATCH 09/23] Update identity handling to use MapType in CalculatedChannel and related components - Changed the identity representation from VARIANT to MapType(string, string) across CalculatedChannel, QueryBuilder, and DefaultSolver, ensuring consistency in identity handling. - Updated schemas and documentation to reflect the new MapType identity structure, enhancing clarity and compatibility with heterogeneous keys. - Refactored tests to validate the new identity handling, ensuring correct behavior and data representation in various scenarios. --- .../query/channels/calculated_channel.py | 9 ++--- .../analyze/query/query_builder.py | 5 +-- .../analyze/query/solvers/default_solver.py | 15 +++----- .../analyze/query/solvers/query_solver.py | 11 +++--- .../channels/calculated_channel.py | 31 +++++----------- .../persist/dimension_schema.py | 8 ++--- ...default_solver_calculated_channels_test.py | 36 +++++++++---------- .../integration/calculated_channel_test.py | 8 ++--- .../unit/channels/calculated_channel_test.py | 2 +- 9 files changed, 51 insertions(+), 74 deletions(-) diff --git a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py index 1056490d..6bac7a78 100644 --- a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py +++ b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py @@ -24,7 +24,8 @@ class CalculatedChannel(TimeSeriesExpression): :meth:`QueryBuilder.solve_calculated_channels` explodes that series into narrow rows matching the silver ``channel_data`` shape (``container_id, channel_id, tstart, tend, value``) plus a single - ``identity`` ``VARIANT`` column holding the whole identity dict. + ``identity`` ``MapType(string, string)`` column holding the whole identity + dict. Parameters ---------- @@ -33,9 +34,9 @@ class CalculatedChannel(TimeSeriesExpression): identity : dict of str Identity for the output rows, e.g. ``{"channel_name": "Eng_RPM", "data_key": "TM"}``. The whole dict is - emitted per row in a single ``identity`` ``VARIANT`` column (keys are - arbitrary). Must be non-empty; the identity also seeds the - deterministic :attr:`channel_id`. + emitted per row in a single ``identity`` ``MapType(string, string)`` + column (keys are arbitrary). Must be non-empty; the identity also seeds + the deterministic :attr:`channel_id`. Notes ----- diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 05e78418..b85e8689 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -295,7 +295,8 @@ def solve_calculated_channels( each calculated channel depends on), then evaluates each calculated channel per container and emits rows in the silver ``channel_data`` shape — ``container_id, channel_id, tstart, tend, value`` — plus a single - ``identity`` ``VARIANT`` column holding each channel's identity dict. + ``identity`` ``MapType(string, string)`` column holding each channel's + identity dict. Parameters ---------- @@ -333,7 +334,7 @@ def _validate_calculated_channels(self) -> None: Every selection must be a ``CalculatedChannel`` and each wrapped expression must evaluate to a ``SampleSeries``. Identity key sets need not match across selections — the identity is emitted as a single - self-describing ``VARIANT`` column, so heterogeneous keys are fine. + self-describing ``MapType`` column, so heterogeneous keys are fine. """ if not self.selections: raise ValueError( diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 8277fafb..839be916 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -1022,10 +1022,8 @@ def _solve_calculated_channels_udf( narrow rows: each ``CalculatedChannel`` builds to a ``SampleSeries`` that is exploded into ``(container_id, channel_id, tstart, tend, value)`` rows, with the channel's whole identity dict attached in a single ``identity`` - map column. Each channel supplies its own deterministic ``channel_id`` - (:attr:`CalculatedChannel.channel_id`). The map is converted to a - ``VARIANT`` by the caller (a pandas UDF cannot emit ``VARIANT`` directly, - since it is not Arrow-representable). + ``MapType(string, string)`` column. Each channel supplies its own + deterministic ``channel_id`` (:attr:`CalculatedChannel.channel_id`). Parameters ---------- @@ -1070,7 +1068,8 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame tail — but the grouped-map UDF emits narrow silver-shaped rows (many per container) instead of one wide row. Output columns are ``[container_id, channel_id, tstart, tend, value, identity]`` where - ``identity`` is a ``VARIANT`` holding the channel's identity dict. + ``identity`` is a ``MapType(string, string)`` holding the channel's + identity dict. Parameters ---------- @@ -1099,8 +1098,4 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame schema, F.PandasUDFType.GROUPED_MAP, ) - res = self._apply_grouped_map(joined_df, container_count, schema, solve_udf) - # The grouped-map UDF emits ``identity`` as a MapType column (Arrow-safe); - # convert it to a self-describing VARIANT here so both the populated and - # empty branches return an identical schema. - return res.withColumn("identity", F.to_variant_object(F.col("identity"))) + return self._apply_grouped_map(joined_df, container_count, schema, solve_udf) diff --git a/src/impulse_query_engine/analyze/query/solvers/query_solver.py b/src/impulse_query_engine/analyze/query/solvers/query_solver.py index 08189ec4..dcafada1 100644 --- a/src/impulse_query_engine/analyze/query/solvers/query_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/query_solver.py @@ -105,11 +105,10 @@ def _build_calculated_channels_output_schema(self, channels: DataFrame) -> T.Str The output mirrors the silver ``channel_data`` table (``container_id, channel_id, tstart, tend, value``) plus a single ``identity`` column holding each channel's identity dict as a - ``MapType(string, string)``. This is the **grouped-map UDF** schema; the - map is Arrow-representable so the pandas UDF can emit it directly. The - solver converts the map to a ``VARIANT`` on the returned DataFrame (via - :func:`pyspark.sql.functions.to_variant_object`), so the identity is - self-describing and no gold column depends on the identity key set. + ``MapType(string, string)``. The map is Arrow-representable so the + grouped-map pandas UDF emits it directly, and it is the DataFrame's final + identity type — self-describing, so no gold column depends on the + identity key set. The ``container_id`` and ``channel_id`` field types are derived from *channels* (the column-mapped channels DataFrame) so they match the @@ -434,7 +433,7 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame ------- pyspark.sql.DataFrame Narrow ``[container_id, channel_id, tstart, tend, value, - ]`` DataFrame. + identity]`` DataFrame (``identity`` is a ``MapType(string, string)``). """ raise NotImplementedError( f"{self.__class__.__name__} does not support calculated channels" diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py index c356caf3..bf16e675 100644 --- a/src/impulse_reporting/channels/calculated_channel.py +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -3,8 +3,6 @@ import hashlib from collections.abc import Mapping -import pyspark.sql.functions as f -import pyspark.sql.types as T from pyspark.sql import DataFrame, Row, SparkSession from impulse_query_engine.analyze.metadata.time_series_expression import ( @@ -44,8 +42,8 @@ class CalculatedChannel: The wrapped expression; must evaluate to a ``SampleSeries``. identity : Mapping[str, str] Output identity. Any non-empty set of keys; the whole dict is emitted - per fact row in a single ``identity`` ``VARIANT`` column and seeds the - deterministic ``channel_id``. + per fact row in a single ``identity`` ``MapType(string, string)`` column + and seeds the deterministic ``channel_id``. desc : str, optional Human-readable description (stored on the dimension row, excluded from the definition hash). @@ -138,9 +136,8 @@ def determine_definition_hash(self) -> int: def as_dict(self) -> dict: """Dictionary representation of the dimension metadata. - ``identity`` is emitted as a plain dict here; ``determine_metadata_df`` - converts it to a ``VARIANT`` column so the dimension identity mirrors the - fact identity (no fixed per-key columns). + ``identity`` is a plain dict, persisted as a ``MapType(string, string)`` + column that mirrors the fact identity (no fixed per-key columns). """ return { "channel_id": self.get_id(), @@ -206,21 +203,9 @@ def determine_calculated_channels( def determine_metadata_df(cls, spark: SparkSession, channels: list[CalculatedChannel]): """Create the dimension DataFrame for the given channels. - ``identity`` in ``CALCULATED_CHANNEL_DIMENSION_SCHEMA`` is a ``VARIANT``, - which ``createDataFrame`` cannot build from a Python dict directly. So - the frame is staged with ``identity`` as a ``MapType`` (matching the plain - dict from :meth:`as_dict`) and then converted to ``VARIANT`` via - :func:`pyspark.sql.functions.to_variant_object` — mirroring the fact-side - conversion in the solver. + ``identity`` is a ``MapType(string, string)`` (same self-describing + representation as the fact table), which ``createDataFrame`` builds + directly from the plain dict returned by :meth:`as_dict`. """ rows = [channel.as_spark_row() for channel in channels] - staging_fields = [ - ( - T.StructField("identity", T.MapType(T.StringType(), T.StringType())) - if field.name == "identity" - else field - ) - for field in CALCULATED_CHANNEL_DIMENSION_SCHEMA.fields - ] - df = spark.createDataFrame(rows, schema=T.StructType(staging_fields)) - return df.withColumn("identity", f.to_variant_object(f.col("identity"))) + return spark.createDataFrame(rows, schema=CALCULATED_CHANNEL_DIMENSION_SCHEMA) diff --git a/src/impulse_reporting/persist/dimension_schema.py b/src/impulse_reporting/persist/dimension_schema.py index 04030eab..388c5bbc 100644 --- a/src/impulse_reporting/persist/dimension_schema.py +++ b/src/impulse_reporting/persist/dimension_schema.py @@ -7,7 +7,6 @@ StringType, StructField, StructType, - VariantType, ) HISTOGRAM_DIMENSION_SCHEMA = StructType( @@ -84,9 +83,8 @@ # Metadata for calculated channels. ``channel_id`` is the deterministic entity id # (identical to the fact's channel_id), ``definition_hash`` drives incremental -# reprocessing, and ``identity`` is a VARIANT holding the full identity dict — -# the same self-describing representation as the fact table (no fixed per-key -# columns). +# reprocessing, and ``identity`` is a ``MapType(string, string)`` holding the full +# identity dict — self-describing, with no fixed per-key columns. CALCULATED_CHANNEL_DIMENSION_SCHEMA = StructType( [ StructField("channel_id", LongType(), False), @@ -94,7 +92,7 @@ StructField("channel_type", StringType(), True), StructField("channel_description", StringType(), True), StructField("channel_expression", StringType(), True), - StructField("identity", VariantType(), True), + StructField("identity", MapType(StringType(), StringType()), True), StructField("definition_hash", LongType(), True), StructField("attributes", MapType(StringType(), StringType()), True), ] diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py index 3ac384b2..e17f97aa 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py @@ -27,11 +27,9 @@ _C1_RPM_VALUE = 1081.0 -def _id_str(col="identity", key=None): - """Extract identity as a JSON string, or a single key's string value, from VARIANT.""" - if key is None: - return F.to_json(F.col(col)) - return F.variant_get(F.col(col), f"$.{key}", "string") +def _id_val(key, col="identity"): + """Extract a single identity key's value from the ``MapType`` identity column.""" + return F.col(col).getItem(key) def _recast_container_id(db: MeasurementDB, cid_type: T.DataType) -> MeasurementDB: @@ -60,8 +58,8 @@ def test_scaling_produces_real_values(self, spark, basic_narrow_db): result.filter((F.col("container_id") == 1) & (F.col("tstart") == _C1_RPM_TSTART)) .select( "value", - _id_str(key="channel_name").alias("cn"), - _id_str(key="data_key").alias("dk"), + _id_val("channel_name").alias("cn"), + _id_val("data_key").alias("dk"), ) .collect() ) @@ -77,21 +75,21 @@ def test_all_rows_carry_identity(self, spark, basic_narrow_db): {"channel_name": "rpm_plus_1", "data_key": "CALC"}, ) result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - distinct = {r["j"] for r in result.select(_id_str().alias("j")).collect()} - assert distinct == {'{"channel_name":"rpm_plus_1","data_key":"CALC"}'} + distinct = {tuple(sorted(r["identity"].items())) for r in result.collect()} + assert distinct == {(("channel_name", "rpm_plus_1"), ("data_key", "CALC"))} assert result.count() > 0 def test_arbitrary_identity_keys_round_trip(self, spark, basic_narrow_db): - # Identity is a self-describing VARIANT, so non-{channel_name,data_key} keys work. + # Identity is a self-describing map, so non-{channel_name,data_key} keys work. q = basic_narrow_db.query cc = CalculatedChannel( q.channel(channel_name="Engine RPM") * 2, {"sensor_id": "s1", "unit": "rpm"}, ) result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - assert result.schema["identity"].dataType == T.VariantType() + assert result.schema["identity"].dataType == T.MapType(T.StringType(), T.StringType()) row = result.select( - _id_str(key="sensor_id").alias("sid"), _id_str(key="unit").alias("unit") + _id_val("sensor_id").alias("sid"), _id_val("unit").alias("unit") ).first() assert row["sid"] == "s1" assert row["unit"] == "rpm" @@ -139,7 +137,7 @@ def test_output_columns_and_order(self, spark, basic_narrow_db): {"channel_name": "rpm_x2", "data_key": "CALC"}, ) result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - # Identity is a single self-describing VARIANT column after the silver columns. + # Identity is a single self-describing map column after the silver columns. assert result.columns == [ "container_id", "channel_id", @@ -150,7 +148,7 @@ def test_output_columns_and_order(self, spark, basic_narrow_db): ] assert result.schema["tstart"].dataType == T.LongType() assert result.schema["value"].dataType == T.DoubleType() - assert result.schema["identity"].dataType == T.VariantType() + assert result.schema["identity"].dataType == T.MapType(T.StringType(), T.StringType()) @pytest.mark.parametrize( "cid_type", [T.StringType(), T.IntegerType(), T.LongType()], ids=lambda t: t.simpleString() @@ -189,7 +187,7 @@ def test_distinct_identities_distinct_ids(self, spark, basic_narrow_db): result = q.select(cc_a, cc_b).solve_calculated_channels(spark, solver=DefaultSolver(spark)) by_name = { r["cn"]: r["channel_id"] - for r in result.select(_id_str(key="channel_name").alias("cn"), "channel_id") + for r in result.select(_id_val("channel_name").alias("cn"), "channel_id") .distinct() .collect() } @@ -224,14 +222,14 @@ def test_aggregation_rejected(self, spark, basic_narrow_db): q.solve_calculated_channels(spark, solver=DefaultSolver(spark)) def test_mismatched_identity_keys_allowed(self, spark, basic_narrow_db): - # Identity is a self-describing VARIANT, so heterogeneous key sets are fine. + # Identity is a self-describing map, so heterogeneous key sets are fine. q = basic_narrow_db.query cc_a = CalculatedChannel(q.channel(channel_name="Engine RPM") * 2, {"channel_name": "a"}) cc_b = CalculatedChannel( q.channel(channel_name="Engine RPM") * 3, {"channel_name": "b", "data_key": "CALC"} ) result = q.select(cc_a, cc_b).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - assert result.schema["identity"].dataType == T.VariantType() + assert result.schema["identity"].dataType == T.MapType(T.StringType(), T.StringType()) assert result.count() > 0 @@ -252,8 +250,8 @@ def test_no_matching_channels_empty_frame(self, spark, basic_narrow_db): "value", "identity", ] - # Empty branch funnels through the same parse → identity is VARIANT too. - assert result.schema["identity"].dataType == T.VariantType() + # Empty branch returns the same schema → identity is a map too. + assert result.schema["identity"].dataType == T.MapType(T.StringType(), T.StringType()) def test_base_solver_not_supported(self, spark, basic_narrow_db): from impulse_query_engine.analyze.query.solvers.blob_solver import BlobSolver diff --git a/tests/impulse_reporting/integration/calculated_channel_test.py b/tests/impulse_reporting/integration/calculated_channel_test.py index 4c4db95c..3928f423 100644 --- a/tests/impulse_reporting/integration/calculated_channel_test.py +++ b/tests/impulse_reporting/integration/calculated_channel_test.py @@ -101,15 +101,15 @@ def test_persist_calculated_channel_full(spark): calc_sum = fact.select(F.sum("value")).first()[0] assert calc_sum == pytest.approx(raw_sum * 3.6) - # The dimension carries the identity (VARIANT), keyed by channel_id — the join + # The dimension carries the identity (a map), keyed by channel_id — the join # target for the fact. Confirm the fact's channel_id resolves to the identity. dim = spark.read.table(_DIM) - assert dim.schema["identity"].dataType == T.VariantType() + assert dim.schema["identity"].dataType == T.MapType(T.StringType(), T.StringType()) dim_row = ( dim.filter(F.col("channel_id") == ch.get_id()) .select( - F.variant_get(F.col("identity"), "$.channel_name", "string").alias("cn"), - F.variant_get(F.col("identity"), "$.data_key", "string").alias("dk"), + F.col("identity").getItem("channel_name").alias("cn"), + F.col("identity").getItem("data_key").alias("dk"), "definition_hash", ) .collect() diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py index 6ed1f5c6..c6697c69 100644 --- a/tests/impulse_reporting/unit/channels/calculated_channel_test.py +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -49,7 +49,7 @@ def test_rejects_non_sample_series_expr(self): ) def test_accepts_arbitrary_identity_keys(self): - # Identity persists as a VARIANT, so any non-empty key set is valid. + # Identity persists as a map, so any non-empty key set is valid. ch = CalculatedChannel( name="ok", expr=(TimeSeriesSelector(None) * 2), From 4cf7d374fa926e17d0a9cc104e168929b214cfb6 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Fri, 17 Jul 2026 15:19:17 +0200 Subject: [PATCH 10/23] Refactor CalculatedChannel to return raw arrays and update dtype handling - Modified the `build` method in `CalculatedChannel` to return raw `(tstarts, tends, values)` arrays instead of a `SampleSeries`, aligning with the narrow output requirements. - Updated the `dtype` method to reflect the new output structure as a `StructType` containing three arrays (long, long, double). - Adjusted related tests to validate the new behavior of `build` and `dtype`, ensuring accurate representation of the output type. --- .../query/channels/calculated_channel.py | 77 +++++++++---------- .../analyze/query/solvers/default_solver.py | 11 +-- .../query/channels/calculated_channel_test.py | 27 +++++-- 3 files changed, 62 insertions(+), 53 deletions(-) diff --git a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py index 6bac7a78..0e425918 100644 --- a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py +++ b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py @@ -13,42 +13,22 @@ class CalculatedChannel(TimeSeriesExpression): - """A derived channel: a wrapped time-series expression plus output identity. - - A ``CalculatedChannel`` wraps an arbitrary :class:`TimeSeriesExpression` - built from the operator DSL (e.g. ``q.channel(channel_name="raw_speed") * 3.6`` - or ``rpm + speed``). Like the other ``TimeSeriesExpression`` leaves/nodes - (``TimeSeriesSelector``, ``TimeSeriesOp``) it ``build()``s to a - :class:`SampleSeries` — it is a *labeled derived signal*, not a reduction, so - it is a plain ``TimeSeriesExpression`` rather than an ``Aggregation``. - :meth:`QueryBuilder.solve_calculated_channels` explodes that series into - narrow rows matching the silver ``channel_data`` shape - (``container_id, channel_id, tstart, tend, value``) plus a single - ``identity`` ``MapType(string, string)`` column holding the whole identity - dict. + """A derived channel: a wrapped ``TimeSeriesExpression`` plus output identity. + + Wraps an expression built from the operator DSL (e.g. ``rpm * 3.6``) that must + evaluate to a :class:`SampleSeries`. ``build()`` returns that series' raw + ``(tstarts, tends, values)`` arrays, which + :meth:`QueryBuilder.solve_calculated_channels` explodes into narrow + silver-shaped rows plus an ``identity`` ``MapType`` column. Parameters ---------- expr : TimeSeriesExpression The wrapped expression; must ``build()`` to a ``SampleSeries``. identity : dict of str - Identity for the output rows, e.g. - ``{"channel_name": "Eng_RPM", "data_key": "TM"}``. The whole dict is - emitted per row in a single ``identity`` ``MapType(string, string)`` - column (keys are arbitrary). Must be non-empty; the identity also seeds - the deterministic :attr:`channel_id`. - - Notes - ----- - ``_alias`` is set in ``__init__`` (via ``super().__init__(alias=...)``) — as - the identity values joined by ``"::"`` (e.g. ``"Eng_RPM::TM"``) — rather than - left for the caller to chain ``.alias(...)``: reading a missing ``_alias`` - would trigger - ``TimeSeriesExpression.__getattr__`` and silently return a callable instead - of raising. The alias is not consumed by the calculated-channels solve path - (that emits the identity column directly); it exists so the object stays a - well-formed ``TimeSeriesExpression``. A later ``.alias(...)`` still overrides - it. + Non-empty identity dict (arbitrary keys, e.g. + ``{"channel_name": "Eng_RPM", "data_key": "TM"}``), emitted per row and + used to seed the deterministic :attr:`channel_id`. Examples -------- @@ -71,6 +51,9 @@ def __init__( "defines the output identifier columns and seeds the " "deterministic channel_id." ) + # Set _alias eagerly (identity values joined by "::"): a missing _alias + # would hit TimeSeriesExpression.__getattr__ and return a callable instead + # of raising. The solve path ignores it; a later .alias(...) overrides it. super().__init__( alias=self._IDENTITY_SEPARATOR.join(str(v) for v in identity.values()), is_single_signal=getattr(expr, "is_single_signal", True), @@ -105,19 +88,33 @@ def channel_id(self) -> int: return zlib.crc32(self.canonical_identity().encode()) & 0x7FFFFFFF def build(self, cache: SeriesCache): - """Evaluate the wrapped expression against the cache (yields a SampleSeries).""" - return self.expr.build(cache) + """Evaluate the wrapped expression and return its raw ``(tstarts, tends, values)``. - def dtype(self) -> T.DataType: - """Spark type of ``build()``'s result: a serialized ``SampleSeries``. + Unlike a typical ``TimeSeriesExpression`` (whose ``build`` yields a + ``SampleSeries``), this returns the three parallel ``float64`` arrays + underlying that series, since the calculated-channels solve path consumes + the raw samples directly. + """ + series = self.expr.build(cache) + return series.tstarts, series.tends, series.values - Matches :meth:`TimeSeriesSelector.dtype` (``BinaryType``), since the - wrapped expression evaluates to a ``SampleSeries``. The narrow - calculated-channels solve path does not consume this — it emits its own - ``container_id, channel_id, tstart, tend, value`` schema — but keeping it - honest makes the expression safe if ever routed through ``solve()``. + def dtype(self) -> T.DataType: + """Spark type of ``build()``'s output: the raw ``(tstarts, tends, values)`` arrays. + + A struct of three arrays mirroring the tuple ``build()`` returns, with + element types matching the narrow calculated-channels output + (``tstart``/``tend`` are ``long``, ``value`` is ``double``). The narrow + solve path does not consume this — it builds its own schema via + ``_build_calculated_channels_output_schema`` — but keeping it honest + describes the expression's actual output type. """ - return T.BinaryType() + return T.StructType( + [ + T.StructField("tstarts", T.ArrayType(T.LongType())), + T.StructField("tends", T.ArrayType(T.LongType())), + T.StructField("values", T.ArrayType(T.DoubleType())), + ] + ) def get_selectors(self) -> list[TimeSeriesSelector]: return self.expr.get_selectors() diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 839be916..d40c9fe0 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -1019,9 +1019,10 @@ def _solve_calculated_channels_udf( UDF to solve calculated channels for a single container. Unlike :meth:`_solve_udf` (one wide row per container), this emits many - narrow rows: each ``CalculatedChannel`` builds to a ``SampleSeries`` that - is exploded into ``(container_id, channel_id, tstart, tend, value)`` rows, - with the channel's whole identity dict attached in a single ``identity`` + narrow rows: each ``CalculatedChannel`` builds to its raw + ``(tstarts, tends, values)`` arrays that are exploded into + ``(container_id, channel_id, tstart, tend, value)`` rows, with the + channel's whole identity dict attached in a single ``identity`` ``MapType(string, string)`` column. Each channel supplies its own deterministic ``channel_id`` (:attr:`CalculatedChannel.channel_id`). @@ -1048,8 +1049,8 @@ def _solve_calculated_channels_udf( cols = {cid_col: [], ch_col: [], "tstart": [], "tend": [], "value": [], "identity": []} for cc in selections: - series = cc.build(cache) - for tstart, tend, value in series.get_data(): + tstarts, tends, values = cc.build(cache) + for tstart, tend, value in zip(tstarts, tends, values, strict=False): cols[cid_col].append(container_id) cols[ch_col].append(cc.channel_id) cols["tstart"].append(int(tstart)) diff --git a/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py b/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py index f7324002..5211c1e0 100644 --- a/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/channels/calculated_channel_test.py @@ -110,10 +110,13 @@ def test_distinct_identities_differ(self): class TestDelegation: - def test_build_returns_wrapped_series(self): + def test_build_returns_raw_arrays(self): series = SampleSeries([0, 100], [100, 200], [3.0, 4.0]) cc = CalculatedChannel(_StubExpr(series), {"channel_name": "x"}) - assert cc.build(cache=None) is series + tstarts, tends, values = cc.build(cache=None) + assert list(tstarts) == [0, 100] + assert list(tends) == [100, 200] + assert list(values) == [3.0, 4.0] def test_get_selectors_delegates(self): expr = _StubExpr() @@ -126,11 +129,19 @@ def test_selector_interface_delegates(self): assert cc.get_required_tag_exprs() == {"tag_expr"} assert cc.required_tags() == {"tag"} - def test_dtype_is_binary(self): - # Builds to a SampleSeries → BinaryType, matching TimeSeriesSelector. + def test_dtype_is_struct_of_arrays(self): + # build() returns the raw (tstarts, tends, values) arrays → a struct of + # three arrays (long, long, double) mirroring the narrow output. cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) - assert cc.dtype() == T.BinaryType() - - def test_evaluation_type_is_sample_series(self): + assert cc.dtype() == T.StructType( + [ + T.StructField("tstarts", T.ArrayType(T.LongType())), + T.StructField("tends", T.ArrayType(T.LongType())), + T.StructField("values", T.ArrayType(T.DoubleType())), + ] + ) + + def test_evaluation_type_is_tuple(self): + # build() now yields a (tstarts, tends, values) tuple, not a SampleSeries. cc = CalculatedChannel(_StubExpr(), {"channel_name": "x"}) - assert cc.evaluation_type() is SampleSeries + assert cc.evaluation_type() is tuple From d322b642135f3e6258a4beaa5a39fc9b20ee95e8 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Fri, 17 Jul 2026 15:37:39 +0200 Subject: [PATCH 11/23] Refactor calculated channel identity handling and output schema - Updated the output schema for calculated channels to exclude the identity column, reflecting that it is now attached post-UDF based on channel_id. - Modified the DefaultSolver to handle identity mapping more efficiently, ensuring it is constant per channel_id and reducing the Arrow payload size. - Adjusted related tests to validate the new identity handling and output structure, ensuring accurate representation of calculated channel data. --- .../query/channels/calculated_channel.py | 5 +-- .../analyze/query/solvers/default_solver.py | 32 +++++++++++---- .../analyze/query/solvers/query_solver.py | 20 +++++----- ...default_solver_calculated_channels_test.py | 40 +++++++++++++++++-- 4 files changed, 70 insertions(+), 27 deletions(-) diff --git a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py index 0e425918..5690033b 100644 --- a/src/impulse_query_engine/analyze/query/channels/calculated_channel.py +++ b/src/impulse_query_engine/analyze/query/channels/calculated_channel.py @@ -103,10 +103,7 @@ def dtype(self) -> T.DataType: A struct of three arrays mirroring the tuple ``build()`` returns, with element types matching the narrow calculated-channels output - (``tstart``/``tend`` are ``long``, ``value`` is ``double``). The narrow - solve path does not consume this — it builds its own schema via - ``_build_calculated_channels_output_schema`` — but keeping it honest - describes the expression's actual output type. + (``tstart``/``tend`` are ``long``, ``value`` is ``double``). """ return T.StructType( [ diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index d40c9fe0..edca0b11 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -1021,10 +1021,11 @@ def _solve_calculated_channels_udf( Unlike :meth:`_solve_udf` (one wide row per container), this emits many narrow rows: each ``CalculatedChannel`` builds to its raw ``(tstarts, tends, values)`` arrays that are exploded into - ``(container_id, channel_id, tstart, tend, value)`` rows, with the - channel's whole identity dict attached in a single ``identity`` - ``MapType(string, string)`` column. Each channel supplies its own - deterministic ``channel_id`` (:attr:`CalculatedChannel.channel_id`). + ``(container_id, channel_id, tstart, tend, value)`` rows. Each channel + supplies its own deterministic ``channel_id`` + (:attr:`CalculatedChannel.channel_id`); the identity map is *not* emitted + here — it is constant per ``channel_id`` and attached afterward by + :meth:`solve_calculated_channels` to keep the UDF's Arrow payload small. Parameters ---------- @@ -1046,7 +1047,7 @@ def _solve_calculated_channels_udf( ch_col = col_map["ch"] container_id = pdf[cid_col].iloc[0] - cols = {cid_col: [], ch_col: [], "tstart": [], "tend": [], "value": [], "identity": []} + cols = {cid_col: [], ch_col: [], "tstart": [], "tend": [], "value": []} for cc in selections: tstarts, tends, values = cc.build(cache) @@ -1056,7 +1057,6 @@ def _solve_calculated_channels_udf( cols["tstart"].append(int(tstart)) cols["tend"].append(int(tend)) cols["value"].append(float(value)) - cols["identity"].append(dict(cc.identity)) return pd.DataFrame(cols) @@ -1089,7 +1089,7 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame col_map = self.config.col_map q, joined_df, container_count = self._prepare_channels_join(query, channels_df) - schema = self._build_calculated_channels_output_schema(q) + schema = self._build_calculated_channels_udf_schema(q) solve_udf = F.pandas_udf( partial( DefaultSolver._solve_calculated_channels_udf, @@ -1099,4 +1099,20 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame schema, F.PandasUDFType.GROUPED_MAP, ) - return self._apply_grouped_map(joined_df, container_count, schema, solve_udf) + res = self._apply_grouped_map(joined_df, container_count, schema, solve_udf) + + # Identity is constant per channel_id, so attach it here via a + # channel_id-keyed CASE instead of the UDF shipping a copy per row. + channel_id_col = self.config.channel_id_col + identity_expr = None + for cc in selections: + id_map = F.create_map( + *[F.lit(x) for k, v in cc.identity.items() for x in (str(k), str(v))] + ) + branch = F.col(channel_id_col) == F.lit(cc.channel_id) + identity_expr = ( + F.when(branch, id_map) + if identity_expr is None + else identity_expr.when(branch, id_map) + ) + return res.withColumn("identity", identity_expr) diff --git a/src/impulse_query_engine/analyze/query/solvers/query_solver.py b/src/impulse_query_engine/analyze/query/solvers/query_solver.py index dcafada1..23fe8a56 100644 --- a/src/impulse_query_engine/analyze/query/solvers/query_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/query_solver.py @@ -99,16 +99,15 @@ def _build_solve_output_schema(self, channels: DataFrame, selections, dtypes) -> entries.append(T.StructField(s._alias, dtype)) return T.StructType(entries) - def _build_calculated_channels_output_schema(self, channels: DataFrame) -> T.StructType: - """Build the narrow grouped-map output schema for calculated channels. + def _build_calculated_channels_udf_schema(self, channels: DataFrame) -> T.StructType: + """Build the grouped-map UDF return schema for calculated channels. - The output mirrors the silver ``channel_data`` table - (``container_id, channel_id, tstart, tend, value``) plus a single - ``identity`` column holding each channel's identity dict as a - ``MapType(string, string)``. The map is Arrow-representable so the - grouped-map pandas UDF emits it directly, and it is the DataFrame's final - identity type — self-describing, so no gold column depends on the - identity key set. + This describes what :meth:`DefaultSolver._solve_calculated_channels_udf` + returns — the narrow silver ``channel_data`` columns + (``container_id, channel_id, tstart, tend, value``). The ``identity`` + map is *not* part of this schema: it is constant per ``channel_id`` and + appended by :meth:`solve_calculated_channels` after the UDF runs, so it + never crosses the Arrow boundary. The ``container_id`` and ``channel_id`` field types are derived from *channels* (the column-mapped channels DataFrame) so they match the @@ -123,7 +122,7 @@ def _build_calculated_channels_output_schema(self, channels: DataFrame) -> T.Str Returns ------- pyspark.sql.types.StructType - ``[container_id, channel_id, tstart, tend, value, identity]``. + ``[container_id, channel_id, tstart, tend, value]``. """ return T.StructType( [ @@ -138,7 +137,6 @@ def _build_calculated_channels_output_schema(self, channels: DataFrame) -> T.Str T.StructField(self.config.tstart_col, T.LongType()), T.StructField(self.config.tend_col, T.LongType()), T.StructField(self.config.value_col, T.DoubleType()), - T.StructField("identity", T.MapType(T.StringType(), T.StringType())), ] ) diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py index e17f97aa..f0e17113 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py @@ -87,7 +87,10 @@ def test_arbitrary_identity_keys_round_trip(self, spark, basic_narrow_db): {"sensor_id": "s1", "unit": "rpm"}, ) result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - assert result.schema["identity"].dataType == T.MapType(T.StringType(), T.StringType()) + # create_map builds a map whose values are never null (valueContainsNull=False). + assert result.schema["identity"].dataType == T.MapType( + T.StringType(), T.StringType(), False + ) row = result.select( _id_val("sensor_id").alias("sid"), _id_val("unit").alias("unit") ).first() @@ -148,7 +151,10 @@ def test_output_columns_and_order(self, spark, basic_narrow_db): ] assert result.schema["tstart"].dataType == T.LongType() assert result.schema["value"].dataType == T.DoubleType() - assert result.schema["identity"].dataType == T.MapType(T.StringType(), T.StringType()) + # create_map builds a map whose values are never null (valueContainsNull=False). + assert result.schema["identity"].dataType == T.MapType( + T.StringType(), T.StringType(), False + ) @pytest.mark.parametrize( "cid_type", [T.StringType(), T.IntegerType(), T.LongType()], ids=lambda t: t.simpleString() @@ -204,6 +210,26 @@ def test_emitted_id_matches_channel_property(self, spark, basic_narrow_db): ids = {r["channel_id"] for r in result.select("channel_id").distinct().collect()} assert ids == {cc.channel_id} + def test_identity_resolves_per_channel_id(self, spark, basic_narrow_db): + # Identity is attached post-UDF via a channel_id-keyed CASE, so each row's + # identity must match its own channel's identity (not another channel's). + q = basic_narrow_db.query + cc_a = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, {"channel_name": "a", "data_key": "CALC"} + ) + cc_b = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 3, {"channel_name": "b", "data_key": "CALC"} + ) + result = q.select(cc_a, cc_b).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + by_id = { + r["channel_id"]: r["cn"] + for r in result.select("channel_id", _id_val("channel_name").alias("cn")) + .distinct() + .collect() + } + assert by_id[cc_a.channel_id] == "a" + assert by_id[cc_b.channel_id] == "b" + class TestValidation: def test_plain_selector_rejected(self, spark, basic_narrow_db): @@ -229,7 +255,10 @@ def test_mismatched_identity_keys_allowed(self, spark, basic_narrow_db): q.channel(channel_name="Engine RPM") * 3, {"channel_name": "b", "data_key": "CALC"} ) result = q.select(cc_a, cc_b).solve_calculated_channels(spark, solver=DefaultSolver(spark)) - assert result.schema["identity"].dataType == T.MapType(T.StringType(), T.StringType()) + # create_map builds a map whose values are never null (valueContainsNull=False). + assert result.schema["identity"].dataType == T.MapType( + T.StringType(), T.StringType(), False + ) assert result.count() > 0 @@ -251,7 +280,10 @@ def test_no_matching_channels_empty_frame(self, spark, basic_narrow_db): "identity", ] # Empty branch returns the same schema → identity is a map too. - assert result.schema["identity"].dataType == T.MapType(T.StringType(), T.StringType()) + # create_map builds a map whose values are never null (valueContainsNull=False). + assert result.schema["identity"].dataType == T.MapType( + T.StringType(), T.StringType(), False + ) def test_base_solver_not_supported(self, spark, basic_narrow_db): from impulse_query_engine.analyze.query.solvers.blob_solver import BlobSolver From dc397632655a345cc9f9547948d941854f620083 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Fri, 17 Jul 2026 16:24:44 +0200 Subject: [PATCH 12/23] Enhance calculated channel processing and utility functions - Updated the `CalculatedChannel` class documentation to clarify the usage of the Spark session in `QueryBuilder.solve_calculated_channels`. - Introduced new utility functions for grouping calculated channels by type, merging changed and unchanged data, and building metadata DataFrames. - Refactored the `Report` class to utilize these new utility functions for improved clarity and maintainability in the persistence of calculated channel facts and dimensions. - Removed redundant methods related to grouping channels by type and direct persistence logic, streamlining the codebase. --- .../channels/calculated_channel.py | 2 +- src/impulse_reporting/core/report.py | 149 +++------- src/impulse_reporting/core/report_utils.py | 278 +++++++++++++++++- 3 files changed, 312 insertions(+), 117 deletions(-) diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py index bf16e675..e996640d 100644 --- a/src/impulse_reporting/channels/calculated_channel.py +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -175,7 +175,7 @@ def determine_calculated_channels( Parameters ---------- spark : SparkSession - Spark session (unused directly; kept for interface parity). + Spark session, forwarded to ``QueryBuilder.solve_calculated_channels``. channels : list of CalculatedChannel Channels to solve; identity keys may differ across channels. query : QueryBuilder diff --git a/src/impulse_reporting/core/report.py b/src/impulse_reporting/core/report.py index 5fdb35e8..36ac59a6 100644 --- a/src/impulse_reporting/core/report.py +++ b/src/impulse_reporting/core/report.py @@ -22,11 +22,18 @@ ) from impulse_reporting.core.page import Page from impulse_reporting.core.report_utils import ( + build_metadata_dfs, cleanup_temp_tables, collect_solvable_expressions, dispatch_aggregations, dispatch_calculated_channels, dispatch_events, + group_selectables_by_type, + merge_changed_unchanged, + persist_dimensions_full, + persist_dimensions_incremental, + persist_facts_full, + persist_facts_incremental, solve_expressions_batched, split_by_hash_change, ) @@ -465,23 +472,6 @@ def get_calculated_channels(self) -> list: """ return self.calculated_channels - def _group_calculated_channels_by_type(self): - """ - Group calculated channels by their type. - - Returns - ------- - dict - Dictionary mapping channel type names to lists of channels. - """ - channel_types = {channel_type.name: [] for channel_type in ChannelType} - for channel in self.calculated_channels: - for channel_type in channel_types.keys(): - if isinstance(channel, ChannelType[channel_type].value): - channel_types[channel_type].append(channel) - break - return channel_types - def _validate_unique_calculated_channels(self): """ Reject calculated channels that share a canonical identity. @@ -685,41 +675,9 @@ def _persist_full(self): schema, uri = writer.extract_metadata_schema_and_output_uri(event_type) writer.write(event_meta_dfs_list, schema=schema, uri=uri) - # calculated channel fact tables - channel_fact_by_table = {} - for channel_type_str, channel_dfs in self.calculated_channel_dfs.items(): - table_name = ChannelType[channel_type_str].get_fact_table_name() - channel_fact_by_table.setdefault(table_name, []) - if isinstance(channel_dfs, dict): - if channel_dfs.get("changed") is not None: - channel_fact_by_table[table_name].append(channel_dfs["changed"]) - if channel_dfs.get("unchanged") is not None: - channel_fact_by_table[table_name].append(channel_dfs["unchanged"]) - elif channel_dfs is not None: - channel_fact_by_table[table_name].append(channel_dfs) - - for table_name, channel_dfs_list in channel_fact_by_table.items(): - if not channel_dfs_list: - continue - channel_type = ChannelType.get_any_for_fact_table(table_name) - writer = storage_factory.create_writer(channel_type) - schema, uri = writer.extract_fact_schema_and_output_uri(channel_type) - writer.write(channel_dfs_list, schema=schema, uri=uri) - - # calculated channel dimension tables - channel_dim_by_table = {} - for channel_type_str, channel_metadata_df in self.calculated_channel_metadata_dfs.items(): - table_name = ChannelType[channel_type_str].get_dimension_table_name() - channel_dim_by_table.setdefault(table_name, []) - channel_dim_by_table[table_name].append(channel_metadata_df) - - for table_name, channel_meta_dfs_list in channel_dim_by_table.items(): - if not channel_meta_dfs_list: - continue - channel_type = ChannelType.get_any_for_dimension_table(table_name) - writer = storage_factory.create_writer(channel_type) - schema, uri = writer.extract_metadata_schema_and_output_uri(channel_type) - writer.write(channel_meta_dfs_list, schema=schema, uri=uri) + # calculated channel fact + dimension tables + persist_facts_full(self.calculated_channel_dfs, ChannelType, storage_factory) + persist_dimensions_full(self.calculated_channel_metadata_dfs, ChannelType, storage_factory) # persist measurement dimensions if self.container_dimension_df: @@ -903,48 +861,26 @@ def _persist_incremental( ], ) - # Persist calculated channel facts - for channel_type_str, channel_data in self.calculated_channel_dfs.items(): - channel_type = ChannelType[channel_type_str] - writer = storage_factory.create_writer(channel_type) - schema, uri = writer.extract_fact_schema_and_output_uri(channel_type) - merge_keys = ["container_id", "channel_id", "tstart"] - - if isinstance(channel_data, dict): - changed_df = channel_data.get("changed") - unchanged_df = channel_data.get("unchanged") - - # Changed definitions: replaceWhere (atomic) over all containers - if changed_df is not None and channel_type_str in changed_channel_ids: - changed_ids = changed_channel_ids[channel_type_str] - df_enriched = self._transform_for_persistence(changed_df, schema, transformer) - self.sink.replace_by_ids( - df=df_enriched, - uri=uri, - id_column="channel_id", - ids_to_replace=changed_ids, - ) + # Persist calculated channel facts + dimensions + def _transform(df, schema): + return self._transform_for_persistence(df, schema, transformer) - # Unchanged definitions: MERGE over the incremental subset - if unchanged_df is not None: - df_enriched = self._transform_for_persistence( - unchanged_df, schema, transformer - ) - self.sink.upsert(df_enriched, uri, merge_keys) - elif channel_data is not None: - df_enriched = self._transform_for_persistence(channel_data, schema, transformer) - self.sink.upsert(df_enriched, uri, merge_keys) - - # Persist calculated channel dimensions (always upsert by channel_id) - for ( - channel_type_str, - channel_metadata_df, - ) in self.calculated_channel_metadata_dfs.items(): - channel_type = ChannelType[channel_type_str] - writer = storage_factory.create_writer(channel_type) - schema, uri = writer.extract_metadata_schema_and_output_uri(channel_type) - df_enriched = self._transform_for_persistence(channel_metadata_df, schema, transformer) - self.sink.upsert(df_enriched, uri, ["channel_id"]) + persist_facts_incremental( + self.calculated_channel_dfs, + ChannelType, + self.sink, + _transform, + id_column="channel_id", + merge_keys=["container_id", "channel_id", "tstart"], + changed_ids=changed_channel_ids, + ) + persist_dimensions_incremental( + self.calculated_channel_metadata_dfs, + ChannelType, + self.sink, + _transform, + merge_keys=["channel_id"], + ) def _transform_for_persistence( self, @@ -1211,7 +1147,7 @@ def determine_report(self, is_incremental: bool = None): # Changed definitions recompute over all containers; unchanged ones over # the incrementally-detected subset. self._validate_unique_calculated_channels() - channels_by_type = self._group_calculated_channels_by_type() + channels_by_type = group_selectables_by_type(self.calculated_channels, ChannelType) changed_channels_by_type, unchanged_channels_by_type, self._changed_channel_ids = ( split_by_hash_change( channels_by_type, @@ -1238,27 +1174,12 @@ def determine_report(self, is_incremental: bool = None): self.solver, pre_filtered_containers_df, ) - calculated_channel_dfs = {} - all_channel_types = set( - list(changed_channel_dfs.keys()) + list(unchanged_channel_dfs.keys()) + self.calculated_channel_dfs = merge_changed_unchanged( + changed_channel_dfs, unchanged_channel_dfs + ) + self.calculated_channel_metadata_dfs = build_metadata_dfs( + channels_by_type, ChannelType, self.spark ) - for t in all_channel_types: - calculated_channel_dfs[t] = { - "changed": changed_channel_dfs.get(t), - "unchanged": unchanged_channel_dfs.get(t), - } - - calculated_channel_metadata_dfs = {} - for channel_name, channel_list in channels_by_type.items(): - if not channel_list: - continue - cls = ChannelType[channel_name].value - calculated_channel_metadata_dfs[channel_name] = cls.determine_metadata_df( - self.spark, channel_list - ) - - self.calculated_channel_dfs = calculated_channel_dfs - self.calculated_channel_metadata_dfs = calculated_channel_metadata_dfs # Determine container dimension self.container_dimension_df = ContainerDimension.get_dimension( diff --git a/src/impulse_reporting/core/report_utils.py b/src/impulse_reporting/core/report_utils.py index 9851cd71..de78883f 100644 --- a/src/impulse_reporting/core/report_utils.py +++ b/src/impulse_reporting/core/report_utils.py @@ -12,6 +12,10 @@ from pyspark.sql import DataFrame, SparkSession if TYPE_CHECKING: + from collections.abc import Callable + + from pyspark.sql.types import StructType + from impulse_query_engine.analyze.metadata.time_series_expression import ( TimeSeriesExpression, ) @@ -20,7 +24,7 @@ from impulse_reporting.incremental.definition_hash_comparator import ( DefinitionHashComparator, ) - from impulse_reporting.persist.report_storage import Sink + from impulse_reporting.persist.report_storage import Sink, WriterFactory def build_batches( @@ -311,7 +315,7 @@ def dispatch_calculated_channels( Unlike events/aggregations, calculated channels never ride the wide ``solved_df`` — each type drives its own narrow solve via ``query.solve_calculated_channels`` (the ``ContainerEvent`` pattern), so this - always passes ``query``/``solver``/``pre_filtered_containers_df``. + always passes ``spark``/``query``/``solver``/``pre_filtered_containers_df``. Parameters ---------- @@ -442,3 +446,273 @@ def cleanup_temp_tables(spark: SparkSession, catalog: str, schema: str) -> None: for row in tables.collect(): table_name = row["tableName"] spark.sql(f"DROP TABLE IF EXISTS `{catalog}`.`{schema}`.`{table_name}`") + + +# --------------------------------------------------------------------------- +# Entity orchestration + persistence helpers +# +# Generic over the entity type-enum (e.g. ``ChannelType``). Currently used only +# by calculated channels; written to be reusable so aggregations can adopt them +# next (``merge_keys`` accepts a per-type callable for that reason). Event +# *incremental facts* group-by-table + union before ``replace_by_ids`` and so do +# not fit ``persist_facts_incremental`` — migrating them needs a separate helper. +# --------------------------------------------------------------------------- + + +def group_selectables_by_type(items: list, type_enum) -> dict[str, list]: + """Bucket *items* into ``{type_enum member name: [items]}`` by ``isinstance``. + + Each item is assigned to the first type whose ``.value`` class it is an + instance of. Every enum member gets a (possibly empty) bucket. + + Parameters + ---------- + items : list + Entities to group (e.g. a report's calculated channels). + type_enum : Enum + The entity type-enum (e.g. ``ChannelType``); each member's ``.value`` is + the entity class. + + Returns + ------- + dict[str, list] + ``{type_name: [items]}``. + """ + by_type: dict[str, list] = {member.name: [] for member in type_enum} + for item in items: + for type_name in by_type: + if isinstance(item, type_enum[type_name].value): + by_type[type_name].append(item) + break + return by_type + + +def merge_changed_unchanged( + changed_by_type: dict[str, DataFrame | None], + unchanged_by_type: dict[str, DataFrame | None], +) -> dict[str, dict[str, DataFrame | None]]: + """Combine changed/unchanged dispatch results into per-type structured dicts. + + Parameters + ---------- + changed_by_type : dict[str, DataFrame | None] + Solved facts for changed definitions, keyed by type name. + unchanged_by_type : dict[str, DataFrame | None] + Solved facts for unchanged definitions, keyed by type name. + + Returns + ------- + dict[str, dict[str, DataFrame | None]] + ``{type_name: {"changed": df, "unchanged": df}}`` for every type present + in either input. + """ + result: dict[str, dict[str, DataFrame | None]] = {} + for type_name in set(changed_by_type) | set(unchanged_by_type): + result[type_name] = { + "changed": changed_by_type.get(type_name), + "unchanged": unchanged_by_type.get(type_name), + } + return result + + +def build_metadata_dfs( + items_by_type: dict[str, list], + type_enum, + spark: SparkSession, +) -> dict[str, DataFrame]: + """Build the dimension (metadata) DataFrame for each non-empty type. + + Parameters + ---------- + items_by_type : dict[str, list] + ``{type_name: [items]}`` as returned by :func:`group_selectables_by_type`. + type_enum : Enum + The entity type-enum; ``type_enum[name].value`` is the entity class whose + ``determine_metadata_df(spark, items)`` classmethod builds the dimension. + spark : SparkSession + Active Spark session. + + Returns + ------- + dict[str, DataFrame] + ``{type_name: metadata_df}`` for each type with at least one item. + """ + metadata_dfs: dict[str, DataFrame] = {} + for type_name, items in items_by_type.items(): + if not items: + continue + cls = type_enum[type_name].value + metadata_dfs[type_name] = cls.determine_metadata_df(spark, items) + return metadata_dfs + + +def _fact_dfs_for_table(dfs) -> list[DataFrame]: + """Flatten a per-type fact value into a list of DataFrames to write. + + Accepts the structured ``{"changed": df, "unchanged": df}`` dict (either may + be ``None``) or a bare DataFrame; ``None`` values are dropped. + """ + if isinstance(dfs, dict): + return [dfs[key] for key in ("changed", "unchanged") if dfs.get(key) is not None] + return [dfs] if dfs is not None else [] + + +def persist_facts_full(dfs_by_type: dict, type_enum, writer_factory: WriterFactory) -> None: + """Full-overwrite persist of fact DataFrames, grouped by output table. + + Groups per-type facts by their fact-table name (so types sharing a table are + written together), then writes each table via the entity writer. + + Parameters + ---------- + dfs_by_type : dict + ``{type_name: value}`` where value is a structured + ``{"changed", "unchanged"}`` dict or a bare DataFrame. + type_enum : Enum + The entity type-enum (resolves fact-table names + writer). + writer_factory : WriterFactory + Factory producing the entity writer. + """ + dfs_by_table: dict[str, list[DataFrame]] = {} + for type_name, dfs in dfs_by_type.items(): + table_name = type_enum[type_name].get_fact_table_name() + dfs_by_table.setdefault(table_name, []).extend(_fact_dfs_for_table(dfs)) + + for table_name, dfs_list in dfs_by_table.items(): + if not dfs_list: + continue + entity_type = type_enum.get_any_for_fact_table(table_name) + writer = writer_factory.create_writer(entity_type) + schema, uri = writer.extract_fact_schema_and_output_uri(entity_type) + writer.write(dfs_list, schema=schema, uri=uri) + + +def persist_dimensions_full( + metadata_dfs_by_type: dict[str, DataFrame], + type_enum, + writer_factory: WriterFactory, +) -> None: + """Full-overwrite persist of dimension DataFrames, grouped by output table. + + Parameters + ---------- + metadata_dfs_by_type : dict[str, DataFrame] + ``{type_name: metadata_df}``. + type_enum : Enum + The entity type-enum (resolves dimension-table names + writer). + writer_factory : WriterFactory + Factory producing the entity writer. + """ + dfs_by_table: dict[str, list[DataFrame]] = {} + for type_name, metadata_df in metadata_dfs_by_type.items(): + table_name = type_enum[type_name].get_dimension_table_name() + dfs_by_table.setdefault(table_name, []).append(metadata_df) + + for table_name, dfs_list in dfs_by_table.items(): + if not dfs_list: + continue + entity_type = type_enum.get_any_for_dimension_table(table_name) + writer = writer_factory.create_writer(entity_type) + schema, uri = writer.extract_metadata_schema_and_output_uri(entity_type) + writer.write(dfs_list, schema=schema, uri=uri) + + +def _resolve_merge_keys(merge_keys, entity_type) -> list[str]: + """Return merge keys, resolving a per-type callable if one was supplied.""" + return merge_keys(entity_type) if callable(merge_keys) else merge_keys + + +def persist_facts_incremental( + dfs_by_type: dict, + type_enum, + sink: Sink, + transform_fn: Callable[[DataFrame, StructType], DataFrame], + *, + id_column: str, + merge_keys: list[str] | Callable[[object], list[str]], + changed_ids: dict[str, list[int]], +) -> None: + """Incremental persist of fact DataFrames, per type. + + Changed definitions are rewritten atomically via ``replace_by_ids`` over all + containers; unchanged definitions are ``upsert``-ed (MERGE) over the + incremental container subset. + + Parameters + ---------- + dfs_by_type : dict + ``{type_name: value}`` where value is a structured + ``{"changed", "unchanged"}`` dict or a bare DataFrame (treated as + unchanged). + type_enum : Enum + The entity type-enum (resolves fact schema/uri + writer). + sink : Sink + Target sink exposing ``replace_by_ids`` / ``upsert``. + transform_fn : Callable[[DataFrame, StructType], DataFrame] + Prepares a DataFrame for persistence (column projection + metadata). + id_column : str + Column ``replace_by_ids`` targets for changed definitions (e.g. + ``"channel_id"``). + merge_keys : list[str] or Callable + MERGE keys for unchanged upserts; a callable is resolved per entity type + (mirrors ``Report._get_aggregation_merge_keys``). + changed_ids : dict[str, list[int]] + ``{type_name: [ids]}`` with changed definitions to replace. + """ + from impulse_reporting.persist.report_storage import WriterFactory + + factory = WriterFactory(sink) + for type_name, data in dfs_by_type.items(): + entity_type = type_enum[type_name] + writer = factory.create_writer(entity_type) + schema, uri = writer.extract_fact_schema_and_output_uri(entity_type) + keys = _resolve_merge_keys(merge_keys, entity_type) + + if isinstance(data, dict): + changed_df = data.get("changed") + unchanged_df = data.get("unchanged") + + if changed_df is not None and type_name in changed_ids: + sink.replace_by_ids( + df=transform_fn(changed_df, schema), + uri=uri, + id_column=id_column, + ids_to_replace=changed_ids[type_name], + ) + if unchanged_df is not None: + sink.upsert(transform_fn(unchanged_df, schema), uri, keys) + elif data is not None: + sink.upsert(transform_fn(data, schema), uri, keys) + + +def persist_dimensions_incremental( + metadata_dfs_by_type: dict[str, DataFrame], + type_enum, + sink: Sink, + transform_fn: Callable[[DataFrame, StructType], DataFrame], + *, + merge_keys: list[str], +) -> None: + """Incremental persist of dimension DataFrames (always ``upsert``), per type. + + Parameters + ---------- + metadata_dfs_by_type : dict[str, DataFrame] + ``{type_name: metadata_df}``. + type_enum : Enum + The entity type-enum (resolves dimension schema/uri + writer). + sink : Sink + Target sink exposing ``upsert``. + transform_fn : Callable[[DataFrame, StructType], DataFrame] + Prepares a DataFrame for persistence (column projection + metadata). + merge_keys : list[str] + MERGE keys for the dimension upsert (e.g. ``["channel_id"]``). + """ + from impulse_reporting.persist.report_storage import WriterFactory + + factory = WriterFactory(sink) + for type_name, metadata_df in metadata_dfs_by_type.items(): + entity_type = type_enum[type_name] + writer = factory.create_writer(entity_type) + schema, uri = writer.extract_metadata_schema_and_output_uri(entity_type) + sink.upsert(transform_fn(metadata_df, schema), uri, merge_keys) From 9753901305ce1e3ad8091ca971dcfcbf16ae5f0f Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Fri, 17 Jul 2026 20:03:11 +0200 Subject: [PATCH 13/23] Add unit tests for DefaultSolver's calculated channels UDF - Introduced a new test class `TestSolveCalculatedChannelsUdf` to validate the behavior of the `_solve_calculated_channels_udf` method. - Added tests to ensure correct handling of input DataFrames, including array explosion into narrow rows, type casting for `tstart` and `tend`, and behavior with empty arrays. - Verified that multiple channels emit the correct number of rows per channel ID, enhancing test coverage for calculated channel processing. - Included a mock class `_MockCalcChannel` to simulate calculated channel behavior in tests, improving isolation and reliability of unit tests. --- ...default_solver_calculated_channels_test.py | 98 ++++++ .../unit/channels/calculated_channel_test.py | 9 + .../unit/core/report_utils_test.py | 285 ++++++++++++++++++ 3 files changed, 392 insertions(+) diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py index f0e17113..e01f5c77 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py @@ -6,6 +6,7 @@ channel_id, dynamic container_id typing, validation, and empty results. """ +import pandas as pd import pyspark.sql.functions as F import pyspark.sql.types as T import pytest @@ -21,6 +22,45 @@ from impulse_query_engine.model.series.sample_series import SampleSeries from tests.conftest import basic_narrow_db, spark # noqa: F401 (pytest fixtures) +_UDF_COL_MAP = { + "cid": "container_id", + "ch": "channel_id", + "ts": "tstart", + "te": "tend", + "val": "value", + "conv": "conversion_factor", +} + + +def _udf_input_pdf(): + """A tiny single-container joined channel frame for the grouped-map UDF.""" + return pd.DataFrame( + { + "container_id": [1, 1], + "channel_id": [10, 10], + "tstart": [0, 100], + "tend": [100, 200], + "value": [3.0, 4.0], + } + ) + + +class _MockCalcChannel: + """Stands in for a query-engine CalculatedChannel in direct UDF tests. + + ``build`` ignores the cache and returns fixed ``(tstarts, tends, values)`` + arrays, mirroring the real channel's raw-array contract. + """ + + def __init__(self, channel_id, arrays, identity=None): + self.channel_id = channel_id + self._arrays = arrays + self.identity = identity or {"channel_name": "mock"} + + def build(self, cache): + return self._arrays + + # Known datum from tests/unit/data/basic_narrow_csv/channel_data.csv: # container 1, channel 5 (Engine RPM), second RLE row. _C1_RPM_TSTART = 1499929245761999 @@ -293,3 +333,61 @@ def test_base_solver_not_supported(self, spark, basic_narrow_db): q.select(cc) with pytest.raises(NotImplementedError, match="calculated channels"): q.solve_calculated_channels(spark, solver=BlobSolver()) + + +class TestSolveCalculatedChannelsUdf: + """Direct unit tests for the grouped-map UDF body. + + Calls ``DefaultSolver._solve_calculated_channels_udf`` with a plain pandas + DataFrame — it normally runs inside a Spark pandas-UDF worker process, so + exercising it directly is what actually covers the explode loop. Identity is + intentionally absent from the UDF output (attached later by + ``solve_calculated_channels``). + """ + + def test_explodes_arrays_into_narrow_rows(self): + cc = _MockCalcChannel(channel_id=10, arrays=([0, 100], [100, 200], [3.0, 4.0])) + result = DefaultSolver._solve_calculated_channels_udf( + _udf_input_pdf(), selections=[cc], col_map=_UDF_COL_MAP + ) + assert list(result.columns) == ["container_id", "channel_id", "tstart", "tend", "value"] + assert "identity" not in result.columns + assert list(result["container_id"]) == [1, 1] + assert list(result["channel_id"]) == [10, 10] + assert list(result["tstart"]) == [0, 100] + assert list(result["tend"]) == [100, 200] + assert list(result["value"]) == [3.0, 4.0] + + def test_tstart_tend_cast_to_int_value_to_float(self): + # Float tstart/tend arrays (as SampleSeries stores them) are truncated to + # integer columns; the value column stays floating point. + cc = _MockCalcChannel(channel_id=7, arrays=([0.5], [100.9], [5])) + result = DefaultSolver._solve_calculated_channels_udf( + _udf_input_pdf(), selections=[cc], col_map=_UDF_COL_MAP + ) + # int(0.5) == 0, int(100.9) == 100 → truncation happened. + assert result["tstart"].iloc[0] == 0 + assert result["tend"].iloc[0] == 100 + assert pd.api.types.is_integer_dtype(result["tstart"]) + assert pd.api.types.is_integer_dtype(result["tend"]) + assert result["value"].iloc[0] == 5.0 + assert pd.api.types.is_float_dtype(result["value"]) + + def test_empty_arrays_produce_zero_rows_full_width(self): + cc = _MockCalcChannel(channel_id=10, arrays=([], [], [])) + result = DefaultSolver._solve_calculated_channels_udf( + _udf_input_pdf(), selections=[cc], col_map=_UDF_COL_MAP + ) + assert len(result) == 0 + assert list(result.columns) == ["container_id", "channel_id", "tstart", "tend", "value"] + + def test_multi_channel_emits_rows_per_channel_id(self): + cc_a = _MockCalcChannel(channel_id=10, arrays=([0], [100], [3.0])) + cc_b = _MockCalcChannel(channel_id=20, arrays=([0, 100], [100, 200], [1.0, 2.0])) + result = DefaultSolver._solve_calculated_channels_udf( + _udf_input_pdf(), selections=[cc_a, cc_b], col_map=_UDF_COL_MAP + ) + by_channel = result.groupby("channel_id").size().to_dict() + assert by_channel == {10: 1, 20: 2} + # Every emitted row carries this container's id. + assert set(result["container_id"]) == {1} diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py index c6697c69..50037331 100644 --- a/tests/impulse_reporting/unit/channels/calculated_channel_test.py +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -38,6 +38,15 @@ def test_get_expression_is_none(self): # Channels drive their own solve, so they are excluded from the batch solve. assert _channel().get_expression() is None + def test_get_expression_str_returns_str_for_expression(self): + assert _channel().get_expression_str() != "NA" + + def test_get_expression_str_na_when_not_expression(self): + # Defensive branch: if the wrapped object isn't a TimeSeriesExpression. + ch = _channel() + ch.expression = object() + assert ch.get_expression_str() == "NA" + def test_channel_type_str(self): assert _channel().get_channel_type_str() == "CALCULATED_CHANNEL" diff --git a/tests/impulse_reporting/unit/core/report_utils_test.py b/tests/impulse_reporting/unit/core/report_utils_test.py index 4977e7d7..d602cd05 100644 --- a/tests/impulse_reporting/unit/core/report_utils_test.py +++ b/tests/impulse_reporting/unit/core/report_utils_test.py @@ -1,5 +1,6 @@ """Unit tests for report_utils helper functions.""" +from enum import Enum from unittest.mock import MagicMock, create_autospec, patch from databricks.sdk import WorkspaceClient @@ -8,7 +9,14 @@ from impulse_reporting.core.report import Report from impulse_reporting.core.report_utils import ( build_batches, + build_metadata_dfs, dispatch_events, + group_selectables_by_type, + merge_changed_unchanged, + persist_dimensions_full, + persist_dimensions_incremental, + persist_facts_full, + persist_facts_incremental, split_by_hash_change, ) @@ -715,3 +723,280 @@ def test_query_select_called_with_batch_expressions(self, spark): report.query.select.assert_called_once_with(expr) report.query.select.return_value.solve.assert_called_once() + + +# ============================================================================ +# Fixtures for the generic entity orchestration/persistence helpers +# ============================================================================ +class _FooEntity: + pass + + +class _BarEntity: + pass + + +class _FakeType(Enum): + """Two-member entity type-enum mirroring ChannelType's resolver shape. + + FOO and BAR intentionally share a fact table (``shared_fact``) but have + distinct dimension tables, so grouping-by-output-table is exercised. + """ + + FOO = _FooEntity + BAR = _BarEntity + + def get_fact_table_name(self): + return "shared_fact" + + def get_dimension_table_name(self): + return {"FOO": "foo_dim", "BAR": "bar_dim"}[self.name] + + @classmethod + def get_any_for_fact_table(cls, table_name): + for member in cls: + if member.get_fact_table_name() == table_name: + return member + raise ValueError(f"no type for fact table {table_name}") + + @classmethod + def get_any_for_dimension_table(cls, table_name): + for member in cls: + if member.get_dimension_table_name() == table_name: + return member + raise ValueError(f"no type for dimension table {table_name}") + + +class TestGroupSelectablesByType: + def test_buckets_by_isinstance(self): + foo1, bar, foo2 = _FooEntity(), _BarEntity(), _FooEntity() + result = group_selectables_by_type([foo1, bar, foo2], _FakeType) + assert result["FOO"] == [foo1, foo2] + assert result["BAR"] == [bar] + + def test_every_member_gets_a_bucket_even_when_empty(self): + result = group_selectables_by_type([], _FakeType) + assert set(result) == {"FOO", "BAR"} + assert result["FOO"] == [] and result["BAR"] == [] + + def test_item_matching_no_type_is_dropped(self): + # An item that isn't an instance of any member's class lands in no bucket. + result = group_selectables_by_type([object()], _FakeType) + assert result == {"FOO": [], "BAR": []} + + +class TestMergeChangedUnchanged: + def test_only_changed(self): + df = MagicMock() + result = merge_changed_unchanged({"FOO": df}, {}) + assert result == {"FOO": {"changed": df, "unchanged": None}} + + def test_only_unchanged(self): + df = MagicMock() + result = merge_changed_unchanged({}, {"FOO": df}) + assert result == {"FOO": {"changed": None, "unchanged": df}} + + def test_both_sides_present(self): + c, u = MagicMock(), MagicMock() + result = merge_changed_unchanged({"FOO": c}, {"FOO": u}) + assert result == {"FOO": {"changed": c, "unchanged": u}} + + +class TestBuildMetadataDfs: + def test_non_empty_types_build_metadata(self): + meta = MagicMock() + cls = MagicMock() + cls.determine_metadata_df.return_value = meta + type_enum = MagicMock() + type_enum.__getitem__.return_value.value = cls + spark = MagicMock() + items = [MagicMock()] + + result = build_metadata_dfs({"FOO": items}, type_enum, spark) + + cls.determine_metadata_df.assert_called_once_with(spark, items) + assert result == {"FOO": meta} + + def test_empty_types_skipped(self): + type_enum = MagicMock() + result = build_metadata_dfs({"FOO": []}, type_enum, MagicMock()) + assert result == {} + + +class TestPersistFactsFull: + def test_groups_by_table_and_flattens_dict_and_bare(self): + bare, changed, unchanged = MagicMock(), MagicMock(), MagicMock() + writer = MagicMock() + writer.extract_fact_schema_and_output_uri.return_value = ("schema", "uri") + factory = MagicMock() + factory.create_writer.return_value = writer + + persist_facts_full( + {"FOO": bare, "BAR": {"changed": changed, "unchanged": unchanged}}, + _FakeType, + factory, + ) + + # FOO + BAR share "shared_fact" → a single write with all three dfs. + writer.write.assert_called_once() + written = writer.write.call_args.args[0] + assert written == [bare, changed, unchanged] + + def test_none_entries_dropped(self): + writer = MagicMock() + writer.extract_fact_schema_and_output_uri.return_value = ("schema", "uri") + factory = MagicMock() + factory.create_writer.return_value = writer + + persist_facts_full({"FOO": {"changed": None, "unchanged": None}}, _FakeType, factory) + # Nothing to write → no write call. + writer.write.assert_not_called() + + +class TestPersistDimensionsFull: + def test_writes_each_dimension_table(self): + foo_meta, bar_meta = MagicMock(), MagicMock() + writer = MagicMock() + writer.extract_metadata_schema_and_output_uri.return_value = ("schema", "uri") + factory = MagicMock() + factory.create_writer.return_value = writer + + persist_dimensions_full({"FOO": foo_meta, "BAR": bar_meta}, _FakeType, factory) + + # Distinct dim tables (foo_dim, bar_dim) → one write each. + assert writer.write.call_count == 2 + + +class TestPersistFactsIncremental: + def _patch_factory(self, writer): + factory = MagicMock() + factory.create_writer.return_value = writer + return patch( + "impulse_reporting.persist.report_storage.WriterFactory", return_value=factory + ) + + def test_changed_replaced_and_unchanged_upserted(self): + changed, unchanged = MagicMock(), MagicMock() + writer = MagicMock() + writer.extract_fact_schema_and_output_uri.return_value = ("schema", "uri") + sink = MagicMock() + + with self._patch_factory(writer): + persist_facts_incremental( + {"FOO": {"changed": changed, "unchanged": unchanged}}, + _FakeType, + sink, + lambda df, _schema: df, + id_column="channel_id", + merge_keys=["container_id", "channel_id", "tstart"], + changed_ids={"FOO": [1, 2]}, + ) + + sink.replace_by_ids.assert_called_once_with( + df=changed, uri="uri", id_column="channel_id", ids_to_replace=[1, 2] + ) + sink.upsert.assert_called_once_with( + unchanged, "uri", ["container_id", "channel_id", "tstart"] + ) + + def test_changed_skipped_when_type_not_in_changed_ids(self): + changed = MagicMock() + writer = MagicMock() + writer.extract_fact_schema_and_output_uri.return_value = ("schema", "uri") + sink = MagicMock() + + with self._patch_factory(writer): + persist_facts_incremental( + {"FOO": {"changed": changed, "unchanged": None}}, + _FakeType, + sink, + lambda df, _schema: df, + id_column="channel_id", + merge_keys=["channel_id"], + changed_ids={}, + ) + + sink.replace_by_ids.assert_not_called() + sink.upsert.assert_not_called() + + def test_none_value_is_a_noop(self): + # A type whose fact value is None triggers neither replace nor upsert. + writer = MagicMock() + writer.extract_fact_schema_and_output_uri.return_value = ("schema", "uri") + sink = MagicMock() + + with self._patch_factory(writer): + persist_facts_incremental( + {"FOO": None}, + _FakeType, + sink, + lambda df, _schema: df, + id_column="channel_id", + merge_keys=["channel_id"], + changed_ids={}, + ) + + sink.replace_by_ids.assert_not_called() + sink.upsert.assert_not_called() + + def test_bare_df_is_upserted(self): + bare = MagicMock() + writer = MagicMock() + writer.extract_fact_schema_and_output_uri.return_value = ("schema", "uri") + sink = MagicMock() + + with self._patch_factory(writer): + persist_facts_incremental( + {"FOO": bare}, + _FakeType, + sink, + lambda df, _schema: df, + id_column="channel_id", + merge_keys=["channel_id"], + changed_ids={}, + ) + + sink.upsert.assert_called_once_with(bare, "uri", ["channel_id"]) + + def test_callable_merge_keys_resolved_per_type(self): + unchanged = MagicMock() + writer = MagicMock() + writer.extract_fact_schema_and_output_uri.return_value = ("schema", "uri") + sink = MagicMock() + + def keys_for(entity_type): + return ["container_id", entity_type.name.lower()] + + with self._patch_factory(writer): + persist_facts_incremental( + {"FOO": {"changed": None, "unchanged": unchanged}}, + _FakeType, + sink, + lambda df, _schema: df, + id_column="id", + merge_keys=keys_for, + changed_ids={}, + ) + + sink.upsert.assert_called_once_with(unchanged, "uri", ["container_id", "foo"]) + + +class TestPersistDimensionsIncremental: + def test_upserts_each_type_with_merge_keys(self): + foo_meta = MagicMock() + writer = MagicMock() + writer.extract_metadata_schema_and_output_uri.return_value = ("schema", "uri") + factory = MagicMock() + factory.create_writer.return_value = writer + sink = MagicMock() + + with patch("impulse_reporting.persist.report_storage.WriterFactory", return_value=factory): + persist_dimensions_incremental( + {"FOO": foo_meta}, + _FakeType, + sink, + lambda df, _schema: df, + merge_keys=["channel_id"], + ) + + sink.upsert.assert_called_once_with(foo_meta, "uri", ["channel_id"]) From 716c24b8548b9e4cbbaf393a1a03b216d1b08a98 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Tue, 21 Jul 2026 20:48:33 +0200 Subject: [PATCH 14/23] Refactor persist_facts_incremental to group by output table for shared types - Enhanced the `persist_facts_incremental` function to group changed and unchanged DataFrames by their output table, allowing for efficient persistence of shared entity types. - Implemented atomic updates for changed definitions using `unionByName` and a single `replace_by_ids` call, while unchanged definitions are upserted individually. - Updated the function's docstring to clarify the new behavior and added unit tests to validate the handling of shared tables in the persistence logic. --- src/impulse_reporting/core/report.py | 310 ++++-------------- src/impulse_reporting/core/report_utils.py | 68 ++-- .../unit/core/report_utils_test.py | 55 ++++ 3 files changed, 166 insertions(+), 267 deletions(-) diff --git a/src/impulse_reporting/core/report.py b/src/impulse_reporting/core/report.py index 36ac59a6..383087a0 100644 --- a/src/impulse_reporting/core/report.py +++ b/src/impulse_reporting/core/report.py @@ -1,6 +1,5 @@ import json import zlib -from functools import reduce from typing import Any from databricks.sdk import WorkspaceClient from pyspark.sql import DataFrame, SparkSession @@ -437,13 +436,7 @@ def _group_events_by_type(self): dict Dictionary mapping event type names to lists of events. """ - event_types = {event_type.name: [] for event_type in EventType} - for event in self.events: - for event_type in event_types.keys(): - if isinstance(event, EventType[event_type].value): - event_types[event_type].append(event) - break - return event_types + return group_selectables_by_type(self.events, EventType) def add_calculated_channel(self, channel): """ @@ -516,14 +509,8 @@ def _group_aggregations_by_type(self): dict Dictionary mapping aggregation type names to lists of aggregations. """ - agg_types = {agg_type.name: [] for agg_type in AggregationType} - for page in self.pages: - for aggregation in page.aggregations: - for agg_type in agg_types.keys(): - if isinstance(aggregation, AggregationType[agg_type].value): - agg_types[agg_type].append(aggregation) - break - return agg_types + aggregations = [agg for page in self.pages for agg in page.aggregations] + return group_selectables_by_type(aggregations, AggregationType) def _validate_aggregation_events(self) -> None: """ @@ -596,84 +583,16 @@ def _persist_full(self): """ storage_factory = WriterFactory(self.sink) - # aggregation fact tables — group by output table to handle shared tables - # (e.g. StatsAggregator and PointValueAggregator both write stats_aggregator_fact) - agg_fact_by_table = {} - for aggregation_type_str, aggregation_dfs in self.aggregation_dfs.items(): - table_name = AggregationType[aggregation_type_str].get_fact_table_name() - agg_fact_by_table.setdefault(table_name, []) - - # Handle both dict format (from incremental mode) and DataFrame format - if isinstance(aggregation_dfs, dict): - if aggregation_dfs.get("changed") is not None: - agg_fact_by_table[table_name].append(aggregation_dfs["changed"]) - if aggregation_dfs.get("unchanged") is not None: - agg_fact_by_table[table_name].append(aggregation_dfs["unchanged"]) - else: - agg_fact_by_table[table_name].append(aggregation_dfs) - - for table_name, agg_dfs_list in agg_fact_by_table.items(): - if not agg_dfs_list: - continue - aggregation_type = AggregationType.get_any_for_fact_table(table_name) - writer = storage_factory.create_writer(aggregation_type) - schema, uri = writer.extract_fact_schema_and_output_uri(aggregation_type) - writer.write(agg_dfs_list, schema=schema, uri=uri) - - # aggregation dimension tables — group by output table to handle shared tables - agg_dim_by_table = {} - for ( - aggregation_type_str, - aggregation_metadata_dfs, - ) in self.aggregation_metadata_dfs.items(): - table_name = AggregationType[aggregation_type_str].get_dimension_table_name() - agg_dim_by_table.setdefault(table_name, []) - agg_dim_by_table[table_name].append(aggregation_metadata_dfs) - - for table_name, agg_meta_dfs_list in agg_dim_by_table.items(): - if not agg_meta_dfs_list: - continue - aggregation_type = AggregationType.get_any_for_dimension_table(table_name) - writer = storage_factory.create_writer(aggregation_type) - schema, uri = writer.extract_metadata_schema_and_output_uri(aggregation_type) - writer.write(agg_meta_dfs_list, schema=schema, uri=uri) - - # event fact tables — group by output table to handle mixed event types - event_fact_by_table = {} - for event_type_str, event_dfs in self.event_dfs.items(): - table_name = EventType[event_type_str].get_fact_table_name() - event_fact_by_table.setdefault(table_name, []) - - if isinstance(event_dfs, dict): - if event_dfs.get("changed") is not None: - event_fact_by_table[table_name].append(event_dfs["changed"]) - if event_dfs.get("unchanged") is not None: - event_fact_by_table[table_name].append(event_dfs["unchanged"]) - else: - event_fact_by_table[table_name].append(event_dfs) - - for table_name, event_dfs_list in event_fact_by_table.items(): - if not event_dfs_list: - continue - event_type = EventType.get_any_for_fact_table(table_name) - writer = storage_factory.create_writer(event_type) - schema, uri = writer.extract_fact_schema_and_output_uri(event_type) - writer.write(event_dfs_list, schema=schema, uri=uri) - - # event dimension tables — group by output table to handle mixed event types - event_dim_by_table = {} - for event_type_str, event_metadata_dfs in self.event_metadata_dfs.items(): - table_name = EventType[event_type_str].get_dimension_table_name() - event_dim_by_table.setdefault(table_name, []) - event_dim_by_table[table_name].append(event_metadata_dfs) - - for table_name, event_meta_dfs_list in event_dim_by_table.items(): - if not event_meta_dfs_list: - continue - event_type = EventType.get_any_for_dimension_table(table_name) - writer = storage_factory.create_writer(event_type) - schema, uri = writer.extract_metadata_schema_and_output_uri(event_type) - writer.write(event_meta_dfs_list, schema=schema, uri=uri) + # aggregation fact + dimension tables — grouped by output table so shared + # tables (e.g. StatsAggregator + PointValueAggregator → stats_aggregator_fact) + # are written together. + persist_facts_full(self.aggregation_dfs, AggregationType, storage_factory) + persist_dimensions_full(self.aggregation_metadata_dfs, AggregationType, storage_factory) + + # event fact + dimension tables — grouped by output table to handle mixed + # event types sharing a table. + persist_facts_full(self.event_dfs, EventType, storage_factory) + persist_dimensions_full(self.event_metadata_dfs, EventType, storage_factory) # calculated channel fact + dimension tables persist_facts_full(self.calculated_channel_dfs, ChannelType, storage_factory) @@ -720,119 +639,48 @@ def _persist_incremental( storage_factory = WriterFactory(self.sink) transformer = ReportEntityTransformer() - # Persist aggregation facts - for aggregation_type_str, agg_data in self.aggregation_dfs.items(): - aggregation_type = AggregationType[aggregation_type_str] - writer = storage_factory.create_writer(aggregation_type) - schema, uri = writer.extract_fact_schema_and_output_uri(aggregation_type) - merge_keys = self._get_aggregation_merge_keys(aggregation_type) - - if isinstance(agg_data, dict): - # Structured format: {'changed': df, 'unchanged': df} - changed_df = agg_data.get("changed") - unchanged_df = agg_data.get("unchanged") - - # Changed definitions: replaceWhere (atomic) - if changed_df is not None and aggregation_type_str in changed_aggregation_ids: - changed_ids = changed_aggregation_ids[aggregation_type_str] - # Transform and enrich the DataFrame before persisting - df_enriched = self._transform_for_persistence(changed_df, schema, transformer) - self.sink.replace_by_ids( - df=df_enriched, - uri=uri, - id_column="visual_id", - ids_to_replace=changed_ids, - ) - - # Unchanged definitions: MERGE - if unchanged_df is not None: - df_enriched = self._transform_for_persistence( - unchanged_df, schema, transformer - ) - self.sink.upsert(df_enriched, uri, merge_keys) - else: - # Backward compatibility: single DataFrame - use MERGE - df_enriched = self._transform_for_persistence(agg_data, schema, transformer) - self.sink.upsert(df_enriched, uri, merge_keys) - - # Persist aggregation dimensions (always upsert by visual_id) - for ( - aggregation_type_str, - aggregation_metadata_df, - ) in self.aggregation_metadata_dfs.items(): - aggregation_type = AggregationType[aggregation_type_str] - writer = storage_factory.create_writer(aggregation_type) - schema, uri = writer.extract_metadata_schema_and_output_uri(aggregation_type) - df_enriched = self._transform_for_persistence( - aggregation_metadata_df, schema, transformer - ) - self.sink.upsert(df_enriched, uri, ["visual_id"]) - - # Persist event facts — group by output table to handle mixed event types - event_fact_changed_by_table: dict[str, list] = {} - event_fact_unchanged_by_table: dict[str, list] = {} - event_changed_ids_by_table: dict[str, list[int]] = {} - for event_type_str, event_data in self.event_dfs.items(): - table_name = EventType[event_type_str].get_fact_table_name() - - if isinstance(event_data, dict): - changed_df = event_data.get("changed") - unchanged_df = event_data.get("unchanged") - - if changed_df is not None and event_type_str in changed_event_ids: - event_fact_changed_by_table.setdefault(table_name, []).append(changed_df) - event_changed_ids_by_table.setdefault(table_name, []).extend( - changed_event_ids[event_type_str] - ) - - if unchanged_df is not None: - event_fact_unchanged_by_table.setdefault(table_name, []).append(unchanged_df) - else: - event_fact_unchanged_by_table.setdefault(table_name, []).append(event_data) - - for table_name in set( - list(event_fact_changed_by_table.keys()) + list(event_fact_unchanged_by_table.keys()) - ): - event_type = EventType.get_any_for_fact_table(table_name) - writer = storage_factory.create_writer(event_type) - schema, uri = writer.extract_fact_schema_and_output_uri(event_type) - merge_keys = ["container_id", "event_id", "event_instance_id"] - - # Changed definitions: replaceWhere (atomic) - changed_dfs = event_fact_changed_by_table.get(table_name, []) - changed_ids = event_changed_ids_by_table.get(table_name, []) - if changed_dfs and changed_ids: - transformed = [ - self._transform_for_persistence(cdf, schema, transformer) - for cdf in changed_dfs - ] - combined_df = reduce(lambda a, b: a.unionByName(b), transformed) - self.sink.replace_by_ids( - df=combined_df, - uri=uri, - id_column="event_id", - ids_to_replace=changed_ids, - ) + def _transform(df, schema): + return self._transform_for_persistence(df, schema, transformer) - # Unchanged definitions: MERGE - unchanged_dfs = event_fact_unchanged_by_table.get(table_name, []) - for udf in unchanged_dfs: - df_enriched = self._transform_for_persistence(udf, schema, transformer) - self.sink.upsert(df_enriched, uri, merge_keys) - - # Persist event dimensions — group by output table to handle mixed event types - event_dim_by_table: dict[str, list] = {} - for event_type_str, event_metadata_df in self.event_metadata_dfs.items(): - table_name = EventType[event_type_str].get_dimension_table_name() - event_dim_by_table.setdefault(table_name, []).append(event_metadata_df) - - for table_name, event_meta_dfs_list in event_dim_by_table.items(): - event_type = EventType.get_any_for_dimension_table(table_name) - writer = storage_factory.create_writer(event_type) - schema, uri = writer.extract_metadata_schema_and_output_uri(event_type) - for mdf in event_meta_dfs_list: - df_enriched = self._transform_for_persistence(mdf, schema, transformer) - self.sink.upsert(df_enriched, uri, ["event_id"]) + # Persist aggregation facts + dimensions. StatsAggregator and + # PointValueAggregator share stats_aggregator_fact, so facts group by + # table; merge keys are per-type (via _get_aggregation_merge_keys). + persist_facts_incremental( + self.aggregation_dfs, + AggregationType, + self.sink, + _transform, + id_column="visual_id", + merge_keys=self._get_aggregation_merge_keys, + changed_ids=changed_aggregation_ids, + ) + persist_dimensions_incremental( + self.aggregation_metadata_dfs, + AggregationType, + self.sink, + _transform, + merge_keys=["visual_id"], + ) + + # Persist event facts + dimensions. Mixed event types share + # event_instance_fact, so facts group by table and union changed defs + # before a single replaceWhere. + persist_facts_incremental( + self.event_dfs, + EventType, + self.sink, + _transform, + id_column="event_id", + merge_keys=["container_id", "event_id", "event_instance_id"], + changed_ids=changed_event_ids, + ) + persist_dimensions_incremental( + self.event_metadata_dfs, + EventType, + self.sink, + _transform, + merge_keys=["event_id"], + ) # Persist measurement dimension (upsert by container_id) if self.container_dimension_df: @@ -862,9 +710,6 @@ def _persist_incremental( ) # Persist calculated channel facts + dimensions - def _transform(df, schema): - return self._transform_for_persistence(df, schema, transformer) - persist_facts_incremental( self.calculated_channel_dfs, ChannelType, @@ -1089,25 +934,10 @@ def determine_report(self, is_incremental: bool = None): ContainerEvent, ) - # Merge event results into {type: {"changed": df, "unchanged": df}} - event_dfs = {} - all_event_types = set(list(changed_event_dfs.keys()) + list(unchanged_event_dfs.keys())) - for t in all_event_types: - event_dfs[t] = { - "changed": changed_event_dfs.get(t), - "unchanged": unchanged_event_dfs.get(t), - } - - # Metadata: merge from all events (changed + unchanged) - event_metadata_dfs = {} - for event_name, events_list in events_by_type.items(): - if not events_list: - continue - cls = EventType[event_name].value - event_metadata_dfs[event_name] = cls.determine_metadata_df(self.spark, events_list) - - self.event_dfs = event_dfs - self.event_metadata_dfs = event_metadata_dfs + # Merge event results into {type: {"changed": df, "unchanged": df}} and + # build event dimensions from all (changed + unchanged) events. + self.event_dfs = merge_changed_unchanged(changed_event_dfs, unchanged_event_dfs) + self.event_metadata_dfs = build_metadata_dfs(events_by_type, EventType, self.spark) # Dispatch aggregations changed_agg_dfs = dispatch_aggregations( @@ -1123,25 +953,11 @@ def determine_report(self, is_incremental: bool = None): unchanged_solved_df, ) - # Merge aggregation results - aggregation_dfs = {} - all_agg_types = set(list(changed_agg_dfs.keys()) + list(unchanged_agg_dfs.keys())) - for t in all_agg_types: - aggregation_dfs[t] = { - "changed": changed_agg_dfs.get(t), - "unchanged": unchanged_agg_dfs.get(t), - } - - # Metadata: merge from all aggregations - aggregation_metadata_dfs = {} - for agg_name, agg_list in aggs_by_type.items(): - if not agg_list: - continue - cls = AggregationType[agg_name].value - aggregation_metadata_dfs[agg_name] = cls.determine_metadata_df(self.spark, agg_list) - - self.aggregation_dfs = aggregation_dfs - self.aggregation_metadata_dfs = aggregation_metadata_dfs + # Merge aggregation results and build aggregation dimensions from all. + self.aggregation_dfs = merge_changed_unchanged(changed_agg_dfs, unchanged_agg_dfs) + self.aggregation_metadata_dfs = build_metadata_dfs( + aggs_by_type, AggregationType, self.spark + ) # Calculated channels: own narrow solve (not the wide solved_df). # Changed definitions recompute over all containers; unchanged ones over diff --git a/src/impulse_reporting/core/report_utils.py b/src/impulse_reporting/core/report_utils.py index de78883f..4e58458d 100644 --- a/src/impulse_reporting/core/report_utils.py +++ b/src/impulse_reporting/core/report_utils.py @@ -7,6 +7,7 @@ from __future__ import annotations import uuid +from functools import reduce from typing import TYPE_CHECKING from pyspark.sql import DataFrame, SparkSession @@ -632,11 +633,18 @@ def persist_facts_incremental( merge_keys: list[str] | Callable[[object], list[str]], changed_ids: dict[str, list[int]], ) -> None: - """Incremental persist of fact DataFrames, per type. + """Incremental persist of fact DataFrames, grouped by output table. - Changed definitions are rewritten atomically via ``replace_by_ids`` over all - containers; unchanged definitions are ``upsert``-ed (MERGE) over the - incremental container subset. + Per-type facts are grouped by their fact-table name, so entity types that + share a table (e.g. ``StatsAggregator`` + ``PointValueAggregator`` → + ``stats_aggregator_fact``, or mixed event types → ``event_instance_fact``) + are persisted together with no clobber. For each table: + + - **Changed** definitions (only types listed in *changed_ids*) are + ``unionByName``-combined and rewritten atomically in a single + ``replace_by_ids`` over all containers. + - **Unchanged** definitions are ``upsert``-ed (MERGE) over the incremental + container subset. Parameters ---------- @@ -652,7 +660,7 @@ def persist_facts_incremental( Prepares a DataFrame for persistence (column projection + metadata). id_column : str Column ``replace_by_ids`` targets for changed definitions (e.g. - ``"channel_id"``). + ``"channel_id"``, ``"visual_id"``, ``"event_id"``). merge_keys : list[str] or Callable MERGE keys for unchanged upserts; a callable is resolved per entity type (mirrors ``Report._get_aggregation_merge_keys``). @@ -661,28 +669,48 @@ def persist_facts_incremental( """ from impulse_reporting.persist.report_storage import WriterFactory - factory = WriterFactory(sink) + # Group per-type facts by output table so shared-table types persist together. + changed_by_table: dict[str, list[DataFrame]] = {} + unchanged_by_table: dict[str, list[DataFrame]] = {} + changed_ids_by_table: dict[str, list[int]] = {} for type_name, data in dfs_by_type.items(): - entity_type = type_enum[type_name] - writer = factory.create_writer(entity_type) - schema, uri = writer.extract_fact_schema_and_output_uri(entity_type) - keys = _resolve_merge_keys(merge_keys, entity_type) - + table_name = type_enum[type_name].get_fact_table_name() if isinstance(data, dict): changed_df = data.get("changed") unchanged_df = data.get("unchanged") - if changed_df is not None and type_name in changed_ids: - sink.replace_by_ids( - df=transform_fn(changed_df, schema), - uri=uri, - id_column=id_column, - ids_to_replace=changed_ids[type_name], - ) + changed_by_table.setdefault(table_name, []).append(changed_df) + changed_ids_by_table.setdefault(table_name, []).extend(changed_ids[type_name]) if unchanged_df is not None: - sink.upsert(transform_fn(unchanged_df, schema), uri, keys) + unchanged_by_table.setdefault(table_name, []).append(unchanged_df) elif data is not None: - sink.upsert(transform_fn(data, schema), uri, keys) + unchanged_by_table.setdefault(table_name, []).append(data) + + factory = WriterFactory(sink) + for table_name in set(changed_by_table) | set(unchanged_by_table): + entity_type = type_enum.get_any_for_fact_table(table_name) + writer = factory.create_writer(entity_type) + schema, uri = writer.extract_fact_schema_and_output_uri(entity_type) + keys = _resolve_merge_keys(merge_keys, entity_type) + + # Changed definitions: union then a single atomic replaceWhere. + changed_dfs = changed_by_table.get(table_name, []) + table_changed_ids = changed_ids_by_table.get(table_name, []) + if changed_dfs and table_changed_ids: + combined = reduce( + lambda a, b: a.unionByName(b), + (transform_fn(cdf, schema) for cdf in changed_dfs), + ) + sink.replace_by_ids( + df=combined, + uri=uri, + id_column=id_column, + ids_to_replace=table_changed_ids, + ) + + # Unchanged definitions: MERGE over the incremental subset. + for udf in unchanged_by_table.get(table_name, []): + sink.upsert(transform_fn(udf, schema), uri, keys) def persist_dimensions_incremental( diff --git a/tests/impulse_reporting/unit/core/report_utils_test.py b/tests/impulse_reporting/unit/core/report_utils_test.py index d602cd05..83ec46af 100644 --- a/tests/impulse_reporting/unit/core/report_utils_test.py +++ b/tests/impulse_reporting/unit/core/report_utils_test.py @@ -980,6 +980,61 @@ def keys_for(entity_type): sink.upsert.assert_called_once_with(unchanged, "uri", ["container_id", "foo"]) + def test_shared_table_changed_types_union_into_one_replace(self): + # FOO and BAR share `shared_fact`: their changed DFs are unioned and + # rewritten in a single replace_by_ids with the combined ids. + foo_changed, bar_changed, unioned = MagicMock(), MagicMock(), MagicMock() + foo_changed.unionByName.return_value = unioned + writer = MagicMock() + writer.extract_fact_schema_and_output_uri.return_value = ("schema", "uri") + sink = MagicMock() + + with self._patch_factory(writer): + persist_facts_incremental( + { + "FOO": {"changed": foo_changed, "unchanged": None}, + "BAR": {"changed": bar_changed, "unchanged": None}, + }, + _FakeType, + sink, + lambda df, _schema: df, + id_column="visual_id", + merge_keys=["visual_id"], + changed_ids={"FOO": [1], "BAR": [2]}, + ) + + # Single replace over the shared table with unioned DF + combined ids. + foo_changed.unionByName.assert_called_once_with(bar_changed) + sink.replace_by_ids.assert_called_once_with( + df=unioned, uri="uri", id_column="visual_id", ids_to_replace=[1, 2] + ) + + def test_shared_table_unchanged_types_each_upserted(self): + # FOO and BAR unchanged dfs both target `shared_fact` → one upsert each. + foo_unchanged, bar_unchanged = MagicMock(), MagicMock() + writer = MagicMock() + writer.extract_fact_schema_and_output_uri.return_value = ("schema", "uri") + sink = MagicMock() + + with self._patch_factory(writer): + persist_facts_incremental( + { + "FOO": {"changed": None, "unchanged": foo_unchanged}, + "BAR": {"changed": None, "unchanged": bar_unchanged}, + }, + _FakeType, + sink, + lambda df, _schema: df, + id_column="visual_id", + merge_keys=["visual_id"], + changed_ids={}, + ) + + sink.replace_by_ids.assert_not_called() + assert sink.upsert.call_count == 2 + upserted = {call.args[0] for call in sink.upsert.call_args_list} + assert upserted == {foo_unchanged, bar_unchanged} + class TestPersistDimensionsIncremental: def test_upserts_each_type_with_merge_keys(self): From e1de28f231ddfb5243017f22e56607a20678d30a Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Tue, 21 Jul 2026 21:07:13 +0200 Subject: [PATCH 15/23] Refactor identity handling and improve query builder logic - Removed redundant iteration over selections in `QueryBuilder` to streamline the evaluation type requirement for `CalculatedChannel`. - Introduced a static method `_identity_map_expr` in `DefaultSolver` to encapsulate the creation of a map from identity dictionaries, enhancing code clarity and reusability. - Simplified the `get_expression_str` method in `CalculatedChannel` to directly return the string representation of the expression, improving maintainability. - Refactored the `DefinitionHashComparator` to generalize the grouping of entities by definition hash, allowing for more flexible handling of different entity types and reducing code duplication. - Updated unit tests to reflect changes in the `get_expression_str` method and ensure accurate behavior across the refactored components. --- .../analyze/query/query_builder.py | 2 - .../analyze/query/solvers/default_solver.py | 10 +- .../channels/calculated_channel.py | 4 +- .../incremental/definition_hash_comparator.py | 195 +++++------------- .../unit/channels/calculated_channel_test.py | 9 +- 5 files changed, 67 insertions(+), 153 deletions(-) diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index b85e8689..26ac47e3 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -347,8 +347,6 @@ def _validate_calculated_channels(self) -> None: "solve_calculated_channels() requires all selections to be " f"CalculatedChannel; got {type(s).__name__} at index {i}." ) - - for s in self.selections: s.expr.require_evaluation_type( SampleSeries, owner="CalculatedChannel", diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index edca0b11..049e3fc6 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -1060,6 +1060,12 @@ def _solve_calculated_channels_udf( return pd.DataFrame(cols) + @staticmethod + def _identity_map_expr(identity: dict[str, str]) -> "F.Column": + """Build a ``map`` literal column from an identity dict.""" + kv_literals = [F.lit(str(x)) for k, v in identity.items() for x in (k, v)] + return F.create_map(*kv_literals) + def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame: """ Solve calculated channels by grouping channels and exploding each result. @@ -1106,10 +1112,8 @@ def solve_calculated_channels(self, query, channels_df, selections) -> DataFrame channel_id_col = self.config.channel_id_col identity_expr = None for cc in selections: - id_map = F.create_map( - *[F.lit(x) for k, v in cc.identity.items() for x in (str(k), str(v))] - ) branch = F.col(channel_id_col) == F.lit(cc.channel_id) + id_map = self._identity_map_expr(cc.identity) identity_expr = ( F.when(branch, id_map) if identity_expr is None diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py index e996640d..3c507543 100644 --- a/src/impulse_reporting/channels/calculated_channel.py +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -119,9 +119,7 @@ def get_expression(self) -> TimeSeriesExpression | None: def get_expression_str(self) -> str: """String form of the wrapped expression (identity + expr, no name/desc).""" - if isinstance(self.expression, TimeSeriesExpression): - return self.expression.__str__() - return "NA" + return str(self.expression) def get_channel_type_str(self) -> str: """Channel type string, matching the ``ChannelType`` enum member name.""" diff --git a/src/impulse_reporting/incremental/definition_hash_comparator.py b/src/impulse_reporting/incremental/definition_hash_comparator.py index ec0b66c9..651d0f36 100644 --- a/src/impulse_reporting/incremental/definition_hash_comparator.py +++ b/src/impulse_reporting/incremental/definition_hash_comparator.py @@ -42,176 +42,95 @@ def __init__(self, spark: SparkSession): """ self.spark = spark - def group_events_by_hash_change( + def _group_by_hash_change( self, - events: list[Event], - event_dimension_table: str, - ) -> tuple[list[Event], list[Event]]: + items: list, + dimension_table: str, + id_column: str, + ) -> tuple[list, list]: """ - Group events into changed and unchanged based on definition hash comparison. + Split entities into changed and unchanged by definition-hash comparison. + + Shared implementation behind :meth:`group_events_by_hash_change`, + :meth:`group_aggregations_by_hash_change`, and + :meth:`group_calculated_channels_by_hash_change` — the three differ only + in the entity type and the id column keyed on in the gold dimension table. - Compares the current definition hash of each event against the stored - hash in the gold layer dimension table. Events with different hashes - (or new events not in gold) are considered "changed" and require full - reprocessing. Events with matching hashes are "unchanged" and can be - processed incrementally. + Compares each item's current ``determine_definition_hash()`` against the + hash stored under its ``get_id()`` in *dimension_table*. Items whose hash + differs, or that are absent from gold, are "changed" (need full + reprocessing of all containers); the rest are "unchanged" (processed + incrementally). When the gold table does not exist yet, everything is + "changed". Parameters ---------- - events : List[Event] - Current event definitions to check. - event_dimension_table : str - URI of the gold layer event dimension table - (e.g., "catalog.gold.report_event_dimension"). + items : list + Current entity definitions to check (events, aggregations, or + calculated channels). + dimension_table : str + URI of the gold layer dimension table. + id_column : str + Entity id column in the dimension table (e.g. ``"event_id"``, + ``"visual_id"``, ``"channel_id"``). Returns ------- - Tuple[List[Event], List[Event]] - A tuple of (changed_events, unchanged_events): - - changed_events: Events with changed definitions that need full - reprocessing of all containers. - - unchanged_events: Events with unchanged definitions that can be - processed incrementally. + tuple[list, list] + ``(changed, unchanged)``. """ + if not self._table_exists(dimension_table): + # No gold table exists - everything is "changed" (needs full processing) + return (items, []) - if not self._table_exists(event_dimension_table): - # No gold table exists - all events are "changed" (need full processing) - return (events, []) - - # Load stored hashes from gold layer stored_hashes = ( - self.spark.read.table(event_dimension_table) - .select("event_id", "definition_hash") - .collect() + self.spark.read.table(dimension_table).select(id_column, "definition_hash").collect() ) - stored_hash_map = {row.event_id: row.definition_hash for row in stored_hashes} - - changed: list[Event] = [] - unchanged: list[Event] = [] - - for event in events: - event_id = event.get_id() - current_hash = event.determine_definition_hash() - stored_hash = stored_hash_map.get(event_id) - - if stored_hash is None or stored_hash != current_hash: - # New event or definition changed - changed.append(event) + stored_hash_map = {row[id_column]: row.definition_hash for row in stored_hashes} + + changed: list = [] + unchanged: list = [] + for item in items: + stored_hash = stored_hash_map.get(item.get_id()) + if stored_hash is None or stored_hash != item.determine_definition_hash(): + changed.append(item) else: - # Definition unchanged - unchanged.append(event) + unchanged.append(item) return (changed, unchanged) + def group_events_by_hash_change( + self, + events: list[Event], + event_dimension_table: str, + ) -> tuple[list[Event], list[Event]]: + """Split events into (changed, unchanged) by definition hash. + + Thin wrapper over :meth:`_group_by_hash_change` keyed on ``event_id``. + """ + return self._group_by_hash_change(events, event_dimension_table, "event_id") + def group_aggregations_by_hash_change( self, aggregations: list[Aggregation], dimension_table: str, ) -> tuple[list[Aggregation], list[Aggregation]]: - """ - Group aggregations into changed and unchanged based on definition hash. - - Compares the current definition hash of each aggregation against the - stored hash in the gold layer dimension table. Aggregations with - different hashes (or new aggregations not in gold) are considered - "changed" and require full reprocessing. + """Split aggregations into (changed, unchanged) by definition hash. - Parameters - ---------- - aggregations : List[Aggregation] - Current aggregation definitions to check. - dimension_table : str - URI of the gold layer aggregation dimension table - (e.g., "catalog.gold.report_histogram_dimension"). - - Returns - ------- - Tuple[List[Aggregation], List[Aggregation]] - A tuple of (changed_aggregations, unchanged_aggregations): - - changed_aggregations: Aggregations with changed definitions that - need full reprocessing of all containers. - - unchanged_aggregations: Aggregations with unchanged definitions - that can be processed incrementally. + Thin wrapper over :meth:`_group_by_hash_change` keyed on ``visual_id``. """ - - if not self._table_exists(dimension_table): - # No gold table exists - all aggregations are "changed" - return (aggregations, []) - - # Load stored hashes from gold layer - stored_hashes = ( - self.spark.read.table(dimension_table).select("visual_id", "definition_hash").collect() - ) - stored_hash_map = {row.visual_id: row.definition_hash for row in stored_hashes} - - changed: list[Aggregation] = [] - unchanged: list[Aggregation] = [] - - for agg in aggregations: - agg_id = agg.get_id() - current_hash = agg.determine_definition_hash() - stored_hash = stored_hash_map.get(agg_id) - - if stored_hash is None or stored_hash != current_hash: - # New aggregation or definition changed - changed.append(agg) - else: - # Definition unchanged - unchanged.append(agg) - - return (changed, unchanged) + return self._group_by_hash_change(aggregations, dimension_table, "visual_id") def group_calculated_channels_by_hash_change( self, channels: list[CalculatedChannel], dimension_table: str, ) -> tuple[list[CalculatedChannel], list[CalculatedChannel]]: - """ - Group calculated channels into changed and unchanged based on definition hash. - - Compares the current definition hash of each channel against the stored - hash in the gold layer dimension table (keyed on ``channel_id``). Channels - with different hashes (or new channels not in gold) are "changed" and need - full reprocessing of all containers. + """Split calculated channels into (changed, unchanged) by definition hash. - Parameters - ---------- - channels : List[CalculatedChannel] - Current calculated-channel definitions to check. - dimension_table : str - URI of the gold layer calculated-channel dimension table. - - Returns - ------- - Tuple[List[CalculatedChannel], List[CalculatedChannel]] - A tuple of (changed_channels, unchanged_channels). + Thin wrapper over :meth:`_group_by_hash_change` keyed on ``channel_id``. """ - - if not self._table_exists(dimension_table): - # No gold table exists - all channels are "changed" (need full processing) - return (channels, []) - - stored_hashes = ( - self.spark.read.table(dimension_table) - .select("channel_id", "definition_hash") - .collect() - ) - stored_hash_map = {row.channel_id: row.definition_hash for row in stored_hashes} - - changed: list[CalculatedChannel] = [] - unchanged: list[CalculatedChannel] = [] - - for channel in channels: - channel_id = channel.get_id() - current_hash = channel.determine_definition_hash() - stored_hash = stored_hash_map.get(channel_id) - - if stored_hash is None or stored_hash != current_hash: - changed.append(channel) - else: - unchanged.append(channel) - - return (changed, unchanged) + return self._group_by_hash_change(channels, dimension_table, "channel_id") def _table_exists(self, table_uri: str) -> bool: """ diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py index 50037331..8487870f 100644 --- a/tests/impulse_reporting/unit/channels/calculated_channel_test.py +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -38,14 +38,9 @@ def test_get_expression_is_none(self): # Channels drive their own solve, so they are excluded from the batch solve. assert _channel().get_expression() is None - def test_get_expression_str_returns_str_for_expression(self): - assert _channel().get_expression_str() != "NA" - - def test_get_expression_str_na_when_not_expression(self): - # Defensive branch: if the wrapped object isn't a TimeSeriesExpression. + def test_get_expression_str_is_wrapped_expression_str(self): ch = _channel() - ch.expression = object() - assert ch.get_expression_str() == "NA" + assert ch.get_expression_str() == str(ch.expression) def test_channel_type_str(self): assert _channel().get_channel_type_str() == "CALCULATED_CHANNEL" From 6d005c75e67ea6c6bf1da3bf6ecbedde2b344223 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Wed, 22 Jul 2026 07:47:14 +0200 Subject: [PATCH 16/23] Add incremental configuration and shared fact table tests - Introduced a new incremental configuration function to facilitate testing of shared fact tables in the PointValueAggregator and StatsAggregator. - Added tests to validate the behavior of aggregators with changed definitions, ensuring that both types persist correctly without clobbering each other's data. - Enhanced the test suite for PointValueAggregator to cover scenarios involving incremental updates and shared fact tables, improving overall test coverage and reliability. - Updated the CalculatedChannel tests to ensure attributes are normalized and default to an empty dictionary when not provided, enhancing robustness in attribute handling. --- .../point_value_aggregator_test.py | 120 ++++++++++++++++++ .../unit/channels/calculated_channel_test.py | 14 ++ .../unit/core/report_utils_test.py | 13 ++ 3 files changed, 147 insertions(+) diff --git a/tests/impulse_reporting/integration/point_value_aggregator_test.py b/tests/impulse_reporting/integration/point_value_aggregator_test.py index 50b0120f..691bb988 100644 --- a/tests/impulse_reporting/integration/point_value_aggregator_test.py +++ b/tests/impulse_reporting/integration/point_value_aggregator_test.py @@ -6,6 +6,12 @@ from impulse_reporting.aggregations.point_value_aggregator import PointValueAggregator from impulse_reporting.aggregations.stats_aggregator import StatsAggregator +from impulse_reporting.config.config_parser import ( + ImpulseConfig, + IncrementalConfig, + Source, + UnitySink, +) from impulse_reporting.core.page import Page from impulse_reporting.core.report import Report from impulse_reporting.events.basic_event import BasicEvent @@ -18,6 +24,27 @@ def _config_path(): return os.path.join(base_path, "tests", "data", "config", "config.json") +def _incremental_config(is_enabled: bool): + """Incremental-capable config writing to the shared ``evaluation_*`` gold tables.""" + return ImpulseConfig( + source=Source( + container_metrics_table="spark_catalog.silver.container_metrics_inc_1_2", + channel_metrics_table="spark_catalog.silver.channel_metrics", + channels_uri="spark_catalog.silver.channels", + ), + unity_sink=UnitySink( + catalog="spark_catalog", + schema="gold", + table_prefix="evaluation", + ), + incremental=IncrementalConfig( + enabled=is_enabled, + silver_last_modified_column="timestamp", + gold_last_modified_column="_created_at", + ), + ) + + def test_persist_point_value_aggregator(spark): """End-to-end: a PointValueAggregator samples a channel at a PointsInTimeEvent's instants, writes to stats_aggregator_fact, and its rows link to the event's instances.""" @@ -163,3 +190,96 @@ def test_stats_and_point_value_share_fact_table_without_clobber(spark): "rpm_stats": "stats_aggregator", "rpm_at_edges": "point_value_aggregator", } + + +def _add_stats_and_pva(report, *, statistics, pva_scale): + """Add a StatsAggregator + PointValueAggregator (both → stats_aggregator_fact). + + Events are identical across runs (stable ``event_instance_fact``); only the + aggregation definitions vary: ``statistics`` changes the stats hash and + ``pva_scale`` changes the PVA's input expression (and thus its hash), so both + aggregation types land in the changed-DF union on the shared fact table. + """ + q = report.get_db().query + eng_rpm = q.channel(channel_name="Engine RPM") + + rpm_event = BasicEvent(name="rpm_gt_0", expr=eng_rpm > 0) + pit_event = PointsInTimeEvent(name="rpm_rising", expr=eng_rpm.rising_edges()) + report.add_event(rpm_event) + report.add_event(pit_event) + + page = Page(page_number=1) + report.add_page(page) + stats = StatsAggregator( + name="rpm_stats", + input_expressions=[eng_rpm], + channel_names=["Engine RPM"], + statistics=statistics, + event=rpm_event, + ) + pva = PointValueAggregator( + name="rpm_at_edges", + input_expressions=[eng_rpm * pva_scale], + channel_names=["Engine RPM"], + event=pit_event, + ) + page.add_aggregation(stats) + page.add_aggregation(pva) + return stats, pva + + +def test_incremental_stats_and_point_value_shared_fact_changed_defs(spark): + """Two aggregation types sharing stats_aggregator_fact, both with CHANGED + definitions, persisted incrementally. + + Exercises the aggregation branch of ``persist_facts_incremental`` where the + changed DataFrames of *different* types on one fact table are ``unionByName`` + -combined into a single ``replace_by_ids(id_column="visual_id")`` — the + cross-type union must not clobber either type's rows. + """ + # --- Run 1: seed both aggregators into the shared fact table --- + report_1 = Report( + name="agg_shared_fact_incremental", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(_incremental_config(is_enabled=False)), + ) + stats_1, pva_1 = _add_stats_and_pva(report_1, statistics=["min", "max"], pva_scale=1.0) + report_1.determine_report() + report_1.persist_results() + + fact_run1 = spark.read.table("spark_catalog.gold.evaluation_stats_aggregator_fact") + assert fact_run1.where(F.col("visual_id") == stats_1.get_id()).count() > 0 + assert fact_run1.where(F.col("visual_id") == pva_1.get_id()).count() > 0 + + # --- Run 2 (incremental): change BOTH definitions --- + # stats: expanded statistics list; PVA: changed event expression threshold. + report_2 = Report( + name="agg_shared_fact_incremental", + spark=spark, + workspace_client=create_autospec(WorkspaceClient), + config=dict(_incremental_config(is_enabled=True)), + ) + stats_2, pva_2 = _add_stats_and_pva(report_2, statistics=["min", "max", "mean"], pva_scale=2.0) + # same identities → same visual_ids across runs + assert stats_2.get_id() == stats_1.get_id() + assert pva_2.get_id() == pva_1.get_id() + + report_2.determine_report() + report_2.persist_results() + + fact_run2 = spark.read.table("spark_catalog.gold.evaluation_stats_aggregator_fact") + stats_rows = fact_run2.where(F.col("visual_id") == stats_2.get_id()) + pva_rows = fact_run2.where(F.col("visual_id") == pva_2.get_id()) + + # Both types survive the cross-type union + single replace_by_ids (no clobber). + assert stats_rows.count() > 0 + assert pva_rows.count() > 0 + # The changed stats definition took effect: the expanded statistics are present. + stats_labels = { + r["aggregation_label"] for r in stats_rows.select("aggregation_label").distinct().collect() + } + assert stats_labels == {"min", "max", "mean"} + assert { + r["aggregation_label"] for r in pva_rows.select("aggregation_label").distinct().collect() + } == {"value"} diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py index 8487870f..008ddd98 100644 --- a/tests/impulse_reporting/unit/channels/calculated_channel_test.py +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -22,6 +22,20 @@ def test_stores_identity(self): ch = _channel() assert ch.identity == _IDENTITY + def test_attributes_normalized_to_str(self): + # attributes are optional; when given they are coerced to a str->str dict. + ch = CalculatedChannel( + name="speed_kmh", + expr=TimeSeriesSelector(None) * 3.6, + identity=dict(_IDENTITY), + attributes={"unit": "kmh", "scale": 3.6}, + ) + assert ch.attributes == {"unit": "kmh", "scale": "3.6"} + + def test_attributes_default_empty(self): + # No attributes → empty dict, not None. + assert _channel().attributes == {} + def test_get_id_deterministic_and_identity_derived(self): # Same identity → same id regardless of name; different identity → different id. a = _channel(name="a") diff --git a/tests/impulse_reporting/unit/core/report_utils_test.py b/tests/impulse_reporting/unit/core/report_utils_test.py index 83ec46af..a42bbaca 100644 --- a/tests/impulse_reporting/unit/core/report_utils_test.py +++ b/tests/impulse_reporting/unit/core/report_utils_test.py @@ -3,6 +3,7 @@ from enum import Enum from unittest.mock import MagicMock, create_autospec, patch +import pytest from databricks.sdk import WorkspaceClient from pyspark.sql import DataFrame @@ -350,6 +351,18 @@ def side_effect(items, _table): assert "BASIC_EVENT" in changed assert "SEQUENCE_OF_EVENTS" in changed + def test_unknown_kind_raises(self): + """An unsupported `kind` is rejected before any comparison runs.""" + with pytest.raises(ValueError, match="Unsupported kind 'bogus'"): + split_by_hash_change( + items_by_type={"BASIC_EVENT": [MagicMock()]}, + type_enum=MagicMock(), + sink=MagicMock(), + spark=MagicMock(), + hash_comparator=MagicMock(), + kind="bogus", + ) + # ============================================================================ # Tests: dispatch_events From 8ae58f1fe4889b5c89b20cc77a74d4df0908fb9e Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Wed, 22 Jul 2026 11:00:57 +0200 Subject: [PATCH 17/23] Refactor calculated channel identity handling and output schema - Updated the `CalculatedChannel` class to clarify that the identity is stored as a `MapType(string, string)` on the dimension and not repeated on fact rows. - Enhanced documentation across various modules to reflect the new identity handling and its implications for calculated channel processing. - Adjusted output schemas in related components to ensure consistency with the updated identity structure, improving clarity and maintainability. - Updated tests to validate the new identity handling and output structure, ensuring accurate representation of calculated channel data. --- .../data_model/gold_layer_event_normalized.md | 5 +- .../query/channels/calculated_channel.md | 63 ++++++++++--------- .../analyze/query/query_builder.md | 12 ++-- .../analyze/query/solvers/default_solver.md | 12 ++-- .../analyze/query/solvers/query_solver.md | 2 +- .../channels/calculated_channel.md | 35 ++++++++--- .../references/query_engine/query_solvers.md | 7 ++- .../impulse/docs/references/report/channel.md | 11 ++-- docs/impulse/docs/tutorial/demo.md | 5 +- skills/impulse-analyze/SKILL.md | 6 +- skills/impulse-config/SKILL.md | 1 + .../channels/calculated_channel.py | 17 ++--- 12 files changed, 103 insertions(+), 73 deletions(-) diff --git a/docs/impulse/docs/data_model/gold_layer_event_normalized.md b/docs/impulse/docs/data_model/gold_layer_event_normalized.md index d7a8cbd1..df45d8b7 100644 --- a/docs/impulse/docs/data_model/gold_layer_event_normalized.md +++ b/docs/impulse/docs/data_model/gold_layer_event_normalized.md @@ -169,7 +169,6 @@ calculated_channel_dimension { long channel_id int report_id string channel_type - string channel_name string channel_description string channel_expression map identity @@ -184,8 +183,6 @@ calculated_channel_fact { long tstart long tend double value - string channel_name - string data_key timestamp _created_at } @@ -224,7 +221,7 @@ guaranteed. | `{prefix}_histogram2d_fact` | `container_id`, `visual_id`, `event_id`, `x_bin_id`, `y_bin_id` | 2D histogram bin values per container. | | `{prefix}_stats_aggregator_fact` | `container_id`, `visual_id`, `event_instance_id`, `channel_name`, `aggregation_label` | Statistics values per signal, event instance, and container. | | `{prefix}_event_instance_fact` | `container_id`, `event_id`, `event_instance_id` | Materialized event occurrences with start/end timestamps. | -| `{prefix}_calculated_channel_fact` | `container_id`, `channel_id`, `tstart` | Materialized derived signal — one row per sample interval, in the silver `channels` shape (`tstart`, `tend`, `value`) plus identity columns. | +| `{prefix}_calculated_channel_fact` | `container_id`, `channel_id`, `tstart` | Materialized derived signal — one row per sample interval, in the silver `channels` shape (`tstart`, `tend`, `value`). The channel's identity lives on `calculated_channel_dimension`, joined via `channel_id`. | --- diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/channels/calculated_channel.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/channels/calculated_channel.md index ff60d241..d0b34f4e 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/channels/calculated_channel.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/channels/calculated_channel.md @@ -12,31 +12,20 @@ CalculatedChannel: a labeled derived time-series channel. class CalculatedChannel(TimeSeriesExpression) ``` -A derived channel: a wrapped time-series expression plus output identity. - -A ``CalculatedChannel`` wraps an arbitrary :class:`TimeSeriesExpression` -built from the operator DSL (e.g. ``q.channel(channel_name="raw_speed") * 3.6`` -or ``rpm + speed``). Like the other ``TimeSeriesExpression`` leaves/nodes -(``TimeSeriesSelector``, ``TimeSeriesOp``) it ``build()``s to a -:class:`SampleSeries` — it is a *labeled derived signal*, not a reduction, so -it is a plain ``TimeSeriesExpression`` rather than an ``Aggregation``. -:meth:`QueryBuilder.solve_calculated_channels` explodes that series into -narrow rows matching the silver ``channel_data`` shape -(``container_id, channel_id, tstart, tend, value``) plus one string column -per identity entry. +A derived channel: a wrapped ``TimeSeriesExpression`` plus output identity. + +Wraps an expression built from the operator DSL (e.g. ``rpm * 3.6``) that must +evaluate to a :class:`SampleSeries`. ``build()`` returns that series' raw +``(tstarts, tends, values)`` arrays, which +:meth:`QueryBuilder.solve_calculated_channels` explodes into narrow +silver-shaped rows plus an ``identity`` ``MapType`` column. **Arguments**: - `expr` (`TimeSeriesExpression`): The wrapped expression; must ``build()`` to a ``SampleSeries``. -- `identity` (`dict of str`): Identity columns for the output rows, e.g. -``{"channel_name": "Eng_RPM", "data_key": "TM"}``. Each key becomes a -``StringType`` output column and each value is emitted as a literal on -every row of this channel. Must be non-empty; the identity also seeds -the deterministic ``channel_id`` hash. -- `channel_id` (`int or None`): Output ``channel_id`` for every emitted row. When omitted (the -``_AUTO`` sentinel) a deterministic id is derived from the identity by -the solver, typed to match the source ``channel_id`` column. Pass an -``int`` to use it verbatim, or ``None`` to emit a SQL null. +- `identity` (`dict of str`): Non-empty identity dict (arbitrary keys, e.g. +``{"channel_name": "Eng_RPM", "data_key": "TM"}``), emitted per row and +used to seed the deterministic :attr:`channel_id`. #### canonical\_identity @@ -50,13 +39,33 @@ Keys are sorted so the encoding (and the derived ``channel_id``) is independent of kwarg order. +#### channel\_id + +```python +def channel_id() -> int +``` + +Deterministic output ``channel_id`` derived from the identity. + +A CRC32 of :meth:`canonical_identity` masked to a positive int32, so the +value is stable across runs/processes and fits both ``IntegerType`` and +``LongType`` source ``channel_id`` columns. Determinism makes writes +idempotent and joins predictable; sharing this one derivation across the +query-engine and reporting layers keeps their ids in lockstep. + + #### build ```python def build(cache: SeriesCache) ``` -Evaluate the wrapped expression against the cache (yields a SampleSeries). +Evaluate the wrapped expression and return its raw ``(tstarts, tends, values)``. + +Unlike a typical ``TimeSeriesExpression`` (whose ``build`` yields a +``SampleSeries``), this returns the three parallel ``float64`` arrays +underlying that series, since the calculated-channels solve path consumes +the raw samples directly. #### dtype @@ -65,12 +74,10 @@ Evaluate the wrapped expression against the cache (yields a SampleSeries). def dtype() -> T.DataType ``` -Spark type of ``build()``'s result: a serialized ``SampleSeries``. +Spark type of ``build()``'s output: the raw ``(tstarts, tends, values)`` arrays. -Matches :meth:`TimeSeriesSelector.dtype` (``BinaryType``), since the -wrapped expression evaluates to a ``SampleSeries``. The narrow -calculated-channels solve path does not consume this — it emits its own -``container_id, channel_id, tstart, tend, value`` schema — but keeping it -honest makes the expression safe if ever routed through ``solve()``. +A struct of three arrays mirroring the tuple ``build()`` returns, with +element types matching the narrow calculated-channels output +(``tstart``/``tend`` are ``long``, ``value`` is ``double``). diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md index ca3f8950..40fffeb0 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md @@ -172,8 +172,9 @@ Every selection must be a :class:`CalculatedChannel`. This runs the same metadata filter pipeline as :meth:`solve` (resolving the input channels each calculated channel depends on), then evaluates each calculated channel per container and emits rows in the silver ``channel_data`` shape -— ``container_id, channel_id, tstart, tend, value`` — plus one string -column per identity kwarg shared by the selections. +— ``container_id, channel_id, tstart, tend, value`` — plus a single +``identity`` ``MapType(string, string)`` column holding each channel's +identity dict. **Arguments**: @@ -186,14 +187,13 @@ containers matching the query filters are processed. **Raises**: -- `ValueError`: If any selection is not a ``CalculatedChannel``, if the selections -declare inconsistent identity-key sets, or if a wrapped expression -does not evaluate to a ``SampleSeries``. +- `ValueError`: If any selection is not a ``CalculatedChannel``, or if a wrapped +expression does not evaluate to a ``SampleSeries``. **Returns**: `pyspark.sql.DataFrame`: Narrow DataFrame ``[container_id, channel_id, tstart, tend, value, -]``. +identity]``. #### toPandas diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/default_solver.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/default_solver.md index 4f29b807..20d75546 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/default_solver.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/default_solver.md @@ -297,11 +297,13 @@ def solve_calculated_channels(query, channels_df, selections) -> DataFrame Solve calculated channels by grouping channels and exploding each result. -Structurally parallels :meth:`solve` — same unit-conversion prelude, -same channel-data read and ``[container_id, channel_id]`` join — but the -grouped-map UDF emits narrow silver-shaped rows (many per container) -instead of one wide row. Output columns are -``[container_id, channel_id, tstart, tend, value, ]``. +Structurally parallels :meth:`solve` — sharing the +:meth:`_prepare_channels_join` prelude and :meth:`_apply_grouped_map` +tail — but the grouped-map UDF emits narrow silver-shaped rows (many per +container) instead of one wide row. Output columns are +``[container_id, channel_id, tstart, tend, value, identity]`` where +``identity`` is a ``MapType(string, string)`` holding the channel's +identity dict. **Arguments**: diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/query_solver.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/query_solver.md index 6c651fe6..df58de0b 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/query_solver.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/query_solver.md @@ -224,5 +224,5 @@ instead of one wide row. Solvers that support calculated channels **Returns**: `pyspark.sql.DataFrame`: Narrow ``[container_id, channel_id, tstart, tend, value, -]`` DataFrame. +identity]`` DataFrame (``identity`` is a ``MapType(string, string)``). diff --git a/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md b/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md index 53de6c14..44947f91 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md +++ b/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md @@ -28,13 +28,26 @@ returns ``None`` so it is excluded from the batch solve. - `name` (`str`): Name of the calculated channel (used as the entity id seed's fallback and stored on the dimension row). - `expr` (`TimeSeriesExpression`): The wrapped expression; must evaluate to a ``SampleSeries``. -- `identity` (`Mapping[str, str]`): Output identity columns. Must contain exactly the keys -``{"channel_name", "data_key"}``; the values are emitted as literals on -every fact row and seed the deterministic ``channel_id``. +- `identity` (`Mapping[str, str]`): Channel identity. Any non-empty set of keys; seeds the deterministic +``channel_id`` and is stored once on ``calculated_channel_dimension`` as a +``MapType(string, string)`` column (joined to the fact via ``channel_id``, +not repeated on fact rows). - `desc` (`str`): Human-readable description (stored on the dimension row, excluded from the definition hash). - `attributes` (`Mapping[str, str]`): Key-value metadata stored on the dimension row. +#### canonical\_identity + +```python +def canonical_identity() -> str +``` + +Public, order-independent identity key. + +Two channels with the same ``identity`` (regardless of key insertion +order) share this value and therefore the same ``channel_id``. Used by + + #### get\_name ```python @@ -100,10 +113,6 @@ def determine_definition_hash() -> int Hash of the computation-affecting definition (expression + identity). -Uses the wrapped-expression string, which encodes identity and the -expression but not name/description/report_id/attributes. SHA-256, first -8 bytes as a signed int (fits ``LongType``) — same technique as events. - #### as\_dict @@ -111,7 +120,10 @@ expression but not name/description/report_id/attributes. SHA-256, first def as_dict() -> dict ``` -Dictionary representation matching ``CALCULATED_CHANNEL_DIMENSION_SCHEMA``. +Dictionary representation of the dimension metadata. + +``identity`` is a plain dict, persisted on the dimension as a +``MapType(string, string)`` column (no fixed per-key columns). #### as\_spark\_row @@ -146,8 +158,8 @@ needed. **Arguments**: -- `spark` (`SparkSession`): Spark session (unused directly; kept for interface parity). -- `channels` (`list of CalculatedChannel`): Channels to solve; all share the same identity key set. +- `spark` (`SparkSession`): Spark session, forwarded to ``QueryBuilder.solve_calculated_channels``. +- `channels` (`list of CalculatedChannel`): Channels to solve; identity keys may differ across channels. - `query` (`QueryBuilder`): Query builder used to select and solve the channels. - `solver` (`QuerySolver`): Solver implementing ``solve_calculated_channels`` (a ``DefaultSolver``). - `pre_filtered_containers_df` (`DataFrame`): Incremental container subset; ``None`` processes all containers. @@ -165,4 +177,7 @@ def determine_metadata_df(cls, spark: SparkSession, Create the dimension DataFrame for the given channels. +``identity`` is a self-describing ``MapType(string, string)`` column, +which ``createDataFrame`` builds directly from the plain dict returned by + diff --git a/docs/impulse/docs/references/query_engine/query_solvers.md b/docs/impulse/docs/references/query_engine/query_solvers.md index 17170f6a..5f3ee0d1 100644 --- a/docs/impulse/docs/references/query_engine/query_solvers.md +++ b/docs/impulse/docs/references/query_engine/query_solvers.md @@ -113,10 +113,11 @@ with `DefaultSolver` and `data_type = "RLE"`. The standard `solve()` produces a **wide** result — one row per container, one column per selection. `DefaultSolver.solve_calculated_channels` is a parallel entry point that produces a **narrow**, silver-shaped result instead: it explodes each selection's `SampleSeries` into one row per sample -interval, emitting `container_id, channel_id, tstart, tend, value` plus the identity columns. +interval, emitting `container_id, channel_id, tstart, tend, value` plus a single self-describing +`identity` `map` column. -Every selection must be a [`CalculatedChannel`](../report/channel.md) (all sharing the same identity -key set), and each wrapped expression must evaluate to a `SampleSeries`. The stage reuses the same +Every selection must be a [`CalculatedChannel`](../report/channel.md) (identity keys are arbitrary and +need not match across selections), and each wrapped expression must evaluate to a `SampleSeries`. The stage reuses the same metadata filter pipeline as `solve()` to resolve the input channels, then runs a grouped-map UDF per container. Only `DefaultSolver` implements it — the base `QuerySolver` raises `NotImplementedError`. diff --git a/docs/impulse/docs/references/report/channel.md b/docs/impulse/docs/references/report/channel.md index 56b513c5..84fa6277 100644 --- a/docs/impulse/docs/references/report/channel.md +++ b/docs/impulse/docs/references/report/channel.md @@ -27,10 +27,11 @@ my_report.add_calculated_channel(my_channel) ## CalculatedChannel -A `CalculatedChannel` couples a TSAL expression with an **identity** — the set of identifier columns -(`channel_name`, `data_key`) emitted on every output row. When the report is solved, the wrapped +A `CalculatedChannel` couples a TSAL expression with an **identity** — an arbitrary key-value dict that +identifies the channel and seeds its deterministic `channel_id`. When the report is solved, the wrapped expression is evaluated per measurement container and exploded into narrow rows matching the silver -`channels` shape. +`channels` shape. The identity itself is stored once on the channel dimension (as a `map`), not repeated +on every fact row. ```python from impulse_reporting.channels.calculated_channel import CalculatedChannel @@ -49,7 +50,7 @@ speed_kmh = CalculatedChannel( |--------------|------------------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `name` | `str` | Yes | Human-readable channel name, stored in the channel dimension table. | | `expr` | `TimeSeriesExpression` | Yes | TSAL expression defining the derived signal. Must evaluate to `SampleSeries`; validated at construction (raises `ValueError` otherwise). | -| `identity` | `dict[str, str]` | Yes | Output identifier columns. Must contain **exactly** the keys `{"channel_name", "data_key"}`. Values are emitted as literals on every fact row and seed the `channel_id`. | +| `identity` | `dict[str, str]` | Yes | Channel identity. Any **non-empty** dict (keys are arbitrary, e.g. `channel_name` / `data_key`). Seeds the deterministic `channel_id` and is stored as a `map` on `calculated_channel_dimension` (joined to the fact via `channel_id`); not repeated on fact rows. | | `desc` | `str` | No | Human-readable description stored in the channel dimension table. | | `attributes` | `Mapping[str, str]` | No | Free-form key-value metadata stored in the channel dimension table. Values are coerced to strings. | @@ -57,7 +58,7 @@ speed_kmh = CalculatedChannel( 1. The `expr` is evaluated per measurement container by the solver, yielding a `SampleSeries` for each container. 2. Calculated channels take their **own narrow solve path** (`QueryBuilder.solve_calculated_channels`), distinct from the wide batch solve used by events and aggregations. Multi-channel expressions (e.g. `rpm + speed`) are time-base synchronized automatically; emitted intervals are the intersection. -3. Each series is exploded into one row per sample interval: `container_id, channel_id, tstart, tend, value`, with the `identity` values attached as columns. +3. Each series is exploded into one row per sample interval: `container_id, channel_id, tstart, tend, value`. The `identity` is **not** on the fact — it lives on the dimension, joined via `channel_id`. 4. The `channel_id` is a deterministic hash of the sorted `identity` — the same value appears in both the fact and dimension tables, so they join directly. 5. The sample rows are written to the `calculated_channel_fact` table; the channel definition (name, description, expression, identity) is written to the `calculated_channel_dimension` table. diff --git a/docs/impulse/docs/tutorial/demo.md b/docs/impulse/docs/tutorial/demo.md index ba4953c8..ee5a9133 100644 --- a/docs/impulse/docs/tutorial/demo.md +++ b/docs/impulse/docs/tutorial/demo.md @@ -277,8 +277,9 @@ my_report.add_calculated_channel(avg_temp_channel) ``` - The wrapped expression must evaluate to a `SampleSeries` (a signal), not `Intervals`. -- `identity` must contain exactly `channel_name` and `data_key`; these become columns on every output row - and seed the deterministic `channel_id`. +- `identity` is any non-empty dict (keys are arbitrary, e.g. `channel_name` / `data_key`); it seeds the + deterministic `channel_id` and is stored on `calculated_channel_dimension` (as a `map`, joined to the + fact via `channel_id`) — not repeated on every fact row. - Registration mirrors events: `add_calculated_channel()` before `determine_report()`. --- diff --git a/skills/impulse-analyze/SKILL.md b/skills/impulse-analyze/SKILL.md index 48d2bdd3..34a677ef 100644 --- a/skills/impulse-analyze/SKILL.md +++ b/skills/impulse-analyze/SKILL.md @@ -92,12 +92,16 @@ pdf = db.query.select(eng_rpm.mean().alias("rpm_mean")).toPandas(spark, solver=D `DefaultSolver(spark)` reads your silver layer. Constructor: ```python -DefaultSolver(spark, config=None, is_raw_data=False, drop_implausible_data=False) +DefaultSolver(spark, config=None, is_raw_data=False, drop_implausible_data=False, raw_encoder=RawEncoder.RLE) ``` - `is_raw_data=True` when `channels` stores raw `(timestamp, value)` samples (default `False` expects RLE `[tstart, tend)` intervals). This is the ad-hoc equivalent of `query_engine.data_type` in a report config. +- `raw_encoder` (`RawEncoder.RLE` default, or `RawEncoder.INTERVAL`) chooses how raw points become + intervals — RLE collapses equal-valued runs, INTERVAL keeps every sample. Only used when + `is_raw_data=True`; the ad-hoc equivalent of `query_engine.raw_encoder`. Import from + `impulse_query_engine.analyze.query.solvers.solver_config`. - `drop_implausible_data=True` drops rows where `is_plausible = false` (requires `is_raw_data=True`). - `config` takes a `SolverConfig` for column-name remapping / project scoping — the same object described under `solver_config` in `impulse-config`. diff --git a/skills/impulse-config/SKILL.md b/skills/impulse-config/SKILL.md index cb299962..912a7be5 100644 --- a/skills/impulse-config/SKILL.md +++ b/skills/impulse-config/SKILL.md @@ -94,6 +94,7 @@ Omit entirely for `DefaultSolver` + `data_type="RLE"`. |-------------------------|--------------------|---------------------------------------------------------------------------------------------------| | `solver` | `"DefaultSolver"` | The solver. `"DeltaSolver"` / `"KeyValueStoreSolver"` are **deprecated aliases** for `DefaultSolver`. | | `data_type` | `"RLE"` | `"RLE"` for interval-encoded `[tstart, tend)` samples; `"RAW"` for `(timestamp, value)` samples. | +| `raw_encoder` | `null` (→ `"RLE"`) | How RAW point data becomes intervals. `"RLE"` collapses equal-valued runs (default); `"INTERVAL"` keeps every sample, only deriving `tend` + dropping exact duplicates. Only used when `data_type="RAW"`. | | `drop_implausible_data` | `false` | Drop `channels` rows where `is_plausible = false`. **Requires `data_type="RAW"`** (RLE raises). | | `batch_size` | `500` | Max selectors solved per batch. | | `solver_config` | `null` | Per-table column mappings, per-table filters, and project scoping (below). | diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py index 3c507543..81935f5f 100644 --- a/src/impulse_reporting/channels/calculated_channel.py +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -41,9 +41,10 @@ class CalculatedChannel: expr : TimeSeriesExpression The wrapped expression; must evaluate to a ``SampleSeries``. identity : Mapping[str, str] - Output identity. Any non-empty set of keys; the whole dict is emitted - per fact row in a single ``identity`` ``MapType(string, string)`` column - and seeds the deterministic ``channel_id``. + Channel identity. Any non-empty set of keys; seeds the deterministic + ``channel_id`` and is stored once on ``calculated_channel_dimension`` as a + ``MapType(string, string)`` column (joined to the fact via ``channel_id``, + not repeated on fact rows). desc : str, optional Human-readable description (stored on the dimension row, excluded from the definition hash). @@ -134,8 +135,8 @@ def determine_definition_hash(self) -> int: def as_dict(self) -> dict: """Dictionary representation of the dimension metadata. - ``identity`` is a plain dict, persisted as a ``MapType(string, string)`` - column that mirrors the fact identity (no fixed per-key columns). + ``identity`` is a plain dict, persisted on the dimension as a + ``MapType(string, string)`` column (no fixed per-key columns). """ return { "channel_id": self.get_id(), @@ -201,9 +202,9 @@ def determine_calculated_channels( def determine_metadata_df(cls, spark: SparkSession, channels: list[CalculatedChannel]): """Create the dimension DataFrame for the given channels. - ``identity`` is a ``MapType(string, string)`` (same self-describing - representation as the fact table), which ``createDataFrame`` builds - directly from the plain dict returned by :meth:`as_dict`. + ``identity`` is a self-describing ``MapType(string, string)`` column, + which ``createDataFrame`` builds directly from the plain dict returned by + :meth:`as_dict`. """ rows = [channel.as_spark_row() for channel in channels] return spark.createDataFrame(rows, schema=CALCULATED_CHANNEL_DIMENSION_SCHEMA) From 6e844e09cddf61b6cfa9c35be19a2b5da591aeb1 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Wed, 22 Jul 2026 11:40:33 +0200 Subject: [PATCH 18/23] Enhance documentation for calculated channels and identity handling - Added notes on when to use calculated channels, emphasizing their role in materializing derived time series for queryable outputs. - Clarified the output schema in documentation, specifying that identity is stored as a map on the dimension and not repeated in fact rows. - Updated examples and descriptions to reflect the flexibility of identity keys across selections, improving clarity for users. --- .../impulse/docs/references/report/channel.md | 8 +++++ skills/impulse-analyze/SKILL.md | 6 ++-- skills/impulse-channels/SKILL.md | 32 ++++++++++++------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/docs/impulse/docs/references/report/channel.md b/docs/impulse/docs/references/report/channel.md index 84fa6277..fb2c6342 100644 --- a/docs/impulse/docs/references/report/channel.md +++ b/docs/impulse/docs/references/report/channel.md @@ -33,6 +33,14 @@ expression is evaluated per measurement container and exploded into narrow rows `channels` shape. The identity itself is stored once on the channel dimension (as a `map`), not repeated on every fact row. +:::note When to use a calculated channel +Reach for a calculated channel **only when the derived time series itself is the deliverable** — you want +it materialized to a queryable gold table, or returned as narrow silver-shaped rows via +`solve_calculated_channels`. To *compute over* a derived signal, you don't need one: aggregations and +events accept any TSAL expression directly (e.g. `StatsAggregator(input_expressions=[rpm * torque], ...)`, +`BasicEvent(expr=speed > 100)`) and derive it on the fly at solve time. +::: + ```python from impulse_reporting.channels.calculated_channel import CalculatedChannel diff --git a/skills/impulse-analyze/SKILL.md b/skills/impulse-analyze/SKILL.md index 34a677ef..47a86d63 100644 --- a/skills/impulse-analyze/SKILL.md +++ b/skills/impulse-analyze/SKILL.md @@ -143,7 +143,9 @@ cc = CalculatedChannel( {"channel_name": "speed_kmh", "data_key": "CALC"}, ) df = db.query.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) -# df: [container_id, channel_id, tstart, tend, value, channel_name, data_key] +# df: [container_id, channel_id, tstart, tend, value, identity] +# identity is a map holding the channel's identity dict ``` -All selections must be `CalculatedChannel`s sharing one identity key set. +All selections must be `CalculatedChannel`s; identity keys are arbitrary and need not match across +selections (each row carries its own identity map). diff --git a/skills/impulse-channels/SKILL.md b/skills/impulse-channels/SKILL.md index 16380019..b568c13b 100644 --- a/skills/impulse-channels/SKILL.md +++ b/skills/impulse-channels/SKILL.md @@ -16,6 +16,13 @@ persisted at the same per-sample grain as the silver `channels` table — unlike `impulse-aggregations`), which summarizes a signal into bins or statistics. It is the *persisted, labeled* form of a virtual signal. +**When to use — only when you want to persist or directly work with the derived time series itself.** +Aggregations and events accept any derived TSAL expression *directly* (e.g. +`StatsAggregator(input_expressions=[rpm * torque], ...)`, `BasicEvent(expr=speed > 100)`) and derive it +**on the fly** at solve time — they do **not** need a calculated channel to compute over a derived signal. +Reach for a `CalculatedChannel` when the derived signal is itself the deliverable: you want it materialized +to a queryable gold table, or returned as narrow silver-shaped rows via `solve_calculated_channels`. + **Every calculated channel must be registered with the report before computing:** ```python @@ -27,8 +34,9 @@ at construction (raises `ValueError`). ## CalculatedChannel -Couples a TSAL expression with an `identity` — the identifier columns emitted on every output row. The -expression is evaluated per container and exploded into narrow rows (one per sample interval). +Couples a TSAL expression with an `identity` — an arbitrary key-value dict identifying the channel (stored +on the dimension, seeds `channel_id`). The expression is evaluated per container and exploded into narrow +rows (one per sample interval). ```python from impulse_reporting.channels.calculated_channel import CalculatedChannel @@ -48,7 +56,7 @@ report.add_calculated_channel(speed_kmh) |--------------|------------------------|----------|---------------------------------------------------------------------------------------------------| | `name` | `str` | Yes | Human-readable channel name; stored in `calculated_channel_dimension`. | | `expr` | `TimeSeriesExpression` | Yes | Derived signal. **Must evaluate to `SampleSeries`.** | -| `identity` | `dict[str, str]` | Yes | Output identifier columns. **Must contain exactly `{"channel_name", "data_key"}`.** Seeds `channel_id`. | +| `identity` | `dict[str, str]` | Yes | Channel identity — any **non-empty** dict (arbitrary keys). Seeds `channel_id`; stored as a `map` on `calculated_channel_dimension`, not on fact rows. | | `desc` | `str` | No | Description stored in `calculated_channel_dimension`. | | `attributes` | `Mapping[str, str]` | No | Free-form metadata; values coerced to strings. | @@ -69,24 +77,26 @@ from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSol cc = CalculatedChannel( db.query.channel(channel_name="Vehicle Speed Sensor") * 3.6, {"channel_name": "speed_kmh", "data_key": "CALC"}, - # channel_id defaults to a deterministic hash; pass an int to override, or None to emit null. + # channel_id is a deterministic hash of the sorted identity. ) df = db.query.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) -df.show() # [container_id, channel_id, tstart, tend, value, channel_name, data_key] +df.show() # [container_id, channel_id, tstart, tend, value, identity] +# identity is a map holding the channel's identity dict ``` -All selections in one call must share the same identity key set (validated up front). +All selections in one call must be `CalculatedChannel`s; identity keys are arbitrary and need not match +across selections. ## Output schema **calculated_channel_dimension** (one row per channel per report) — key columns: `channel_id`, -`report_id`, `channel_type` (`"CALCULATED_CHANNEL"`), `channel_name`, `channel_description`, -`channel_expression` (TSAL string), `identity` (map), `definition_hash`, `attributes` (map). +`report_id`, `channel_type` (`"CALCULATED_CHANNEL"`), `channel_description`, `channel_expression` +(TSAL string), `identity` (map), `definition_hash`, `attributes` (map). **calculated_channel_fact** (one row per sample interval per container) — `container_id`, `channel_id`, -`tstart`, `tend`, `value`, `channel_name`, `data_key`. Same narrow, run-length-encoded shape as the silver -`channels` table. Join to the dimension on `channel_id`, and to `measurement_dimension` on `container_id` -(see `impulse-data-model`). +`tstart`, `tend`, `value`. Same narrow, run-length-encoded shape as the silver `channels` table. The +identity is **not** on the fact — it lives on the dimension; join on `channel_id` (and to +`measurement_dimension` on `container_id`; see `impulse-data-model`). ## Incremental From 9c1cb8e0baf7f1ffc19251863ca8a1468efc65e0 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Wed, 22 Jul 2026 12:54:27 +0200 Subject: [PATCH 19/23] Update DefaultSolver to preserve native types for tstart, tend, and value fields - Modified the DefaultSolver to append tstart, tend, and value fields without casting to int or float, ensuring that original data types are maintained. - Enhanced the QuerySolver documentation to reflect the authoritative types for tstart and tend fields. - Added tests to validate that double precision for tstart and tend is preserved during calculations, preventing truncation and ensuring accurate output. --- .../analyze/query/solvers/default_solver.py | 6 +- .../analyze/query/solvers/query_solver.py | 18 +++-- ...default_solver_calculated_channels_test.py | 66 ++++++++++++++++--- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 8d4e2fb8..5e2e5d7d 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -1103,9 +1103,9 @@ def _solve_calculated_channels_udf( for tstart, tend, value in zip(tstarts, tends, values, strict=False): cols[cid_col].append(container_id) cols[ch_col].append(cc.channel_id) - cols["tstart"].append(int(tstart)) - cols["tend"].append(int(tend)) - cols["value"].append(float(value)) + cols["tstart"].append(tstart) + cols["tend"].append(tend) + cols["value"].append(value) return pd.DataFrame(cols) diff --git a/src/impulse_query_engine/analyze/query/solvers/query_solver.py b/src/impulse_query_engine/analyze/query/solvers/query_solver.py index 23fe8a56..bfa35049 100644 --- a/src/impulse_query_engine/analyze/query/solvers/query_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/query_solver.py @@ -109,15 +109,15 @@ def _build_calculated_channels_udf_schema(self, channels: DataFrame) -> T.Struct appended by :meth:`solve_calculated_channels` after the UDF runs, so it never crosses the Arrow boundary. - The ``container_id`` and ``channel_id`` field types are derived from - *channels* (the column-mapped channels DataFrame) so they match the - physical source types instead of being hardcoded. + The ``container_id``, ``channel_id``, ``tstart``, and ``tend`` field types + are derived from *channels* (the column-mapped channels DataFrame) so they + match the physical source types. Parameters ---------- channels : pyspark.sql.DataFrame The column-mapped channels DataFrame whose ``container_id`` / - ``channel_id`` column types are authoritative. + ``channel_id`` / ``tstart`` / ``tend`` column types are authoritative. Returns ------- @@ -134,8 +134,14 @@ def _build_calculated_channels_udf_schema(self, channels: DataFrame) -> T.Struct self.config.channel_id_col, channels.schema[self.config.channel_id_col].dataType, ), - T.StructField(self.config.tstart_col, T.LongType()), - T.StructField(self.config.tend_col, T.LongType()), + T.StructField( + self.config.tstart_col, + channels.schema[self.config.tstart_col].dataType, + ), + T.StructField( + self.config.tend_col, + channels.schema[self.config.tend_col].dataType, + ), T.StructField(self.config.value_col, T.DoubleType()), ] ) diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py index e01f5c77..bcd56c1c 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_calculated_channels_test.py @@ -85,6 +85,24 @@ def _recast_container_id(db: MeasurementDB, cid_type: T.DataType) -> Measurement return MeasurementDB(MeasurementDBConfig.for_debug(tables), ws=db.ws) +def _recast_channel_time( + db: MeasurementDB, ts_type: T.DataType, offset: float = 0.0 +) -> MeasurementDB: + """Clone a ``for_debug`` db, casting ``tstart``/``tend`` on the channels table. + + ``offset`` is added after the cast — use a fractional value (e.g. ``0.5``) to + inject sub-unit precision that a long output would truncate away. + """ + tables = {} + for name, df in db.config.debug_tables.items(): + if "tstart" in df.columns and "tend" in df.columns: + df = df.withColumn("tstart", F.col("tstart").cast(ts_type) + F.lit(offset)).withColumn( + "tend", F.col("tend").cast(ts_type) + F.lit(offset) + ) + tables[name] = df + return MeasurementDB(MeasurementDBConfig.for_debug(tables), ws=db.ws) + + class TestCalculatedChannelValues: def test_scaling_produces_real_values(self, spark, basic_narrow_db): q = basic_narrow_db.query @@ -211,6 +229,36 @@ def test_dynamic_container_id_type(self, spark, basic_narrow_db, cid_type): assert result.schema["channel_id"].dataType == T.IntegerType() assert result.count() > 0 + def test_double_tstart_tend_preserved(self, spark, basic_narrow_db): + # A silver channels table with double tstart/tend (e.g. relative seconds) + # must be preserved on the output, not truncated to long. + db = _recast_channel_time(basic_narrow_db, T.DoubleType()) + q = db.query + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, {"channel_name": "rpm_x2"} + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + assert result.schema["tstart"].dataType == T.DoubleType() + assert result.schema["tend"].dataType == T.DoubleType() + assert result.count() > 0 + + def test_fractional_tstart_survives(self, spark, basic_narrow_db): + # Inject a fractional offset into double tstart/tend and confirm it + # round-trips end to end (a long output would truncate the .5 to 0). + db = _recast_channel_time(basic_narrow_db, T.DoubleType(), offset=0.5) + q = db.query + cc = CalculatedChannel( + q.channel(channel_name="Engine RPM") * 2, {"channel_name": "rpm_x2"} + ) + result = q.select(cc).solve_calculated_channels(spark, solver=DefaultSolver(spark)) + assert result.schema["tstart"].dataType == T.DoubleType() + # Every tstart keeps its fractional part (== .5), proving no truncation. + fracs = { + round(r["frac"], 6) + for r in result.select((F.col("tstart") % 1).alias("frac")).distinct().collect() + } + assert fracs == {0.5} + class TestChannelId: def test_deterministic_across_runs(self, spark, basic_narrow_db): @@ -358,20 +406,18 @@ def test_explodes_arrays_into_narrow_rows(self): assert list(result["tend"]) == [100, 200] assert list(result["value"]) == [3.0, 4.0] - def test_tstart_tend_cast_to_int_value_to_float(self): - # Float tstart/tend arrays (as SampleSeries stores them) are truncated to - # integer columns; the value column stays floating point. - cc = _MockCalcChannel(channel_id=7, arrays=([0.5], [100.9], [5])) + def test_udf_preserves_native_values_no_truncation(self): + # The UDF no longer casts: it emits the native SampleSeries values, and the + # grouped-map output schema (source-derived) drives the final coercion. + # So fractional tstart/tend survive here — truncation, if any, happens at + # the Spark schema boundary, not in the UDF. + cc = _MockCalcChannel(channel_id=7, arrays=([0.5], [100.9], [5.0])) result = DefaultSolver._solve_calculated_channels_udf( _udf_input_pdf(), selections=[cc], col_map=_UDF_COL_MAP ) - # int(0.5) == 0, int(100.9) == 100 → truncation happened. - assert result["tstart"].iloc[0] == 0 - assert result["tend"].iloc[0] == 100 - assert pd.api.types.is_integer_dtype(result["tstart"]) - assert pd.api.types.is_integer_dtype(result["tend"]) + assert result["tstart"].iloc[0] == 0.5 + assert result["tend"].iloc[0] == 100.9 assert result["value"].iloc[0] == 5.0 - assert pd.api.types.is_float_dtype(result["value"]) def test_empty_arrays_produce_zero_rows_full_width(self): cc = _MockCalcChannel(channel_id=10, arrays=([], [], [])) From 9be220e7ef18567fd5078c1e1eecebd1db0394ad Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Wed, 22 Jul 2026 13:54:29 +0200 Subject: [PATCH 20/23] Refactor CalculatedChannel to return wrapped expression - Updated the `get_expression` method in the `CalculatedChannel` class to return the wrapped query-engine expression instead of `None`, aligning with its role in driving a narrow solve. - Adjusted related unit tests to verify that the `get_expression` method correctly returns the wrapped expression, enhancing clarity on the channel's behavior in the solving process. --- .../channels/calculated_channel.py | 14 ++++++-------- .../unit/channels/calculated_channel_test.py | 9 ++++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py index 81935f5f..4bba5319 100644 --- a/src/impulse_reporting/channels/calculated_channel.py +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -30,8 +30,8 @@ class CalculatedChannel: Structurally parallels :class:`BasicEvent` (holds an aliased expression, name-derived id, SHA-256 definition hash) but — like ``ContainerEvent`` — it drives its own solve via ``QueryBuilder.solve_calculated_channels`` rather than - riding the centralized wide ``solved_df``. Accordingly :meth:`get_expression` - returns ``None`` so it is excluded from the batch solve. + riding the centralized wide ``solved_df``. It is dispatched separately from + the batch solve (never passed to ``collect_solvable_expressions``). Parameters ---------- @@ -110,13 +110,11 @@ def get_id(self) -> int: """Return the deterministic entity id (also the fact/dimension ``channel_id``).""" return self.expression.channel_id - def get_expression(self) -> TimeSeriesExpression | None: - """Return ``None`` — calculated channels drive their own narrow solve. - - Returning ``None`` keeps this channel out of the centralized wide batch - solve (``collect_solvable_expressions``), mirroring ``ContainerEvent``. + def get_expression(self) -> TimeSeriesExpression: + """ + Return the wrapped query-engine ``CalculatedChannel`` expression. """ - return None + return self.expression def get_expression_str(self) -> str: """String form of the wrapped expression (identity + expr, no name/desc).""" diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py index 008ddd98..8d5856e3 100644 --- a/tests/impulse_reporting/unit/channels/calculated_channel_test.py +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -48,9 +48,12 @@ def test_get_id_positive_int32(self): ch = _channel() assert 0 <= ch.get_id() <= 0x7FFFFFFF - def test_get_expression_is_none(self): - # Channels drive their own solve, so they are excluded from the batch solve. - assert _channel().get_expression() is None + def test_get_expression_returns_wrapped_expression(self): + # Returns the wrapped query-engine CalculatedChannel expression. Channels are + # dispatched via their own narrow solve, not collect_solvable_expressions, so + # this is never passed to the wide batch solve. + ch = _channel() + assert ch.get_expression() is ch.expression def test_get_expression_str_is_wrapped_expression_str(self): ch = _channel() From 17c2a98aa28ae688b7c977007cf049293cda3d5e Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Wed, 22 Jul 2026 14:09:04 +0200 Subject: [PATCH 21/23] Refactor documentation for entity orchestration helpers - Updated comments in `report_utils.py` to clarify the purpose and usage of entity orchestration and persistence helpers, emphasizing their shared functionality across different entity types. - Enhanced descriptions of the `persist_facts_incremental` and `merge_keys` methods to improve understanding of their roles in handling incremental facts and aggregations. --- src/impulse_reporting/core/report_utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/impulse_reporting/core/report_utils.py b/src/impulse_reporting/core/report_utils.py index 4e58458d..ec893d3e 100644 --- a/src/impulse_reporting/core/report_utils.py +++ b/src/impulse_reporting/core/report_utils.py @@ -452,11 +452,10 @@ def cleanup_temp_tables(spark: SparkSession, catalog: str, schema: str) -> None: # --------------------------------------------------------------------------- # Entity orchestration + persistence helpers # -# Generic over the entity type-enum (e.g. ``ChannelType``). Currently used only -# by calculated channels; written to be reusable so aggregations can adopt them -# next (``merge_keys`` accepts a per-type callable for that reason). Event -# *incremental facts* group-by-table + union before ``replace_by_ids`` and so do -# not fit ``persist_facts_incremental`` — migrating them needs a separate helper. +# Generic over the entity type-enum (``EventType`` / ``AggregationType`` / +# ``ChannelType``); shared by events, aggregations, and calculated channels. +# ``persist_facts_incremental`` groups by output table and unions shared-table +# types before ``replace_by_ids``; ``merge_keys`` accepts a per-type callable. # --------------------------------------------------------------------------- From f9805225aab7958b3faed750c8fe56ccd67070a6 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Thu, 23 Jul 2026 09:22:08 +0200 Subject: [PATCH 22/23] ci: re-trigger coverage upload From d918b1d09b54c948c9aa72212c6c2b1ea6abe2e1 Mon Sep 17 00:00:00 2001 From: "tom.bonfert" Date: Thu, 23 Jul 2026 11:03:04 +0200 Subject: [PATCH 23/23] Enhance calculated channel tests to validate passthrough behavior - Added a new passthrough channel with a factor of 1.0 to the `test_persist_calculated_channel_full` test. - Updated assertions to ensure both the original and passthrough channel IDs are present in the fact table. - Implemented checks to confirm that the gold fact rows for the passthrough channel match the corresponding silver channel rows, validating end-to-end passthrough invariants. --- .../integration/calculated_channel_test.py | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/impulse_reporting/integration/calculated_channel_test.py b/tests/impulse_reporting/integration/calculated_channel_test.py index 3928f423..d5c7df4d 100644 --- a/tests/impulse_reporting/integration/calculated_channel_test.py +++ b/tests/impulse_reporting/integration/calculated_channel_test.py @@ -69,6 +69,9 @@ def test_persist_calculated_channel_full(spark): config=dict(_config(is_enabled=False)), ) ch = _add_channel(report, factor=3.6) + # A *1.0 passthrough channel materializes the physical signal unchanged, so its + # gold fact rows must be identical to the silver channel rows (see below). + ch_pass = _add_channel(report, factor=1.0, name="speed_raw") report.determine_report() report.persist_results() @@ -78,14 +81,14 @@ def test_persist_calculated_channel_full(spark): fact = spark.read.table(_FACT) assert fact.count() > 0 - # channel_id in the fact matches the reporting entity id. + # Both channels' ids appear in the fact and match their reporting entity ids. ids = {r["channel_id"] for r in fact.select("channel_id").distinct().collect()} - assert ids == {ch.get_id()} + assert ids == {ch.get_id(), ch_pass.get_id()} # Identity is NOT on the fact — it lives on the dimension, joined via channel_id. assert "identity" not in fact.columns assert fact.columns == ["container_id", "channel_id", "tstart", "tend", "value", "_created_at"] - # Values are the derived signal: compare against the raw source scaled by 3.6. + # The silver source rows for the physical channel this derived signal is built from. raw = ( report.get_db() .channels(spark) @@ -98,9 +101,26 @@ def test_persist_calculated_channel_full(spark): ) ) raw_sum = raw.select(F.sum("value")).first()[0] - calc_sum = fact.select(F.sum("value")).first()[0] + calc_sum = fact.filter(F.col("channel_id") == ch.get_id()).select(F.sum("value")).first()[0] assert calc_sum == pytest.approx(raw_sum * 3.6) + # End-to-end passthrough invariant: a *1.0 calculated channel reproduces the + # silver signal exactly. Compare the gold fact rows to the silver channel rows + # on (container_id, tstart, tend, value). channel_id is intentionally excluded + # (the derived channel has its own identity-derived id); value is compared + # numerically since silver stores int and the derived signal is float64. + gold_pass = { + (r["container_id"], r["tstart"], r["tend"], float(r["value"])) + for r in fact.filter(F.col("channel_id") == ch_pass.get_id()) + .select("container_id", "tstart", "tend", "value") + .collect() + } + silver_rows = { + (r["container_id"], r["tstart"], r["tend"], float(r["value"])) + for r in raw.select("container_id", "tstart", "tend", "value").collect() + } + assert gold_pass == silver_rows + # The dimension carries the identity (a map), keyed by channel_id — the join # target for the fact. Confirm the fact's channel_id resolves to the identity. dim = spark.read.table(_DIM)