diff --git a/converters/microsoft/src/ossie_microsoft/_dax_to_sql.py b/converters/microsoft/src/ossie_microsoft/_dax_to_sql.py new file mode 100644 index 00000000..d82bea9f --- /dev/null +++ b/converters/microsoft/src/ossie_microsoft/_dax_to_sql.py @@ -0,0 +1,193 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A deliberately narrow DAX-to-OSSIE_SQL translator for Power BI measures. + +Power BI measures are DAX. On import, the Microsoft converter keeps them as DAX, +so a measure that leaves Power BI arrives at every other engine as an opaque +vendor string it cannot evaluate. This module is the opt-in reverse of +``_sql_to_dax``: it recognises the single-column aggregate measures whose +``OSSIE_SQL_2026`` equivalent is unambiguous, and refuses everything else. + +The governing rule is the one ``_sql_to_dax`` states and ``#437`` argues for: +never emit a plausible-looking expression that was not authored for the target. +A measure this module declines is kept verbatim as DAX with a diagnostic; a +measure it mistranslated would deploy cleanly on another engine and quietly +return a wrong number. Refusal is the safe default; translation is the exception +that must be proven unambiguous. + +What is translated +------------------ +A single DAX aggregate over one fully qualified column, for the six aggregates +that reverse ``_sql_to_dax`` exactly, so an Ossie SQL model exported to Power BI +and re-imported round-trips to the same expression:: + + SUM('Sales'[Amount]) -> SUM(Sales.Amount) + AVERAGE('Sales'[Amount]) -> AVG(Sales.Amount) + MIN('Sales'[Amount]) -> MIN(Sales.Amount) + MAX('Sales'[Amount]) -> MAX(Sales.Amount) + COUNTA('Sales'[Customer]) -> COUNT(Sales.Customer) + DISTINCTCOUNTNOBLANK('Sales'[Cust]) -> COUNT(DISTINCT Sales.Cust) + +What is refused (and why, so the refusal is defensible) +------------------------------------------------------- +* ``COUNT`` and ``DISTINCTCOUNT``. DAX ``COUNT`` documents ``TRUE``/``FALSE`` + values as unsupported, so it is not the equivalent of SQL ``COUNT(x)`` for an + arbitrary column, and ``DISTINCTCOUNT`` counts BLANK as a distinct value where + SQL ``COUNT(DISTINCT x)`` excludes NULL. These are refused rather than guessed. + Their counterparts ``COUNTA`` and ``DISTINCTCOUNTNOBLANK`` are what + ``_sql_to_dax`` emits for SQL ``COUNT`` and ``COUNT(DISTINCT)``, and are translated. + As ``_sql_to_dax`` notes, a DAX count over an empty set is BLANK where SQL gives 0; + that convention is accepted in both directions. +* ``COUNTROWS``, ``MEDIAN``, the statistical aggregates, iterators (``SUMX``), + ``CALCULATE`` and the filter-context family, ``DIVIDE``, ``IF``, arithmetic, + time intelligence, and anything with more than one argument or a nested call. + These are later tiers or have no declaration-scoped portable form. +* A column that is not fully qualified as ``Table[Column]``, or a table or + column name that is not a simple identifier. A name needing SQL quoting is + left to a later tier so this first pass never has to get a quoting rule right. + +This module performs no I/O and holds no model state. The caller supplies a +``resolve`` callback mapping a DAX (table, column) to the Ossie (dataset, field) +names, which is where model knowledge lives. +""" + +from __future__ import annotations + +import re + +#: DAX aggregate -> OSSIE_SQL aggregate, for the exact reverses of _sql_to_dax. +#: COUNTA and DISTINCTCOUNTNOBLANK are what _sql_to_dax emits for SQL COUNT and +#: COUNT(DISTINCT); reversing those same functions keeps the round-trip exact. +_SIMPLE_AGGREGATES = { + "SUM": "SUM", + "AVERAGE": "AVG", + "MIN": "MIN", + "MAX": "MAX", + "COUNTA": "COUNT", +} + +#: DAX functions we recognise well enough to refuse with a specific reason, +#: rather than a generic "not translated", because they look translatable. +_KNOWN_UNSAFE = { + "COUNT": "DAX COUNT does not support TRUE/FALSE values, so it is not the equivalent " + "of SQL COUNT(x) for an arbitrary column (COUNTA is)", + "DISTINCTCOUNT": "DAX DISTINCTCOUNT counts BLANK as a distinct value; SQL " + "COUNT(DISTINCT x) excludes NULL, so the two differ by one on a nullable column", + "COUNTROWS": "COUNTROWS counts rows, not a single column; it is a later tier", + "MEDIAN": "MEDIAN is a later tier", + "SUMX": "iterators evaluate per row and are a later tier", + "AVERAGEX": "iterators evaluate per row and are a later tier", + "CALCULATE": "CALCULATE manipulates filter context, which has no declaration-scoped form", + "DIVIDE": "division between measures is a later tier", +} + +# A simple identifier: a letter or underscore, then letters, digits, underscores. +_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") + +# One DAX function call: NAME ( ARG ). ARG is captured whole and validated +# separately so nested parentheses or extra arguments fail the ARG match. +_CALL = re.compile(r"^([A-Za-z_][A-Za-z0-9_.]*)\s*\((.*)\)$", re.DOTALL) + +# A fully qualified DAX column: Table[Column] or 'Table Name'[Column], where a +# quoted table escapes ' as '' and a column escapes ] as ]]. +_QUALIFIED_COLUMN = re.compile( + r"""^\s* + (?:'((?:[^']|'')*)'|([A-Za-z_][A-Za-z0-9_]*)) # 1 quoted or 2 bare table + \s* + \[((?:[^\]]|\]\])*)\] # 3 column, ] escaped as ]] + \s*$""", + re.VERBOSE, +) + + +def _strip_measure(expression): + """Normalise a measure body: drop a leading '=' and outer whitespace.""" + text = (expression or "").strip() + if text.startswith("="): + text = text[1:].strip() + return text + + +def _parse_qualified_column(argument): + """Return (table, column) for a fully qualified DAX column, else None.""" + match = _QUALIFIED_COLUMN.match(argument) + if match is None: + return None + quoted_table, bare_table, column = match.groups() + table = quoted_table.replace("''", "'") if quoted_table is not None else bare_table + column = column.replace("]]", "]") + return table, column + + +def _is_simple_identifier(name): + return bool(name) and _IDENTIFIER.fullmatch(name) is not None + + +def _qualify(dataset, field): + """Render an OSSIE_SQL ``dataset.field`` reference, or None if either name + is not a simple identifier (a name needing quoting is a later tier).""" + if not _is_simple_identifier(dataset) or not _is_simple_identifier(field): + return None + return f"{dataset}.{field}" + + +def translate_measure(expression, resolve): + """Translate a single-column DAX aggregate measure to an OSSIE_SQL expression. + + ``resolve`` maps a DAX ``(table, column)`` to the Ossie ``(dataset, field)`` + names, or returns None when the column does not resolve to exactly one field. + + Returns ``(sql, None)`` on success or ``(None, reason)`` on refusal. The + caller keeps the original DAX and attaches ``reason`` as a diagnostic when a + measure is refused. + """ + text = _strip_measure(expression) + if not text: + return None, "measure has no expression" + + call = _CALL.match(text) + if call is None: + return None, "only a single aggregate function call is translated" + + function = call.group(1).upper() + argument = call.group(2).strip() + + distinct_target = function == "DISTINCTCOUNTNOBLANK" + if function not in _SIMPLE_AGGREGATES and not distinct_target: + reason = _KNOWN_UNSAFE.get(function, f"'{function}' is not a translated aggregate") + return None, reason + + parsed = _parse_qualified_column(argument) + if parsed is None: + return None, ( + "only an aggregate over one fully qualified column (Table[Column]) is translated" + ) + table, column = parsed + + resolved = resolve(table, column) + if resolved is None: + return None, f"column '{table}'[{column}] does not resolve to exactly one dataset field" + dataset, field = resolved + + reference = _qualify(dataset, field) + if reference is None: + return None, "a dataset or field name needing SQL quoting is a later tier" + + if distinct_target: + return f"COUNT(DISTINCT {reference})", None + return f"{_SIMPLE_AGGREGATES[function]}({reference})", None diff --git a/converters/microsoft/src/ossie_microsoft/cli.py b/converters/microsoft/src/ossie_microsoft/cli.py index dd1550ae..3b5fb02b 100644 --- a/converters/microsoft/src/ossie_microsoft/cli.py +++ b/converters/microsoft/src/ossie_microsoft/cli.py @@ -100,6 +100,13 @@ def _build_parser(): ) imp.add_argument("-i", "--input", required=True, help="model.bim (TMSL JSON) file") imp.add_argument("-o", "--output", help="output Apache Ossie YAML (default: stdout)") + imp.add_argument( + "--translate-dax", + action="store_true", + default=False, + help="translate single-column aggregate DAX measures to OSSIE_SQL_2026 where " + "unambiguous, keeping the original DAX as a fallback (default: off, keep DAX only)", + ) exp = sub.add_parser( "export", @@ -113,7 +120,7 @@ def _build_parser(): def _run_import(args): with open(args.input, encoding="utf-8-sig") as fh: - return convert_semantic_model_to_ossie(json.load(fh)) + return convert_semantic_model_to_ossie(json.load(fh), translate_dax=args.translate_dax) def _run_export(args): diff --git a/converters/microsoft/src/ossie_microsoft/semantic_model_to_ossie.py b/converters/microsoft/src/ossie_microsoft/semantic_model_to_ossie.py index 52286a20..e3de9ca1 100644 --- a/converters/microsoft/src/ossie_microsoft/semantic_model_to_ossie.py +++ b/converters/microsoft/src/ossie_microsoft/semantic_model_to_ossie.py @@ -39,6 +39,7 @@ AUTO_DATE_TABLE_RE, DIALECT_ANSI, DIALECT_DAX, + DIALECT_OSSIE_SQL, IDENTIFIER_RE, OSSIE_TO_TMSL_DATATYPE, OSSIE_VERSION, @@ -58,6 +59,7 @@ warn_unsupported, write_stash, ) +from ._dax_to_sql import translate_measure # What happens to a TMSL construct Apache Ossie cannot express: it survives a round trip # back to Power BI, but it is invisible to any other consumer of the document. @@ -102,13 +104,18 @@ "isActive", } ) -def convert_semantic_model_to_ossie(semantic_model: dict | str) -> str: +def convert_semantic_model_to_ossie(semantic_model: dict | str, translate_dax: bool = False) -> str: """Convert a Power BI semantic model into an Apache Ossie semantic model. Args: semantic_model: A ``model.bim`` document, either parsed (a TMSL mapping) or as JSON text, or a single TMDL document as text. TMDL input requires the optional ``tom`` extra. + translate_dax: When true, DAX measures whose single-column aggregate has an + unambiguous ``OSSIE_SQL_2026`` form are translated, with the original DAX + kept as a fallback dialect; measures that cannot be translated safely are + kept as DAX with a diagnostic. Off by default, so the default output is + unchanged. Returns: The Apache Ossie document serialized as YAML. @@ -117,7 +124,7 @@ def convert_semantic_model_to_ossie(semantic_model: dict | str) -> str: TypeError: if `semantic_model` is neither a parsed TMSL document nor text. ValueError: if the document has no ``model`` object. """ - return dump_yaml(build_ossie_document(_as_tmsl(semantic_model))) + return dump_yaml(build_ossie_document(_as_tmsl(semantic_model), translate_dax=translate_dax)) def _as_tmsl(semantic_model): @@ -139,8 +146,13 @@ def _as_tmsl(semantic_model): return deserialize_tmdl(semantic_model) -def build_ossie_document(bim_file): - """Convert a parsed ``model.bim`` document into an Apache Ossie document (dict).""" +def build_ossie_document(bim_file, translate_dax=False): + """Convert a parsed ``model.bim`` document into an Apache Ossie document (dict). + + When ``translate_dax`` is true, single-column aggregate measures are translated + to ``OSSIE_SQL_2026`` where unambiguous (see ``_dax_to_sql``); otherwise every + measure is kept as DAX, which is the default. + """ if not isinstance(bim_file, dict): raise TypeError("bim_file must be a parsed model.bim document (dict)") @@ -176,7 +188,10 @@ def build_ossie_document(bim_file): semantic_model["relationships"] = relationships metric_tables = [table for table in all_tables if _is_measure_table(table)] - metrics, excluded_measures = _convert_metrics(metric_tables) + resolve_field = _build_field_resolver(semantic_model["datasets"]) if translate_dax else None + metrics, excluded_measures = _convert_metrics( + metric_tables, translate_dax=translate_dax, resolve_field=resolve_field + ) excluded_tables = _strip_converted_measures(excluded_tables, metrics) if metrics: semantic_model["metrics"] = metrics @@ -491,7 +506,56 @@ def _map_datatype(tmsl_type, format_string, scope): # --------------------------------------------------------------------------- -def _convert_metrics(tables): +def _build_field_resolver(datasets): + """Return a resolver mapping a DAX (table, column) to an Ossie (dataset, field). + + A Power BI measure addresses a column as ``'Table'[Column]`` by the model names, + which the converter uses verbatim as the dataset and field names. So a reference + resolves exactly when that dataset carries that field; anything else (an unknown + column, a measure table, a column that was not exported) does not resolve, and the + translator keeps the measure as DAX. + """ + known = set() + for dataset in datasets or []: + dataset_name = dataset.get("name") + if not dataset_name: + continue + for field in dataset.get("fields") or []: + field_name = field.get("name") + if field_name: + known.add((dataset_name, field_name)) + + def resolve(table, column): + return (table, column) if (table, column) in known else None + + return resolve + + +def _measure_expression(dax_expression, scope, translate_dax, resolve_field): + """Build a metric's expression from a DAX measure body. + + By default, and whenever a measure cannot be translated safely, the DAX is kept + verbatim as the sole dialect. When translation is on and succeeds, the metric + carries ``OSSIE_SQL_2026`` as the primary dialect with the original DAX kept as a + fallback, so no information is lost and a reader that trusts only DAX still has it. + """ + if not translate_dax or resolve_field is None: + return make_expression(dax_expression, DIALECT_DAX) + + sql, reason = translate_measure(dax_expression, resolve_field) + if sql is None: + warn(scope, f"DAX measure kept as-is; not translated to OSSIE_SQL_2026 ({reason})") + return make_expression(dax_expression, DIALECT_DAX) + + return { + "dialects": [ + {"dialect": DIALECT_OSSIE_SQL, "expression": sql}, + {"dialect": DIALECT_DAX, "expression": dax_expression}, + ] + } + + +def _convert_metrics(tables, translate_dax=False, resolve_field=None): metrics = [] excluded = [] seen = set() @@ -526,7 +590,12 @@ def _convert_metrics(tables): warn(scope, f"duplicate measure name; renamed to '{name}'") seen.add(name) - metric = {"name": name, "expression": make_expression(expression, DIALECT_DAX)} + metric = { + "name": name, + "expression": _measure_expression( + expression, scope, translate_dax, resolve_field + ), + } if measure.get("description"): metric["description"] = text(measure["description"]) # A measure's result type is inferred from its DAX, so it is reported for diff --git a/converters/microsoft/tests/test_dax_to_sql.py b/converters/microsoft/tests/test_dax_to_sql.py new file mode 100644 index 00000000..0305ef0b --- /dev/null +++ b/converters/microsoft/tests/test_dax_to_sql.py @@ -0,0 +1,177 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the opt-in DAX-to-OSSIE_SQL measure translator. + +The translator must translate exactly the six single-column aggregates whose +SQL equivalent is unambiguous, and refuse everything else with a reason. The +refusals matter as much as the translations: a wrong translation is a silent +wrong number on another engine. +""" + +import warnings + +import pytest + +from ossie_microsoft._dax_to_sql import translate_measure +from ossie_microsoft.semantic_model_to_ossie import build_ossie_document + + +def _dialects(metric): + return {d["dialect"]: d["expression"] for d in metric["expression"]["dialects"]} + + +def _metric(document, name): + return next(m for m in document["metrics"] if m["name"] == name) + + +def resolve(table, column): + """A resolver over a small two-dataset model, mirroring the real converter: + the DAX table name is the dataset name and the DAX column name is the field + name. Unknown table/column pairs do not resolve.""" + model = { + ("Sales", "Amount"): ("Sales", "Amount"), + ("Sales", "Customer"): ("Sales", "Customer"), + ("Sales", "Status"): ("Sales", "Status"), + ("Order Lines", "Qty"): ("Order Lines", "Qty"), # table name needs quoting + } + return model.get((table, column)) + + +@pytest.mark.parametrize( + "dax, expected", + [ + ("SUM('Sales'[Amount])", "SUM(Sales.Amount)"), + ("AVERAGE('Sales'[Amount])", "AVG(Sales.Amount)"), + ("MIN('Sales'[Amount])", "MIN(Sales.Amount)"), + ("MAX('Sales'[Amount])", "MAX(Sales.Amount)"), + ("COUNTA('Sales'[Customer])", "COUNT(Sales.Customer)"), + ("DISTINCTCOUNTNOBLANK('Sales'[Customer])", "COUNT(DISTINCT Sales.Customer)"), + ], +) +def test_translates_the_six_aggregates(dax, expected): + sql, reason = translate_measure(dax, resolve) + assert reason is None + assert sql == expected + + +def test_bare_table_reference_is_accepted(): + sql, reason = translate_measure("SUM(Sales[Amount])", resolve) + assert reason is None + assert sql == "SUM(Sales.Amount)" + + +def test_leading_equals_and_whitespace_are_tolerated(): + sql, reason = translate_measure(" = SUM( 'Sales'[Amount] ) ", resolve) + assert reason is None + assert sql == "SUM(Sales.Amount)" + + +@pytest.mark.parametrize( + "dax", + [ + "COUNT('Sales'[Amount])", # TRUE/FALSE unsupported in DAX + "DISTINCTCOUNT('Sales'[Customer])", # BLANK counted as distinct + "COUNTROWS('Sales')", # row count, not single column + "MEDIAN('Sales'[Amount])", # later tier + "SUMX('Sales', 'Sales'[Amount])", # iterator, two args + "CALCULATE(SUM('Sales'[Amount]), 'Sales'[Status] = \"done\")", # filter context + "DIVIDE(SUM('Sales'[Amount]), COUNTA('Sales'[Customer]))", # division + "SUM('Sales'[Amount]) + SUM('Sales'[Amount])", # arithmetic + "SUM('Sales'[Amount]) * 1.1", # arithmetic + "IF('Sales'[Amount] > 0, 1, 0)", # scalar logic + ], +) +def test_refuses_everything_outside_the_safe_set(dax): + sql, reason = translate_measure(dax, resolve) + assert sql is None + assert reason # a non-empty reason is always given + + +def test_count_and_distinctcount_refusals_name_the_semantic_difference(): + _, count_reason = translate_measure("COUNT('Sales'[Amount])", resolve) + _, distinct_reason = translate_measure("DISTINCTCOUNT('Sales'[Customer])", resolve) + assert "TRUE/FALSE" in count_reason + assert "BLANK" in distinct_reason + + +def test_unqualified_column_is_refused(): + sql, reason = translate_measure("SUM([Amount])", resolve) + assert sql is None + assert "fully qualified" in reason + + +def test_unresolved_column_is_refused(): + sql, reason = translate_measure("SUM('Ghost'[Nope])", resolve) + assert sql is None + assert "does not resolve" in reason + + +def test_name_needing_quoting_is_refused_in_this_tier(): + # 'Order Lines' resolves, but the dataset name has a space and would need + # SQL quoting, which this first tier declines rather than risk getting wrong. + sql, reason = translate_measure("SUM('Order Lines'[Qty])", resolve) + assert sql is None + assert "quoting" in reason + + +def test_empty_measure_is_refused(): + sql, reason = translate_measure(" ", resolve) + assert sql is None + assert reason + + +def test_nested_call_does_not_parse_as_single_aggregate(): + sql, reason = translate_measure("SUM(ROUND('Sales'[Amount], 2))", resolve) + assert sql is None + assert reason + + +# --- Integration: the opt-in flag wired through build_ossie_document --------- + + +def _build(bim, translate_dax): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return build_ossie_document(bim, translate_dax=translate_dax) + + +def test_default_keeps_every_measure_as_dax_only(bim): + document = _build(bim, translate_dax=False) + for metric in document["metrics"]: + assert list(_dialects(metric)) == ["DAX"], metric["name"] + + +def test_flag_translates_a_single_column_aggregate_and_keeps_dax_fallback(bim): + document = _build(bim, translate_dax=True) + dialects = _dialects(_metric(document, "Total Sales")) + # OSSIE_SQL_2026 is added as the portable rendering ... + assert dialects["OSSIE_SQL_2026"] == "SUM(Sales.Amount)" + # ... and the original DAX is preserved as a fallback, so nothing is lost. + assert dialects["DAX"] == "SUM ( Sales[Amount] )" + + +def test_flag_refuses_countrows_and_leaves_it_as_dax(bim): + document = _build(bim, translate_dax=True) + dialects = _dialects(_metric(document, "Order Count")) + assert list(dialects) == ["DAX"] + + +def test_flag_is_order_preserving_portable_first(bim): + document = _build(bim, translate_dax=True) + first = _metric(document, "Total Sales")["expression"]["dialects"][0] + assert first["dialect"] == "OSSIE_SQL_2026" diff --git a/converters/microsoft/tests/test_packaging.py b/converters/microsoft/tests/test_packaging.py index 6dcd86be..a9ca7a4f 100644 --- a/converters/microsoft/tests/test_packaging.py +++ b/converters/microsoft/tests/test_packaging.py @@ -54,6 +54,7 @@ MODULES = frozenset( { "_common", + "_dax_to_sql", "_sql_to_dax", "cli", "engine",