feat(microsoft): opt-in DAX to OSSIE_SQL_2026 translation for single-column aggregate measures (#584)

converters/microsoft/src/ossie_microsoft/_dax_to_sql.py

@@ -0,0 +1,193 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""A deliberately narrow DAX-to-OSSIE_SQL translator for Power BI measures.
+
+Power BI measures are DAX. On import, the Microsoft converter keeps them as DAX,
+so a measure that leaves Power BI arrives at every other engine as an opaque
+vendor string it cannot evaluate. This module is the opt-in reverse of
+``_sql_to_dax``: it recognises the single-column aggregate measures whose
+``OSSIE_SQL_2026`` equivalent is unambiguous, and refuses everything else.
+
+The governing rule is the one ``_sql_to_dax`` states and ``#437`` argues for:
+never emit a plausible-looking expression that was not authored for the target.
+A measure this module declines is kept verbatim as DAX with a diagnostic; a
+measure it mistranslated would deploy cleanly on another engine and quietly
+return a wrong number. Refusal is the safe default; translation is the exception
+that must be proven unambiguous.
+
+What is translated
+------------------
+A single DAX aggregate over one fully qualified column, for the six aggregates
+that reverse ``_sql_to_dax`` exactly, so an Ossie SQL model exported to Power BI
+and re-imported round-trips to the same expression::
+
+    SUM('Sales'[Amount])                 ->  SUM(Sales.Amount)
+    AVERAGE('Sales'[Amount])             ->  AVG(Sales.Amount)
+    MIN('Sales'[Amount])                 ->  MIN(Sales.Amount)
+    MAX('Sales'[Amount])                 ->  MAX(Sales.Amount)
+    COUNTA('Sales'[Customer])            ->  COUNT(Sales.Customer)
+    DISTINCTCOUNTNOBLANK('Sales'[Cust])  ->  COUNT(DISTINCT Sales.Cust)
+
+What is refused (and why, so the refusal is defensible)
+-------------------------------------------------------
+* ``COUNT`` and ``DISTINCTCOUNT``. DAX ``COUNT`` documents ``TRUE``/``FALSE``
+  values as unsupported, so it is not the equivalent of SQL ``COUNT(x)`` for an
+  arbitrary column, and ``DISTINCTCOUNT`` counts BLANK as a distinct value where
+  SQL ``COUNT(DISTINCT x)`` excludes NULL. These are refused rather than guessed.
+  Their counterparts ``COUNTA`` and ``DISTINCTCOUNTNOBLANK`` are what
+  ``_sql_to_dax`` emits for SQL ``COUNT`` and ``COUNT(DISTINCT)``, and are translated.
+  As ``_sql_to_dax`` notes, a DAX count over an empty set is BLANK where SQL gives 0;
+  that convention is accepted in both directions.
+* ``COUNTROWS``, ``MEDIAN``, the statistical aggregates, iterators (``SUMX``),
+  ``CALCULATE`` and the filter-context family, ``DIVIDE``, ``IF``, arithmetic,
+  time intelligence, and anything with more than one argument or a nested call.
+  These are later tiers or have no declaration-scoped portable form.
+* A column that is not fully qualified as ``Table[Column]``, or a table or
+  column name that is not a simple identifier. A name needing SQL quoting is
+  left to a later tier so this first pass never has to get a quoting rule right.
+
+This module performs no I/O and holds no model state. The caller supplies a
+``resolve`` callback mapping a DAX (table, column) to the Ossie (dataset, field)
+names, which is where model knowledge lives.
+"""
+
+from __future__ import annotations
+
+import re
+
+#: DAX aggregate -> OSSIE_SQL aggregate, for the exact reverses of _sql_to_dax.
+#: COUNTA and DISTINCTCOUNTNOBLANK are what _sql_to_dax emits for SQL COUNT and
+#: COUNT(DISTINCT); reversing those same functions keeps the round-trip exact.
+_SIMPLE_AGGREGATES = {
+    "SUM": "SUM",
+    "AVERAGE": "AVG",
+    "MIN": "MIN",
+    "MAX": "MAX",
+    "COUNTA": "COUNT",
+}
+
+#: DAX functions we recognise well enough to refuse with a specific reason,
+#: rather than a generic "not translated", because they look translatable.
+_KNOWN_UNSAFE = {
+    "COUNT": "DAX COUNT does not support TRUE/FALSE values, so it is not the equivalent "
+    "of SQL COUNT(x) for an arbitrary column (COUNTA is)",
+    "DISTINCTCOUNT": "DAX DISTINCTCOUNT counts BLANK as a distinct value; SQL "
+    "COUNT(DISTINCT x) excludes NULL, so the two differ by one on a nullable column",
+    "COUNTROWS": "COUNTROWS counts rows, not a single column; it is a later tier",
+    "MEDIAN": "MEDIAN is a later tier",
+    "SUMX": "iterators evaluate per row and are a later tier",
+    "AVERAGEX": "iterators evaluate per row and are a later tier",
+    "CALCULATE": "CALCULATE manipulates filter context, which has no declaration-scoped form",
+    "DIVIDE": "division between measures is a later tier",
+}
+
+# A simple identifier: a letter or underscore, then letters, digits, underscores.
+_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+
+# One DAX function call: NAME ( ARG ). ARG is captured whole and validated
+# separately so nested parentheses or extra arguments fail the ARG match.
+_CALL = re.compile(r"^([A-Za-z_][A-Za-z0-9_.]*)\s*\((.*)\)$", re.DOTALL)
+
+# A fully qualified DAX column: Table[Column] or 'Table Name'[Column], where a
+# quoted table escapes ' as '' and a column escapes ] as ]].
+_QUALIFIED_COLUMN = re.compile(
+    r"""^\s*
+        (?:'((?:[^']|'')*)'|([A-Za-z_][A-Za-z0-9_]*))   # 1 quoted or 2 bare table
+        \s*
+        \[((?:[^\]]|\]\])*)\]                            # 3 column, ] escaped as ]]
+        \s*$""",
+    re.VERBOSE,
+)
+
+
+def _strip_measure(expression):
+    """Normalise a measure body: drop a leading '=' and outer whitespace."""
+    text = (expression or "").strip()
+    if text.startswith("="):
+        text = text[1:].strip()
+    return text
+
+
+def _parse_qualified_column(argument):
+    """Return (table, column) for a fully qualified DAX column, else None."""
+    match = _QUALIFIED_COLUMN.match(argument)
+    if match is None:
+        return None
+    quoted_table, bare_table, column = match.groups()
+    table = quoted_table.replace("''", "'") if quoted_table is not None else bare_table
+    column = column.replace("]]", "]")
+    return table, column
+
+
+def _is_simple_identifier(name):
+    return bool(name) and _IDENTIFIER.fullmatch(name) is not None
+
+
+def _qualify(dataset, field):
+    """Render an OSSIE_SQL ``dataset.field`` reference, or None if either name
+    is not a simple identifier (a name needing quoting is a later tier)."""
+    if not _is_simple_identifier(dataset) or not _is_simple_identifier(field):
+        return None
+    return f"{dataset}.{field}"
+
+
+def translate_measure(expression, resolve):
+    """Translate a single-column DAX aggregate measure to an OSSIE_SQL expression.
+
+    ``resolve`` maps a DAX ``(table, column)`` to the Ossie ``(dataset, field)``
+    names, or returns None when the column does not resolve to exactly one field.
+
+    Returns ``(sql, None)`` on success or ``(None, reason)`` on refusal. The
+    caller keeps the original DAX and attaches ``reason`` as a diagnostic when a
+    measure is refused.
+    """
+    text = _strip_measure(expression)
+    if not text:
+        return None, "measure has no expression"
+
+    call = _CALL.match(text)
+    if call is None:
+        return None, "only a single aggregate function call is translated"
+
+    function = call.group(1).upper()
+    argument = call.group(2).strip()
+
+    distinct_target = function == "DISTINCTCOUNTNOBLANK"
+    if function not in _SIMPLE_AGGREGATES and not distinct_target:
+        reason = _KNOWN_UNSAFE.get(function, f"'{function}' is not a translated aggregate")
+        return None, reason
+
+    parsed = _parse_qualified_column(argument)
+    if parsed is None:
+        return None, (
+            "only an aggregate over one fully qualified column (Table[Column]) is translated"
+        )
+    table, column = parsed
+
+    resolved = resolve(table, column)
+    if resolved is None:
+        return None, f"column '{table}'[{column}] does not resolve to exactly one dataset field"
+    dataset, field = resolved
+
+    reference = _qualify(dataset, field)
+    if reference is None:
+        return None, "a dataset or field name needing SQL quoting is a later tier"
+
+    if distinct_target:
+        return f"COUNT(DISTINCT {reference})", None
+    return f"{_SIMPLE_AGGREGATES[function]}({reference})", None

converters/microsoft/src/ossie_microsoft/cli.py

@@ -100,6 +100,13 @@ def _build_parser():
     )
     imp.add_argument("-i", "--input", required=True, help="model.bim (TMSL JSON) file")
     imp.add_argument("-o", "--output", help="output Apache Ossie YAML (default: stdout)")
+    imp.add_argument(
+        "--translate-dax",
+        action="store_true",
+        default=False,
+        help="translate single-column aggregate DAX measures to OSSIE_SQL_2026 where "
+        "unambiguous, keeping the original DAX as a fallback (default: off, keep DAX only)",
+    )
 
     exp = sub.add_parser(
         "export",
@@ -113,7 +120,7 @@ def _build_parser():
 
 def _run_import(args):
     with open(args.input, encoding="utf-8-sig") as fh:
-        return convert_semantic_model_to_ossie(json.load(fh))
+        return convert_semantic_model_to_ossie(json.load(fh), translate_dax=args.translate_dax)
 
 
 def _run_export(args):

converters/microsoft/src/ossie_microsoft/semantic_model_to_ossie.py

@@ -39,6 +39,7 @@
     AUTO_DATE_TABLE_RE,
     DIALECT_ANSI,
     DIALECT_DAX,
+    DIALECT_OSSIE_SQL,
     IDENTIFIER_RE,
     OSSIE_TO_TMSL_DATATYPE,
     OSSIE_VERSION,
@@ -58,6 +59,7 @@
     warn_unsupported,
     write_stash,
 )
+from ._dax_to_sql import translate_measure
 
 # What happens to a TMSL construct Apache Ossie cannot express: it survives a round trip
 # back to Power BI, but it is invisible to any other consumer of the document.
@@ -102,13 +104,18 @@
         "isActive",
     }
 )
-def convert_semantic_model_to_ossie(semantic_model: dict | str) -> str:
+def convert_semantic_model_to_ossie(semantic_model: dict | str, translate_dax: bool = False) -> str:
     """Convert a Power BI semantic model into an Apache Ossie semantic model.
 
     Args:
         semantic_model: A ``model.bim`` document, either parsed (a TMSL mapping) or as
             JSON text, or a single TMDL document as text. TMDL input requires the
             optional ``tom`` extra.
+        translate_dax: When true, DAX measures whose single-column aggregate has an
+            unambiguous ``OSSIE_SQL_2026`` form are translated, with the original DAX
+            kept as a fallback dialect; measures that cannot be translated safely are
+            kept as DAX with a diagnostic. Off by default, so the default output is
+            unchanged.
 
     Returns:
         The Apache Ossie document serialized as YAML.
@@ -117,7 +124,7 @@ def convert_semantic_model_to_ossie(semantic_model: dict | str) -> str:
         TypeError: if `semantic_model` is neither a parsed TMSL document nor text.
         ValueError: if the document has no ``model`` object.
     """
-    return dump_yaml(build_ossie_document(_as_tmsl(semantic_model)))
+    return dump_yaml(build_ossie_document(_as_tmsl(semantic_model), translate_dax=translate_dax))
 
 
 def _as_tmsl(semantic_model):
@@ -139,8 +146,13 @@ def _as_tmsl(semantic_model):
     return deserialize_tmdl(semantic_model)
 
 
-def build_ossie_document(bim_file):
-    """Convert a parsed ``model.bim`` document into an Apache Ossie document (dict)."""
+def build_ossie_document(bim_file, translate_dax=False):
+    """Convert a parsed ``model.bim`` document into an Apache Ossie document (dict).
+
+    When ``translate_dax`` is true, single-column aggregate measures are translated
+    to ``OSSIE_SQL_2026`` where unambiguous (see ``_dax_to_sql``); otherwise every
+    measure is kept as DAX, which is the default.
+    """
     if not isinstance(bim_file, dict):
         raise TypeError("bim_file must be a parsed model.bim document (dict)")
 
@@ -176,7 +188,10 @@ def build_ossie_document(bim_file):
         semantic_model["relationships"] = relationships
 
     metric_tables = [table for table in all_tables if _is_measure_table(table)]
-    metrics, excluded_measures = _convert_metrics(metric_tables)
+    resolve_field = _build_field_resolver(semantic_model["datasets"]) if translate_dax else None
+    metrics, excluded_measures = _convert_metrics(
+        metric_tables, translate_dax=translate_dax, resolve_field=resolve_field
+    )
     excluded_tables = _strip_converted_measures(excluded_tables, metrics)
     if metrics:
         semantic_model["metrics"] = metrics
@@ -491,7 +506,56 @@ def _map_datatype(tmsl_type, format_string, scope):
 # ---------------------------------------------------------------------------
 
 
-def _convert_metrics(tables):
+def _build_field_resolver(datasets):
+    """Return a resolver mapping a DAX (table, column) to an Ossie (dataset, field).
+
+    A Power BI measure addresses a column as ``'Table'[Column]`` by the model names,
+    which the converter uses verbatim as the dataset and field names. So a reference
+    resolves exactly when that dataset carries that field; anything else (an unknown
+    column, a measure table, a column that was not exported) does not resolve, and the
+    translator keeps the measure as DAX.
+    """
+    known = set()
+    for dataset in datasets or []:
+        dataset_name = dataset.get("name")
+        if not dataset_name:
+            continue
+        for field in dataset.get("fields") or []:
+            field_name = field.get("name")
+            if field_name:
+                known.add((dataset_name, field_name))
+
+    def resolve(table, column):
+        return (table, column) if (table, column) in known else None
+
+    return resolve
+
+
+def _measure_expression(dax_expression, scope, translate_dax, resolve_field):
+    """Build a metric's expression from a DAX measure body.
+
+    By default, and whenever a measure cannot be translated safely, the DAX is kept
+    verbatim as the sole dialect. When translation is on and succeeds, the metric
+    carries ``OSSIE_SQL_2026`` as the primary dialect with the original DAX kept as a
+    fallback, so no information is lost and a reader that trusts only DAX still has it.
+    """
+    if not translate_dax or resolve_field is None:
+        return make_expression(dax_expression, DIALECT_DAX)
+
+    sql, reason = translate_measure(dax_expression, resolve_field)
+    if sql is None:
+        warn(scope, f"DAX measure kept as-is; not translated to OSSIE_SQL_2026 ({reason})")
+        return make_expression(dax_expression, DIALECT_DAX)
+
+    return {
+        "dialects": [
+            {"dialect": DIALECT_OSSIE_SQL, "expression": sql},
+            {"dialect": DIALECT_DAX, "expression": dax_expression},
+        ]
+    }
+
+
+def _convert_metrics(tables, translate_dax=False, resolve_field=None):
     metrics = []
     excluded = []
     seen = set()
@@ -526,7 +590,12 @@ def _convert_metrics(tables):
                 warn(scope, f"duplicate measure name; renamed to '{name}'")
             seen.add(name)
 
-            metric = {"name": name, "expression": make_expression(expression, DIALECT_DAX)}
+            metric = {
+                "name": name,
+                "expression": _measure_expression(
+                    expression, scope, translate_dax, resolve_field
+                ),
+            }
             if measure.get("description"):
                 metric["description"] = text(measure["description"])
             # A measure's result type is inferred from its DAX, so it is reported for