From f8d3508c9a53f20f75902e564c5c5bbd9eb0980f Mon Sep 17 00:00:00 2001 From: Chris Eubank <108756251+christianeu-db@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:06:49 +0000 Subject: [PATCH 1/9] feat(core-spec): decouple core spec from specific vendor dialects Make the core specification vendor-neutral: - `Dialect` becomes a free-form string (was a closed enum), mirroring the already-free-form `Vendor`; generic schema `examples` replace concrete product names. - Move the known-dialect and known-vendor catalogs out of the normative spec into a new top-level `extensions/` folder (README, dialects.md, vendors.md), framed as a community registry whose listings do not imply endorsement. - Add an optional top-level portable `expression` field (OSSIE_SQL_2026) to the expression object; when both it and an OSSIE_SQL_2026 `dialects` entry are present, the top-level field is authoritative. Demonstrated on `customer_full_name` in examples/tpcds_semantic_model.yaml. - Keep the normative text that non-OSSIE dialects are opaque and declaration-only. Kept in sync: the enum-sync generator no longer emits `dialects` (free-form, like vendors); the Python SDK types `dialect` as a free-form string and gains the optional portable `expression`; the reference validator treats unregistered dialects as opaque (skipped) and validates the portable expression. Converters are otherwise unchanged except where the now-string dialect required a fix (the dbt dialect comparison uses `==`, and the sigma and hex converters no longer call `.value` on a dialect) and where the new portable `expression` is handled: the ontology converter carries it through its model and both conversion directions, and omni's round-trip test documents the portable marker as a normalization (its text survives as the dialect entry). Co-authored-by: Isaac --- converters/dbt/src/ossie_dbt/ossie_to_msi.py | 4 +- .../ossie_to_hex/load_ossie_expression.py | 4 +- .../pick_ossie_dialect_expression.py | 4 +- .../test_pick_ossie_dialect_expression.py | 4 +- .../tests/test_semantic_model_to_ossie.py | 6 - converters/omni/tests/_util.py | 9 ++ .../converter/ossie_to_spec/converter.py | 1 + .../converter/spec_to_ossie/converter.py | 1 + .../ontology/src/ossie_ontology/model.py | 3 +- .../ontology/src/ossie_ontology/spec.py | 4 +- .../sigma/src/ossie_sigma/ossie_to_sigma.py | 2 +- .../wisdom/tests/test_ossie_to_wisdom.py | 24 ++++ .../wisdom/tests/test_wisdom_to_ossie.py | 8 +- core-spec/expression_language.md | 2 + core-spec/ossie-schema.json | 12 +- core-spec/spec.md | 105 ++++-------------- core-spec/spec.yaml | 29 ++--- docs/index.md | 13 +-- examples/tpcds_semantic_model.yaml | 2 + extensions/README.md | 39 +++++++ extensions/dialects.md | 41 +++++++ extensions/vendors.md | 98 ++++++++++++++++ python/src/ossie/models.py | 11 +- python/tests/test_models.py | 46 ++++++-- python/tests/test_serialization.py | 5 +- python/tests/test_validation.py | 7 +- scripts/generate-spec-types.py | 20 ---- validation/tests/test_generate_spec_types.py | 72 ++++++------ validation/tests/test_validate.py | 36 ++++++ validation/validate.py | 19 ++++ 30 files changed, 428 insertions(+), 203 deletions(-) create mode 100644 extensions/README.md create mode 100644 extensions/dialects.md create mode 100644 extensions/vendors.md diff --git a/converters/dbt/src/ossie_dbt/ossie_to_msi.py b/converters/dbt/src/ossie_dbt/ossie_to_msi.py index bcb69a6e4..e83ca997d 100644 --- a/converters/dbt/src/ossie_dbt/ossie_to_msi.py +++ b/converters/dbt/src/ossie_dbt/ossie_to_msi.py @@ -478,9 +478,9 @@ def _get_expression(self, ossie_expr: OssieExpression) -> str: ossie_sql_expr: Optional[str] = None for dialect_expr in ossie_expr.dialects: - if dialect_expr.dialect is self._dialect: + if dialect_expr.dialect == self._dialect: return dialect_expr.expression - if dialect_expr.dialect is OssieDialect.OSSIE_SQL_2026 and ossie_sql_expr is None: + if dialect_expr.dialect == OssieDialect.OSSIE_SQL_2026 and ossie_sql_expr is None: ossie_sql_expr = dialect_expr.expression if ossie_sql_expr is not None: diff --git a/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_expression.py b/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_expression.py index ab722fdaa..c1a202551 100644 --- a/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_expression.py +++ b/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_expression.py @@ -43,7 +43,7 @@ def load_ossie_field_expression( with ctx.problem_scope("dialects"): choice = pick_ossie_dialect_expression(ossie_expression.dialects, ctx=ctx) if choice is not None: - with ctx.problem_scope(choice.dialect.value): + with ctx.problem_scope(choice.dialect): validate_ossie_field_dialect_expression(choice, ctx=ctx) return ossie_expression @@ -66,7 +66,7 @@ def load_ossie_metric_expression( with ctx.problem_scope("dialects"): choice = pick_ossie_dialect_expression(expression.dialects, ctx=ctx) if choice is not None: - with ctx.problem_scope(choice.dialect.value): + with ctx.problem_scope(choice.dialect): validate_ossie_metric_dialect_expression( choice, field_names=field_names, ctx=ctx ) diff --git a/converters/hex/src/ossie_hex/ossie_to_hex/pick_ossie_dialect_expression.py b/converters/hex/src/ossie_hex/ossie_to_hex/pick_ossie_dialect_expression.py index b98532ab8..56198b865 100644 --- a/converters/hex/src/ossie_hex/ossie_to_hex/pick_ossie_dialect_expression.py +++ b/converters/hex/src/ossie_hex/ossie_to_hex/pick_ossie_dialect_expression.py @@ -41,10 +41,10 @@ def pick_ossie_dialect_expression( if ctx.current_phase_name == "load": if fallback is not None: ctx.warn( - f"Preferred dialect {ctx.ossie_dialect} not found for expression; using fallback dialect {fallback.dialect}" + f"Preferred dialect {ctx.ossie_dialect.value} not found for expression; using fallback dialect {fallback.dialect}" ) else: ctx.error( - f"Preferred dialect {ctx.ossie_dialect} not found for expression and no fallback dialect found" + f"Preferred dialect {ctx.ossie_dialect.value} not found for expression and no fallback dialect found" ) return fallback diff --git a/converters/hex/tests/ossie_to_hex/test_pick_ossie_dialect_expression.py b/converters/hex/tests/ossie_to_hex/test_pick_ossie_dialect_expression.py index 1d25b46ba..6a2e93fa2 100644 --- a/converters/hex/tests/ossie_to_hex/test_pick_ossie_dialect_expression.py +++ b/converters/hex/tests/ossie_to_hex/test_pick_ossie_dialect_expression.py @@ -72,7 +72,7 @@ def test_prefers_ansi_fallback_over_other_variants(ctx: ExportContext) -> None: assert result is expression.dialects[-1] assert problems_snapshot(ctx.problems) == snapshot( - "[WARNING] Preferred dialect OssieDialect.SNOWFLAKE not found for expression; using fallback dialect OssieDialect.ANSI_SQL" + "[WARNING] Preferred dialect SNOWFLAKE not found for expression; using fallback dialect ANSI_SQL" ) @@ -85,5 +85,5 @@ def test_falls_back_to_first_variant_when_preferred_and_ansi_are_missing( assert result is expression.dialects[0] assert problems_snapshot(ctx.problems) == snapshot( - "[WARNING] Preferred dialect OssieDialect.ANSI_SQL not found for expression; using fallback dialect OssieDialect.BIGQUERY" + "[WARNING] Preferred dialect ANSI_SQL not found for expression; using fallback dialect BIGQUERY" ) diff --git a/converters/microsoft/tests/test_semantic_model_to_ossie.py b/converters/microsoft/tests/test_semantic_model_to_ossie.py index 6742ea97f..ece3c180c 100644 --- a/converters/microsoft/tests/test_semantic_model_to_ossie.py +++ b/converters/microsoft/tests/test_semantic_model_to_ossie.py @@ -755,12 +755,6 @@ def test_output_validates_against_core_spec_schema(osi): jsonschema.validate(osi, schema) -def test_dax_is_a_spec_dialect(): - with open(REPO_ROOT / "core-spec" / "ossie-schema.json", encoding="utf-8") as fh: - schema = json.load(fh) - assert "DAX" in schema["$defs"]["Dialect"]["enum"] - - # --- lossless preservation (POWER_BI stash) -------------------------------- diff --git a/converters/omni/tests/_util.py b/converters/omni/tests/_util.py index b9d808ead..9ec36decf 100644 --- a/converters/omni/tests/_util.py +++ b/converters/omni/tests/_util.py @@ -86,6 +86,9 @@ def strip_normalized(ossie): (only `is_time: true` has an Omni form -- `timeframes`). - a primary-key column no field covers materializes as a hidden dimension on export, so it comes back as an extra (stash-only) field. + - an expression's top-level `expression` (the portable OSSIE_SQL_2026 form): + Omni emits SQL per dialect, so the portable marker is normalized away on + export; its text survives as the `ANSI_SQL` dialect entry. """ ossie = copy.deepcopy(ossie) model = ossie @@ -107,6 +110,9 @@ def strip_normalized(ossie): fields = ds.get("fields", []) or [] for field in fields: field.pop("custom_extensions", None) + fexpr = field.get("expression") + if isinstance(fexpr, dict): + fexpr.pop("expression", None) dimension = field.get("dimension") is_time = False if dimension is not None: @@ -138,6 +144,9 @@ def strip_normalized(ossie): for metric in model.get("metrics", []) or []: metric.pop("custom_extensions", None) metric.pop("datatype", None) + mexpr = metric.get("expression") + if isinstance(mexpr, dict): + mexpr.pop("expression", None) # Metric order is not semantic; the import regroups metrics by the view # their measure lives on. if model.get("metrics"): diff --git a/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py b/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py index c367285f5..d60d82c43 100644 --- a/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py +++ b/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py @@ -185,6 +185,7 @@ def _convert_dataset_field(fl: DatasetField) -> SpecDatasetField: def _convert_expression(es: DialectExpressionSet) -> SpecExpression: return SpecExpression( + expression=es.expression, dialects=[SpecDialectExpression(dialect=d.dialect, expression=d.expression) for d in es.dialects] ) diff --git a/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py b/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py index 776a2dcae..6eebddd13 100644 --- a/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py +++ b/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py @@ -461,6 +461,7 @@ def _convert_custom_extension(ce: SpecCustomExtension) -> CustomExtension: def _convert_expression(expr: SpecExpression) -> DialectExpressionSet: return DialectExpressionSet( + expression=expr.expression, dialects=[_convert_dialect_expression(d) for d in expr.dialects] ) diff --git a/converters/ontology/src/ossie_ontology/model.py b/converters/ontology/src/ossie_ontology/model.py index e1563cb6f..b5d3d3244 100644 --- a/converters/ontology/src/ossie_ontology/model.py +++ b/converters/ontology/src/ossie_ontology/model.py @@ -467,8 +467,9 @@ class DialectExpression: @dataclass class DialectExpressionSet: """Runtime equivalent of spec.Expression — same logical expression rendered - in one or more dialects.""" + in one or more dialects, with an optional primary, portable OSSIE_SQL_2026 form.""" dialects: list[DialectExpression] = field(default_factory=list) + expression: str | None = None def by_dialect(self, dialect: str) -> DialectExpression | None: for d in self.dialects: diff --git a/converters/ontology/src/ossie_ontology/spec.py b/converters/ontology/src/ossie_ontology/spec.py index 3c4d34679..43d9b3bc1 100644 --- a/converters/ontology/src/ossie_ontology/spec.py +++ b/converters/ontology/src/ossie_ontology/spec.py @@ -119,7 +119,9 @@ class DialectExpression(OssieObject): class Expression(OssieObject): """Multi-dialect expression carrier — same logical expression rendered in - one or more dialects (e.g. ANSI_SQL + SNOWFLAKE).""" + one or more dialects (e.g. ANSI_SQL + SNOWFLAKE), with an optional primary, + portable form in OSSIE_SQL_2026.""" + expression: str | None = None dialects: list[DialectExpression] = Field(default_factory=list) diff --git a/converters/sigma/src/ossie_sigma/ossie_to_sigma.py b/converters/sigma/src/ossie_sigma/ossie_to_sigma.py index 913a31a27..edc0e36fc 100644 --- a/converters/sigma/src/ossie_sigma/ossie_to_sigma.py +++ b/converters/sigma/src/ossie_sigma/ossie_to_sigma.py @@ -122,7 +122,7 @@ def _resolve_formula( return translated detail = f"ANSI SQL expression {sql!r} has no Sigma formula equivalent" else: - dialects = ", ".join(d.dialect.value for d in expression.dialects) or "none" + dialects = ", ".join(d.dialect for d in expression.dialects) or "none" detail = f"no SIGMA or ANSI_SQL dialect expression to translate from (have: {dialects})" issues.append( diff --git a/converters/wisdom/tests/test_ossie_to_wisdom.py b/converters/wisdom/tests/test_ossie_to_wisdom.py index 3d4abe049..c375dec3e 100644 --- a/converters/wisdom/tests/test_ossie_to_wisdom.py +++ b/converters/wisdom/tests/test_ossie_to_wisdom.py @@ -345,6 +345,30 @@ def test_field_only_ossie_sql_2026_is_exported_without_missing_issue(): assert _issues_of(result, ConverterIssueType.MISSING_DIALECT_EXPRESSION) == [] +def test_field_in_an_unknown_dialect_is_carried_through_without_crashing(): + """A field in an unregistered dialect still exports (carried through), with a + MISSING_DIALECT_EXPRESSION issue since no preferred SQL dialect is present.""" + document = OssieDocument( + name="m", + datasets=[ + OssieDataset( + name="orders", + source="analytics.sales.orders", + fields=[OssieField(name="region", expression=_expr((OssieDialect.ANSI_SQL, "ANSI_region")))], + ), + OssieDataset( + name="events", + source="analytics.sales.events", + fields=[OssieField(name="kind", expression=_expr(("DB_DIALECT", "opaque_kind_expr")))], + ), + ], + ) + result = OssieToWisdomConverter().convert(document, exported_at="2026-07-10T00:00:00+00:00") + formulas = {formula["name"]: formula["expression"] for formula in _table(result.output, "events").get("formulas", [])} + assert formulas == {"kind": "opaque_kind_expr"} + assert _issues_of(result, ConverterIssueType.MISSING_DIALECT_EXPRESSION) + + def test_metric_only_ossie_sql_2026_is_exported_without_missing_issue(): """A metric expressed only in OSSIE_SQL_2026 is exported as a measure, with no spurious MISSING_DIALECT_EXPRESSION issue (#442).""" diff --git a/converters/wisdom/tests/test_wisdom_to_ossie.py b/converters/wisdom/tests/test_wisdom_to_ossie.py index a770f2369..148827cfe 100644 --- a/converters/wisdom/tests/test_wisdom_to_ossie.py +++ b/converters/wisdom/tests/test_wisdom_to_ossie.py @@ -70,7 +70,7 @@ def test_columns_become_fields(model): orders = next(dataset for dataset in model.datasets if dataset.name == "orders") fields = {field.name: field for field in orders.fields} assert fields["order_id"].expression.dialects[0].expression == "order_id" - assert fields["order_id"].expression.dialects[0].dialect is OssieDialect.SNOWFLAKE + assert fields["order_id"].expression.dialects[0].dialect == OssieDialect.SNOWFLAKE assert fields["order_date"].dimension.is_time is True assert fields["order_id"].dimension is None assert fields["status"].label == "Order Status" @@ -86,7 +86,7 @@ def test_formulas_become_fields(model): fields = {field.name: field for field in orders.fields} is_large = fields["is_large"] assert is_large.expression.dialects[0].expression == 'CASE WHEN "orders"."amount" > 100 THEN TRUE ELSE FALSE END' - assert is_large.expression.dialects[0].dialect is OssieDialect.SNOWFLAKE + assert is_large.expression.dialects[0].dialect == OssieDialect.SNOWFLAKE assert is_large.label == "Is Large" assert is_large.description == "true when the order amount exceeds 100" @@ -100,7 +100,7 @@ def test_formula_colliding_with_column_is_dropped(result, model): def test_unsupported_dialect_falls_back_to_ansi(result, model): tags = next(dataset for dataset in model.datasets if dataset.name == "tags") - assert all(field.expression.dialects[0].dialect is OssieDialect.ANSI_SQL for field in tags.fields) + assert all(field.expression.dialects[0].dialect == OssieDialect.ANSI_SQL for field in tags.fields) unsupported = _issues_of(result, ConverterIssueType.UNSUPPORTED_DIALECT) assert len(unsupported) == 1 assert "postgres" in unsupported[0].element_name @@ -154,7 +154,7 @@ def test_measures_become_metrics(result, model): assert set(metrics) == {"total_amount", "customers_total_amount"} total = metrics["total_amount"] assert total.expression.dialects[0].expression == 'SUM("orders"."amount")' - assert total.expression.dialects[0].dialect is OssieDialect.SNOWFLAKE + assert total.expression.dialects[0].dialect == OssieDialect.SNOWFLAKE assert total.description == "Total order amount" collisions = _issues_of(result, ConverterIssueType.METRIC_NAME_COLLISION) assert [issue.element_name for issue in collisions] == ["customers.total_amount"] diff --git a/core-spec/expression_language.md b/core-spec/expression_language.md index 698690a82..1635c0581 100644 --- a/core-spec/expression_language.md +++ b/core-spec/expression_language.md @@ -665,6 +665,8 @@ Ossie implementations MAY support additional functions through dialect-specific The Ossie dialect should always be supported. Other dialects MAY be ignored. There is no guarantee that all different dialects for an expression will act the same, so implementations should be consistent with their dialect handling. This means that if an Ossie model has an expression written in two dialects, the implementation should deterministically choose which dialect to use. +The `dialect` field is a free-form string; the core specification does not enumerate a closed set. For a non-normative catalog of dialects well-known tools use, see [extensions/dialects.md](../extensions/dialects.md). + ### Declaring Dialect-Specific Expressions ``` diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index 6cc259e98..a159ef71d 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -37,13 +37,13 @@ "$defs": { "Dialect": { "type": "string", - "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY", "SIGMA", "THOUGHTSPOT", "DAX", "HOLISTICS_AQL", "OSSIE_SQL_2026"], - "description": "Supported SQL and expression language dialects" + "examples": ["OSSIE_SQL_2026", "ANSI_SQL", "DB_DIALECT", "BI_EXPRESSION", "SEMANTIC_MODEL_EXPRESSION", "PROGRAMMING_LANGUAGE"], + "description": "Expression language a metric or field expression is declared in. Free-form: any string is accepted. OSSIE_SQL_2026 is the only dialect this specification defines; others are opaque to Ossie. See extensions/dialects.md." }, "Vendor": { "type": "string", - "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM", "POWER_BI", "HEX"], - "description": "Vendor name for custom extensions. Any string value is accepted." + "examples": ["COMMON", "BI_TOOL", "DB_VENDOR", "SEMANTIC_LAYER"], + "description": "Vendor name for custom extensions. Free-form: any string is accepted. See extensions/vendors.md." }, "AIContext": { "description": "Additional context for AI tools", @@ -111,6 +111,10 @@ "type": "object", "description": "Expression definition with multi-dialect support", "properties": { + "expression": { + "type": "string", + "description": "Primary, portable expression written in OSSIE_SQL_2026. When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative." + }, "dialects": { "type": "array", "items": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 99a39e329..038acf153 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -47,30 +47,18 @@ Standard enumeration values used throughout the specification. ### Dialects -Dialects in which metric and field expressions can be declared. - -| Dialect | Description | -|---------|-------------| -| `ANSI_SQL` | Standard SQL dialect | -| `SNOWFLAKE` | Snowflake SQL | -| `MDX` | Multi-Dimensional Expressions | -| `TABLEAU` | Tableau calculations | -| `DATABRICKS` | Databricks SQL | -| `MAQL` | GoodData MAQL (Multi-Dimensional Analytical Query Language) | -| `BIGQUERY` | Google BigQuery (GoogleSQL) | -| `SIGMA` | Sigma Computing's spreadsheet-style formula language | -| `THOUGHTSPOT` | ThoughtSpot formula language | -| `DAX` | Data Analysis Expressions (Power BI / Analysis Services) | -| `HOLISTICS_AQL` | Holistics AQL (Analytical Query Language) | -| `OSSIE_SQL_2026` | Ossie's portable SQL expression language, defined in `expression_language.md` | - -`OSSIE_SQL_2026` is the only dialect whose semantics this specification defines. The other values -identify vendor or third-party expression languages. Ossie treats expressions in those dialects as -opaque, and implementations MAY ignore them (see -[Dialect Extensions](expression_language.md#dialect-extensions)). Registering a dialect makes it -valid for declaring a model; it does not mean Ossie supports querying in that dialect (see +The `dialect` field is a free-form string, allowing any expression language to be used +without requiring changes to the core specification. + +`OSSIE_SQL_2026` is the only dialect whose semantics this specification defines (Ossie's +portable SQL expression language, see `expression_language.md`). All other values identify +vendor or third-party expression languages, which Ossie treats as opaque and implementations +MAY ignore. Registering a dialect does not mean Ossie supports querying in it (see [Model Declaration vs. Querying](expression_language.md#model-declaration-vs-querying)). +For a non-normative catalog of dialects well-known tools use, see +[extensions/dialects.md](../extensions/dialects.md). + ### Data types `DataType` declares the logical value type of a field or metric independently @@ -295,8 +283,9 @@ The expression object supports multiple SQL dialects for cross-platform compatib ```yaml expression: + expression: "customer_id" # Optional: primary, portable form (OSSIE_SQL_2026) dialects: - - dialect: ANSI_SQL # Must be one of the dialects enum values + - dialect: ANSI_SQL # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: "customer_id" # Scalar SQL expression ``` @@ -304,6 +293,8 @@ expression: - Use scalar SQL expressions (no aggregations) - Can be simple column references (e.g., `customer_id`) or computed expressions (e.g., `first_name || ' ' || last_name`) +- The optional top-level `expression` is the primary, portable form, written in OSSIE_SQL_2026 +- If both the top-level `expression` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `expression` is authoritative and consumers MUST use it as the OSSIE_SQL_2026 form - Multiple dialect versions can be provided for the same field ### Dimension Object @@ -425,6 +416,7 @@ The expression object supports multiple dialects ```yaml expression: + expression: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" # Optional: primary, portable form (OSSIE_SQL_2026) dialects: - dialect: ANSI_SQL # Default expression: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" @@ -482,70 +474,19 @@ custom_extensions: The `vendor_name` field is a free-form string, allowing any vendor or organization to define custom extensions without requiring changes to the core specification. -The following are well-known examples: - -| Vendor | Description | -|--------|-------------| -| `COMMON` | Common/standard extensions | -| `SNOWFLAKE` | Snowflake-specific attributes | -| `SALESFORCE` | Salesforce/Tableau-specific attributes | -| `DBT` | dbt-specific attributes | -| `DATABRICKS` | Databricks-specific attributes | -| `GOODDATA` | GoodData-specific attributes | -| `HONEYDEW` | Honeydew-specific attributes | -| `WISDOM` | WisdomAI-specific attributes | -| `POWER_BI` | Microsoft Power BI-specific attributes | -| `SIGMA` | Sigma Computing-specific attributes | -| `THOUGHTSPOT` | ThoughtSpot-specific attributes | -| `HEX` | Hex-specific attributes | - -### Examples - -**Snowflake Extension:** - -```yaml -- vendor_name: SNOWFLAKE - data: '{ - "warehouse": "ANALYTICS_WH", - "database": "PROD", - "schema": "PUBLIC" - }' -``` - -**Salesforce Extension:** - -```yaml -- vendor_name: SALESFORCE - data: '{ - "tableau_workbook_id": "sales_dashboard", - "einstein_enabled": true, - "crm_sync": { - "enabled": true, - "sync_frequency": "daily" - } - }' -``` +For a non-normative catalog of `vendor_name` values well-known tools use, see +[extensions/vendors.md](../extensions/vendors.md). -**DBT Extension:** +### Example ```yaml -- vendor_name: DBT - data: '{ - "project_name": "analytics", - "materialized": "table", - "tags": ["daily", "core"] - }' +- vendor_name: DB_VENDOR + data: '{"some_setting": "value", "nested": {"option": true}}' ``` -**Databricks Extension:** - -```yaml -- vendor_name: Databricks - data: '{ - "default_catalog": "finance", - "default_schema": "gold" - }' -``` +The `data` payload is an opaque JSON string that Ossie carries but does not +interpret. For vendor-specific extension examples and the known-vendor catalog, +see [extensions/vendors.md](../extensions/vendors.md). --- diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 887ee86de..1541bb09d 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -28,21 +28,8 @@ version: 0.2.0.dev0 # Enumerations # Standard enums used throughout the specification -# Supported expression language dialects -# Auto-generated from ossie-schema.json ($defs.Dialect.enum). -dialects: - - "ANSI_SQL" # Standard SQL dialect - - "SNOWFLAKE" # Snowflake - - "MDX" # Multi-Dimensional Expressions - - "TABLEAU" # Tableau - - "DATABRICKS" # Databricks SQL - - "MAQL" # GoodData MAQL (Multi-Dimensional Analytical Query Language) - - "BIGQUERY" # Google BigQuery GoogleSQL - - "SIGMA" # Sigma Computing spreadsheet-style formula language - - "THOUGHTSPOT" # ThoughtSpot formula language (not SQL) - - "DAX" # Data Analysis Expressions (Power BI / Analysis Services) - - "HOLISTICS_AQL" # Holistics AQL (Analytical Query Language) - - "OSSIE_SQL_2026" # Ossie portable SQL expression language (see expression_language.md) +# The `dialect` field is a free-form string, not a closed set. OSSIE_SQL_2026 is +# the only dialect this spec defines. See extensions/dialects.md. # Supported logical data types for fields and metrics # Auto-generated from ossie-schema.json ($defs.DataType.enum). @@ -58,8 +45,8 @@ datatypes: - "DateTimeTz" # Instant identified using offset or timezone context - "Opaque" # Known type outside the portable vocabulary -# Vendor name for custom extensions (free-form string) -# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM", "POWER_BI", "HEX" +# Vendor name for custom extensions (free-form string). +# See the known-vendor catalog in extensions/vendors.md. vendor_name: string @@ -206,8 +193,10 @@ fields: # Each field can have expressions in different dialects for portability # Can be a simple column reference or a complex scalar expression expression: + # Optional: primary, portable expression written in OSSIE_SQL_2026 + expression: string # OSSIE_SQL_2026 form, e.g., "customer_id", "first_name || ' ' || last_name" dialects: - - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + - dialect: string # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: string # SQL scalar expression, e.g., "customer_id", "first_name || ' ' || last_name", "UPPER(email)" # Optional: Dimension metadata @@ -261,8 +250,10 @@ metrics: # Supports multiple SQL dialects for cross-platform compatibility # Each metric can have expressions in different dialects for portability expression: + # Optional: primary, portable expression written in OSSIE_SQL_2026 + expression: string # OSSIE_SQL_2026 form, e.g., "SUM(orders.sales)", "AVG(orders.amount)" dialects: - - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + - dialect: string # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: string # Full SQL expression with aggregate functions, e.g., "SUM(orders.sales)", "AVG(orders.amount)" # Optional: Human-readable description of the metric diff --git a/docs/index.md b/docs/index.md index fa71bb83c..439f4bd75 100644 --- a/docs/index.md +++ b/docs/index.md @@ -59,7 +59,7 @@ The Ossie core specification (current version: **0.2.0.dev0**, latest released: | **Custom Extensions** | Vendor-specific metadata stored as JSON, allowing platforms to carry additional information without breaking core compatibility. | | **AI Context** | Optional annotations at every level (model, dataset, field, relationship, metric) to help AI tools understand business meaning — including instructions, synonyms, and example queries. | -The specification supports multiple SQL dialects (`ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `MDX`, `TABLEAU`) so that expressions can be tailored to each platform while maintaining a common model structure. +An expression carries a primary, portable form in `OSSIE_SQL_2026` (the dialect Ossie defines) and may also carry vendor-specific variants. The `dialect` label is a free-form string, so expressions can be tailored to any platform while maintaining a common model structure. For the full specification, see [core-spec/spec.md](../core-spec/spec.md). For validation tooling, see [validation/validate.py](../validation/validate.py). For a complete example, see the [TPC-DS semantic model](../examples/tpcds_semantic_model.yaml). @@ -195,20 +195,19 @@ A typical Ossie-based workflow looks like this: ### Multi-Dialect Expression System -A key architectural feature of Ossie is its multi-dialect expression system. Fields and metrics can carry expressions in multiple SQL dialects simultaneously: +A key architectural feature of Ossie is its multi-dialect expression system. An expression carries a primary, portable form in `OSSIE_SQL_2026`, and fields and metrics can also carry vendor-specific variants in multiple dialects simultaneously: ```yaml expression: + expression: LOWER(email) # primary, portable form (OSSIE_SQL_2026) dialects: - - dialect: ANSI_SQL - expression: LOWER(email) - dialect: SNOWFLAKE expression: LOWER(email)::VARCHAR - dialect: DATABRICKS expression: lower(email) ``` -This allows a single semantic model to be consumed natively by different platforms without expression translation. Each converter selects the dialect that matches its target platform, falling back to `ANSI_SQL` when a platform-specific dialect is not available. +This allows a single semantic model to be consumed natively by different platforms without expression translation. Each converter selects the dialect that matches its target platform, falling back to the portable `OSSIE_SQL_2026` form when a platform-specific dialect is not available. The `dialect` label is a free-form string; `OSSIE_SQL_2026` is the only dialect Ossie defines, and all others are opaque. --- @@ -251,7 +250,7 @@ Through `custom_extensions`. Each vendor can store arbitrary JSON metadata in ex Yes. Metrics are defined at the semantic model level (not within a dataset) and can reference fields from multiple datasets. **What SQL dialects are supported?** -The current specification supports `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `MDX`, and `TABLEAU`. New dialects can be proposed through the standard specification change process. +The `dialect` label is a free-form string, so any dialect can be used without a specification change. `OSSIE_SQL_2026` is the only dialect Ossie defines; all others are opaque and an implementation MAY ignore them. For a non-normative catalog of dialects well-known tools use, see [extensions/dialects.md](../extensions/dialects.md). --- @@ -320,7 +319,7 @@ A practical guide for organizations looking to adopt Ossie. | **Dimension** | A categorical attribute used to slice and filter data (e.g., region, product category, date). In Ossie, dimensions are represented as fields with optional metadata such as `is_time`. | | **Metric** | A quantitative measure computed by aggregating data across one or more datasets (e.g., total revenue, average order value). Metrics are defined at the semantic model level. | | **Relationship** | A foreign key connection between two datasets, defining how they can be joined. Relationships are always many-to-one (from the referencing dataset to the referenced dataset). | -| **Dialect** | A specific SQL or expression language variant (e.g., `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`). Ossie supports multiple dialects so expressions can be tailored to each platform. | +| **Dialect** | A SQL or expression language variant an expression is written in (e.g., `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`). A free-form string; `OSSIE_SQL_2026` is the only dialect Ossie defines, and all others are opaque. | | **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as a JSON string. Extensions allow platforms to carry additional information without modifying the core specification. | | **AI Context** | Optional annotations on any Ossie construct (model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | | **Converter** | A tool that translates between the Ossie format and a specific vendor's semantic model format. Converters come in pairs: import (vendor → Ossie) and export (Ossie → vendor). | diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index e2578f758..02fc85a76 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -287,6 +287,8 @@ datasets: - name: customer_full_name expression: + # Primary, portable form (OSSIE_SQL_2026) + expression: c_first_name || ' ' || c_last_name dialects: - dialect: ANSI_SQL expression: c_first_name || ' ' || c_last_name diff --git a/extensions/README.md b/extensions/README.md new file mode 100644 index 000000000..cdb0e7cd0 --- /dev/null +++ b/extensions/README.md @@ -0,0 +1,39 @@ + + +# Apache Ossie - Extensions + +The core specification ([`core-spec/`](../core-spec/)) is vendor-neutral. It +defines one expression language, `OSSIE_SQL_2026`, and treats every other dialect +and every `custom_extensions` vendor as extension metadata that Ossie carries but +does not interpret. + +This folder is where the community aligns on the meaning of specific dialect and +vendor tokens, so that producers and consumers spell them the same way. The lists +here are a shared registry, not part of the normative spec. **Presence in a list +does not imply endorsement of a tool or vendor**, and the lists are not +exhaustive: `dialect` and `vendor_name` are free-form strings, so any value is +valid whether or not it appears here. + +## Contents + +| File | Purpose | +|------|---------| +| [`dialects.md`](dialects.md) | Dialect tokens used to declare metric and field expressions. | +| [`vendors.md`](vendors.md) | `vendor_name` tokens, and the payload convention, for `custom_extensions`. | diff --git a/extensions/dialects.md b/extensions/dialects.md new file mode 100644 index 000000000..2c2ab5c59 --- /dev/null +++ b/extensions/dialects.md @@ -0,0 +1,41 @@ + + +# Dialect Extensions Catalog + +A registry of `dialect` tokens the community uses (see +[`README.md`](README.md) for how these lists are used). `OSSIE_SQL_2026` is the +only dialect the specification defines; all others are opaque to Ossie. + +## Known dialects + +| Dialect | Description | +|---------|-------------| +| `OSSIE_SQL_2026` | Ossie's portable SQL expression language, defined in [`expression_language.md`](../core-spec/expression_language.md) | +| `ANSI_SQL` | Standard SQL dialect | +| `SNOWFLAKE` | Snowflake SQL | +| `MDX` | Multi-Dimensional Expressions | +| `TABLEAU` | Tableau calculations | +| `DATABRICKS` | Databricks SQL | +| `MAQL` | GoodData MAQL (Multi-Dimensional Analytical Query Language) | +| `BIGQUERY` | Google BigQuery (GoogleSQL) | +| `SIGMA` | Sigma Computing's spreadsheet-style formula language | +| `THOUGHTSPOT` | ThoughtSpot formula language | +| `DAX` | Data Analysis Expressions (Power BI / Analysis Services) | +| `HOLISTICS_AQL` | Holistics AQL (Analytical Query Language) | diff --git a/extensions/vendors.md b/extensions/vendors.md new file mode 100644 index 000000000..90a739bde --- /dev/null +++ b/extensions/vendors.md @@ -0,0 +1,98 @@ + + +# Vendor Extensions Catalog + +Custom extensions attach platform-specific metadata to a model (see +[`core-spec/spec.md`](../core-spec/spec.md#custom-extensions)): + +```yaml +custom_extensions: + - vendor_name: string # Free-form string identifying the vendor + data: string # JSON string containing vendor-specific data +``` + +`vendor_name` is a free-form string and `data` is an opaque JSON string that Ossie +carries but does not interpret. This is a registry of `vendor_name` tokens the +community uses (see [`README.md`](README.md) for how these lists are used). + +## Known vendors + +| Vendor | Description | +|--------|-------------| +| `COMMON` | Common/standard extensions | +| `SNOWFLAKE` | Snowflake-specific attributes | +| `SALESFORCE` | Salesforce/Tableau-specific attributes | +| `DBT` | dbt-specific attributes | +| `DATABRICKS` | Databricks-specific attributes | +| `GOODDATA` | GoodData-specific attributes | +| `HONEYDEW` | Honeydew-specific attributes | +| `WISDOM` | WisdomAI-specific attributes | +| `POWER_BI` | Microsoft Power BI-specific attributes | +| `SIGMA` | Sigma Computing-specific attributes | +| `THOUGHTSPOT` | ThoughtSpot-specific attributes | +| `HEX` | Hex-specific attributes | + +## Examples + +**Snowflake Extension:** + +```yaml +- vendor_name: SNOWFLAKE + data: '{ + "warehouse": "ANALYTICS_WH", + "database": "PROD", + "schema": "PUBLIC" + }' +``` + +**Salesforce Extension:** + +```yaml +- vendor_name: SALESFORCE + data: '{ + "tableau_workbook_id": "sales_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily" + } + }' +``` + +**DBT Extension:** + +```yaml +- vendor_name: DBT + data: '{ + "project_name": "analytics", + "materialized": "table", + "tags": ["daily", "core"] + }' +``` + +**Databricks Extension:** + +```yaml +- vendor_name: Databricks + data: '{ + "default_catalog": "finance", + "default_schema": "gold" + }' +``` diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 357a1446b..619e8976f 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -106,7 +106,8 @@ def _invalid_properties(node: Any) -> list[tuple[str, str]]: class OssieDialect(str, Enum): - """Supported SQL and expression language dialects.""" + """Well-known dialect constants. `dialect` is a free-form string, so this is a + convenience set, not a closed one. See `extensions/dialects.md`.""" ANSI_SQL = "ANSI_SQL" SNOWFLAKE = "SNOWFLAKE" @@ -148,7 +149,8 @@ class OssieDataType(str, Enum): class OssieVendor(str, Enum): - """Well-known vendor names for custom extensions.""" + """Well-known vendor constants. `vendor_name` is a free-form string, so this is + a convenience set, not a closed one. See `extensions/vendors.md`.""" COMMON = "COMMON" SNOWFLAKE = "SNOWFLAKE" @@ -189,7 +191,8 @@ class OssieDialectExpression(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - dialect: OssieDialect + # Free-form; see OssieDialect for the well-known values. + dialect: str expression: str @@ -198,6 +201,8 @@ class OssieExpression(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") + # Primary, portable expression written in OSSIE_SQL_2026. + expression: Optional[str] = None dialects: list[OssieDialectExpression] = Field(..., min_length=1) diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 6b34f55cf..3062270f8 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -45,13 +45,15 @@ # --------------------------------------------------------------------------- -def test_dialect_enum_matches_core_schema() -> None: +def test_dialect_is_free_form_in_core_schema() -> None: + # `dialect` is a free-form string in the core schema (no closed enum). schema_path = Path(__file__).parents[2] / "core-spec" / "ossie-schema.json" schema = json.loads(schema_path.read_text()) - assert {member.value for member in OssieDialect} == set( - schema["$defs"]["Dialect"]["enum"] - ) + dialect = schema["$defs"]["Dialect"] + assert "enum" not in dialect + assert dialect["type"] == "string" + assert "OSSIE_SQL_2026" in dialect["examples"] @pytest.mark.parametrize( @@ -64,14 +66,16 @@ def test_dialect_accepts_string_and_enum_input(dialect: str | OssieDialect) -> N {"dialects": [{"dialect": dialect, "expression": "value"}]} ) - assert expression.dialects[0].dialect is OssieDialect.ANSI_SQL + assert expression.dialects[0].dialect == "ANSI_SQL" -def test_dialect_rejects_unknown_string() -> None: - with pytest.raises(ValidationError): - OssieExpression.model_validate( - {"dialects": [{"dialect": "NOT_A_DIALECT", "expression": "value"}]} - ) +def test_dialect_accepts_unknown_string() -> None: + # Free-form dialect: an unregistered value is accepted and preserved. + expression = OssieExpression.model_validate( + {"dialects": [{"dialect": "DB_DIALECT", "expression": "value"}]} + ) + + assert expression.dialects[0].dialect == "DB_DIALECT" def test_ossie_sql_2026_dialect_survives_serialization(document_data: dict) -> None: @@ -84,12 +88,32 @@ def test_ossie_sql_2026_dialect_survives_serialization(document_data: dict) -> N document = OssieDocument.model_validate(data) for item in (document.datasets[0].fields[0], document.metrics[0]): - assert item.expression.dialects[0].dialect is OssieDialect.OSSIE_SQL_2026 + assert item.expression.dialects[0].dialect == "OSSIE_SQL_2026" + + as_json = json.loads(document.to_ossie_json()) + as_yaml = yaml.safe_load(document.to_ossie_yaml()) + for serialized in (as_json, as_yaml): + assert serialized == data + assert OssieDocument.model_validate(serialized) == document + + +def test_portable_expression_round_trips_and_is_omitted_when_absent( + document_data: dict, +) -> None: + # The optional portable `expression` round-trips when set and is omitted when unset. + data = document_data + data["datasets"][0]["fields"][0]["expression"]["expression"] = "customer_id" + + document = OssieDocument.model_validate(data) + assert document.datasets[0].fields[0].expression.expression == "customer_id" + # A field whose expression omits the portable form keeps it unset. + assert document.metrics[0].expression.expression is None as_json = json.loads(document.to_ossie_json()) as_yaml = yaml.safe_load(document.to_ossie_yaml()) for serialized in (as_json, as_yaml): assert serialized == data + assert "expression" not in serialized["metrics"][0]["expression"] assert OssieDocument.model_validate(serialized) == document diff --git a/python/tests/test_serialization.py b/python/tests/test_serialization.py index 62d8e2197..ff105f32a 100644 --- a/python/tests/test_serialization.py +++ b/python/tests/test_serialization.py @@ -284,7 +284,10 @@ def _missing_properties(model: type[BaseModel], sample: object, where: str) -> l def _fully_populated_document() -> dict: """A document that sets every optional property, so each can be nulled in turn.""" - expression = {"dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}]} + expression = { + "expression": "amount", + "dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}], + } ai_context = {"instructions": "i", "synonyms": ["s"], "examples": ["e"]} extensions = [{"vendor_name": "COMMON", "data": "{}"}] return { diff --git a/python/tests/test_validation.py b/python/tests/test_validation.py index e98fb9258..f010de1a0 100644 --- a/python/tests/test_validation.py +++ b/python/tests/test_validation.py @@ -153,9 +153,10 @@ def test_expression_missing_dialects() -> None: OssieExpression() -def test_invalid_dialect_in_expression() -> None: - with pytest.raises(ValidationError): - OssieDialectExpression(dialect="INVALID", expression="order_id") +def test_free_form_dialect_in_expression() -> None: + # `dialect` is free-form: an unregistered value is accepted and preserved. + dialect_expr = OssieDialectExpression(dialect="DB_DIALECT", expression="order_id") + assert dialect_expr.dialect == "DB_DIALECT" # --------------------------------------------------------------------------- diff --git a/scripts/generate-spec-types.py b/scripts/generate-spec-types.py index 4b8362ba0..6c3f4e2fe 100755 --- a/scripts/generate-spec-types.py +++ b/scripts/generate-spec-types.py @@ -50,26 +50,6 @@ SPEC = REPO / "core-spec" / "spec.yaml" SECTIONS = [ - { - "key": "dialects", - "schema_path": ["$defs", "Dialect", "enum"], - "comment_col": 28, - "header": "# Supported expression language dialects", - "descriptions": { - "ANSI_SQL": "Standard SQL dialect", - "SNOWFLAKE": "Snowflake", - "MDX": "Multi-Dimensional Expressions", - "TABLEAU": "Tableau", - "DATABRICKS": "Databricks SQL", - "MAQL": "GoodData MAQL (Multi-Dimensional Analytical Query Language)", - "BIGQUERY": "Google BigQuery GoogleSQL", - "SIGMA": "Sigma Computing spreadsheet-style formula language", - "THOUGHTSPOT": "ThoughtSpot formula language (not SQL)", - "DAX": "Data Analysis Expressions (Power BI / Analysis Services)", - "HOLISTICS_AQL": "Holistics AQL (Analytical Query Language)", - "OSSIE_SQL_2026": "Ossie portable SQL expression language (see expression_language.md)", - }, - }, { "key": "datatypes", "schema_path": ["$defs", "DataType", "enum"], diff --git a/validation/tests/test_generate_spec_types.py b/validation/tests/test_generate_spec_types.py index 414ba4374..19bdad633 100644 --- a/validation/tests/test_generate_spec_types.py +++ b/validation/tests/test_generate_spec_types.py @@ -32,22 +32,23 @@ """ -_DIALECT_HEADER = "# Supported expression language dialects" -_DIALECT_MARKER = "# Auto-generated from ossie-schema.json ($defs.Dialect.enum)." - -_DIALECTS = """# Supported expression language dialects -# Auto-generated from ossie-schema.json ($defs.Dialect.enum). -dialects: - - "ANSI_SQL" # Standard SQL dialect - -""" +# `datatypes` is the one managed section: generate-spec-types.py syncs it from +# ossie-schema.json ($defs.DataType.enum). `dialects` is intentionally NOT +# generated -- `dialect` is a free-form string -- so it appears below only as an +# unmanaged, hand-authored section the generator must leave untouched. +_HEADER = "# Supported logical data types for fields and metrics" +_MARKER = "# Auto-generated from ossie-schema.json ($defs.DataType.enum)." _DATATYPES = """# Supported logical data types for fields and metrics # Auto-generated from ossie-schema.json ($defs.DataType.enum). datatypes: - "String" # Variable-length Unicode character data + """ +# A hand-authored section the generator does not manage. +_UNMANAGED = 'dialects:\n - "ANSI_SQL"\n\n' + @pytest.fixture def generator(): @@ -67,11 +68,11 @@ def _write_spec(tmp_path: Path, text: str) -> Path: def test_apply_keeps_hand_written_comments_above_a_section(generator, monkeypatch, tmp_path): """A comment not separated by a blank line used to be swallowed by --apply. - The walk back from `dialects:` consumed every contiguous comment, so the lines - above the generated header went with it, and --check then reported the file as in - sync, hiding the loss from CI. + The walk back from `datatypes:` must consume only this script's own header lines, + not the hand-written comments above them, so the enumerations preamble survives and + --check still reports the file as in sync. """ - path = _write_spec(tmp_path, _SPEC_HEAD.rstrip("\n") + "\n" + _DIALECTS + _DATATYPES) + path = _write_spec(tmp_path, _SPEC_HEAD.rstrip("\n") + "\n" + _DATATYPES) monkeypatch.setattr(generator, "SPEC", path) generator._apply() @@ -81,17 +82,17 @@ def test_apply_keeps_hand_written_comments_above_a_section(generator, monkeypatc assert "# Standard enums used throughout the specification" in written # Keeping the comments is not enough: the block must have been rewritten once, # and the result must satisfy the check CI runs. - assert written.count(_DIALECT_HEADER) == 1 - assert written.count("dialects:") == 1 + assert written.count(_HEADER) == 1 + assert written.count("datatypes:") == 1 assert generator._check() == 0 @pytest.mark.parametrize( ("above_key", "label"), [ - (_DIALECT_HEADER + "\n" + _DIALECT_MARKER + "\n", "header-and-marker"), - (_DIALECT_HEADER + "\n", "header-without-marker"), - (_DIALECT_MARKER + "\n", "marker-without-header"), + (_HEADER + "\n" + _MARKER + "\n", "header-and-marker"), + (_HEADER + "\n", "header-without-marker"), + (_MARKER + "\n", "marker-without-header"), ("", "neither"), ], ) @@ -103,15 +104,15 @@ def test_apply_rewrites_a_section_however_its_header_survives( Matching the marker alone would leave a stale header in place and insert a second copy above the key, which `--check` then accepts. """ - spec = _SPEC_HEAD + above_key + 'dialects:\n - "ANSI_SQL"\n\n' + _DATATYPES + spec = _SPEC_HEAD + above_key + 'datatypes:\n - "String"\n' path = _write_spec(tmp_path, spec) monkeypatch.setattr(generator, "SPEC", path) generator._apply() written = path.read_text() - assert written.count(_DIALECT_HEADER) == 1 - assert written.count(_DIALECT_MARKER) == 1 + assert written.count(_HEADER) == 1 + assert written.count(_MARKER) == 1 assert "# Enumerations" in written assert generator._check() == 0 @@ -128,22 +129,22 @@ def test_apply_steps_over_blank_lines_in_any_position( """ above_key = ( "\n" * above - + _DIALECT_HEADER + + _HEADER + "\n" + "\n" * between - + _DIALECT_MARKER + + _MARKER + "\n" + "\n" * below ) - spec = _SPEC_HEAD + above_key + 'dialects:\n - "ANSI_SQL"\n\n' + _DATATYPES + spec = _SPEC_HEAD + above_key + 'datatypes:\n - "String"\n' path = _write_spec(tmp_path, spec) monkeypatch.setattr(generator, "SPEC", path) generator._apply() written = path.read_text() - assert written.count(_DIALECT_HEADER) == 1 - assert written.count(_DIALECT_MARKER) == 1 + assert written.count(_HEADER) == 1 + assert written.count(_MARKER) == 1 assert "# Enumerations" in written assert generator._check() == 0 @@ -151,21 +152,24 @@ def test_apply_steps_over_blank_lines_in_any_position( def test_apply_does_not_eat_a_blank_line_it_did_not_write(generator, monkeypatch, tmp_path): """Reaching past blanks is only correct when they separate this script's own output. - A hand-authored section has no generated header, so the blank above its key is the - author's and must survive. + A hand-authored, unmanaged section has no generated header, so the blank above its + key is the author's and must survive. """ - spec = _SPEC_HEAD + 'dialects:\n - "ANSI_SQL"\n\n' + _DATATYPES + spec = _SPEC_HEAD + _UNMANAGED + _DATATYPES path = _write_spec(tmp_path, spec) monkeypatch.setattr(generator, "SPEC", path) generator._apply() + written = path.read_text() - assert path.read_text().startswith(_SPEC_HEAD) + assert written.startswith(_SPEC_HEAD) + # The unmanaged section is left exactly as the author wrote it. + assert _UNMANAGED in written assert generator._check() == 0 def test_apply_is_idempotent(generator, monkeypatch, tmp_path): - path = _write_spec(tmp_path, _SPEC_HEAD + _DIALECTS + _DATATYPES) + path = _write_spec(tmp_path, _SPEC_HEAD + _DATATYPES) monkeypatch.setattr(generator, "SPEC", path) generator._apply() @@ -176,7 +180,11 @@ def test_apply_is_idempotent(generator, monkeypatch, tmp_path): def test_check_reports_a_file_that_is_out_of_sync(generator, monkeypatch, tmp_path, capsys): - path = _write_spec(tmp_path, _SPEC_HEAD + _DIALECTS.replace("ANSI_SQL", "MADE_UP") + _DATATYPES) + # Seed the full generated block, then mutate one value, so a single wrong + # value is the only thing out of sync (not a missing/incomplete section). + section = generator.SECTIONS[0] + in_sync = generator._generate_block(section, generator._schema_values(section)) + path = _write_spec(tmp_path, _SPEC_HEAD + in_sync.replace("String", "MADE_UP", 1)) monkeypatch.setattr(generator, "SPEC", path) assert generator._check() == 1 diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index edf2f67e5..88aebde6b 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -375,6 +375,42 @@ def test_flags_an_invalid_ossie_sql_2026_expression() -> None: assert error.startswith("[SQL] ctx:") +@pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") +def test_unknown_dialect_is_skipped_as_opaque() -> None: + # An unregistered dialect is opaque: skipped, not parsed as ANSI. + assert "DB_DIALECT" not in _VALIDATE.DIALECT_MAP + error = _VALIDATE.validate_sql_expression( + "this is not sql @ all", "DB_DIALECT", "ctx" + ) + + assert error is None + + +def test_portable_expression_is_validated_as_ossie_sql_2026( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The portable `expression` is validated as OSSIE_SQL_2026, alongside `dialects`. + seen = [] + + def record_expression(expression: str, dialect: str, context: str) -> None: + seen.append((expression, dialect, context)) + + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + monkeypatch.setattr(_VALIDATE, "validate_sql_expression", record_expression) + expression = { + "expression": "amount", + "dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}], + } + dataset = {**_ORDERS, "fields": [{"name": "value", "expression": expression}]} + document = _document([dataset], []) + + assert _VALIDATE.validate_sql(document) == [] + assert seen == [ + ("amount", "OSSIE_SQL_2026", "Field 'orders.value' in model 'm' (OSSIE_SQL_2026)"), + ("amount", "ANSI_SQL", "Field 'orders.value' in model 'm' (ANSI_SQL)"), + ] + + @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") @pytest.mark.parametrize( "expression", diff --git a/validation/validate.py b/validation/validate.py index 548bf127c..ea34c0f61 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -366,6 +366,11 @@ def validate_sql_expression(expr: str, dialect: str, context: str) -> str | None if dialect in SKIP_SQL_VALIDATION: return None + # An unregistered dialect is opaque: skip it rather than parse it as ANSI and + # report spurious errors. Only dialects mapped below are parsed. + if dialect not in DIALECT_MAP: + return None + sqlglot_dialect = DIALECT_MAP.get(dialect) def statement_count_error(sql: str) -> str | None: @@ -453,6 +458,13 @@ def validate_sql(data: dict) -> list[str]: for field in dataset.get("fields", []): field_name = field.get("name", "") expression = field.get("expression", {}) + # The optional top-level expression is the portable OSSIE_SQL_2026 form. + portable = expression.get("expression") + if portable is not None: + context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' (OSSIE_SQL_2026)" + error = validate_sql_expression(portable, "OSSIE_SQL_2026", context) + if error: + errors.append(error) for dialect_expr in expression.get("dialects", []): dialect = dialect_expr.get("dialect", "ANSI_SQL") # Only a missing expression is skipped; an empty one is reported. @@ -467,6 +479,13 @@ def validate_sql(data: dict) -> list[str]: for metric in model.get("metrics", []): metric_name = metric.get("name", "") expression = metric.get("expression", {}) + # The optional top-level expression is the portable OSSIE_SQL_2026 form. + portable = expression.get("expression") + if portable is not None: + context = f"Metric '{metric_name}' in model '{model_name}' (OSSIE_SQL_2026)" + error = validate_sql_expression(portable, "OSSIE_SQL_2026", context) + if error: + errors.append(error) for dialect_expr in expression.get("dialects", []): dialect = dialect_expr.get("dialect", "ANSI_SQL") expr = dialect_expr.get("expression") From adb7c9bf68bd6d937c5783571ad0a8f00c2dc3a5 Mon Sep 17 00:00:00 2001 From: Chris Eubank <108756251+christianeu-db@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:50:59 +0000 Subject: [PATCH 2/9] fix(core-spec): allow portable-only expressions without dialects An Expression required `dialects` on both the JSON Schema and the SDK, so a document carrying only the portable top-level `expression` (OSSIE_SQL_2026) was rejected, contradicting the spec's stated semantics. - Schema: replace `required: [dialects]` with an anyOf requiring at least one of `expression` or `dialects`; `dialects` keeps `minItems: 1` when present. - SDK: make `OssieExpression.dialects` optional and add a model_validator that rejects an expression carrying neither form. - Tests: cover portable-only, dialects-only, both-present, and the empty cases on both the schema and the SDK. - Docs: note the "at least one" rule in spec.md and spec.yaml. Co-authored-by: Isaac --- core-spec/ossie-schema.json | 5 ++++- core-spec/spec.md | 3 +++ core-spec/spec.yaml | 2 ++ python/src/ossie/models.py | 22 +++++++++++++++++++-- python/tests/test_validation.py | 33 ++++++++++++++++++++++++++++++- validation/tests/test_validate.py | 26 ++++++++++++++++++++++++ 6 files changed, 87 insertions(+), 4 deletions(-) diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index a159ef71d..650524cf5 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -123,7 +123,10 @@ "minItems": 1 } }, - "required": ["dialects"], + "anyOf": [ + { "required": ["expression"] }, + { "required": ["dialects"] } + ], "additionalProperties": false }, "DataType": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 038acf153..de488522e 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -294,6 +294,7 @@ expression: - Use scalar SQL expressions (no aggregations) - Can be simple column references (e.g., `customer_id`) or computed expressions (e.g., `first_name || ' ' || last_name`) - The optional top-level `expression` is the primary, portable form, written in OSSIE_SQL_2026 +- At least one of the top-level `expression` or `dialects` MUST be present; either MAY stand alone - If both the top-level `expression` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `expression` is authoritative and consumers MUST use it as the OSSIE_SQL_2026 form - Multiple dialect versions can be provided for the same field @@ -422,6 +423,8 @@ expression: expression: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" ``` +At least one of the top-level `expression` or `dialects` MUST be present; either MAY stand alone. When both a top-level `expression` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `expression` is authoritative. + ### Examples **Simple Aggregation:** diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 1541bb09d..acc46ce40 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -192,6 +192,7 @@ fields: # Supports multiple SQL dialects for cross-platform compatibility # Each field can have expressions in different dialects for portability # Can be a simple column reference or a complex scalar expression + # At least one of the portable `expression` or `dialects` must be present; either may stand alone expression: # Optional: primary, portable expression written in OSSIE_SQL_2026 expression: string # OSSIE_SQL_2026 form, e.g., "customer_id", "first_name || ' ' || last_name" @@ -249,6 +250,7 @@ metrics: # Required: Expression definition with dialect support # Supports multiple SQL dialects for cross-platform compatibility # Each metric can have expressions in different dialects for portability + # At least one of the portable `expression` or `dialects` must be present; either may stand alone expression: # Optional: primary, portable expression written in OSSIE_SQL_2026 expression: string # OSSIE_SQL_2026 form, e.g., "SUM(orders.sales)", "AVG(orders.amount)" diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 619e8976f..b5afe19da 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -21,7 +21,15 @@ from typing import Annotated, Any, Literal, Optional, Union import yaml -from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, StrictBool, model_serializer +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializerFunctionWrapHandler, + StrictBool, + model_serializer, + model_validator, +) _AI_CONTEXT_FIELDS = frozenset({"instructions", "synonyms", "examples"}) @@ -203,7 +211,17 @@ class OssieExpression(BaseModel): # Primary, portable expression written in OSSIE_SQL_2026. expression: Optional[str] = None - dialects: list[OssieDialectExpression] = Field(..., min_length=1) + # When present, a non-empty list; the portable `expression` may stand in for it. + dialects: Optional[list[OssieDialectExpression]] = Field(default=None, min_length=1) + + @model_validator(mode="after") + def _require_expression_or_dialects(self) -> "OssieExpression": + # Mirrors the schema anyOf: at least one of `expression` or `dialects` + # must be present. The `dialects` min_length rule already rejects an + # empty list, so here only the both-absent case remains. + if self.expression is None and not self.dialects: + raise ValueError("expression requires at least one of 'expression' or 'dialects'") + return self class OssieDimension(BaseModel): diff --git a/python/tests/test_validation.py b/python/tests/test_validation.py index f010de1a0..e8f252362 100644 --- a/python/tests/test_validation.py +++ b/python/tests/test_validation.py @@ -148,11 +148,42 @@ def test_dialect_expression_missing_expression() -> None: OssieDialectExpression(dialect=OssieDialect.ANSI_SQL) -def test_expression_missing_dialects() -> None: +def test_expression_requires_expression_or_dialects() -> None: + # Neither the portable `expression` nor `dialects`: rejected (schema anyOf). with pytest.raises(ValidationError): OssieExpression() +def test_expression_portable_only_is_valid() -> None: + # The portable `expression` may stand alone, with no `dialects`. + expr = OssieExpression(expression="c_first_name || ' ' || c_last_name") + assert expr.expression == "c_first_name || ' ' || c_last_name" + assert expr.dialects is None + + +def test_expression_dialects_only_is_valid() -> None: + # `dialects` alone, with no portable `expression`, remains valid. + expr = OssieExpression(dialects=[OssieDialectExpression(dialect="ANSI_SQL", expression="x")]) + assert expr.expression is None + assert expr.dialects is not None and len(expr.dialects) == 1 + + +def test_expression_both_present_is_valid() -> None: + # Both forms together: valid; the portable form is authoritative. + expr = OssieExpression( + expression="x", + dialects=[OssieDialectExpression(dialect="ANSI_SQL", expression="x")], + ) + assert expr.expression == "x" + assert expr.dialects is not None and len(expr.dialects) == 1 + + +def test_expression_empty_dialects_is_rejected() -> None: + # An explicitly empty `dialects` list fails min_length even without a portable form. + with pytest.raises(ValidationError): + OssieExpression(dialects=[]) + + def test_free_form_dialect_in_expression() -> None: # `dialect` is free-form: an unregistered value is accepted and preserved. dialect_expr = OssieDialectExpression(dialect="DB_DIALECT", expression="order_id") diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 88aebde6b..7e61fbc42 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -411,6 +411,32 @@ def record_expression(expression: str, dialect: str, context: str) -> None: ] +def test_schema_accepts_portable_only_expression(core_schema: dict) -> None: + # The portable `expression` may stand alone, with no `dialects` (schema anyOf). + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {"expression": "amount"}}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [] + + +def test_schema_rejects_expression_with_neither_form(core_schema: dict) -> None: + # An expression object carrying neither `expression` nor `dialects` is invalid. + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {}}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression: " + "{} is not valid under any of the given schemas" + ] + + +def test_schema_rejects_empty_dialects_without_portable_form(core_schema: dict) -> None: + # An explicitly empty `dialects` list still fails min_length. + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {"dialects": []}}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> dialects: [] should be non-empty" + ] + + @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") @pytest.mark.parametrize( "expression", From e656b3b50f76952a9a9debf506fc2c701988b669 Mon Sep 17 00:00:00 2001 From: Chris Eubank <108756251+christianeu-db@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:06:29 +0000 Subject: [PATCH 3/9] feat(validation): warn on unknown dialects instead of skipping silently An unrecognized dialect is opaque and legitimate under the free-form `Dialect` model, so its SQL cannot be parse-checked. The validator skipped it silently, which also hid typos such as `SNOWFLKE`. `validate_sql` now tallies expressions whose dialect is not in `DIALECT_MAP` and emits one nonfatal `ValidationWarning` per distinct unknown dialect, with the count of affected expressions. Validation still passes. Known-but-unparseable dialects (MDX, DAX, HOLISTICS_AQL, ...) stay silent, and the per-call `validate_sql_expression` is unchanged. Co-authored-by: Isaac --- validation/tests/test_validate.py | 42 +++++++++++++++++++++++++++++++ validation/validate.py | 16 ++++++++++++ 2 files changed, 58 insertions(+) diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 7e61fbc42..731c08157 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -437,6 +437,48 @@ def test_schema_rejects_empty_dialects_without_portable_form(core_schema: dict) ] +def test_unknown_dialect_emits_one_deduped_warning(monkeypatch: pytest.MonkeyPatch) -> None: + # An unrecognized dialect is nonfatal, but one warning per dialect reports that + # its expressions went unvalidated. The field and metric below share the dialect, + # so they collapse into a single warning carrying the count. + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + expression = {"dialects": [{"dialect": "ACME_SQL", "expression": "x"}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + document = _document([dataset], []) + document["metrics"] = [{"name": "total", "expression": expression}] + + result = _VALIDATE.validate_sql(document) + + assert result == ["[SQL] Warning: unknown dialect 'ACME_SQL'; 2 expressions not validated"] + assert all(isinstance(item, _VALIDATE.ValidationWarning) for item in result) + + +def test_distinct_unknown_dialects_each_warn_once(monkeypatch: pytest.MonkeyPatch) -> None: + # Two different unrecognized dialects produce two warnings, sorted by name. + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + fields = [ + {"name": "a", "expression": {"dialects": [{"dialect": "ZETA_SQL", "expression": "x"}]}}, + {"name": "b", "expression": {"dialects": [{"dialect": "ACME_SQL", "expression": "y"}]}}, + ] + dataset = {**_ORDERS, "fields": fields} + + assert _VALIDATE.validate_sql(_document([dataset], [])) == [ + "[SQL] Warning: unknown dialect 'ACME_SQL'; 1 expression not validated", + "[SQL] Warning: unknown dialect 'ZETA_SQL'; 1 expression not validated", + ] + + +def test_known_unparseable_dialect_is_not_flagged_as_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # MDX is recognized but unparseable: it is skipped silently, not warned as unknown. + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + expression = {"dialects": [{"dialect": "MDX", "expression": "[Measures].[Sales]"}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + + assert _VALIDATE.validate_sql(_document([dataset], [])) == [] + + @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") @pytest.mark.parametrize( "expression", diff --git a/validation/validate.py b/validation/validate.py index ea34c0f61..91a8ed5b1 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -449,6 +449,10 @@ def validate_sql(data: dict) -> list[str]: model = data errors = [] + # Count expressions whose dialect this validator does not recognize, so a single + # nonfatal warning per unknown dialect reports what went unvalidated. A free-form + # dialect is legitimate, but silence would also hide a typo like `SNOWFLKE`. + unknown_dialects: Counter = Counter() model_name = model.get("name", "") @@ -470,6 +474,8 @@ def validate_sql(data: dict) -> list[str]: # Only a missing expression is skipped; an empty one is reported. expr = dialect_expr.get("expression") if expr is not None: + if dialect not in DIALECT_MAP: + unknown_dialects[dialect] += 1 context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' ({dialect})" error = validate_sql_expression(expr, dialect, context) if error: @@ -490,11 +496,21 @@ def validate_sql(data: dict) -> list[str]: dialect = dialect_expr.get("dialect", "ANSI_SQL") expr = dialect_expr.get("expression") if expr is not None: + if dialect not in DIALECT_MAP: + unknown_dialects[dialect] += 1 context = f"Metric '{metric_name}' in model '{model_name}' ({dialect})" error = validate_sql_expression(expr, dialect, context) if error: errors.append(error) + # One nonfatal warning per unrecognized dialect, sorted for deterministic output. + for dialect in sorted(unknown_dialects): + count = unknown_dialects[dialect] + plural = "s" if count != 1 else "" + errors.append(ValidationWarning( + f"[SQL] Warning: unknown dialect '{dialect}'; {count} expression{plural} not validated" + )) + return errors From a91207618e71d6f76b73646e557d88aa7210e928 Mon Sep 17 00:00:00 2001 From: Chris Eubank <108756251+christianeu-db@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:54:51 +0000 Subject: [PATCH 4/9] refactor(core-spec): rename portable expression key to 'sql' The portable top-level key of the Expression object was `expression`, nested under a field/metric's own `expression:` block -- the doubled `expression.expression` Jochen flagged on the thread. Rename it to `sql` so the key reads cleanly and names what it holds: the portable OSSIE_SQL (OSSIE_SQL_2026) form. Scope is the portable key only. The field/metric `expression:` key (which holds the Expression object) and the per-dialect `dialects[].expression` key (already qualified by its `dialect:`) are unchanged. This key was introduced by this same unreleased PR, so the rename carries no migration burden; no released spec ever had `expression.expression`. No back-compat alias -- the old key is rejected (schema additionalProperties:false, SDK extra=forbid). Touches the schema anyOf + property, the SDK model and validator, the validator's portable reads, spec.md/spec.yaml/docs prose and examples, the tpcds portable example, and the mirrored tests. Co-authored-by: Isaac --- core-spec/ossie-schema.json | 6 +++--- core-spec/spec.md | 12 ++++++------ core-spec/spec.yaml | 12 ++++++------ docs/index.md | 2 +- examples/tpcds_semantic_model.yaml | 4 ++-- python/src/ossie/models.py | 14 +++++++------- python/tests/test_models.py | 10 +++++----- python/tests/test_serialization.py | 2 +- python/tests/test_validation.py | 18 +++++++++--------- validation/tests/test_validate.py | 8 ++++---- validation/validate.py | 8 ++++---- 11 files changed, 48 insertions(+), 48 deletions(-) diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index 35ee38e97..7a2b13e5c 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -111,9 +111,9 @@ "type": "object", "description": "Expression definition with multi-dialect support", "properties": { - "expression": { + "sql": { "type": "string", - "description": "Primary, portable expression written in OSSIE_SQL_2026. When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative." + "description": "Primary, portable expression written in OSSIE_SQL (the OSSIE_SQL_2026 dialect). When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative." }, "dialects": { "type": "array", @@ -124,7 +124,7 @@ } }, "anyOf": [ - { "required": ["expression"] }, + { "required": ["sql"] }, { "required": ["dialects"] } ], "additionalProperties": false diff --git a/core-spec/spec.md b/core-spec/spec.md index 69bdb3e93..407c0b310 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -298,7 +298,7 @@ The expression object supports multiple SQL dialects for cross-platform compatib ```yaml expression: - expression: "customer_id" # Optional: primary, portable form (OSSIE_SQL_2026) + sql: "customer_id" # Optional: primary, portable OSSIE_SQL form (OSSIE_SQL_2026) dialects: - dialect: ANSI_SQL # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: "customer_id" # Scalar SQL expression @@ -308,9 +308,9 @@ expression: - Use scalar SQL expressions (no aggregations) - Can be simple column references (e.g., `customer_id`) or computed expressions (e.g., `first_name || ' ' || last_name`) -- The optional top-level `expression` is the primary, portable form, written in OSSIE_SQL_2026 -- At least one of the top-level `expression` or `dialects` MUST be present; either MAY stand alone -- If both the top-level `expression` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `expression` is authoritative and consumers MUST use it as the OSSIE_SQL_2026 form +- The optional top-level `sql` is the primary, portable form, written in OSSIE_SQL (the OSSIE_SQL_2026 dialect) +- At least one of the top-level `sql` or `dialects` MUST be present; either MAY stand alone +- If both the top-level `sql` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `sql` is authoritative and consumers MUST use it as the OSSIE_SQL_2026 form - Identifiers refer to columns of the dataset's `source`, not to other fields. See [Name Resolution](#name-resolution). - Multiple dialect versions can be provided for the same field @@ -440,13 +440,13 @@ The expression object supports multiple dialects ```yaml expression: - expression: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" # Optional: primary, portable form (OSSIE_SQL_2026) + sql: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" # Optional: primary, portable OSSIE_SQL form (OSSIE_SQL_2026) dialects: - dialect: ANSI_SQL # Default expression: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" ``` -At least one of the top-level `expression` or `dialects` MUST be present; either MAY stand alone. When both a top-level `expression` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `expression` is authoritative. +At least one of the top-level `sql` or `dialects` MUST be present; either MAY stand alone. When both a top-level `sql` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `sql` is authoritative. ### Examples diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 021fa5ad4..097e7b7a1 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -217,10 +217,10 @@ fields: # Each field can have expressions in different dialects for portability # Can be a simple column reference or a complex scalar expression # Identifiers refer to columns of the dataset's source, not to other fields - # At least one of the portable `expression` or `dialects` must be present; either may stand alone + # At least one of the portable `sql` or `dialects` must be present; either may stand alone expression: - # Optional: primary, portable expression written in OSSIE_SQL_2026 - expression: string # OSSIE_SQL_2026 form, e.g., "customer_id", "first_name || ' ' || last_name" + # Optional: primary, portable expression written in OSSIE_SQL (the OSSIE_SQL_2026 dialect) + sql: string # OSSIE_SQL_2026 form, e.g., "customer_id", "first_name || ' ' || last_name" dialects: - dialect: string # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: string # SQL scalar expression, e.g., "customer_id", "first_name || ' ' || last_name", "UPPER(email)" @@ -283,10 +283,10 @@ metrics: # Model-scoped metrics qualify them as dataset.field; dataset-scoped # metrics use unqualified field names # Metrics cannot reference other metrics - # At least one of the portable `expression` or `dialects` must be present; either may stand alone + # At least one of the portable `sql` or `dialects` must be present; either may stand alone expression: - # Optional: primary, portable expression written in OSSIE_SQL_2026 - expression: string # OSSIE_SQL_2026 form, e.g., "SUM(orders.sales)", "AVG(orders.amount)" + # Optional: primary, portable expression written in OSSIE_SQL (the OSSIE_SQL_2026 dialect) + sql: string # OSSIE_SQL_2026 form, e.g., "SUM(orders.sales)", "AVG(orders.amount)" dialects: - dialect: string # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: string # Full SQL expression with aggregate functions, e.g., "SUM(orders.sales)", "AVG(orders.amount)" diff --git a/docs/index.md b/docs/index.md index 86ba11ada..e4bf2bdfa 100644 --- a/docs/index.md +++ b/docs/index.md @@ -199,7 +199,7 @@ A key architectural feature of Ossie is its multi-dialect expression system. An ```yaml expression: - expression: LOWER(email) # primary, portable form (OSSIE_SQL_2026) + sql: LOWER(email) # primary, portable OSSIE_SQL form (OSSIE_SQL_2026) dialects: - dialect: SNOWFLAKE expression: LOWER(email)::VARCHAR diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index 0beba6d30..dfd5ec267 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -309,8 +309,8 @@ datasets: - name: customer_full_name expression: - # Primary, portable form (OSSIE_SQL_2026) - expression: c_first_name || ' ' || c_last_name + # Primary, portable OSSIE_SQL form (OSSIE_SQL_2026) + sql: c_first_name || ' ' || c_last_name dialects: - dialect: ANSI_SQL expression: c_first_name || ' ' || c_last_name diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 3cfa4705e..ccb1a2c83 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -226,18 +226,18 @@ class OssieExpression(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - # Primary, portable expression written in OSSIE_SQL_2026. - expression: Optional[str] = None - # When present, a non-empty list; the portable `expression` may stand in for it. + # Primary, portable expression written in OSSIE_SQL (the OSSIE_SQL_2026 dialect). + sql: Optional[str] = None + # When present, a non-empty list; the portable `sql` may stand in for it. dialects: Optional[list[OssieDialectExpression]] = Field(default=None, min_length=1) @model_validator(mode="after") - def _require_expression_or_dialects(self) -> "OssieExpression": - # Mirrors the schema anyOf: at least one of `expression` or `dialects` + def _require_sql_or_dialects(self) -> "OssieExpression": + # Mirrors the schema anyOf: at least one of `sql` or `dialects` # must be present. The `dialects` min_length rule already rejects an # empty list, so here only the both-absent case remains. - if self.expression is None and not self.dialects: - raise ValueError("expression requires at least one of 'expression' or 'dialects'") + if self.sql is None and not self.dialects: + raise ValueError("expression requires at least one of 'sql' or 'dialects'") return self diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 3062270f8..6cce62867 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -100,20 +100,20 @@ def test_ossie_sql_2026_dialect_survives_serialization(document_data: dict) -> N def test_portable_expression_round_trips_and_is_omitted_when_absent( document_data: dict, ) -> None: - # The optional portable `expression` round-trips when set and is omitted when unset. + # The optional portable `sql` round-trips when set and is omitted when unset. data = document_data - data["datasets"][0]["fields"][0]["expression"]["expression"] = "customer_id" + data["datasets"][0]["fields"][0]["expression"]["sql"] = "customer_id" document = OssieDocument.model_validate(data) - assert document.datasets[0].fields[0].expression.expression == "customer_id" + assert document.datasets[0].fields[0].expression.sql == "customer_id" # A field whose expression omits the portable form keeps it unset. - assert document.metrics[0].expression.expression is None + assert document.metrics[0].expression.sql is None as_json = json.loads(document.to_ossie_json()) as_yaml = yaml.safe_load(document.to_ossie_yaml()) for serialized in (as_json, as_yaml): assert serialized == data - assert "expression" not in serialized["metrics"][0]["expression"] + assert "sql" not in serialized["metrics"][0]["expression"] assert OssieDocument.model_validate(serialized) == document diff --git a/python/tests/test_serialization.py b/python/tests/test_serialization.py index 12827c7b1..4cf576bb2 100644 --- a/python/tests/test_serialization.py +++ b/python/tests/test_serialization.py @@ -285,7 +285,7 @@ def _missing_properties(model: type[BaseModel], sample: object, where: str) -> l def _fully_populated_document() -> dict: """A document that sets every optional property, so each can be nulled in turn.""" expression = { - "expression": "amount", + "sql": "amount", "dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}], } ai_context = {"instructions": "i", "synonyms": ["s"], "examples": ["e"]} diff --git a/python/tests/test_validation.py b/python/tests/test_validation.py index e8f252362..6251841fc 100644 --- a/python/tests/test_validation.py +++ b/python/tests/test_validation.py @@ -148,33 +148,33 @@ def test_dialect_expression_missing_expression() -> None: OssieDialectExpression(dialect=OssieDialect.ANSI_SQL) -def test_expression_requires_expression_or_dialects() -> None: - # Neither the portable `expression` nor `dialects`: rejected (schema anyOf). +def test_expression_requires_sql_or_dialects() -> None: + # Neither the portable `sql` nor `dialects`: rejected (schema anyOf). with pytest.raises(ValidationError): OssieExpression() def test_expression_portable_only_is_valid() -> None: - # The portable `expression` may stand alone, with no `dialects`. - expr = OssieExpression(expression="c_first_name || ' ' || c_last_name") - assert expr.expression == "c_first_name || ' ' || c_last_name" + # The portable `sql` may stand alone, with no `dialects`. + expr = OssieExpression(sql="c_first_name || ' ' || c_last_name") + assert expr.sql == "c_first_name || ' ' || c_last_name" assert expr.dialects is None def test_expression_dialects_only_is_valid() -> None: - # `dialects` alone, with no portable `expression`, remains valid. + # `dialects` alone, with no portable `sql`, remains valid. expr = OssieExpression(dialects=[OssieDialectExpression(dialect="ANSI_SQL", expression="x")]) - assert expr.expression is None + assert expr.sql is None assert expr.dialects is not None and len(expr.dialects) == 1 def test_expression_both_present_is_valid() -> None: # Both forms together: valid; the portable form is authoritative. expr = OssieExpression( - expression="x", + sql="x", dialects=[OssieDialectExpression(dialect="ANSI_SQL", expression="x")], ) - assert expr.expression == "x" + assert expr.sql == "x" assert expr.dialects is not None and len(expr.dialects) == 1 diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index b3021cf96..c430b1024 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -389,7 +389,7 @@ def test_unknown_dialect_is_skipped_as_opaque() -> None: def test_portable_expression_is_validated_as_ossie_sql_2026( monkeypatch: pytest.MonkeyPatch, ) -> None: - # The portable `expression` is validated as OSSIE_SQL_2026, alongside `dialects`. + # The portable `sql` is validated as OSSIE_SQL_2026, alongside `dialects`. seen = [] def record_expression(expression: str, dialect: str, context: str) -> None: @@ -398,7 +398,7 @@ def record_expression(expression: str, dialect: str, context: str) -> None: monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) monkeypatch.setattr(_VALIDATE, "validate_sql_expression", record_expression) expression = { - "expression": "amount", + "sql": "amount", "dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}], } dataset = {**_ORDERS, "fields": [{"name": "value", "expression": expression}]} @@ -412,8 +412,8 @@ def record_expression(expression: str, dialect: str, context: str) -> None: def test_schema_accepts_portable_only_expression(core_schema: dict) -> None: - # The portable `expression` may stand alone, with no `dialects` (schema anyOf). - dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {"expression": "amount"}}]} + # The portable `sql` may stand alone, with no `dialects` (schema anyOf). + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {"sql": "amount"}}]} assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [] diff --git a/validation/validate.py b/validation/validate.py index 4528764d2..296b68b7d 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -499,8 +499,8 @@ def validate_sql(data: dict) -> list[str]: for field in dataset.get("fields", []): field_name = field.get("name", "") expression = field.get("expression", {}) - # The optional top-level expression is the portable OSSIE_SQL_2026 form. - portable = expression.get("expression") + # The optional top-level `sql` is the portable OSSIE_SQL_2026 form. + portable = expression.get("sql") if portable is not None: context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' (OSSIE_SQL_2026)" error = validate_sql_expression(portable, "OSSIE_SQL_2026", context) @@ -535,8 +535,8 @@ def validate_sql(data: dict) -> list[str]: for metric in model.get("metrics", []): metric_name = metric.get("name", "") expression = metric.get("expression", {}) - # The optional top-level expression is the portable OSSIE_SQL_2026 form. - portable = expression.get("expression") + # The optional top-level `sql` is the portable OSSIE_SQL_2026 form. + portable = expression.get("sql") if portable is not None: context = f"Metric '{metric_name}' in model '{model_name}' (OSSIE_SQL_2026)" error = validate_sql_expression(portable, "OSSIE_SQL_2026", context) From 7169bfffc242a6b25a4b2d6baffc9f0dc2e3ce6a Mon Sep 17 00:00:00 2001 From: Chris Eubank <108756251+christianeu-db@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:46:07 +0000 Subject: [PATCH 5/9] fix(validation): check a dataset-scoped metric's portable sql A dataset-scoped metric may be written with only the portable `sql` form (no dialects). Such a metric escaped both checks: `validate_sql` only walked its `dialects`, so the portable form was never parsed; and `validate_metric_scoping` likewise only walked `dialects`, so the single-namespace scoping rules (no qualified references, declared fields only) were never applied to it. Both now handle the portable `sql` exactly like a dialect expression: it is parsed as OSSIE_SQL_2026 and scope-checked. A portable-only dataset-scoped metric with a cross-dataset reference or an undeclared field is now reported, as a dialect form already was. Adds four tests covering the portable form: a declared single-part field passes, a cross-dataset reference and an undeclared field are rejected, and an invalid portable expression is reported. Co-authored-by: Isaac --- validation/tests/test_validate.py | 62 ++++++++++++++++++++ validation/validate.py | 94 ++++++++++++++++++++----------- 2 files changed, 122 insertions(+), 34 deletions(-) diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index c430b1024..7f3128185 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -1286,6 +1286,11 @@ def _ds_expr(sql: str, dialect: str = "OSSIE_SQL_2026") -> dict: return {"dialects": [{"dialect": dialect, "expression": sql}]} +def _ds_portable(sql: str) -> dict: + # A dataset-scoped metric written with only the portable `sql` form, no dialects. + return {"sql": sql} + + def _orders_with(fields=None, metrics=None) -> dict: dataset = {"name": "orders", "source": "db.s.orders"} if fields is not None: @@ -1399,3 +1404,60 @@ def test_qualified_dataset_scoped_metric_fails_document_validation(core_schema, scope_errors = [error for error in errors if error.startswith("[Scope]")] assert scope_errors assert all(not isinstance(error, _VALIDATE.ValidationWarning) for error in scope_errors) + + +def test_dataset_scoped_metric_portable_sql_accepts_declared_field() -> None: + # The portable `sql` form is scope-checked like a dialect: a single-part + # declared field passes. + doc = _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[{"name": "total_amount", "expression": _ds_portable("SUM(amount)")}], + ) + + assert _VALIDATE.validate_metric_scoping(doc) == [] + + +def test_dataset_scoped_metric_portable_sql_rejects_a_cross_dataset_reference() -> None: + # A portable-only dataset-scoped metric no longer escapes scope-checking; the + # qualified reference is reported against the OSSIE_SQL_2026 context. + doc = _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[ + {"name": "bad", "expression": _ds_portable("SUM(amount) / COUNT(customers.id)")} + ], + ) + + errors = _VALIDATE.validate_metric_scoping(doc) + + assert any( + error.startswith("[Scope]") and "'customers'" in error and "(OSSIE_SQL_2026)" in error + for error in errors + ) + + +def test_dataset_scoped_metric_portable_sql_rejects_an_undeclared_field() -> None: + doc = _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[{"name": "total_qty", "expression": _ds_portable("SUM(quantity)")}], + ) + + errors = _VALIDATE.validate_metric_scoping(doc) + + assert any( + error.startswith("[Scope]") and "'quantity'" in error and "(OSSIE_SQL_2026)" in error + for error in errors + ) + + +def test_dataset_scoped_metric_portable_sql_is_sql_validated() -> None: + # The portable `sql` of a dataset-scoped metric is now parsed as OSSIE_SQL_2026, + # so a syntactically invalid portable form is reported (previously skipped). + doc = _orders_with( + metrics=[{"name": "broken", "expression": _ds_portable("SUM(")}], + ) + + errors = _VALIDATE.validate_sql(doc) + + assert any( + "Metric 'orders.broken'" in error and "(OSSIE_SQL_2026)" in error for error in errors + ) diff --git a/validation/validate.py b/validation/validate.py index 296b68b7d..264a28b4d 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -522,6 +522,13 @@ def validate_sql(data: dict) -> list[str]: for metric in dataset.get("metrics", []): metric_name = metric.get("name", "") expression = metric.get("expression", {}) + # The optional top-level `sql` is the portable OSSIE_SQL_2026 form. + portable = expression.get("sql") + if portable is not None: + context = f"Metric '{dataset_name}.{metric_name}' in model '{model_name}' (OSSIE_SQL_2026)" + error = validate_sql_expression(portable, "OSSIE_SQL_2026", context) + if error: + errors.append(error) for dialect_expr in expression.get("dialects", []): dialect = dialect_expr.get("dialect", "ANSI_SQL") expr = dialect_expr.get("expression") @@ -587,6 +594,49 @@ def validate_metric_scoping(data: dict) -> list[str]: errors = [] model_name = model.get("name", "") + def scope_errors( + expr: str, dialect: str, dataset_name: str, metric_name: str, field_names: set + ) -> list[str]: + # Parse once; an empty, unparseable, or unknown-dialect expression yields no + # tree and is skipped (reported, if at all, by `validate_sql`). + tree, _ = _parse_sql_expression( + expr, + dialect, + f"Metric '{dataset_name}.{metric_name}' in model '{model_name}'", + ) + if tree is None: + return [] + + qualifiers = set() + undeclared = set() + for col in tree.find_all(exp.Column): + parts = [part.name for part in col.parts] + if len(parts) > 1 and parts[0]: + qualifiers.add(parts[0]) + elif parts and parts[0].casefold() not in field_names: + undeclared.add(parts[0]) + + prefix = ( + f"[Scope] Dataset-scoped metric '{dataset_name}.{metric_name}' in model " + f"'{model_name}' ({dialect})" + ) + out = [] + if qualifiers: + names = ", ".join(repr(q) for q in sorted(qualifiers)) + out.append( + f"{prefix} uses qualified reference(s) {names}. A dataset-scoped metric " + f"expression must reference the dataset's fields by single-part " + f"(unqualified) name; a metric that must reference another dataset " + f"belongs in the model-root metrics." + ) + if undeclared: + names = ", ".join(repr(n) for n in sorted(undeclared)) + out.append( + f"{prefix} references undeclared field(s) {names}. A dataset-scoped " + f"metric may reference only the dataset's declared fields." + ) + return out + for dataset in model.get("datasets", []): dataset_name = dataset.get("name", "") field_names = { @@ -595,46 +645,22 @@ def validate_metric_scoping(data: dict) -> list[str]: for metric in dataset.get("metrics", []): metric_name = metric.get("name", "") expression = metric.get("expression") or {} + # The portable OSSIE_SQL_2026 form is scope-checked like its dialect siblings. + portable = expression.get("sql") + if portable: + errors.extend( + scope_errors( + portable, "OSSIE_SQL_2026", dataset_name, metric_name, field_names + ) + ) for dialect_expr in expression.get("dialects", []): dialect = dialect_expr.get("dialect", "ANSI_SQL") expr = dialect_expr.get("expression") if not expr: continue - tree, _ = _parse_sql_expression( - expr, - dialect, - f"Metric '{dataset_name}.{metric_name}' in model '{model_name}'", - ) - if tree is None: - continue - - qualifiers = set() - undeclared = set() - for col in tree.find_all(exp.Column): - parts = [part.name for part in col.parts] - if len(parts) > 1 and parts[0]: - qualifiers.add(parts[0]) - elif parts and parts[0].casefold() not in field_names: - undeclared.add(parts[0]) - - prefix = ( - f"[Scope] Dataset-scoped metric '{dataset_name}.{metric_name}' in model " - f"'{model_name}' ({dialect})" + errors.extend( + scope_errors(expr, dialect, dataset_name, metric_name, field_names) ) - if qualifiers: - names = ", ".join(repr(q) for q in sorted(qualifiers)) - errors.append( - f"{prefix} uses qualified reference(s) {names}. A dataset-scoped metric " - f"expression must reference the dataset's fields by single-part " - f"(unqualified) name; a metric that must reference another dataset " - f"belongs in the model-root metrics." - ) - if undeclared: - names = ", ".join(repr(n) for n in sorted(undeclared)) - errors.append( - f"{prefix} references undeclared field(s) {names}. A dataset-scoped " - f"metric may reference only the dataset's declared fields." - ) return errors From bd2969267126e4f29385d252f5947af803b28080 Mon Sep 17 00:00:00 2001 From: Chris Eubank <108756251+christianeu-db@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:10:48 +0000 Subject: [PATCH 6/9] refactor(validation): walk expressions through one iterator `validate_sql` and `validate_metric_scoping` each walked the document to find every expression. Introduce `iter_expressions`, the single traversal both consume, yielding an `Expression` record per portable `sql` form and per dialect expression, keyed by `kind` (field / dataset_metric / model_metric). Also folded in while here: - Split the overloaded DIALECT_MAP into SQLGLOT_DIALECTS (parseable, mapped to the sqlglot dialect) and NON_SQL_DIALECTS (carried but not parsed); one guard in _parse_sql_expression indexes SQLGLOT_DIALECTS. - Name the PORTABLE_DIALECT constant; drop the `.get("dialect", "ANSI_SQL")` fallback, since the schema requires `dialect`. - A dialect this validator cannot parse is reported as a nonfatal warning that its expressions were not syntax-checked; it may be a legitimate entry from extensions/dialects.md, not a mistake. - Honor the spec's precedence rule: a top-level `sql` outranks an OSSIE_SQL_2026 dialect entry, so that entry is non-authoritative. It is not scope-checked, and an invalid one warns instead of failing the document, since a client may still read it. - Spec/docs wording: write OSSIE_SQL_2026 directly (no parenthetical), "Any string is accepted", drop "Optional:" from the portable-form comments, describe `data` as an opaque string (not JSON), and give the glossary Dialect entry only ANSI_SQL as an example. - models.py: optional `dialects` comment and the "convenience set, not a closed one" phrasing removed; docstrings state any string is accepted. - Tests track the split, assert exact scope messages, cover the precedence behavior, and compare dialects by value. Co-authored-by: Isaac --- core-spec/ossie-schema.json | 6 +- core-spec/spec.md | 8 +- core-spec/spec.yaml | 6 +- docs/index.md | 8 +- examples/tpcds_semantic_model.yaml | 2 +- extensions/vendors.md | 2 +- python/src/ossie/models.py | 14 +- python/tests/test_models.py | 4 +- python/tests/test_validation.py | 10 +- validation/tests/test_validate.py | 126 ++++++++----- validation/validate.py | 286 +++++++++++++++-------------- 11 files changed, 254 insertions(+), 218 deletions(-) diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index 7a2b13e5c..93c405191 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -38,12 +38,12 @@ "Dialect": { "type": "string", "examples": ["OSSIE_SQL_2026", "ANSI_SQL", "DB_DIALECT", "BI_EXPRESSION", "SEMANTIC_MODEL_EXPRESSION", "PROGRAMMING_LANGUAGE"], - "description": "Expression language a metric or field expression is declared in. Free-form: any string is accepted. OSSIE_SQL_2026 is the only dialect this specification defines; others are opaque to Ossie. See extensions/dialects.md." + "description": "Expression language a metric or field expression is declared in. Any string is accepted. OSSIE_SQL_2026 is the only dialect this specification defines; others are opaque to Ossie. See extensions/dialects.md." }, "Vendor": { "type": "string", "examples": ["COMMON", "BI_TOOL", "DB_VENDOR", "SEMANTIC_LAYER"], - "description": "Vendor name for custom extensions. Free-form: any string is accepted. See extensions/vendors.md." + "description": "Vendor name for custom extensions. Any string is accepted. See extensions/vendors.md." }, "AIContext": { "description": "Additional context for AI tools", @@ -113,7 +113,7 @@ "properties": { "sql": { "type": "string", - "description": "Primary, portable expression written in OSSIE_SQL (the OSSIE_SQL_2026 dialect). When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative." + "description": "Primary, portable expression written in OSSIE_SQL_2026. When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative." }, "dialects": { "type": "array", diff --git a/core-spec/spec.md b/core-spec/spec.md index 407c0b310..259adc436 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -298,7 +298,7 @@ The expression object supports multiple SQL dialects for cross-platform compatib ```yaml expression: - sql: "customer_id" # Optional: primary, portable OSSIE_SQL form (OSSIE_SQL_2026) + sql: "customer_id" # Primary, portable OSSIE_SQL_2026 form dialects: - dialect: ANSI_SQL # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: "customer_id" # Scalar SQL expression @@ -308,7 +308,7 @@ expression: - Use scalar SQL expressions (no aggregations) - Can be simple column references (e.g., `customer_id`) or computed expressions (e.g., `first_name || ' ' || last_name`) -- The optional top-level `sql` is the primary, portable form, written in OSSIE_SQL (the OSSIE_SQL_2026 dialect) +- The optional top-level `sql` is the primary, portable form, written in OSSIE_SQL_2026 - At least one of the top-level `sql` or `dialects` MUST be present; either MAY stand alone - If both the top-level `sql` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `sql` is authoritative and consumers MUST use it as the OSSIE_SQL_2026 form - Identifiers refer to columns of the dataset's `source`, not to other fields. See [Name Resolution](#name-resolution). @@ -440,7 +440,7 @@ The expression object supports multiple dialects ```yaml expression: - sql: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" # Optional: primary, portable OSSIE_SQL form (OSSIE_SQL_2026) + sql: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" # Primary, portable OSSIE_SQL_2026 form dialects: - dialect: ANSI_SQL # Default expression: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" @@ -594,7 +594,7 @@ For a non-normative catalog of `vendor_name` values well-known tools use, see data: '{"some_setting": "value", "nested": {"option": true}}' ``` -The `data` payload is an opaque JSON string that Ossie carries but does not +The `data` payload is an opaque string that Ossie carries but does not interpret. For vendor-specific extension examples and the known-vendor catalog, see [extensions/vendors.md](../extensions/vendors.md). diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 097e7b7a1..58c66569e 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -219,8 +219,7 @@ fields: # Identifiers refer to columns of the dataset's source, not to other fields # At least one of the portable `sql` or `dialects` must be present; either may stand alone expression: - # Optional: primary, portable expression written in OSSIE_SQL (the OSSIE_SQL_2026 dialect) - sql: string # OSSIE_SQL_2026 form, e.g., "customer_id", "first_name || ' ' || last_name" + sql: string # Primary, portable OSSIE_SQL_2026 form, e.g., "customer_id", "first_name || ' ' || last_name" dialects: - dialect: string # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: string # SQL scalar expression, e.g., "customer_id", "first_name || ' ' || last_name", "UPPER(email)" @@ -285,8 +284,7 @@ metrics: # Metrics cannot reference other metrics # At least one of the portable `sql` or `dialects` must be present; either may stand alone expression: - # Optional: primary, portable expression written in OSSIE_SQL (the OSSIE_SQL_2026 dialect) - sql: string # OSSIE_SQL_2026 form, e.g., "SUM(orders.sales)", "AVG(orders.amount)" + sql: string # Primary, portable OSSIE_SQL_2026 form, e.g., "SUM(orders.sales)", "AVG(orders.amount)" dialects: - dialect: string # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: string # Full SQL expression with aggregate functions, e.g., "SUM(orders.sales)", "AVG(orders.amount)" diff --git a/docs/index.md b/docs/index.md index e4bf2bdfa..9bc4e26ce 100644 --- a/docs/index.md +++ b/docs/index.md @@ -59,7 +59,7 @@ The Ossie core specification (current version: **0.2.0.dev0**, latest released: | **Custom Extensions** | Vendor-specific metadata stored as JSON, allowing platforms to carry additional information without breaking core compatibility. | | **AI Context** | Optional annotations at every level (model, dataset, field, relationship, metric) to help AI tools understand business meaning — including instructions, synonyms, and example queries. | -An expression carries a primary, portable form in `OSSIE_SQL_2026` (the dialect Ossie defines) and may also carry vendor-specific variants. The `dialect` label is a free-form string, so expressions can be tailored to any platform while maintaining a common model structure. +An expression carries a primary, portable form in `OSSIE_SQL_2026` and may also carry vendor-specific variants. The `dialect` label is a free-form string, so expressions can be tailored to any platform while maintaining a common model structure. For the full specification, see [core-spec/spec.md](../core-spec/spec.md). For validation tooling, see [validation/validate.py](../validation/validate.py). For a complete example, see the [TPC-DS semantic model](../examples/tpcds_semantic_model.yaml). @@ -199,7 +199,7 @@ A key architectural feature of Ossie is its multi-dialect expression system. An ```yaml expression: - sql: LOWER(email) # primary, portable OSSIE_SQL form (OSSIE_SQL_2026) + sql: LOWER(email) # primary, portable OSSIE_SQL_2026 form dialects: - dialect: SNOWFLAKE expression: LOWER(email)::VARCHAR @@ -319,8 +319,8 @@ A practical guide for organizations looking to adopt Ossie. | **Dimension** | A categorical attribute used to slice and filter data (e.g., region, product category, date). In Ossie, dimensions are represented as fields with optional metadata such as `is_time`. | | **Metric** | A quantitative measure computed by aggregating fields (e.g., total revenue, average order value). A dataset-scoped metric is defined on an individual dataset, for a measure that semantically belongs to it and is evaluated at the dataset's row level; a model-scoped metric is defined at the model root and may span datasets. | | **Relationship** | A foreign key connection between two datasets, defining how they can be joined. Relationships are always many-to-one (from the referencing dataset to the referenced dataset). | -| **Dialect** | A SQL or expression language variant an expression is written in (e.g., `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`). A free-form string; `OSSIE_SQL_2026` is the only dialect Ossie defines, and all others are opaque. | -| **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as a JSON string. Extensions allow platforms to carry additional information without modifying the core specification. | +| **Dialect** | A SQL or expression language variant an expression is written in (e.g., `ANSI_SQL`). A free-form string; `OSSIE_SQL_2026` is the only dialect Ossie defines, and all others are opaque. | +| **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as an opaque string. Extensions allow platforms to carry additional information without modifying the core specification. | | **AI Context** | Optional annotations on any Ossie construct (model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | | **Converter** | A tool that translates between the Ossie format and a specific vendor's semantic model format. Converters come in pairs: import (vendor → Ossie) and export (Ossie → vendor). | | **Hub-and-Spoke** | The architectural pattern used by Ossie, where the specification acts as the central format (hub) and vendor converters act as spokes, avoiding the need for point-to-point integrations. | diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index dfd5ec267..347a7e2a4 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -309,7 +309,7 @@ datasets: - name: customer_full_name expression: - # Primary, portable OSSIE_SQL form (OSSIE_SQL_2026) + # Primary, portable OSSIE_SQL_2026 form sql: c_first_name || ' ' || c_last_name dialects: - dialect: ANSI_SQL diff --git a/extensions/vendors.md b/extensions/vendors.md index 90a739bde..6daf4d825 100644 --- a/extensions/vendors.md +++ b/extensions/vendors.md @@ -28,7 +28,7 @@ custom_extensions: data: string # JSON string containing vendor-specific data ``` -`vendor_name` is a free-form string and `data` is an opaque JSON string that Ossie +`vendor_name` is a free-form string and `data` is an opaque string that Ossie carries but does not interpret. This is a registry of `vendor_name` tokens the community uses (see [`README.md`](README.md) for how these lists are used). diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index ccb1a2c83..207ac3d0b 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -117,8 +117,8 @@ def _invalid_properties(node: Any) -> list[tuple[str, str]]: class OssieDialect(str, Enum): - """Well-known dialect constants. `dialect` is a free-form string, so this is a - convenience set, not a closed one. See `extensions/dialects.md`.""" + """Well-known dialect values. The field accepts any string; see + `extensions/dialects.md`.""" ANSI_SQL = "ANSI_SQL" SNOWFLAKE = "SNOWFLAKE" @@ -160,8 +160,8 @@ class OssieDataType(str, Enum): class OssieVendor(str, Enum): - """Well-known vendor constants. `vendor_name` is a free-form string, so this is - a convenience set, not a closed one. See `extensions/vendors.md`.""" + """Well-known vendor values. The field accepts any string; see + `extensions/vendors.md`.""" COMMON = "COMMON" SNOWFLAKE = "SNOWFLAKE" @@ -226,16 +226,12 @@ class OssieExpression(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - # Primary, portable expression written in OSSIE_SQL (the OSSIE_SQL_2026 dialect). + # Primary, portable expression written in OSSIE_SQL_2026. sql: Optional[str] = None - # When present, a non-empty list; the portable `sql` may stand in for it. dialects: Optional[list[OssieDialectExpression]] = Field(default=None, min_length=1) @model_validator(mode="after") def _require_sql_or_dialects(self) -> "OssieExpression": - # Mirrors the schema anyOf: at least one of `sql` or `dialects` - # must be present. The `dialects` min_length rule already rejects an - # empty list, so here only the both-absent case remains. if self.sql is None and not self.dialects: raise ValueError("expression requires at least one of 'sql' or 'dialects'") return self diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 6cce62867..040e5f4c9 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -70,7 +70,7 @@ def test_dialect_accepts_string_and_enum_input(dialect: str | OssieDialect) -> N def test_dialect_accepts_unknown_string() -> None: - # Free-form dialect: an unregistered value is accepted and preserved. + # Free-form dialect: an unknown value is accepted and preserved. expression = OssieExpression.model_validate( {"dialects": [{"dialect": "DB_DIALECT", "expression": "value"}]} ) @@ -106,7 +106,7 @@ def test_portable_expression_round_trips_and_is_omitted_when_absent( document = OssieDocument.model_validate(data) assert document.datasets[0].fields[0].expression.sql == "customer_id" - # A field whose expression omits the portable form keeps it unset. + # A metric whose expression omits the portable form keeps it unset. assert document.metrics[0].expression.sql is None as_json = json.loads(document.to_ossie_json()) diff --git a/python/tests/test_validation.py b/python/tests/test_validation.py index 6251841fc..99df25f64 100644 --- a/python/tests/test_validation.py +++ b/python/tests/test_validation.py @@ -155,27 +155,25 @@ def test_expression_requires_sql_or_dialects() -> None: def test_expression_portable_only_is_valid() -> None: - # The portable `sql` may stand alone, with no `dialects`. expr = OssieExpression(sql="c_first_name || ' ' || c_last_name") assert expr.sql == "c_first_name || ' ' || c_last_name" assert expr.dialects is None def test_expression_dialects_only_is_valid() -> None: - # `dialects` alone, with no portable `sql`, remains valid. + # `dialects` alone, with no portable `sql`, is accepted. expr = OssieExpression(dialects=[OssieDialectExpression(dialect="ANSI_SQL", expression="x")]) assert expr.sql is None - assert expr.dialects is not None and len(expr.dialects) == 1 + assert expr.dialects == [OssieDialectExpression(dialect="ANSI_SQL", expression="x")] def test_expression_both_present_is_valid() -> None: - # Both forms together: valid; the portable form is authoritative. expr = OssieExpression( sql="x", dialects=[OssieDialectExpression(dialect="ANSI_SQL", expression="x")], ) assert expr.sql == "x" - assert expr.dialects is not None and len(expr.dialects) == 1 + assert expr.dialects == [OssieDialectExpression(dialect="ANSI_SQL", expression="x")] def test_expression_empty_dialects_is_rejected() -> None: @@ -185,7 +183,7 @@ def test_expression_empty_dialects_is_rejected() -> None: def test_free_form_dialect_in_expression() -> None: - # `dialect` is free-form: an unregistered value is accepted and preserved. + # `dialect` is free-form: an unknown value is accepted and preserved. dialect_expr = OssieDialectExpression(dialect="DB_DIALECT", expression="order_id") assert dialect_expr.dialect == "DB_DIALECT" diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 7f3128185..9df6460f3 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -330,23 +330,22 @@ def record_expression(expression: str, dialect: str, context: str) -> None: ] -def test_ossie_sql_2026_maps_to_the_ansi_default_explicitly() -> None: - # OSSIE_SQL_2026 is ANSI-SQL-compatible, so it must resolve to the same - # sqlglot default (None) as ANSI_SQL. Assert it via an explicit key rather - # than DIALECT_MAP.get()'s default, which happened to return the same None. - assert "OSSIE_SQL_2026" in _VALIDATE.DIALECT_MAP - assert _VALIDATE.DIALECT_MAP["OSSIE_SQL_2026"] is _VALIDATE.DIALECT_MAP["ANSI_SQL"] - # It is parseable as ANSI SQL, so it stays validated rather than skipped. - assert "OSSIE_SQL_2026" not in _VALIDATE.SKIP_SQL_VALIDATION +def test_ossie_sql_2026_maps_to_the_ansi_default() -> None: + # OSSIE_SQL_2026 is ANSI-SQL-compatible, so it resolves to the same sqlglot + # default (None) as ANSI_SQL, and is parseable rather than a non-SQL dialect. + assert ( + _VALIDATE.SQLGLOT_DIALECTS["OSSIE_SQL_2026"] + is _VALIDATE.SQLGLOT_DIALECTS["ANSI_SQL"] + ) + assert "OSSIE_SQL_2026" not in _VALIDATE.NON_SQL_DIALECTS def test_holistics_aql_is_skipped_rather_than_parsed_as_sql() -> None: # AQL is relationship-aware and is not SQL, so sqlglot cannot parse it. # Skipping it is what lets a Holistics field carry its body at all, the way # MDX, MAQL, THOUGHTSPOT and DAX already do. - assert "HOLISTICS_AQL" in _VALIDATE.DIALECT_MAP - assert _VALIDATE.DIALECT_MAP["HOLISTICS_AQL"] is None - assert "HOLISTICS_AQL" in _VALIDATE.SKIP_SQL_VALIDATION + assert "HOLISTICS_AQL" in _VALIDATE.NON_SQL_DIALECTS + assert "HOLISTICS_AQL" not in _VALIDATE.SQLGLOT_DIALECTS def test_a_holistics_aql_expression_is_not_reported_as_invalid_sql() -> None: @@ -376,9 +375,9 @@ def test_flags_an_invalid_ossie_sql_2026_expression() -> None: @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") -def test_unknown_dialect_is_skipped_as_opaque() -> None: - # An unregistered dialect is opaque: skipped, not parsed as ANSI. - assert "DB_DIALECT" not in _VALIDATE.DIALECT_MAP +def test_unsupported_dialect_is_skipped() -> None: + # A dialect this validator cannot parse is skipped, not parsed as ANSI. + assert "DB_DIALECT" not in _VALIDATE.SQLGLOT_DIALECTS error = _VALIDATE.validate_sql_expression( "this is not sql @ all", "DB_DIALECT", "ctx" ) @@ -419,7 +418,7 @@ def test_schema_accepts_portable_only_expression(core_schema: dict) -> None: def test_schema_rejects_expression_with_neither_form(core_schema: dict) -> None: - # An expression object carrying neither `expression` nor `dialects` is invalid. + # An expression object carrying neither `sql` nor `dialects` is invalid. dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {}}]} assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ @@ -437,10 +436,10 @@ def test_schema_rejects_empty_dialects_without_portable_form(core_schema: dict) ] -def test_unknown_dialect_emits_one_deduped_warning(monkeypatch: pytest.MonkeyPatch) -> None: - # An unrecognized dialect is nonfatal, but one warning per dialect reports that - # its expressions went unvalidated. The field and metric below share the dialect, - # so they collapse into a single warning carrying the count. +def test_unsupported_dialect_emits_one_deduped_warning(monkeypatch: pytest.MonkeyPatch) -> None: + # A dialect this validator cannot parse is nonfatal, but one warning per dialect + # reports that its expressions went unchecked. The field and metric below share + # the dialect, so they collapse into a single warning carrying the count. monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) expression = {"dialects": [{"dialect": "ACME_SQL", "expression": "x"}]} dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} @@ -449,12 +448,15 @@ def test_unknown_dialect_emits_one_deduped_warning(monkeypatch: pytest.MonkeyPat result = _VALIDATE.validate_sql(document) - assert result == ["[SQL] Warning: unknown dialect 'ACME_SQL'; 2 expressions not validated"] + assert result == [ + "[SQL] Warning: dialect 'ACME_SQL' is not supported by this validator; " + "2 expressions not syntax-checked" + ] assert all(isinstance(item, _VALIDATE.ValidationWarning) for item in result) -def test_distinct_unknown_dialects_each_warn_once(monkeypatch: pytest.MonkeyPatch) -> None: - # Two different unrecognized dialects produce two warnings, sorted by name. +def test_distinct_unsupported_dialects_each_warn_once(monkeypatch: pytest.MonkeyPatch) -> None: + # Two different unsupported dialects produce two warnings, sorted by name. monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) fields = [ {"name": "a", "expression": {"dialects": [{"dialect": "ZETA_SQL", "expression": "x"}]}}, @@ -463,15 +465,17 @@ def test_distinct_unknown_dialects_each_warn_once(monkeypatch: pytest.MonkeyPatc dataset = {**_ORDERS, "fields": fields} assert _VALIDATE.validate_sql(_document([dataset], [])) == [ - "[SQL] Warning: unknown dialect 'ACME_SQL'; 1 expression not validated", - "[SQL] Warning: unknown dialect 'ZETA_SQL'; 1 expression not validated", + "[SQL] Warning: dialect 'ACME_SQL' is not supported by this validator; " + "1 expression not syntax-checked", + "[SQL] Warning: dialect 'ZETA_SQL' is not supported by this validator; " + "1 expression not syntax-checked", ] -def test_known_unparseable_dialect_is_not_flagged_as_unknown( +def test_known_unparseable_dialect_is_not_flagged( monkeypatch: pytest.MonkeyPatch, ) -> None: - # MDX is recognized but unparseable: it is skipped silently, not warned as unknown. + # MDX is a known non-SQL dialect: it is skipped silently, not warned as unsupported. monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) expression = {"dialects": [{"dialect": "MDX", "expression": "[Measures].[Sales]"}]} dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} @@ -1407,8 +1411,6 @@ def test_qualified_dataset_scoped_metric_fails_document_validation(core_schema, def test_dataset_scoped_metric_portable_sql_accepts_declared_field() -> None: - # The portable `sql` form is scope-checked like a dialect: a single-part - # declared field passes. doc = _orders_with( fields=[{"name": "amount", "expression": _ds_expr("amount")}], metrics=[{"name": "total_amount", "expression": _ds_portable("SUM(amount)")}], @@ -1418,8 +1420,6 @@ def test_dataset_scoped_metric_portable_sql_accepts_declared_field() -> None: def test_dataset_scoped_metric_portable_sql_rejects_a_cross_dataset_reference() -> None: - # A portable-only dataset-scoped metric no longer escapes scope-checking; the - # qualified reference is reported against the OSSIE_SQL_2026 context. doc = _orders_with( fields=[{"name": "amount", "expression": _ds_expr("amount")}], metrics=[ @@ -1427,12 +1427,12 @@ def test_dataset_scoped_metric_portable_sql_rejects_a_cross_dataset_reference() ], ) - errors = _VALIDATE.validate_metric_scoping(doc) - - assert any( - error.startswith("[Scope]") and "'customers'" in error and "(OSSIE_SQL_2026)" in error - for error in errors - ) + assert _VALIDATE.validate_metric_scoping(doc) == [ + "[Scope] Dataset-scoped metric 'orders.bad' in model 'm' (OSSIE_SQL_2026) uses " + "qualified reference(s) 'customers'. A dataset-scoped metric expression must " + "reference the dataset's fields by single-part (unqualified) name; a metric that " + "must reference another dataset belongs in the model-root metrics." + ] def test_dataset_scoped_metric_portable_sql_rejects_an_undeclared_field() -> None: @@ -1441,23 +1441,57 @@ def test_dataset_scoped_metric_portable_sql_rejects_an_undeclared_field() -> Non metrics=[{"name": "total_qty", "expression": _ds_portable("SUM(quantity)")}], ) - errors = _VALIDATE.validate_metric_scoping(doc) - - assert any( - error.startswith("[Scope]") and "'quantity'" in error and "(OSSIE_SQL_2026)" in error - for error in errors - ) + assert _VALIDATE.validate_metric_scoping(doc) == [ + "[Scope] Dataset-scoped metric 'orders.total_qty' in model 'm' (OSSIE_SQL_2026) " + "references undeclared field(s) 'quantity'. A dataset-scoped metric may reference " + "only the dataset's declared fields." + ] def test_dataset_scoped_metric_portable_sql_is_sql_validated() -> None: - # The portable `sql` of a dataset-scoped metric is now parsed as OSSIE_SQL_2026, - # so a syntactically invalid portable form is reported (previously skipped). doc = _orders_with( metrics=[{"name": "broken", "expression": _ds_portable("SUM(")}], ) errors = _VALIDATE.validate_sql(doc) - assert any( - "Metric 'orders.broken'" in error and "(OSSIE_SQL_2026)" in error for error in errors + # The sqlglot error text varies by version, so assert the context prefix only. + assert len(errors) == 1 + assert errors[0].startswith("[SQL] Metric 'orders.broken' in model 'm' (OSSIE_SQL_2026):") + + +def test_invalid_superseded_ossie_sql_2026_dialect_entry_warns_not_fails() -> None: + # The top-level `sql` is authoritative, so an invalid OSSIE_SQL_2026 dialect + # entry beside it is reported as a nonfatal warning, not a hard error. + expression = { + "sql": "SUM(amount)", + "dialects": [{"dialect": "OSSIE_SQL_2026", "expression": "SUM("}], + } + doc = _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[{"name": "total", "expression": expression}], ) + + errors = _VALIDATE.validate_sql(doc) + + assert errors == [ + "[SQL] Warning: Metric 'orders.total' in model 'm' (OSSIE_SQL_2026): not valid " + "SQL; ignored because the top-level 'sql' is authoritative" + ] + assert all(isinstance(e, _VALIDATE.ValidationWarning) for e in errors) + + +def test_superseded_ossie_sql_2026_dialect_entry_is_not_scope_checked() -> None: + # The authoritative top-level `sql` is scope-checked; a superseded + # OSSIE_SQL_2026 dialect entry beside it is not, so its cross-dataset + # reference does not fail the document. + expression = { + "sql": "SUM(amount)", + "dialects": [{"dialect": "OSSIE_SQL_2026", "expression": "SUM(customers.id)"}], + } + doc = _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[{"name": "total", "expression": expression}], + ) + + assert _VALIDATE.validate_metric_scoping(doc) == [] diff --git a/validation/validate.py b/validation/validate.py index 264a28b4d..68dd64170 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -57,8 +57,10 @@ import sys import traceback from collections import Counter -from collections.abc import Hashable +from collections.abc import Hashable, Iterator +from dataclasses import dataclass from pathlib import Path +from typing import Literal try: import yaml @@ -81,24 +83,23 @@ except ImportError: SQLGLOT_AVAILABLE = False -# Map Ossie dialects to sqlglot dialects -DIALECT_MAP = { - "ANSI_SQL": None, # sqlglot default - "OSSIE_SQL_2026": None, # ANSI-SQL-compatible; parse with the sqlglot default +# The dialect the portable `sql` form is written in; the only one this +# specification defines. +PORTABLE_DIALECT = "OSSIE_SQL_2026" + +# Ossie dialects sqlglot can parse, each mapped to the sqlglot dialect to parse it +# as. None is sqlglot's default (ANSI) dialect. +SQLGLOT_DIALECTS = { + "ANSI_SQL": None, + PORTABLE_DIALECT: None, "SNOWFLAKE": "snowflake", "DATABRICKS": "databricks", "BIGQUERY": "bigquery", - "MDX": None, # Not supported by sqlglot, skip validation - "TABLEAU": None, # Not supported by sqlglot, skip validation - "MAQL": None, # Not supported by sqlglot, skip validation - "SIGMA": None, # Sigma's spreadsheet-style formula language, not SQL; skip validation - "THOUGHTSPOT": None, # Not supported by sqlglot, skip validation - "DAX": None, # Not supported by sqlglot, skip validation - "HOLISTICS_AQL": None, # Holistics AQL, not SQL; skip validation } -# Dialects that sqlglot cannot parse -SKIP_SQL_VALIDATION = { +# Known non-SQL dialects: sqlglot cannot parse them, so their expressions are +# carried but not validated. +NON_SQL_DIALECTS = { "MDX", "TABLEAU", "MAQL", @@ -392,17 +393,10 @@ def _parse_sql_expression( Returns `(None, None)` when there is nothing to parse: sqlglot is unavailable or the dialect is one sqlglot cannot parse. """ - if not SQLGLOT_AVAILABLE or dialect in SKIP_SQL_VALIDATION: - return None, None - - # An unregistered dialect is opaque: skip it rather than parse it as ANSI and - # report spurious errors. Only dialects mapped below are parsed. Both callers - # then see no tree and no error, so an unknown dialect is counted and warned - # once by `validate_sql` rather than validated or scope-checked here. - if dialect not in DIALECT_MAP: + if not SQLGLOT_AVAILABLE or dialect not in SQLGLOT_DIALECTS: return None, None - sqlglot_dialect = DIALECT_MAP.get(dialect) + sqlglot_dialect = SQLGLOT_DIALECTS[dialect] def statement_count_error(sql: str) -> str | None: """Reject a parseable string that holds more than one statement. @@ -473,6 +467,89 @@ def validate_sql_expression(expr: str, dialect: str, context: str) -> str | None return error +ExpressionKind = Literal["field", "dataset_metric", "model_metric"] + + +@dataclass(frozen=True) +class Expression: + """An expression and its context within a document. + + `kind` identifies the declaring element, and `name` identifies the field + or metric. `dialect` is OSSIE_SQL_2026 for portable `sql` expressions, + or the dialect's label otherwise. `field_names` contains the enclosing + dataset's field names, or is empty for model-scoped metrics. `superseded` + marks an OSSIE_SQL_2026 dialect entry that a top-level `sql` outranks, so + it is non-authoritative: a client reads the `sql` instead. + """ + + kind: ExpressionKind + dataset_name: str + name: str + dialect: str + text: str + field_names: frozenset[str] + superseded: bool = False + + def context(self, model_name: str) -> str: + if self.kind == "field": + label = f"Field '{self.dataset_name}.{self.name}'" + elif self.kind == "dataset_metric": + label = f"Metric '{self.dataset_name}.{self.name}'" + else: + label = f"Metric '{self.name}'" + return f"{label} in model '{model_name}' ({self.dialect})" + + +def _expressions_in( + member: dict, kind: ExpressionKind, dataset_name: str, field_names: frozenset[str] +) -> Iterator[Expression]: + expression = member.get("expression") or {} + name = member.get("name", "") + sql = expression.get("sql") + if sql is not None: + yield Expression( + kind=kind, + dataset_name=dataset_name, + name=name, + dialect=PORTABLE_DIALECT, + text=sql, + field_names=field_names, + ) + for dialect_expr in expression.get("dialects", []): + text = dialect_expr.get("expression") + if text is None: + continue + dialect = dialect_expr["dialect"] + yield Expression( + kind=kind, + dataset_name=dataset_name, + name=name, + dialect=dialect, + text=text, + field_names=field_names, + # A top-level `sql` is the authoritative OSSIE_SQL_2026 form, so an + # OSSIE_SQL_2026 dialect entry alongside it is non-authoritative. + superseded=sql is not None and dialect == PORTABLE_DIALECT, + ) + + +def iter_expressions(data: dict) -> Iterator[Expression]: + """Yield every expression in a core semantic model document. A missing + expression is skipped; an empty one is yielded, so a caller decides whether to + report it.""" + for dataset in data.get("datasets", []): + dataset_name = dataset.get("name", "") + field_names = frozenset( + f["name"].casefold() for f in dataset.get("fields", []) if f.get("name") + ) + for field in dataset.get("fields", []): + yield from _expressions_in(field, "field", dataset_name, field_names) + for metric in dataset.get("metrics", []): + yield from _expressions_in(metric, "dataset_metric", dataset_name, field_names) + for metric in data.get("metrics", []): + yield from _expressions_in(metric, "model_metric", "", frozenset()) + + def validate_sql(data: dict) -> list[str]: """Validate SQL expressions in fields and metrics.""" # Only core semantic model documents contain a root datasets property. @@ -484,88 +561,42 @@ def validate_sql(data: dict) -> list[str]: "[SQL] Warning: sqlglot not installed, skipping SQL validation. Install with: pip install sqlglot" )] - model = data + model_name = data.get("name", "") errors = [] - # Count expressions whose dialect this validator does not recognize, so a single - # nonfatal warning per unknown dialect reports what went unvalidated. A free-form - # dialect is legitimate, but silence would also hide a typo like `SNOWFLKE`. - unknown_dialects: Counter = Counter() - - model_name = model.get("name", "") + # A dialect outside both sets is one this validator cannot parse. It may be a + # legitimate entry from extensions/dialects.md, so warn once that its + # expressions went unchecked rather than treat it as an error. + unsupported_dialects = Counter() + + for expression in iter_expressions(data): + dialect = expression.dialect + if dialect in NON_SQL_DIALECTS: + continue + if dialect not in SQLGLOT_DIALECTS: + unsupported_dialects[dialect] += 1 + continue + error = validate_sql_expression( + expression.text, dialect, expression.context(model_name) + ) + if not error: + continue + if expression.superseded: + # A top-level `sql` is authoritative, so an invalid OSSIE_SQL_2026 + # dialect entry does not fail the document; warn, since a client may + # still read the dialect entry instead of `sql`. + errors.append(ValidationWarning( + f"[SQL] Warning: {expression.context(model_name)}: not valid SQL; " + f"ignored because the top-level 'sql' is authoritative" + )) + else: + errors.append(error) - # Validate field expressions - for dataset in model.get("datasets", []): - dataset_name = dataset.get("name", "") - for field in dataset.get("fields", []): - field_name = field.get("name", "") - expression = field.get("expression", {}) - # The optional top-level `sql` is the portable OSSIE_SQL_2026 form. - portable = expression.get("sql") - if portable is not None: - context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' (OSSIE_SQL_2026)" - error = validate_sql_expression(portable, "OSSIE_SQL_2026", context) - if error: - errors.append(error) - for dialect_expr in expression.get("dialects", []): - dialect = dialect_expr.get("dialect", "ANSI_SQL") - # Only a missing expression is skipped; an empty one is reported. - expr = dialect_expr.get("expression") - if expr is not None: - if dialect not in DIALECT_MAP: - unknown_dialects[dialect] += 1 - context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' ({dialect})" - error = validate_sql_expression(expr, dialect, context) - if error: - errors.append(error) - - # Validate dataset-scoped metric expressions - for metric in dataset.get("metrics", []): - metric_name = metric.get("name", "") - expression = metric.get("expression", {}) - # The optional top-level `sql` is the portable OSSIE_SQL_2026 form. - portable = expression.get("sql") - if portable is not None: - context = f"Metric '{dataset_name}.{metric_name}' in model '{model_name}' (OSSIE_SQL_2026)" - error = validate_sql_expression(portable, "OSSIE_SQL_2026", context) - if error: - errors.append(error) - for dialect_expr in expression.get("dialects", []): - dialect = dialect_expr.get("dialect", "ANSI_SQL") - expr = dialect_expr.get("expression") - if expr is not None: - context = f"Metric '{dataset_name}.{metric_name}' in model '{model_name}' ({dialect})" - error = validate_sql_expression(expr, dialect, context) - if error: - errors.append(error) - - # Validate metric expressions - for metric in model.get("metrics", []): - metric_name = metric.get("name", "") - expression = metric.get("expression", {}) - # The optional top-level `sql` is the portable OSSIE_SQL_2026 form. - portable = expression.get("sql") - if portable is not None: - context = f"Metric '{metric_name}' in model '{model_name}' (OSSIE_SQL_2026)" - error = validate_sql_expression(portable, "OSSIE_SQL_2026", context) - if error: - errors.append(error) - for dialect_expr in expression.get("dialects", []): - dialect = dialect_expr.get("dialect", "ANSI_SQL") - expr = dialect_expr.get("expression") - if expr is not None: - if dialect not in DIALECT_MAP: - unknown_dialects[dialect] += 1 - context = f"Metric '{metric_name}' in model '{model_name}' ({dialect})" - error = validate_sql_expression(expr, dialect, context) - if error: - errors.append(error) - - # One nonfatal warning per unrecognized dialect, sorted for deterministic output. - for dialect in sorted(unknown_dialects): - count = unknown_dialects[dialect] + for dialect in sorted(unsupported_dialects): + count = unsupported_dialects[dialect] plural = "s" if count != 1 else "" errors.append(ValidationWarning( - f"[SQL] Warning: unknown dialect '{dialect}'; {count} expression{plural} not validated" + f"[SQL] Warning: dialect '{dialect}' is not supported by this validator; " + f"{count} expression{plural} not syntax-checked" )) return errors @@ -590,19 +621,12 @@ def validate_metric_scoping(data: dict) -> list[str]: if not isinstance(data, dict) or "datasets" not in data: return [] - model = data + model_name = data.get("name", "") errors = [] - model_name = model.get("name", "") - def scope_errors( - expr: str, dialect: str, dataset_name: str, metric_name: str, field_names: set - ) -> list[str]: - # Parse once; an empty, unparseable, or unknown-dialect expression yields no - # tree and is skipped (reported, if at all, by `validate_sql`). + def scope_errors(expression: Expression) -> list[str]: tree, _ = _parse_sql_expression( - expr, - dialect, - f"Metric '{dataset_name}.{metric_name}' in model '{model_name}'", + expression.text, expression.dialect, expression.context(model_name) ) if tree is None: return [] @@ -613,12 +637,12 @@ def scope_errors( parts = [part.name for part in col.parts] if len(parts) > 1 and parts[0]: qualifiers.add(parts[0]) - elif parts and parts[0].casefold() not in field_names: + elif parts and parts[0].casefold() not in expression.field_names: undeclared.add(parts[0]) prefix = ( - f"[Scope] Dataset-scoped metric '{dataset_name}.{metric_name}' in model " - f"'{model_name}' ({dialect})" + f"[Scope] Dataset-scoped metric '{expression.dataset_name}.{expression.name}' " + f"in model '{model_name}' ({expression.dialect})" ) out = [] if qualifiers: @@ -637,30 +661,16 @@ def scope_errors( ) return out - for dataset in model.get("datasets", []): - dataset_name = dataset.get("name", "") - field_names = { - f["name"].casefold() for f in dataset.get("fields", []) if f.get("name") - } - for metric in dataset.get("metrics", []): - metric_name = metric.get("name", "") - expression = metric.get("expression") or {} - # The portable OSSIE_SQL_2026 form is scope-checked like its dialect siblings. - portable = expression.get("sql") - if portable: - errors.extend( - scope_errors( - portable, "OSSIE_SQL_2026", dataset_name, metric_name, field_names - ) - ) - for dialect_expr in expression.get("dialects", []): - dialect = dialect_expr.get("dialect", "ANSI_SQL") - expr = dialect_expr.get("expression") - if not expr: - continue - errors.extend( - scope_errors(expr, dialect, dataset_name, metric_name, field_names) - ) + for expression in iter_expressions(data): + # A superseded OSSIE_SQL_2026 dialect entry is non-authoritative; the + # top-level `sql` it defers to is scope-checked in its place. + if ( + expression.kind != "dataset_metric" + or not expression.text + or expression.superseded + ): + continue + errors.extend(scope_errors(expression)) return errors From d996f3ee20c118525fc3330647d93e065e6c35dc Mon Sep 17 00:00:00 2001 From: Chris Eubank <108756251+christianeu-db@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:24:29 +0000 Subject: [PATCH 7/9] fix(validation): tighten sql/dialect handling per PR #617 review Addresses review feedback on the decouple-core-spec PR (apache/ossie#617): - Require non-empty expression and identity strings: add minLength 1 to `sql`, `dialect`, the per-dialect `expression`, and `vendor_name` in both the schema and the SDK, so an empty value is rejected there rather than slipping through -- an empty `sql`/`expression` reached the SQL validator, an empty `dialect` produced an "is not supported" warning, and an empty `vendor_name` was unchecked. Suggested by @abolfazlmadanii. - Warn when a non-authoritative OSSIE_SQL_2026 dialect entry drifts from the authoritative top-level `sql`: a differing or invalid entry is dead text a client might still read, so report it as a nonfatal warning; an identical one stays silent. - Suggest the closest known dialect on an unsupported one (difflib, cutoff 0.8), so `SNOWFLKE` says "Did you mean 'SNOWFLAKE'?" without flagging a genuinely new dialect. - extensions/README.md: note that dialect and vendor tokens are matched exactly and are case-sensitive, so `snowflake` is distinct from `SNOWFLAKE`. - Describe custom-extension `data` as an opaque string, not a JSON string, in the schema and SDK, matching the spec/docs wording. Co-authored-by: Isaac --- core-spec/ossie-schema.json | 10 ++- extensions/README.md | 4 +- python/src/ossie/models.py | 10 +-- python/tests/test_validation.py | 24 +++++++ validation/tests/test_validate.py | 116 ++++++++++++++++++++++++++---- validation/validate.py | 50 ++++++++----- 6 files changed, 174 insertions(+), 40 deletions(-) diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index 93c405191..cf7949b19 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -37,13 +37,15 @@ "$defs": { "Dialect": { "type": "string", + "minLength": 1, "examples": ["OSSIE_SQL_2026", "ANSI_SQL", "DB_DIALECT", "BI_EXPRESSION", "SEMANTIC_MODEL_EXPRESSION", "PROGRAMMING_LANGUAGE"], - "description": "Expression language a metric or field expression is declared in. Any string is accepted. OSSIE_SQL_2026 is the only dialect this specification defines; others are opaque to Ossie. See extensions/dialects.md." + "description": "Expression language a metric or field expression is declared in. Any non-empty string is accepted. OSSIE_SQL_2026 is the only dialect this specification defines; others are opaque to Ossie. See extensions/dialects.md." }, "Vendor": { "type": "string", + "minLength": 1, "examples": ["COMMON", "BI_TOOL", "DB_VENDOR", "SEMANTIC_LAYER"], - "description": "Vendor name for custom extensions. Any string is accepted. See extensions/vendors.md." + "description": "Vendor name for custom extensions. Any non-empty string is accepted. See extensions/vendors.md." }, "AIContext": { "description": "Additional context for AI tools", @@ -86,7 +88,7 @@ }, "data": { "type": "string", - "description": "JSON string containing vendor-specific data" + "description": "Opaque string of vendor-specific data that Ossie carries but does not interpret" } }, "required": ["vendor_name", "data"], @@ -101,6 +103,7 @@ }, "expression": { "type": "string", + "minLength": 1, "description": "SQL or dialect-specific expression" } }, @@ -113,6 +116,7 @@ "properties": { "sql": { "type": "string", + "minLength": 1, "description": "Primary, portable expression written in OSSIE_SQL_2026. When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative." }, "dialects": { diff --git a/extensions/README.md b/extensions/README.md index cdb0e7cd0..59a04d941 100644 --- a/extensions/README.md +++ b/extensions/README.md @@ -29,7 +29,9 @@ vendor tokens, so that producers and consumers spell them the same way. The list here are a shared registry, not part of the normative spec. **Presence in a list does not imply endorsement of a tool or vendor**, and the lists are not exhaustive: `dialect` and `vendor_name` are free-form strings, so any value is -valid whether or not it appears here. +valid whether or not it appears here. Tokens are matched exactly and are +case-sensitive, so register the canonical upper-case form (`SNOWFLAKE`, not +`snowflake`) to keep producers and consumers aligned. ## Contents diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 207ac3d0b..6c96c3c71 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -203,11 +203,11 @@ def _serialize_context(self, handler: SerializerFunctionWrapHandler, info: Seria class OssieCustomExtension(BaseModel): - """Vendor-specific metadata as a serialized JSON string.""" + """Vendor-specific metadata carried as an opaque string.""" model_config = ConfigDict(frozen=True, extra="forbid") - vendor_name: str + vendor_name: str = Field(min_length=1) data: str @@ -217,8 +217,8 @@ class OssieDialectExpression(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") # Free-form; see OssieDialect for the well-known values. - dialect: str - expression: str + dialect: str = Field(min_length=1) + expression: str = Field(min_length=1) class OssieExpression(BaseModel): @@ -227,7 +227,7 @@ class OssieExpression(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") # Primary, portable expression written in OSSIE_SQL_2026. - sql: Optional[str] = None + sql: Optional[str] = Field(default=None, min_length=1) dialects: Optional[list[OssieDialectExpression]] = Field(default=None, min_length=1) @model_validator(mode="after") diff --git a/python/tests/test_validation.py b/python/tests/test_validation.py index 99df25f64..f23353254 100644 --- a/python/tests/test_validation.py +++ b/python/tests/test_validation.py @@ -182,6 +182,30 @@ def test_expression_empty_dialects_is_rejected() -> None: OssieExpression(dialects=[]) +def test_expression_empty_sql_is_rejected() -> None: + # `sql` has min_length 1, matching the schema, so an empty portable form is rejected. + with pytest.raises(ValidationError): + OssieExpression(sql="") + + +def test_dialect_expression_empty_dialect_is_rejected() -> None: + # `dialect` has min_length 1, matching the schema, so an empty dialect is rejected. + with pytest.raises(ValidationError): + OssieDialectExpression(dialect="", expression="order_id") + + +def test_dialect_expression_empty_expression_is_rejected() -> None: + # The per-dialect `expression` has min_length 1, matching the schema. + with pytest.raises(ValidationError): + OssieDialectExpression(dialect="ANSI_SQL", expression="") + + +def test_custom_extension_empty_vendor_name_is_rejected() -> None: + # `vendor_name` has min_length 1, matching the schema. + with pytest.raises(ValidationError): + OssieCustomExtension(vendor_name="", data="{}") + + def test_free_form_dialect_in_expression() -> None: # `dialect` is free-form: an unknown value is accepted and preserved. dialect_expr = OssieDialectExpression(dialect="DB_DIALECT", expression="order_id") diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 9df6460f3..2f2e14b1a 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -436,6 +436,49 @@ def test_schema_rejects_empty_dialects_without_portable_form(core_schema: dict) ] +def test_schema_rejects_empty_portable_sql(core_schema: dict) -> None: + # `sql` requires minLength 1, so an empty portable form fails the schema + # rather than slipping through to the SQL validator. + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {"sql": ""}}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> sql: '' should be non-empty" + ] + + +def test_schema_rejects_empty_dialect(core_schema: dict) -> None: + # `dialect` requires minLength 1, so an empty dialect fails the schema rather + # than producing an "is not supported by this validator" warning. + expression = {"dialects": [{"dialect": "", "expression": "x"}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> dialects -> 0 -> dialect: " + "'' should be non-empty" + ] + + +def test_schema_rejects_empty_dialect_expression(core_schema: dict) -> None: + # The per-dialect `expression` requires minLength 1, like `sql`. + expression = {"dialects": [{"dialect": "ANSI_SQL", "expression": ""}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> dialects -> 0 -> " + "expression: '' should be non-empty" + ] + + +def test_schema_rejects_empty_vendor_name(core_schema: dict) -> None: + # `vendor_name` requires minLength 1. + dataset = {**_ORDERS, "custom_extensions": [{"vendor_name": "", "data": "{}"}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> custom_extensions -> 0 -> vendor_name: " + "'' should be non-empty" + ] + + def test_unsupported_dialect_emits_one_deduped_warning(monkeypatch: pytest.MonkeyPatch) -> None: # A dialect this validator cannot parse is nonfatal, but one warning per dialect # reports that its expressions went unchecked. The field and metric below share @@ -483,6 +526,20 @@ def test_known_unparseable_dialect_is_not_flagged( assert _VALIDATE.validate_sql(_document([dataset], [])) == [] +def test_unsupported_dialect_warning_suggests_a_close_match( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A near-miss of a known dialect gets a "did you mean" hint. + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + expression = {"dialects": [{"dialect": "SNOWFLKE", "expression": "x"}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + + assert _VALIDATE.validate_sql(_document([dataset], [])) == [ + "[SQL] Warning: dialect 'SNOWFLKE' is not supported by this validator; " + "1 expression not syntax-checked. Did you mean 'SNOWFLAKE'?" + ] + + @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") @pytest.mark.parametrize( "expression", @@ -769,8 +826,9 @@ def test_sql_error_in_metric_with_warning_in_name_is_an_error(run_validator): assert "[SQL] Metric 'Warning: broken_metric'" in output -@pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") def test_empty_field_expression_fails_validation(run_validator): + # An empty per-dialect expression now fails the schema (minLength 1), before + # the SQL check runs. dataset = {**_ORDERS, "fields": [{ "name": "blank", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": ""}]}, @@ -780,7 +838,10 @@ def test_empty_field_expression_fails_validation(run_validator): exit_code, output = run_validator(document) assert exit_code == 1 - assert "[SQL] Field 'orders.blank' in model 'm' (ANSI_SQL): expression is empty" in output + assert ( + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> dialects -> 0 -> " + "expression: '' should be non-empty" in output + ) @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") @@ -1460,27 +1521,54 @@ def test_dataset_scoped_metric_portable_sql_is_sql_validated() -> None: assert errors[0].startswith("[SQL] Metric 'orders.broken' in model 'm' (OSSIE_SQL_2026):") -def test_invalid_superseded_ossie_sql_2026_dialect_entry_warns_not_fails() -> None: - # The top-level `sql` is authoritative, so an invalid OSSIE_SQL_2026 dialect - # entry beside it is reported as a nonfatal warning, not a hard error. - expression = { - "sql": "SUM(amount)", - "dialects": [{"dialect": "OSSIE_SQL_2026", "expression": "SUM("}], - } - doc = _orders_with( +def _metric_with_sql_and_ossie_dialect(sql: str, dialect_sql: str) -> dict: + return _orders_with( fields=[{"name": "amount", "expression": _ds_expr("amount")}], - metrics=[{"name": "total", "expression": expression}], + metrics=[ + { + "name": "total", + "expression": { + "sql": sql, + "dialects": [{"dialect": "OSSIE_SQL_2026", "expression": dialect_sql}], + }, + } + ], ) - errors = _VALIDATE.validate_sql(doc) + +def test_invalid_superseded_ossie_sql_2026_dialect_entry_warns_not_fails() -> None: + # The top-level `sql` is authoritative, so an invalid OSSIE_SQL_2026 dialect + # entry beside it is reported as a nonfatal warning, not a hard error. + errors = _VALIDATE.validate_sql(_metric_with_sql_and_ossie_dialect("SUM(amount)", "SUM(")) assert errors == [ - "[SQL] Warning: Metric 'orders.total' in model 'm' (OSSIE_SQL_2026): not valid " - "SQL; ignored because the top-level 'sql' is authoritative" + "[SQL] Warning: Metric 'orders.total' in model 'm' (OSSIE_SQL_2026): dialect entry " + "is not valid SQL and is ignored; the top-level 'sql' is authoritative" ] assert all(isinstance(e, _VALIDATE.ValidationWarning) for e in errors) +def test_differing_superseded_ossie_sql_2026_dialect_entry_warns() -> None: + # A valid OSSIE_SQL_2026 dialect entry that differs from the authoritative + # top-level `sql` is dead text; warn so the drift is caught. + errors = _VALIDATE.validate_sql( + _metric_with_sql_and_ossie_dialect("SUM(amount)", "AVG(amount)") + ) + + assert errors == [ + "[SQL] Warning: Metric 'orders.total' in model 'm' (OSSIE_SQL_2026): dialect entry " + "differs from the top-level 'sql' and is ignored; the top-level 'sql' is authoritative" + ] + + +def test_identical_superseded_ossie_sql_2026_dialect_entry_is_silent() -> None: + # An OSSIE_SQL_2026 dialect entry identical to the top-level `sql` is harmless + # redundancy, not drift, so it is not warned. + assert _VALIDATE.validate_sql( + _metric_with_sql_and_ossie_dialect("SUM(amount)", "SUM(amount)") + ) == [] + + def test_superseded_ossie_sql_2026_dialect_entry_is_not_scope_checked() -> None: # The authoritative top-level `sql` is scope-checked; a superseded # OSSIE_SQL_2026 dialect entry beside it is not, so its cross-dataset diff --git a/validation/validate.py b/validation/validate.py index 68dd64170..f212f59ae 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -52,6 +52,7 @@ or --schema at a copy of the schema. """ +import difflib import json import os import sys @@ -109,6 +110,9 @@ "HOLISTICS_AQL", } +# Every dialect this validator knows, for close-match suggestions on a typo. +KNOWN_DIALECTS = (*SQLGLOT_DIALECTS, *NON_SQL_DIALECTS) + # Concepts every ontology includes implicitly (ontology.md, "Built-in concepts") BUILT_IN_CONCEPTS = {"Any", "Boolean", "Date", "DateTime", "Decimal", "Float", "Integer", "String"} @@ -477,9 +481,10 @@ class Expression: `kind` identifies the declaring element, and `name` identifies the field or metric. `dialect` is OSSIE_SQL_2026 for portable `sql` expressions, or the dialect's label otherwise. `field_names` contains the enclosing - dataset's field names, or is empty for model-scoped metrics. `superseded` - marks an OSSIE_SQL_2026 dialect entry that a top-level `sql` outranks, so - it is non-authoritative: a client reads the `sql` instead. + dataset's field names, or is empty for model-scoped metrics. + `superseded_by` is the authoritative top-level `sql` text when this is an + OSSIE_SQL_2026 dialect entry that `sql` outranks, else None: such an entry + is non-authoritative, since a client reads the `sql` instead. """ kind: ExpressionKind @@ -488,7 +493,7 @@ class Expression: dialect: str text: str field_names: frozenset[str] - superseded: bool = False + superseded_by: str | None = None def context(self, model_name: str) -> str: if self.kind == "field": @@ -529,7 +534,7 @@ def _expressions_in( field_names=field_names, # A top-level `sql` is the authoritative OSSIE_SQL_2026 form, so an # OSSIE_SQL_2026 dialect entry alongside it is non-authoritative. - superseded=sql is not None and dialect == PORTABLE_DIALECT, + superseded_by=sql if dialect == PORTABLE_DIALECT and sql is not None else None, ) @@ -578,26 +583,37 @@ def validate_sql(data: dict) -> list[str]: error = validate_sql_expression( expression.text, dialect, expression.context(model_name) ) - if not error: - continue - if expression.superseded: - # A top-level `sql` is authoritative, so an invalid OSSIE_SQL_2026 - # dialect entry does not fail the document; warn, since a client may - # still read the dialect entry instead of `sql`. + if expression.superseded_by is not None: + # A top-level `sql` is authoritative, so this OSSIE_SQL_2026 dialect + # entry does not fail the document. Warn if it is dead text -- invalid, + # or differing from the `sql` a client reads instead -- and stay silent + # when it merely duplicates the `sql`. + if error: + reason = "is not valid SQL" + elif expression.text != expression.superseded_by: + reason = "differs from the top-level 'sql'" + else: + continue errors.append(ValidationWarning( - f"[SQL] Warning: {expression.context(model_name)}: not valid SQL; " - f"ignored because the top-level 'sql' is authoritative" + f"[SQL] Warning: {expression.context(model_name)}: dialect entry {reason} " + f"and is ignored; the top-level 'sql' is authoritative" )) - else: + elif error: errors.append(error) for dialect in sorted(unsupported_dialects): count = unsupported_dialects[dialect] plural = "s" if count != 1 else "" - errors.append(ValidationWarning( + message = ( f"[SQL] Warning: dialect '{dialect}' is not supported by this validator; " f"{count} expression{plural} not syntax-checked" - )) + ) + # Only a close match is suggested, so a legitimate new dialect gets no + # spurious "did you mean". + suggestion = difflib.get_close_matches(dialect, KNOWN_DIALECTS, n=1, cutoff=0.8) + if suggestion: + message += f". Did you mean '{suggestion[0]}'?" + errors.append(ValidationWarning(message)) return errors @@ -667,7 +683,7 @@ def scope_errors(expression: Expression) -> list[str]: if ( expression.kind != "dataset_metric" or not expression.text - or expression.superseded + or expression.superseded_by is not None ): continue errors.extend(scope_errors(expression)) From 43d3e9102c885d9a9d4f0588892b6dc52b8909cf Mon Sep 17 00:00:00 2001 From: Chris Eubank <108756251+christianeu-db@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:01:44 +0000 Subject: [PATCH 8/9] docs(extensions): register ATACCAMA in the vendor catalog Add the ATACCAMA token to the known-vendor table, per apache/ossie#629. Co-authored-by: Isaac --- extensions/vendors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions/vendors.md b/extensions/vendors.md index 6daf4d825..c0e84377a 100644 --- a/extensions/vendors.md +++ b/extensions/vendors.md @@ -48,6 +48,7 @@ community uses (see [`README.md`](README.md) for how these lists are used). | `SIGMA` | Sigma Computing-specific attributes | | `THOUGHTSPOT` | ThoughtSpot-specific attributes | | `HEX` | Hex-specific attributes | +| `ATACCAMA` | Ataccama-specific attributes | ## Examples From 747f36fbef26c8da06d2e6f828f81fa94a9ec136 Mon Sep 17 00:00:00 2001 From: Chris Eubank <108756251+christianeu-db@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:59:54 +0000 Subject: [PATCH 9/9] fix(ci): point #630's precedence tests at SQLGLOT_DIALECTS #630 added OSSIE_SQL_2026 precedence tests whose helper parses through `DIALECT_MAP["OSSIE_SQL_2026"]`. This branch split that map into `SQLGLOT_DIALECTS` (parseable) and `NON_SQL_DIALECTS`, so the merged-in helper hit `AttributeError: module 'ossie_validate' has no attribute 'DIALECT_MAP'`. Point it at `SQLGLOT_DIALECTS[PORTABLE_DIALECT]`, the same None (sqlglot default) entry, so the three tests pass unchanged. Co-authored-by: Isaac --- validation/tests/test_validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 38e2e132a..ece083f1e 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -377,7 +377,7 @@ def test_flags_an_invalid_ossie_sql_2026_expression() -> None: def _parse_ossie_sql(expression: str): """Parse *expression* the way the validator parses an OSSIE_SQL_2026 body.""" return _VALIDATE.sqlglot.parse_one( - expression, dialect=_VALIDATE.DIALECT_MAP["OSSIE_SQL_2026"] + expression, dialect=_VALIDATE.SQLGLOT_DIALECTS[_VALIDATE.PORTABLE_DIALECT] )