diff --git a/core-spec/expression_language.md b/core-spec/expression_language.md index 9cf1580d..a673fdcc 100644 --- a/core-spec/expression_language.md +++ b/core-spec/expression_language.md @@ -705,6 +705,8 @@ Ossie implementations MAY support additional functions through dialect-specific The Ossie dialect should always be supported. Other dialects MAY be ignored. There is no guarantee that all different dialects for an expression will act the same, so implementations should be consistent with their dialect handling. This means that if an Ossie model has an expression written in two dialects, the implementation should deterministically choose which dialect to use. +The `dialect` field is a free-form string; the core specification does not enumerate a closed set. For a non-normative catalog of dialects well-known tools use, see [extensions/dialects.md](../extensions/dialects.md). + ### Declaring Dialect-Specific Expressions ``` diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index a2faf7b0..cf7949b1 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -37,13 +37,15 @@ "$defs": { "Dialect": { "type": "string", - "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY", "SIGMA", "THOUGHTSPOT", "DAX", "HOLISTICS_AQL", "OSSIE_SQL_2026"], - "description": "Supported SQL and expression language dialects" + "minLength": 1, + "examples": ["OSSIE_SQL_2026", "ANSI_SQL", "DB_DIALECT", "BI_EXPRESSION", "SEMANTIC_MODEL_EXPRESSION", "PROGRAMMING_LANGUAGE"], + "description": "Expression language a metric or field expression is declared in. Any non-empty string is accepted. OSSIE_SQL_2026 is the only dialect this specification defines; others are opaque to Ossie. See extensions/dialects.md." }, "Vendor": { "type": "string", - "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM", "POWER_BI", "HEX"], - "description": "Vendor name for custom extensions. Any string value is accepted." + "minLength": 1, + "examples": ["COMMON", "BI_TOOL", "DB_VENDOR", "SEMANTIC_LAYER"], + "description": "Vendor name for custom extensions. Any non-empty string is accepted. See extensions/vendors.md." }, "AIContext": { "description": "Additional context for AI tools", @@ -86,7 +88,7 @@ }, "data": { "type": "string", - "description": "JSON string containing vendor-specific data" + "description": "Opaque string of vendor-specific data that Ossie carries but does not interpret" } }, "required": ["vendor_name", "data"], @@ -101,6 +103,7 @@ }, "expression": { "type": "string", + "minLength": 1, "description": "SQL or dialect-specific expression" } }, @@ -111,6 +114,11 @@ "type": "object", "description": "Expression definition with multi-dialect support", "properties": { + "sql": { + "type": "string", + "minLength": 1, + "description": "Primary, portable expression written in OSSIE_SQL_2026. When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative." + }, "dialects": { "type": "array", "items": { @@ -119,7 +127,10 @@ "minItems": 1 } }, - "required": ["dialects"], + "anyOf": [ + { "required": ["sql"] }, + { "required": ["dialects"] } + ], "additionalProperties": false }, "DataType": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 3afae56d..259adc43 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -48,30 +48,18 @@ Standard enumeration values used throughout the specification. ### Dialects -Dialects in which metric and field expressions can be declared. - -| Dialect | Description | -|---------|-------------| -| `ANSI_SQL` | Standard SQL dialect | -| `SNOWFLAKE` | Snowflake SQL | -| `MDX` | Multi-Dimensional Expressions | -| `TABLEAU` | Tableau calculations | -| `DATABRICKS` | Databricks SQL | -| `MAQL` | GoodData MAQL (Multi-Dimensional Analytical Query Language) | -| `BIGQUERY` | Google BigQuery (GoogleSQL) | -| `SIGMA` | Sigma Computing's spreadsheet-style formula language | -| `THOUGHTSPOT` | ThoughtSpot formula language | -| `DAX` | Data Analysis Expressions (Power BI / Analysis Services) | -| `HOLISTICS_AQL` | Holistics AQL (Analytical Query Language) | -| `OSSIE_SQL_2026` | Ossie's portable SQL expression language, defined in `expression_language.md` | - -`OSSIE_SQL_2026` is the only dialect whose semantics this specification defines. The other values -identify vendor or third-party expression languages. Ossie treats expressions in those dialects as -opaque, and implementations MAY ignore them (see -[Dialect Extensions](expression_language.md#dialect-extensions)). Registering a dialect makes it -valid for declaring a model; it does not mean Ossie supports querying in that dialect (see +The `dialect` field is a free-form string, allowing any expression language to be used +without requiring changes to the core specification. + +`OSSIE_SQL_2026` is the only dialect whose semantics this specification defines (Ossie's +portable SQL expression language, see `expression_language.md`). All other values identify +vendor or third-party expression languages, which Ossie treats as opaque and implementations +MAY ignore. Registering a dialect does not mean Ossie supports querying in it (see [Model Declaration vs. Querying](expression_language.md#model-declaration-vs-querying)). +For a non-normative catalog of dialects well-known tools use, see +[extensions/dialects.md](../extensions/dialects.md). + ### Data types `DataType` declares the logical value type of a field or metric independently @@ -310,8 +298,9 @@ The expression object supports multiple SQL dialects for cross-platform compatib ```yaml expression: + sql: "customer_id" # Primary, portable OSSIE_SQL_2026 form dialects: - - dialect: ANSI_SQL # Must be one of the dialects enum values + - dialect: ANSI_SQL # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: "customer_id" # Scalar SQL expression ``` @@ -319,6 +308,9 @@ expression: - Use scalar SQL expressions (no aggregations) - Can be simple column references (e.g., `customer_id`) or computed expressions (e.g., `first_name || ' ' || last_name`) +- The optional top-level `sql` is the primary, portable form, written in OSSIE_SQL_2026 +- At least one of the top-level `sql` or `dialects` MUST be present; either MAY stand alone +- If both the top-level `sql` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `sql` is authoritative and consumers MUST use it as the OSSIE_SQL_2026 form - Identifiers refer to columns of the dataset's `source`, not to other fields. See [Name Resolution](#name-resolution). - Multiple dialect versions can be provided for the same field @@ -448,11 +440,14 @@ The expression object supports multiple dialects ```yaml expression: + sql: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" # Primary, portable OSSIE_SQL_2026 form dialects: - dialect: ANSI_SQL # Default expression: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" ``` +At least one of the top-level `sql` or `dialects` MUST be present; either MAY stand alone. When both a top-level `sql` and an `OSSIE_SQL_2026` entry in `dialects` are present, the top-level `sql` is authoritative. + ### Examples **Simple Aggregation:** @@ -589,70 +584,19 @@ custom_extensions: The `vendor_name` field is a free-form string, allowing any vendor or organization to define custom extensions without requiring changes to the core specification. -The following are well-known examples: - -| Vendor | Description | -|--------|-------------| -| `COMMON` | Common/standard extensions | -| `SNOWFLAKE` | Snowflake-specific attributes | -| `SALESFORCE` | Salesforce/Tableau-specific attributes | -| `DBT` | dbt-specific attributes | -| `DATABRICKS` | Databricks-specific attributes | -| `GOODDATA` | GoodData-specific attributes | -| `HONEYDEW` | Honeydew-specific attributes | -| `WISDOM` | WisdomAI-specific attributes | -| `POWER_BI` | Microsoft Power BI-specific attributes | -| `SIGMA` | Sigma Computing-specific attributes | -| `THOUGHTSPOT` | ThoughtSpot-specific attributes | -| `HEX` | Hex-specific attributes | - -### Examples - -**Snowflake Extension:** - -```yaml -- vendor_name: SNOWFLAKE - data: '{ - "warehouse": "ANALYTICS_WH", - "database": "PROD", - "schema": "PUBLIC" - }' -``` +For a non-normative catalog of `vendor_name` values well-known tools use, see +[extensions/vendors.md](../extensions/vendors.md). -**Salesforce Extension:** - -```yaml -- vendor_name: SALESFORCE - data: '{ - "tableau_workbook_id": "sales_dashboard", - "einstein_enabled": true, - "crm_sync": { - "enabled": true, - "sync_frequency": "daily" - } - }' -``` - -**DBT Extension:** +### Example ```yaml -- vendor_name: DBT - data: '{ - "project_name": "analytics", - "materialized": "table", - "tags": ["daily", "core"] - }' +- vendor_name: DB_VENDOR + data: '{"some_setting": "value", "nested": {"option": true}}' ``` -**Databricks Extension:** - -```yaml -- vendor_name: Databricks - data: '{ - "default_catalog": "finance", - "default_schema": "gold" - }' -``` +The `data` payload is an opaque string that Ossie carries but does not +interpret. For vendor-specific extension examples and the known-vendor catalog, +see [extensions/vendors.md](../extensions/vendors.md). --- diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index d56629c5..58c66569 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -28,21 +28,8 @@ version: 0.2.0.dev0 # Enumerations # Standard enums used throughout the specification -# Supported expression language dialects -# Auto-generated from ossie-schema.json ($defs.Dialect.enum). -dialects: - - "ANSI_SQL" # Standard SQL dialect - - "SNOWFLAKE" # Snowflake - - "MDX" # Multi-Dimensional Expressions - - "TABLEAU" # Tableau - - "DATABRICKS" # Databricks SQL - - "MAQL" # GoodData MAQL (Multi-Dimensional Analytical Query Language) - - "BIGQUERY" # Google BigQuery GoogleSQL - - "SIGMA" # Sigma Computing spreadsheet-style formula language - - "THOUGHTSPOT" # ThoughtSpot formula language (not SQL) - - "DAX" # Data Analysis Expressions (Power BI / Analysis Services) - - "HOLISTICS_AQL" # Holistics AQL (Analytical Query Language) - - "OSSIE_SQL_2026" # Ossie portable SQL expression language (see expression_language.md) +# The `dialect` field is a free-form string, not a closed set. OSSIE_SQL_2026 is +# the only dialect this spec defines. See extensions/dialects.md. # Supported logical data types for fields and metrics # Auto-generated from ossie-schema.json ($defs.DataType.enum). @@ -58,8 +45,8 @@ datatypes: - "DateTimeTz" # Instant identified using offset or timezone context - "Opaque" # Known type outside the portable vocabulary -# Vendor name for custom extensions (free-form string) -# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM", "POWER_BI", "HEX" +# Vendor name for custom extensions (free-form string). +# See the known-vendor catalog in extensions/vendors.md. vendor_name: string @@ -230,9 +217,11 @@ fields: # Each field can have expressions in different dialects for portability # Can be a simple column reference or a complex scalar expression # Identifiers refer to columns of the dataset's source, not to other fields + # At least one of the portable `sql` or `dialects` must be present; either may stand alone expression: + sql: string # Primary, portable OSSIE_SQL_2026 form, e.g., "customer_id", "first_name || ' ' || last_name" dialects: - - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + - dialect: string # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: string # SQL scalar expression, e.g., "customer_id", "first_name || ' ' || last_name", "UPPER(email)" # Optional: Dimension metadata @@ -293,9 +282,11 @@ metrics: # Model-scoped metrics qualify them as dataset.field; dataset-scoped # metrics use unqualified field names # Metrics cannot reference other metrics + # At least one of the portable `sql` or `dialects` must be present; either may stand alone expression: + sql: string # Primary, portable OSSIE_SQL_2026 form, e.g., "SUM(orders.sales)", "AVG(orders.amount)" dialects: - - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + - dialect: string # Free-form string; OSSIE_SQL_2026 is the only dialect the spec defines expression: string # Full SQL expression with aggregate functions, e.g., "SUM(orders.sales)", "AVG(orders.amount)" # Optional: Human-readable description of the metric diff --git a/docs/index.md b/docs/index.md index 0e67d847..9bc4e26c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -59,7 +59,7 @@ The Ossie core specification (current version: **0.2.0.dev0**, latest released: | **Custom Extensions** | Vendor-specific metadata stored as JSON, allowing platforms to carry additional information without breaking core compatibility. | | **AI Context** | Optional annotations at every level (model, dataset, field, relationship, metric) to help AI tools understand business meaning — including instructions, synonyms, and example queries. | -The specification supports multiple SQL dialects (`ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `MDX`, `TABLEAU`) so that expressions can be tailored to each platform while maintaining a common model structure. +An expression carries a primary, portable form in `OSSIE_SQL_2026` and may also carry vendor-specific variants. The `dialect` label is a free-form string, so expressions can be tailored to any platform while maintaining a common model structure. For the full specification, see [core-spec/spec.md](../core-spec/spec.md). For validation tooling, see [validation/validate.py](../validation/validate.py). For a complete example, see the [TPC-DS semantic model](../examples/tpcds_semantic_model.yaml). @@ -195,20 +195,19 @@ A typical Ossie-based workflow looks like this: ### Multi-Dialect Expression System -A key architectural feature of Ossie is its multi-dialect expression system. Fields and metrics can carry expressions in multiple SQL dialects simultaneously: +A key architectural feature of Ossie is its multi-dialect expression system. An expression carries a primary, portable form in `OSSIE_SQL_2026`, and fields and metrics can also carry vendor-specific variants in multiple dialects simultaneously: ```yaml expression: + sql: LOWER(email) # primary, portable OSSIE_SQL_2026 form dialects: - - dialect: ANSI_SQL - expression: LOWER(email) - dialect: SNOWFLAKE expression: LOWER(email)::VARCHAR - dialect: DATABRICKS expression: lower(email) ``` -This allows a single semantic model to be consumed natively by different platforms without expression translation. Each converter selects the dialect that matches its target platform, falling back to `ANSI_SQL` when a platform-specific dialect is not available. +This allows a single semantic model to be consumed natively by different platforms without expression translation. Each converter selects the dialect that matches its target platform, falling back to the portable `OSSIE_SQL_2026` form when a platform-specific dialect is not available. The `dialect` label is a free-form string; `OSSIE_SQL_2026` is the only dialect Ossie defines, and all others are opaque. --- @@ -251,7 +250,7 @@ Through `custom_extensions`. Each vendor can store arbitrary JSON metadata in ex In the expression, only a model-scoped metric can — it uses multi-part `dataset.field` references and combines datasets. A dataset-scoped metric's expression stays within its own dataset. See [Metric Scoping](https://github.com/apache/ossie/blob/main/core-spec/spec.md#metric-scoping). **What SQL dialects are supported?** -The current specification supports `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `MDX`, and `TABLEAU`. New dialects can be proposed through the standard specification change process. +The `dialect` label is a free-form string, so any dialect can be used without a specification change. `OSSIE_SQL_2026` is the only dialect Ossie defines; all others are opaque and an implementation MAY ignore them. For a non-normative catalog of dialects well-known tools use, see [extensions/dialects.md](../extensions/dialects.md). --- @@ -320,8 +319,8 @@ A practical guide for organizations looking to adopt Ossie. | **Dimension** | A categorical attribute used to slice and filter data (e.g., region, product category, date). In Ossie, dimensions are represented as fields with optional metadata such as `is_time`. | | **Metric** | A quantitative measure computed by aggregating fields (e.g., total revenue, average order value). A dataset-scoped metric is defined on an individual dataset, for a measure that semantically belongs to it and is evaluated at the dataset's row level; a model-scoped metric is defined at the model root and may span datasets. | | **Relationship** | A foreign key connection between two datasets, defining how they can be joined. Relationships are always many-to-one (from the referencing dataset to the referenced dataset). | -| **Dialect** | A specific SQL or expression language variant (e.g., `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`). Ossie supports multiple dialects so expressions can be tailored to each platform. | -| **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as a JSON string. Extensions allow platforms to carry additional information without modifying the core specification. | +| **Dialect** | A SQL or expression language variant an expression is written in (e.g., `ANSI_SQL`). A free-form string; `OSSIE_SQL_2026` is the only dialect Ossie defines, and all others are opaque. | +| **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as an opaque string. Extensions allow platforms to carry additional information without modifying the core specification. | | **AI Context** | Optional annotations on any Ossie construct (model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | | **Converter** | A tool that translates between the Ossie format and a specific vendor's semantic model format. Converters come in pairs: import (vendor → Ossie) and export (Ossie → vendor). | | **Hub-and-Spoke** | The architectural pattern used by Ossie, where the specification acts as the central format (hub) and vendor converters act as spokes, avoiding the need for point-to-point integrations. | diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index 7837032f..347a7e2a 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -309,6 +309,8 @@ datasets: - name: customer_full_name expression: + # Primary, portable OSSIE_SQL_2026 form + sql: c_first_name || ' ' || c_last_name dialects: - dialect: ANSI_SQL expression: c_first_name || ' ' || c_last_name diff --git a/extensions/README.md b/extensions/README.md new file mode 100644 index 00000000..59a04d94 --- /dev/null +++ b/extensions/README.md @@ -0,0 +1,41 @@ + + +# Apache Ossie - Extensions + +The core specification ([`core-spec/`](../core-spec/)) is vendor-neutral. It +defines one expression language, `OSSIE_SQL_2026`, and treats every other dialect +and every `custom_extensions` vendor as extension metadata that Ossie carries but +does not interpret. + +This folder is where the community aligns on the meaning of specific dialect and +vendor tokens, so that producers and consumers spell them the same way. The lists +here are a shared registry, not part of the normative spec. **Presence in a list +does not imply endorsement of a tool or vendor**, and the lists are not +exhaustive: `dialect` and `vendor_name` are free-form strings, so any value is +valid whether or not it appears here. Tokens are matched exactly and are +case-sensitive, so register the canonical upper-case form (`SNOWFLAKE`, not +`snowflake`) to keep producers and consumers aligned. + +## Contents + +| File | Purpose | +|------|---------| +| [`dialects.md`](dialects.md) | Dialect tokens used to declare metric and field expressions. | +| [`vendors.md`](vendors.md) | `vendor_name` tokens, and the payload convention, for `custom_extensions`. | diff --git a/extensions/dialects.md b/extensions/dialects.md new file mode 100644 index 00000000..2c2ab5c5 --- /dev/null +++ b/extensions/dialects.md @@ -0,0 +1,41 @@ + + +# Dialect Extensions Catalog + +A registry of `dialect` tokens the community uses (see +[`README.md`](README.md) for how these lists are used). `OSSIE_SQL_2026` is the +only dialect the specification defines; all others are opaque to Ossie. + +## Known dialects + +| Dialect | Description | +|---------|-------------| +| `OSSIE_SQL_2026` | Ossie's portable SQL expression language, defined in [`expression_language.md`](../core-spec/expression_language.md) | +| `ANSI_SQL` | Standard SQL dialect | +| `SNOWFLAKE` | Snowflake SQL | +| `MDX` | Multi-Dimensional Expressions | +| `TABLEAU` | Tableau calculations | +| `DATABRICKS` | Databricks SQL | +| `MAQL` | GoodData MAQL (Multi-Dimensional Analytical Query Language) | +| `BIGQUERY` | Google BigQuery (GoogleSQL) | +| `SIGMA` | Sigma Computing's spreadsheet-style formula language | +| `THOUGHTSPOT` | ThoughtSpot formula language | +| `DAX` | Data Analysis Expressions (Power BI / Analysis Services) | +| `HOLISTICS_AQL` | Holistics AQL (Analytical Query Language) | diff --git a/extensions/vendors.md b/extensions/vendors.md new file mode 100644 index 00000000..c0e84377 --- /dev/null +++ b/extensions/vendors.md @@ -0,0 +1,99 @@ + + +# Vendor Extensions Catalog + +Custom extensions attach platform-specific metadata to a model (see +[`core-spec/spec.md`](../core-spec/spec.md#custom-extensions)): + +```yaml +custom_extensions: + - vendor_name: string # Free-form string identifying the vendor + data: string # JSON string containing vendor-specific data +``` + +`vendor_name` is a free-form string and `data` is an opaque string that Ossie +carries but does not interpret. This is a registry of `vendor_name` tokens the +community uses (see [`README.md`](README.md) for how these lists are used). + +## Known vendors + +| Vendor | Description | +|--------|-------------| +| `COMMON` | Common/standard extensions | +| `SNOWFLAKE` | Snowflake-specific attributes | +| `SALESFORCE` | Salesforce/Tableau-specific attributes | +| `DBT` | dbt-specific attributes | +| `DATABRICKS` | Databricks-specific attributes | +| `GOODDATA` | GoodData-specific attributes | +| `HONEYDEW` | Honeydew-specific attributes | +| `WISDOM` | WisdomAI-specific attributes | +| `POWER_BI` | Microsoft Power BI-specific attributes | +| `SIGMA` | Sigma Computing-specific attributes | +| `THOUGHTSPOT` | ThoughtSpot-specific attributes | +| `HEX` | Hex-specific attributes | +| `ATACCAMA` | Ataccama-specific attributes | + +## Examples + +**Snowflake Extension:** + +```yaml +- vendor_name: SNOWFLAKE + data: '{ + "warehouse": "ANALYTICS_WH", + "database": "PROD", + "schema": "PUBLIC" + }' +``` + +**Salesforce Extension:** + +```yaml +- vendor_name: SALESFORCE + data: '{ + "tableau_workbook_id": "sales_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily" + } + }' +``` + +**DBT Extension:** + +```yaml +- vendor_name: DBT + data: '{ + "project_name": "analytics", + "materialized": "table", + "tags": ["daily", "core"] + }' +``` + +**Databricks Extension:** + +```yaml +- vendor_name: Databricks + data: '{ + "default_catalog": "finance", + "default_schema": "gold" + }' +``` diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index fe5efac3..a7bc947b 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -30,6 +30,7 @@ StrictBool, TypeAdapter, model_serializer, + model_validator, ) from ossie._loading import UniqueKeyLoader, unique_json_object @@ -118,7 +119,8 @@ def _invalid_properties(node: Any) -> list[tuple[str, str]]: class OssieDialect(str, Enum): - """Supported SQL and expression language dialects.""" + """Well-known dialect values. The field accepts any string; see + `extensions/dialects.md`.""" ANSI_SQL = "ANSI_SQL" SNOWFLAKE = "SNOWFLAKE" @@ -160,7 +162,8 @@ class OssieDataType(str, Enum): class OssieVendor(str, Enum): - """Well-known vendor names for custom extensions.""" + """Well-known vendor values. The field accepts any string; see + `extensions/vendors.md`.""" COMMON = "COMMON" SNOWFLAKE = "SNOWFLAKE" @@ -203,11 +206,11 @@ def _serialize_context(self, handler: SerializerFunctionWrapHandler, info: Seria class OssieCustomExtension(BaseModel): - """Vendor-specific metadata as a serialized JSON string.""" + """Vendor-specific metadata carried as an opaque string.""" model_config = ConfigDict(frozen=True, extra="forbid") - vendor_name: str + vendor_name: str = Field(min_length=1) data: str @@ -216,8 +219,9 @@ class OssieDialectExpression(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - dialect: OssieDialect - expression: str + # Free-form; see OssieDialect for the well-known values. + dialect: str = Field(min_length=1) + expression: str = Field(min_length=1) class OssieExpression(BaseModel): @@ -225,7 +229,15 @@ class OssieExpression(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - dialects: list[OssieDialectExpression] = Field(..., min_length=1) + # Primary, portable expression written in OSSIE_SQL_2026. + sql: Optional[str] = Field(default=None, min_length=1) + dialects: Optional[list[OssieDialectExpression]] = Field(default=None, min_length=1) + + @model_validator(mode="after") + def _require_sql_or_dialects(self) -> "OssieExpression": + if self.sql is None and not self.dialects: + raise ValueError("expression requires at least one of 'sql' or 'dialects'") + return self class OssieDimension(BaseModel): diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 6b34f55c..040e5f4c 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -45,13 +45,15 @@ # --------------------------------------------------------------------------- -def test_dialect_enum_matches_core_schema() -> None: +def test_dialect_is_free_form_in_core_schema() -> None: + # `dialect` is a free-form string in the core schema (no closed enum). schema_path = Path(__file__).parents[2] / "core-spec" / "ossie-schema.json" schema = json.loads(schema_path.read_text()) - assert {member.value for member in OssieDialect} == set( - schema["$defs"]["Dialect"]["enum"] - ) + dialect = schema["$defs"]["Dialect"] + assert "enum" not in dialect + assert dialect["type"] == "string" + assert "OSSIE_SQL_2026" in dialect["examples"] @pytest.mark.parametrize( @@ -64,14 +66,16 @@ def test_dialect_accepts_string_and_enum_input(dialect: str | OssieDialect) -> N {"dialects": [{"dialect": dialect, "expression": "value"}]} ) - assert expression.dialects[0].dialect is OssieDialect.ANSI_SQL + assert expression.dialects[0].dialect == "ANSI_SQL" -def test_dialect_rejects_unknown_string() -> None: - with pytest.raises(ValidationError): - OssieExpression.model_validate( - {"dialects": [{"dialect": "NOT_A_DIALECT", "expression": "value"}]} - ) +def test_dialect_accepts_unknown_string() -> None: + # Free-form dialect: an unknown value is accepted and preserved. + expression = OssieExpression.model_validate( + {"dialects": [{"dialect": "DB_DIALECT", "expression": "value"}]} + ) + + assert expression.dialects[0].dialect == "DB_DIALECT" def test_ossie_sql_2026_dialect_survives_serialization(document_data: dict) -> None: @@ -84,12 +88,32 @@ def test_ossie_sql_2026_dialect_survives_serialization(document_data: dict) -> N document = OssieDocument.model_validate(data) for item in (document.datasets[0].fields[0], document.metrics[0]): - assert item.expression.dialects[0].dialect is OssieDialect.OSSIE_SQL_2026 + assert item.expression.dialects[0].dialect == "OSSIE_SQL_2026" + + as_json = json.loads(document.to_ossie_json()) + as_yaml = yaml.safe_load(document.to_ossie_yaml()) + for serialized in (as_json, as_yaml): + assert serialized == data + assert OssieDocument.model_validate(serialized) == document + + +def test_portable_expression_round_trips_and_is_omitted_when_absent( + document_data: dict, +) -> None: + # The optional portable `sql` round-trips when set and is omitted when unset. + data = document_data + data["datasets"][0]["fields"][0]["expression"]["sql"] = "customer_id" + + document = OssieDocument.model_validate(data) + assert document.datasets[0].fields[0].expression.sql == "customer_id" + # A metric whose expression omits the portable form keeps it unset. + assert document.metrics[0].expression.sql is None as_json = json.loads(document.to_ossie_json()) as_yaml = yaml.safe_load(document.to_ossie_yaml()) for serialized in (as_json, as_yaml): assert serialized == data + assert "sql" not in serialized["metrics"][0]["expression"] assert OssieDocument.model_validate(serialized) == document diff --git a/python/tests/test_serialization.py b/python/tests/test_serialization.py index a7c2b46d..4cf576bb 100644 --- a/python/tests/test_serialization.py +++ b/python/tests/test_serialization.py @@ -284,7 +284,10 @@ def _missing_properties(model: type[BaseModel], sample: object, where: str) -> l def _fully_populated_document() -> dict: """A document that sets every optional property, so each can be nulled in turn.""" - expression = {"dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}]} + expression = { + "sql": "amount", + "dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}], + } ai_context = {"instructions": "i", "synonyms": ["s"], "examples": ["e"]} extensions = [{"vendor_name": "COMMON", "data": "{}"}] return { diff --git a/python/tests/test_validation.py b/python/tests/test_validation.py index e98fb925..f2335325 100644 --- a/python/tests/test_validation.py +++ b/python/tests/test_validation.py @@ -148,14 +148,68 @@ def test_dialect_expression_missing_expression() -> None: OssieDialectExpression(dialect=OssieDialect.ANSI_SQL) -def test_expression_missing_dialects() -> None: +def test_expression_requires_sql_or_dialects() -> None: + # Neither the portable `sql` nor `dialects`: rejected (schema anyOf). with pytest.raises(ValidationError): OssieExpression() -def test_invalid_dialect_in_expression() -> None: +def test_expression_portable_only_is_valid() -> None: + expr = OssieExpression(sql="c_first_name || ' ' || c_last_name") + assert expr.sql == "c_first_name || ' ' || c_last_name" + assert expr.dialects is None + + +def test_expression_dialects_only_is_valid() -> None: + # `dialects` alone, with no portable `sql`, is accepted. + expr = OssieExpression(dialects=[OssieDialectExpression(dialect="ANSI_SQL", expression="x")]) + assert expr.sql is None + assert expr.dialects == [OssieDialectExpression(dialect="ANSI_SQL", expression="x")] + + +def test_expression_both_present_is_valid() -> None: + expr = OssieExpression( + sql="x", + dialects=[OssieDialectExpression(dialect="ANSI_SQL", expression="x")], + ) + assert expr.sql == "x" + assert expr.dialects == [OssieDialectExpression(dialect="ANSI_SQL", expression="x")] + + +def test_expression_empty_dialects_is_rejected() -> None: + # An explicitly empty `dialects` list fails min_length even without a portable form. with pytest.raises(ValidationError): - OssieDialectExpression(dialect="INVALID", expression="order_id") + OssieExpression(dialects=[]) + + +def test_expression_empty_sql_is_rejected() -> None: + # `sql` has min_length 1, matching the schema, so an empty portable form is rejected. + with pytest.raises(ValidationError): + OssieExpression(sql="") + + +def test_dialect_expression_empty_dialect_is_rejected() -> None: + # `dialect` has min_length 1, matching the schema, so an empty dialect is rejected. + with pytest.raises(ValidationError): + OssieDialectExpression(dialect="", expression="order_id") + + +def test_dialect_expression_empty_expression_is_rejected() -> None: + # The per-dialect `expression` has min_length 1, matching the schema. + with pytest.raises(ValidationError): + OssieDialectExpression(dialect="ANSI_SQL", expression="") + + +def test_custom_extension_empty_vendor_name_is_rejected() -> None: + # `vendor_name` has min_length 1, matching the schema. + with pytest.raises(ValidationError): + OssieCustomExtension(vendor_name="", data="{}") + + +def test_free_form_dialect_in_expression() -> None: + # `dialect` is free-form: an unknown value is accepted and preserved. + dialect_expr = OssieDialectExpression(dialect="DB_DIALECT", expression="order_id") + assert dialect_expr.dialect == "DB_DIALECT" # --------------------------------------------------------------------------- diff --git a/scripts/generate-spec-types.py b/scripts/generate-spec-types.py index 4b8362ba..6c3f4e2f 100755 --- a/scripts/generate-spec-types.py +++ b/scripts/generate-spec-types.py @@ -50,26 +50,6 @@ SPEC = REPO / "core-spec" / "spec.yaml" SECTIONS = [ - { - "key": "dialects", - "schema_path": ["$defs", "Dialect", "enum"], - "comment_col": 28, - "header": "# Supported expression language dialects", - "descriptions": { - "ANSI_SQL": "Standard SQL dialect", - "SNOWFLAKE": "Snowflake", - "MDX": "Multi-Dimensional Expressions", - "TABLEAU": "Tableau", - "DATABRICKS": "Databricks SQL", - "MAQL": "GoodData MAQL (Multi-Dimensional Analytical Query Language)", - "BIGQUERY": "Google BigQuery GoogleSQL", - "SIGMA": "Sigma Computing spreadsheet-style formula language", - "THOUGHTSPOT": "ThoughtSpot formula language (not SQL)", - "DAX": "Data Analysis Expressions (Power BI / Analysis Services)", - "HOLISTICS_AQL": "Holistics AQL (Analytical Query Language)", - "OSSIE_SQL_2026": "Ossie portable SQL expression language (see expression_language.md)", - }, - }, { "key": "datatypes", "schema_path": ["$defs", "DataType", "enum"], diff --git a/validation/tests/test_generate_spec_types.py b/validation/tests/test_generate_spec_types.py index 414ba437..19bdad63 100644 --- a/validation/tests/test_generate_spec_types.py +++ b/validation/tests/test_generate_spec_types.py @@ -32,22 +32,23 @@ """ -_DIALECT_HEADER = "# Supported expression language dialects" -_DIALECT_MARKER = "# Auto-generated from ossie-schema.json ($defs.Dialect.enum)." - -_DIALECTS = """# Supported expression language dialects -# Auto-generated from ossie-schema.json ($defs.Dialect.enum). -dialects: - - "ANSI_SQL" # Standard SQL dialect - -""" +# `datatypes` is the one managed section: generate-spec-types.py syncs it from +# ossie-schema.json ($defs.DataType.enum). `dialects` is intentionally NOT +# generated -- `dialect` is a free-form string -- so it appears below only as an +# unmanaged, hand-authored section the generator must leave untouched. +_HEADER = "# Supported logical data types for fields and metrics" +_MARKER = "# Auto-generated from ossie-schema.json ($defs.DataType.enum)." _DATATYPES = """# Supported logical data types for fields and metrics # Auto-generated from ossie-schema.json ($defs.DataType.enum). datatypes: - "String" # Variable-length Unicode character data + """ +# A hand-authored section the generator does not manage. +_UNMANAGED = 'dialects:\n - "ANSI_SQL"\n\n' + @pytest.fixture def generator(): @@ -67,11 +68,11 @@ def _write_spec(tmp_path: Path, text: str) -> Path: def test_apply_keeps_hand_written_comments_above_a_section(generator, monkeypatch, tmp_path): """A comment not separated by a blank line used to be swallowed by --apply. - The walk back from `dialects:` consumed every contiguous comment, so the lines - above the generated header went with it, and --check then reported the file as in - sync, hiding the loss from CI. + The walk back from `datatypes:` must consume only this script's own header lines, + not the hand-written comments above them, so the enumerations preamble survives and + --check still reports the file as in sync. """ - path = _write_spec(tmp_path, _SPEC_HEAD.rstrip("\n") + "\n" + _DIALECTS + _DATATYPES) + path = _write_spec(tmp_path, _SPEC_HEAD.rstrip("\n") + "\n" + _DATATYPES) monkeypatch.setattr(generator, "SPEC", path) generator._apply() @@ -81,17 +82,17 @@ def test_apply_keeps_hand_written_comments_above_a_section(generator, monkeypatc assert "# Standard enums used throughout the specification" in written # Keeping the comments is not enough: the block must have been rewritten once, # and the result must satisfy the check CI runs. - assert written.count(_DIALECT_HEADER) == 1 - assert written.count("dialects:") == 1 + assert written.count(_HEADER) == 1 + assert written.count("datatypes:") == 1 assert generator._check() == 0 @pytest.mark.parametrize( ("above_key", "label"), [ - (_DIALECT_HEADER + "\n" + _DIALECT_MARKER + "\n", "header-and-marker"), - (_DIALECT_HEADER + "\n", "header-without-marker"), - (_DIALECT_MARKER + "\n", "marker-without-header"), + (_HEADER + "\n" + _MARKER + "\n", "header-and-marker"), + (_HEADER + "\n", "header-without-marker"), + (_MARKER + "\n", "marker-without-header"), ("", "neither"), ], ) @@ -103,15 +104,15 @@ def test_apply_rewrites_a_section_however_its_header_survives( Matching the marker alone would leave a stale header in place and insert a second copy above the key, which `--check` then accepts. """ - spec = _SPEC_HEAD + above_key + 'dialects:\n - "ANSI_SQL"\n\n' + _DATATYPES + spec = _SPEC_HEAD + above_key + 'datatypes:\n - "String"\n' path = _write_spec(tmp_path, spec) monkeypatch.setattr(generator, "SPEC", path) generator._apply() written = path.read_text() - assert written.count(_DIALECT_HEADER) == 1 - assert written.count(_DIALECT_MARKER) == 1 + assert written.count(_HEADER) == 1 + assert written.count(_MARKER) == 1 assert "# Enumerations" in written assert generator._check() == 0 @@ -128,22 +129,22 @@ def test_apply_steps_over_blank_lines_in_any_position( """ above_key = ( "\n" * above - + _DIALECT_HEADER + + _HEADER + "\n" + "\n" * between - + _DIALECT_MARKER + + _MARKER + "\n" + "\n" * below ) - spec = _SPEC_HEAD + above_key + 'dialects:\n - "ANSI_SQL"\n\n' + _DATATYPES + spec = _SPEC_HEAD + above_key + 'datatypes:\n - "String"\n' path = _write_spec(tmp_path, spec) monkeypatch.setattr(generator, "SPEC", path) generator._apply() written = path.read_text() - assert written.count(_DIALECT_HEADER) == 1 - assert written.count(_DIALECT_MARKER) == 1 + assert written.count(_HEADER) == 1 + assert written.count(_MARKER) == 1 assert "# Enumerations" in written assert generator._check() == 0 @@ -151,21 +152,24 @@ def test_apply_steps_over_blank_lines_in_any_position( def test_apply_does_not_eat_a_blank_line_it_did_not_write(generator, monkeypatch, tmp_path): """Reaching past blanks is only correct when they separate this script's own output. - A hand-authored section has no generated header, so the blank above its key is the - author's and must survive. + A hand-authored, unmanaged section has no generated header, so the blank above its + key is the author's and must survive. """ - spec = _SPEC_HEAD + 'dialects:\n - "ANSI_SQL"\n\n' + _DATATYPES + spec = _SPEC_HEAD + _UNMANAGED + _DATATYPES path = _write_spec(tmp_path, spec) monkeypatch.setattr(generator, "SPEC", path) generator._apply() + written = path.read_text() - assert path.read_text().startswith(_SPEC_HEAD) + assert written.startswith(_SPEC_HEAD) + # The unmanaged section is left exactly as the author wrote it. + assert _UNMANAGED in written assert generator._check() == 0 def test_apply_is_idempotent(generator, monkeypatch, tmp_path): - path = _write_spec(tmp_path, _SPEC_HEAD + _DIALECTS + _DATATYPES) + path = _write_spec(tmp_path, _SPEC_HEAD + _DATATYPES) monkeypatch.setattr(generator, "SPEC", path) generator._apply() @@ -176,7 +180,11 @@ def test_apply_is_idempotent(generator, monkeypatch, tmp_path): def test_check_reports_a_file_that_is_out_of_sync(generator, monkeypatch, tmp_path, capsys): - path = _write_spec(tmp_path, _SPEC_HEAD + _DIALECTS.replace("ANSI_SQL", "MADE_UP") + _DATATYPES) + # Seed the full generated block, then mutate one value, so a single wrong + # value is the only thing out of sync (not a missing/incomplete section). + section = generator.SECTIONS[0] + in_sync = generator._generate_block(section, generator._schema_values(section)) + path = _write_spec(tmp_path, _SPEC_HEAD + in_sync.replace("String", "MADE_UP", 1)) monkeypatch.setattr(generator, "SPEC", path) assert generator._check() == 1 diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 96b0a236..ece083f1 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -330,23 +330,22 @@ def record_expression(expression: str, dialect: str, context: str) -> None: ] -def test_ossie_sql_2026_maps_to_the_ansi_default_explicitly() -> None: - # OSSIE_SQL_2026 is ANSI-SQL-compatible, so it must resolve to the same - # sqlglot default (None) as ANSI_SQL. Assert it via an explicit key rather - # than DIALECT_MAP.get()'s default, which happened to return the same None. - assert "OSSIE_SQL_2026" in _VALIDATE.DIALECT_MAP - assert _VALIDATE.DIALECT_MAP["OSSIE_SQL_2026"] is _VALIDATE.DIALECT_MAP["ANSI_SQL"] - # It is parseable as ANSI SQL, so it stays validated rather than skipped. - assert "OSSIE_SQL_2026" not in _VALIDATE.SKIP_SQL_VALIDATION +def test_ossie_sql_2026_maps_to_the_ansi_default() -> None: + # OSSIE_SQL_2026 is ANSI-SQL-compatible, so it resolves to the same sqlglot + # default (None) as ANSI_SQL, and is parseable rather than a non-SQL dialect. + assert ( + _VALIDATE.SQLGLOT_DIALECTS["OSSIE_SQL_2026"] + is _VALIDATE.SQLGLOT_DIALECTS["ANSI_SQL"] + ) + assert "OSSIE_SQL_2026" not in _VALIDATE.NON_SQL_DIALECTS def test_holistics_aql_is_skipped_rather_than_parsed_as_sql() -> None: # AQL is relationship-aware and is not SQL, so sqlglot cannot parse it. # Skipping it is what lets a Holistics field carry its body at all, the way # MDX, MAQL, THOUGHTSPOT and DAX already do. - assert "HOLISTICS_AQL" in _VALIDATE.DIALECT_MAP - assert _VALIDATE.DIALECT_MAP["HOLISTICS_AQL"] is None - assert "HOLISTICS_AQL" in _VALIDATE.SKIP_SQL_VALIDATION + assert "HOLISTICS_AQL" in _VALIDATE.NON_SQL_DIALECTS + assert "HOLISTICS_AQL" not in _VALIDATE.SQLGLOT_DIALECTS def test_a_holistics_aql_expression_is_not_reported_as_invalid_sql() -> None: @@ -378,7 +377,7 @@ def test_flags_an_invalid_ossie_sql_2026_expression() -> None: def _parse_ossie_sql(expression: str): """Parse *expression* the way the validator parses an OSSIE_SQL_2026 body.""" return _VALIDATE.sqlglot.parse_one( - expression, dialect=_VALIDATE.DIALECT_MAP["OSSIE_SQL_2026"] + expression, dialect=_VALIDATE.SQLGLOT_DIALECTS[_VALIDATE.PORTABLE_DIALECT] ) @@ -417,6 +416,172 @@ def test_not_binds_tighter_than_and() -> None: assert isinstance(tree.this, exp.Not) +@pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") +def test_unsupported_dialect_is_skipped() -> None: + # A dialect this validator cannot parse is skipped, not parsed as ANSI. + assert "DB_DIALECT" not in _VALIDATE.SQLGLOT_DIALECTS + error = _VALIDATE.validate_sql_expression( + "this is not sql @ all", "DB_DIALECT", "ctx" + ) + + assert error is None + + +def test_portable_expression_is_validated_as_ossie_sql_2026( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The portable `sql` is validated as OSSIE_SQL_2026, alongside `dialects`. + seen = [] + + def record_expression(expression: str, dialect: str, context: str) -> None: + seen.append((expression, dialect, context)) + + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + monkeypatch.setattr(_VALIDATE, "validate_sql_expression", record_expression) + expression = { + "sql": "amount", + "dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}], + } + dataset = {**_ORDERS, "fields": [{"name": "value", "expression": expression}]} + document = _document([dataset], []) + + assert _VALIDATE.validate_sql(document) == [] + assert seen == [ + ("amount", "OSSIE_SQL_2026", "Field 'orders.value' in model 'm' (OSSIE_SQL_2026)"), + ("amount", "ANSI_SQL", "Field 'orders.value' in model 'm' (ANSI_SQL)"), + ] + + +def test_schema_accepts_portable_only_expression(core_schema: dict) -> None: + # The portable `sql` may stand alone, with no `dialects` (schema anyOf). + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {"sql": "amount"}}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [] + + +def test_schema_rejects_expression_with_neither_form(core_schema: dict) -> None: + # An expression object carrying neither `sql` nor `dialects` is invalid. + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {}}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression: " + "{} is not valid under any of the given schemas" + ] + + +def test_schema_rejects_empty_dialects_without_portable_form(core_schema: dict) -> None: + # An explicitly empty `dialects` list still fails min_length. + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {"dialects": []}}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> dialects: [] should be non-empty" + ] + + +def test_schema_rejects_empty_portable_sql(core_schema: dict) -> None: + # `sql` requires minLength 1, so an empty portable form fails the schema + # rather than slipping through to the SQL validator. + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": {"sql": ""}}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> sql: '' should be non-empty" + ] + + +def test_schema_rejects_empty_dialect(core_schema: dict) -> None: + # `dialect` requires minLength 1, so an empty dialect fails the schema rather + # than producing an "is not supported by this validator" warning. + expression = {"dialects": [{"dialect": "", "expression": "x"}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> dialects -> 0 -> dialect: " + "'' should be non-empty" + ] + + +def test_schema_rejects_empty_dialect_expression(core_schema: dict) -> None: + # The per-dialect `expression` requires minLength 1, like `sql`. + expression = {"dialects": [{"dialect": "ANSI_SQL", "expression": ""}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> dialects -> 0 -> " + "expression: '' should be non-empty" + ] + + +def test_schema_rejects_empty_vendor_name(core_schema: dict) -> None: + # `vendor_name` requires minLength 1. + dataset = {**_ORDERS, "custom_extensions": [{"vendor_name": "", "data": "{}"}]} + + assert _VALIDATE.validate_schema(_document([dataset], []), core_schema) == [ + "[Schema] datasets -> 0 -> custom_extensions -> 0 -> vendor_name: " + "'' should be non-empty" + ] + + +def test_unsupported_dialect_emits_one_deduped_warning(monkeypatch: pytest.MonkeyPatch) -> None: + # A dialect this validator cannot parse is nonfatal, but one warning per dialect + # reports that its expressions went unchecked. The field and metric below share + # the dialect, so they collapse into a single warning carrying the count. + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + expression = {"dialects": [{"dialect": "ACME_SQL", "expression": "x"}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + document = _document([dataset], []) + document["metrics"] = [{"name": "total", "expression": expression}] + + result = _VALIDATE.validate_sql(document) + + assert result == [ + "[SQL] Warning: dialect 'ACME_SQL' is not supported by this validator; " + "2 expressions not syntax-checked" + ] + assert all(isinstance(item, _VALIDATE.ValidationWarning) for item in result) + + +def test_distinct_unsupported_dialects_each_warn_once(monkeypatch: pytest.MonkeyPatch) -> None: + # Two different unsupported dialects produce two warnings, sorted by name. + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + fields = [ + {"name": "a", "expression": {"dialects": [{"dialect": "ZETA_SQL", "expression": "x"}]}}, + {"name": "b", "expression": {"dialects": [{"dialect": "ACME_SQL", "expression": "y"}]}}, + ] + dataset = {**_ORDERS, "fields": fields} + + assert _VALIDATE.validate_sql(_document([dataset], [])) == [ + "[SQL] Warning: dialect 'ACME_SQL' is not supported by this validator; " + "1 expression not syntax-checked", + "[SQL] Warning: dialect 'ZETA_SQL' is not supported by this validator; " + "1 expression not syntax-checked", + ] + + +def test_known_unparseable_dialect_is_not_flagged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # MDX is a known non-SQL dialect: it is skipped silently, not warned as unsupported. + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + expression = {"dialects": [{"dialect": "MDX", "expression": "[Measures].[Sales]"}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + + assert _VALIDATE.validate_sql(_document([dataset], [])) == [] + + +def test_unsupported_dialect_warning_suggests_a_close_match( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A near-miss of a known dialect gets a "did you mean" hint. + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + expression = {"dialects": [{"dialect": "SNOWFLKE", "expression": "x"}]} + dataset = {**_ORDERS, "fields": [{"name": "f", "expression": expression}]} + + assert _VALIDATE.validate_sql(_document([dataset], [])) == [ + "[SQL] Warning: dialect 'SNOWFLKE' is not supported by this validator; " + "1 expression not syntax-checked. Did you mean 'SNOWFLAKE'?" + ] + + @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") @pytest.mark.parametrize( "expression", @@ -703,8 +868,9 @@ def test_sql_error_in_metric_with_warning_in_name_is_an_error(run_validator): assert "[SQL] Metric 'Warning: broken_metric'" in output -@pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") def test_empty_field_expression_fails_validation(run_validator): + # An empty per-dialect expression now fails the schema (minLength 1), before + # the SQL check runs. dataset = {**_ORDERS, "fields": [{ "name": "blank", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": ""}]}, @@ -714,7 +880,10 @@ def test_empty_field_expression_fails_validation(run_validator): exit_code, output = run_validator(document) assert exit_code == 1 - assert "[SQL] Field 'orders.blank' in model 'm' (ANSI_SQL): expression is empty" in output + assert ( + "[Schema] datasets -> 0 -> fields -> 0 -> expression -> dialects -> 0 -> " + "expression: '' should be non-empty" in output + ) @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") @@ -1224,6 +1393,11 @@ def _ds_expr(sql: str, dialect: str = "OSSIE_SQL_2026") -> dict: return {"dialects": [{"dialect": dialect, "expression": sql}]} +def _ds_portable(sql: str) -> dict: + # A dataset-scoped metric written with only the portable `sql` form, no dialects. + return {"sql": sql} + + def _orders_with(fields=None, metrics=None) -> dict: dataset = {"name": "orders", "source": "db.s.orders"} if fields is not None: @@ -1337,3 +1511,117 @@ def test_qualified_dataset_scoped_metric_fails_document_validation(core_schema, scope_errors = [error for error in errors if error.startswith("[Scope]")] assert scope_errors assert all(not isinstance(error, _VALIDATE.ValidationWarning) for error in scope_errors) + + +def test_dataset_scoped_metric_portable_sql_accepts_declared_field() -> None: + doc = _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[{"name": "total_amount", "expression": _ds_portable("SUM(amount)")}], + ) + + assert _VALIDATE.validate_metric_scoping(doc) == [] + + +def test_dataset_scoped_metric_portable_sql_rejects_a_cross_dataset_reference() -> None: + doc = _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[ + {"name": "bad", "expression": _ds_portable("SUM(amount) / COUNT(customers.id)")} + ], + ) + + assert _VALIDATE.validate_metric_scoping(doc) == [ + "[Scope] Dataset-scoped metric 'orders.bad' in model 'm' (OSSIE_SQL_2026) uses " + "qualified reference(s) 'customers'. A dataset-scoped metric expression must " + "reference the dataset's fields by single-part (unqualified) name; a metric that " + "must reference another dataset belongs in the model-root metrics." + ] + + +def test_dataset_scoped_metric_portable_sql_rejects_an_undeclared_field() -> None: + doc = _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[{"name": "total_qty", "expression": _ds_portable("SUM(quantity)")}], + ) + + assert _VALIDATE.validate_metric_scoping(doc) == [ + "[Scope] Dataset-scoped metric 'orders.total_qty' in model 'm' (OSSIE_SQL_2026) " + "references undeclared field(s) 'quantity'. A dataset-scoped metric may reference " + "only the dataset's declared fields." + ] + + +def test_dataset_scoped_metric_portable_sql_is_sql_validated() -> None: + doc = _orders_with( + metrics=[{"name": "broken", "expression": _ds_portable("SUM(")}], + ) + + errors = _VALIDATE.validate_sql(doc) + + # The sqlglot error text varies by version, so assert the context prefix only. + assert len(errors) == 1 + assert errors[0].startswith("[SQL] Metric 'orders.broken' in model 'm' (OSSIE_SQL_2026):") + + +def _metric_with_sql_and_ossie_dialect(sql: str, dialect_sql: str) -> dict: + return _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[ + { + "name": "total", + "expression": { + "sql": sql, + "dialects": [{"dialect": "OSSIE_SQL_2026", "expression": dialect_sql}], + }, + } + ], + ) + + +def test_invalid_superseded_ossie_sql_2026_dialect_entry_warns_not_fails() -> None: + # The top-level `sql` is authoritative, so an invalid OSSIE_SQL_2026 dialect + # entry beside it is reported as a nonfatal warning, not a hard error. + errors = _VALIDATE.validate_sql(_metric_with_sql_and_ossie_dialect("SUM(amount)", "SUM(")) + + assert errors == [ + "[SQL] Warning: Metric 'orders.total' in model 'm' (OSSIE_SQL_2026): dialect entry " + "is not valid SQL and is ignored; the top-level 'sql' is authoritative" + ] + assert all(isinstance(e, _VALIDATE.ValidationWarning) for e in errors) + + +def test_differing_superseded_ossie_sql_2026_dialect_entry_warns() -> None: + # A valid OSSIE_SQL_2026 dialect entry that differs from the authoritative + # top-level `sql` is dead text; warn so the drift is caught. + errors = _VALIDATE.validate_sql( + _metric_with_sql_and_ossie_dialect("SUM(amount)", "AVG(amount)") + ) + + assert errors == [ + "[SQL] Warning: Metric 'orders.total' in model 'm' (OSSIE_SQL_2026): dialect entry " + "differs from the top-level 'sql' and is ignored; the top-level 'sql' is authoritative" + ] + + +def test_identical_superseded_ossie_sql_2026_dialect_entry_is_silent() -> None: + # An OSSIE_SQL_2026 dialect entry identical to the top-level `sql` is harmless + # redundancy, not drift, so it is not warned. + assert _VALIDATE.validate_sql( + _metric_with_sql_and_ossie_dialect("SUM(amount)", "SUM(amount)") + ) == [] + + +def test_superseded_ossie_sql_2026_dialect_entry_is_not_scope_checked() -> None: + # The authoritative top-level `sql` is scope-checked; a superseded + # OSSIE_SQL_2026 dialect entry beside it is not, so its cross-dataset + # reference does not fail the document. + expression = { + "sql": "SUM(amount)", + "dialects": [{"dialect": "OSSIE_SQL_2026", "expression": "SUM(customers.id)"}], + } + doc = _orders_with( + fields=[{"name": "amount", "expression": _ds_expr("amount")}], + metrics=[{"name": "total", "expression": expression}], + ) + + assert _VALIDATE.validate_metric_scoping(doc) == [] diff --git a/validation/validate.py b/validation/validate.py index d94ce468..8d705e85 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -53,13 +53,17 @@ OSSIE_SCHEMA or --schema at a copy of the schema. """ +import difflib import importlib.util import json import os import sys import traceback from collections import Counter +from collections.abc import Hashable, Iterator +from dataclasses import dataclass from pathlib import Path +from typing import Literal try: import yaml @@ -92,24 +96,23 @@ except ImportError: SQLGLOT_AVAILABLE = False -# Map Ossie dialects to sqlglot dialects -DIALECT_MAP = { - "ANSI_SQL": None, # sqlglot default - "OSSIE_SQL_2026": None, # ANSI-SQL-compatible; parse with the sqlglot default +# The dialect the portable `sql` form is written in; the only one this +# specification defines. +PORTABLE_DIALECT = "OSSIE_SQL_2026" + +# Ossie dialects sqlglot can parse, each mapped to the sqlglot dialect to parse it +# as. None is sqlglot's default (ANSI) dialect. +SQLGLOT_DIALECTS = { + "ANSI_SQL": None, + PORTABLE_DIALECT: None, "SNOWFLAKE": "snowflake", "DATABRICKS": "databricks", "BIGQUERY": "bigquery", - "MDX": None, # Not supported by sqlglot, skip validation - "TABLEAU": None, # Not supported by sqlglot, skip validation - "MAQL": None, # Not supported by sqlglot, skip validation - "SIGMA": None, # Sigma's spreadsheet-style formula language, not SQL; skip validation - "THOUGHTSPOT": None, # Not supported by sqlglot, skip validation - "DAX": None, # Not supported by sqlglot, skip validation - "HOLISTICS_AQL": None, # Holistics AQL, not SQL; skip validation } -# Dialects that sqlglot cannot parse -SKIP_SQL_VALIDATION = { +# Known non-SQL dialects: sqlglot cannot parse them, so their expressions are +# carried but not validated. +NON_SQL_DIALECTS = { "MDX", "TABLEAU", "MAQL", @@ -119,6 +122,9 @@ "HOLISTICS_AQL", } +# Every dialect this validator knows, for close-match suggestions on a typo. +KNOWN_DIALECTS = (*SQLGLOT_DIALECTS, *NON_SQL_DIALECTS) + # Concepts every ontology includes implicitly (ontology.md, "Built-in concepts") BUILT_IN_CONCEPTS = {"Any", "Boolean", "Date", "DateTime", "Decimal", "Float", "Integer", "String"} @@ -343,10 +349,10 @@ def _parse_sql_expression( Returns `(None, None)` when there is nothing to parse: sqlglot is unavailable or the dialect is one sqlglot cannot parse. """ - if not SQLGLOT_AVAILABLE or dialect in SKIP_SQL_VALIDATION: + if not SQLGLOT_AVAILABLE or dialect not in SQLGLOT_DIALECTS: return None, None - sqlglot_dialect = DIALECT_MAP.get(dialect) + sqlglot_dialect = SQLGLOT_DIALECTS[dialect] def statement_count_error(sql: str) -> str | None: """Reject a parseable string that holds more than one statement. @@ -417,6 +423,90 @@ def validate_sql_expression(expr: str, dialect: str, context: str) -> str | None return error +ExpressionKind = Literal["field", "dataset_metric", "model_metric"] + + +@dataclass(frozen=True) +class Expression: + """An expression and its context within a document. + + `kind` identifies the declaring element, and `name` identifies the field + or metric. `dialect` is OSSIE_SQL_2026 for portable `sql` expressions, + or the dialect's label otherwise. `field_names` contains the enclosing + dataset's field names, or is empty for model-scoped metrics. + `superseded_by` is the authoritative top-level `sql` text when this is an + OSSIE_SQL_2026 dialect entry that `sql` outranks, else None: such an entry + is non-authoritative, since a client reads the `sql` instead. + """ + + kind: ExpressionKind + dataset_name: str + name: str + dialect: str + text: str + field_names: frozenset[str] + superseded_by: str | None = None + + def context(self, model_name: str) -> str: + if self.kind == "field": + label = f"Field '{self.dataset_name}.{self.name}'" + elif self.kind == "dataset_metric": + label = f"Metric '{self.dataset_name}.{self.name}'" + else: + label = f"Metric '{self.name}'" + return f"{label} in model '{model_name}' ({self.dialect})" + + +def _expressions_in( + member: dict, kind: ExpressionKind, dataset_name: str, field_names: frozenset[str] +) -> Iterator[Expression]: + expression = member.get("expression") or {} + name = member.get("name", "") + sql = expression.get("sql") + if sql is not None: + yield Expression( + kind=kind, + dataset_name=dataset_name, + name=name, + dialect=PORTABLE_DIALECT, + text=sql, + field_names=field_names, + ) + for dialect_expr in expression.get("dialects", []): + text = dialect_expr.get("expression") + if text is None: + continue + dialect = dialect_expr["dialect"] + yield Expression( + kind=kind, + dataset_name=dataset_name, + name=name, + dialect=dialect, + text=text, + field_names=field_names, + # A top-level `sql` is the authoritative OSSIE_SQL_2026 form, so an + # OSSIE_SQL_2026 dialect entry alongside it is non-authoritative. + superseded_by=sql if dialect == PORTABLE_DIALECT and sql is not None else None, + ) + + +def iter_expressions(data: dict) -> Iterator[Expression]: + """Yield every expression in a core semantic model document. A missing + expression is skipped; an empty one is yielded, so a caller decides whether to + report it.""" + for dataset in data.get("datasets", []): + dataset_name = dataset.get("name", "") + field_names = frozenset( + f["name"].casefold() for f in dataset.get("fields", []) if f.get("name") + ) + for field in dataset.get("fields", []): + yield from _expressions_in(field, "field", dataset_name, field_names) + for metric in dataset.get("metrics", []): + yield from _expressions_in(metric, "dataset_metric", dataset_name, field_names) + for metric in data.get("metrics", []): + yield from _expressions_in(metric, "model_metric", "", frozenset()) + + def validate_sql(data: dict) -> list[str]: """Validate SQL expressions in fields and metrics.""" # Only core semantic model documents contain a root datasets property. @@ -428,52 +518,54 @@ def validate_sql(data: dict) -> list[str]: "[SQL] Warning: sqlglot not installed, skipping SQL validation. Install with: pip install sqlglot" )] - model = data + model_name = data.get("name", "") errors = [] - - model_name = model.get("name", "") - - # Validate field expressions - for dataset in model.get("datasets", []): - dataset_name = dataset.get("name", "") - for field in dataset.get("fields", []): - field_name = field.get("name", "") - expression = field.get("expression", {}) - for dialect_expr in expression.get("dialects", []): - dialect = dialect_expr.get("dialect", "ANSI_SQL") - # Only a missing expression is skipped; an empty one is reported. - expr = dialect_expr.get("expression") - if expr is not None: - context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' ({dialect})" - error = validate_sql_expression(expr, dialect, context) - if error: - errors.append(error) - - # Validate dataset-scoped metric expressions - for metric in dataset.get("metrics", []): - metric_name = metric.get("name", "") - expression = metric.get("expression", {}) - for dialect_expr in expression.get("dialects", []): - dialect = dialect_expr.get("dialect", "ANSI_SQL") - expr = dialect_expr.get("expression") - if expr is not None: - context = f"Metric '{dataset_name}.{metric_name}' in model '{model_name}' ({dialect})" - error = validate_sql_expression(expr, dialect, context) - if error: - errors.append(error) - - # Validate metric expressions - for metric in model.get("metrics", []): - metric_name = metric.get("name", "") - expression = metric.get("expression", {}) - for dialect_expr in expression.get("dialects", []): - dialect = dialect_expr.get("dialect", "ANSI_SQL") - expr = dialect_expr.get("expression") - if expr is not None: - context = f"Metric '{metric_name}' in model '{model_name}' ({dialect})" - error = validate_sql_expression(expr, dialect, context) - if error: - errors.append(error) + # A dialect outside both sets is one this validator cannot parse. It may be a + # legitimate entry from extensions/dialects.md, so warn once that its + # expressions went unchecked rather than treat it as an error. + unsupported_dialects = Counter() + + for expression in iter_expressions(data): + dialect = expression.dialect + if dialect in NON_SQL_DIALECTS: + continue + if dialect not in SQLGLOT_DIALECTS: + unsupported_dialects[dialect] += 1 + continue + error = validate_sql_expression( + expression.text, dialect, expression.context(model_name) + ) + if expression.superseded_by is not None: + # A top-level `sql` is authoritative, so this OSSIE_SQL_2026 dialect + # entry does not fail the document. Warn if it is dead text -- invalid, + # or differing from the `sql` a client reads instead -- and stay silent + # when it merely duplicates the `sql`. + if error: + reason = "is not valid SQL" + elif expression.text != expression.superseded_by: + reason = "differs from the top-level 'sql'" + else: + continue + errors.append(ValidationWarning( + f"[SQL] Warning: {expression.context(model_name)}: dialect entry {reason} " + f"and is ignored; the top-level 'sql' is authoritative" + )) + elif error: + errors.append(error) + + for dialect in sorted(unsupported_dialects): + count = unsupported_dialects[dialect] + plural = "s" if count != 1 else "" + message = ( + f"[SQL] Warning: dialect '{dialect}' is not supported by this validator; " + f"{count} expression{plural} not syntax-checked" + ) + # Only a close match is suggested, so a legitimate new dialect gets no + # spurious "did you mean". + suggestion = difflib.get_close_matches(dialect, KNOWN_DIALECTS, n=1, cutoff=0.8) + if suggestion: + message += f". Did you mean '{suggestion[0]}'?" + errors.append(ValidationWarning(message)) return errors @@ -497,58 +589,56 @@ def validate_metric_scoping(data: dict) -> list[str]: if not isinstance(data, dict) or "datasets" not in data: return [] - model = data + model_name = data.get("name", "") errors = [] - model_name = model.get("name", "") - for dataset in model.get("datasets", []): - dataset_name = dataset.get("name", "") - field_names = { - f["name"].casefold() for f in dataset.get("fields", []) if f.get("name") - } - for metric in dataset.get("metrics", []): - metric_name = metric.get("name", "") - expression = metric.get("expression") or {} - for dialect_expr in expression.get("dialects", []): - dialect = dialect_expr.get("dialect", "ANSI_SQL") - expr = dialect_expr.get("expression") - if not expr: - continue - tree, _ = _parse_sql_expression( - expr, - dialect, - f"Metric '{dataset_name}.{metric_name}' in model '{model_name}'", - ) - if tree is None: - continue - - qualifiers = set() - undeclared = set() - for col in tree.find_all(exp.Column): - parts = [part.name for part in col.parts] - if len(parts) > 1 and parts[0]: - qualifiers.add(parts[0]) - elif parts and parts[0].casefold() not in field_names: - undeclared.add(parts[0]) - - prefix = ( - f"[Scope] Dataset-scoped metric '{dataset_name}.{metric_name}' in model " - f"'{model_name}' ({dialect})" - ) - if qualifiers: - names = ", ".join(repr(q) for q in sorted(qualifiers)) - errors.append( - f"{prefix} uses qualified reference(s) {names}. A dataset-scoped metric " - f"expression must reference the dataset's fields by single-part " - f"(unqualified) name; a metric that must reference another dataset " - f"belongs in the model-root metrics." - ) - if undeclared: - names = ", ".join(repr(n) for n in sorted(undeclared)) - errors.append( - f"{prefix} references undeclared field(s) {names}. A dataset-scoped " - f"metric may reference only the dataset's declared fields." - ) + def scope_errors(expression: Expression) -> list[str]: + tree, _ = _parse_sql_expression( + expression.text, expression.dialect, expression.context(model_name) + ) + if tree is None: + return [] + + qualifiers = set() + undeclared = set() + for col in tree.find_all(exp.Column): + parts = [part.name for part in col.parts] + if len(parts) > 1 and parts[0]: + qualifiers.add(parts[0]) + elif parts and parts[0].casefold() not in expression.field_names: + undeclared.add(parts[0]) + + prefix = ( + f"[Scope] Dataset-scoped metric '{expression.dataset_name}.{expression.name}' " + f"in model '{model_name}' ({expression.dialect})" + ) + out = [] + if qualifiers: + names = ", ".join(repr(q) for q in sorted(qualifiers)) + out.append( + f"{prefix} uses qualified reference(s) {names}. A dataset-scoped metric " + f"expression must reference the dataset's fields by single-part " + f"(unqualified) name; a metric that must reference another dataset " + f"belongs in the model-root metrics." + ) + if undeclared: + names = ", ".join(repr(n) for n in sorted(undeclared)) + out.append( + f"{prefix} references undeclared field(s) {names}. A dataset-scoped " + f"metric may reference only the dataset's declared fields." + ) + return out + + for expression in iter_expressions(data): + # A superseded OSSIE_SQL_2026 dialect entry is non-authoritative; the + # top-level `sql` it defers to is scope-checked in its place. + if ( + expression.kind != "dataset_metric" + or not expression.text + or expression.superseded_by is not None + ): + continue + errors.extend(scope_errors(expression)) return errors