diff --git a/core-spec/expression_language.md b/core-spec/expression_language.md index 67098000..9cf1580d 100644 --- a/core-spec/expression_language.md +++ b/core-spec/expression_language.md @@ -105,9 +105,18 @@ Sometimes, we may refer to a **normalized identifier**. This is a form the iden #### Name Spaces -Namespaces define how an identifier is looked up in an expression. They are covered in the semantics document. Identifiers +Namespaces define how an identifier is looked up in an expression. Identifiers will be able to be multi-part and the parts will be separated by the '.' characters, E.g. `dataset.field` This matches SQL conventions. +What an identifier refers to depends on where the expression appears: + +* In a **field expression**, identifiers refer to columns of the dataset's `source`. A field expression cannot refer to other fields. +* In a **dataset-scoped metric expression**, identifiers are unqualified and refer to fields declared in that dataset. +* In a **model-scoped metric expression**, identifiers must be qualified as `dataset.field` and refer to a field declared in that dataset. +* Metric expressions cannot refer to columns of the dataset's `source` or to other metrics. If a field and a source column share a name, a metric expression refers to the field. + +See [Name Resolution](spec.md#name-resolution) in the core specification. + ## SQL Language Subset ### Supported SQL Constructs @@ -116,7 +125,7 @@ Ossie expressions support the following SQL constructs within any expression: | Construct | Notes | | :---- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Column and Metric references | Varies based on whether in Ontology or Semantic models. See namespaceing in [Ossie Discussion Point: Core Analytic Abstractions](https://docs.google.com/document/d/1si8DqU4arG18ZgX4HnRG5D_zS2X7V1s-vgNY35rvxhM/edit?tab=t.0#heading=h.le505t8uoyfy) And future Ontology documentation | +| Column and Metric references | Varies based on whether in Ontology or Semantic models. For semantic models, see [Name Resolution](spec.md#name-resolution). See also namespacing in [Ossie Discussion Point: Core Analytic Abstractions](https://docs.google.com/document/d/1si8DqU4arG18ZgX4HnRG5D_zS2X7V1s-vgNY35rvxhM/edit?tab=t.0#heading=h.le505t8uoyfy) And future Ontology documentation | | Arithmetic operators | `+`, `-`, `*`, `/`, `%` (modulo) | | Comparison operators | `=`, `<>`, `!=`, `<`, `>`, `<=`, `>=` | | Logical operators | `AND`, `OR`, `NOT` | diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index 174b985c..a2faf7b0 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -208,7 +208,7 @@ "type": "string" }, "minItems": 1, - "description": "Primary key columns (single or composite)" + "description": "Dataset fields that uniquely identify rows" }, "unique_keys": { "type": "array", @@ -220,7 +220,7 @@ "minItems": 1 }, "minItems": 1, - "description": "Array of unique key definitions (each can be single or composite)" + "description": "Array of unique key definitions, each a list of dataset fields" }, "description": { "type": "string", @@ -278,7 +278,7 @@ "type": "string" }, "minItems": 1, - "description": "Foreign key columns in the 'from' dataset" + "description": "Array of field names in the 'from' dataset (foreign key fields)" }, "to_columns": { "type": "array", @@ -286,7 +286,7 @@ "type": "string" }, "minItems": 1, - "description": "Primary/unique key columns in the 'to' dataset" + "description": "Array of field names in the 'to' dataset (primary or unique key fields)" }, "ai_context": { "$ref": "#/$defs/AIContext" diff --git a/core-spec/spec.md b/core-spec/spec.md index b1f06e35..3afae56d 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -37,7 +37,8 @@ 4. [Relationships](#relationships) 5. [Fields](#fields) 6. [Metrics](#metrics) -7. [Examples](#examples) +7. [Name Resolution](#name-resolution) +8. [Examples](#examples) --- @@ -173,8 +174,8 @@ Logical datasets represent business entities or concepts (fact and dimension tab |-------|------|----------|-------------| | `name` | string | Yes | Unique identifier for the dataset | | `source` | string | Yes | Reference to underlying physical table/view (e.g., `database.schema.table`) or query | -| `primary_key` | array | No | Primary key columns that uniquely identify rows (single or composite) | -| `unique_keys` | array of arrays | No | Array of unique key definitions (each can be single or composite) | +| `primary_key` | array | No | Dataset fields that uniquely identify rows | +| `unique_keys` | array of arrays | No | Array of unique key definitions, each a list of dataset fields | | `description` | string | No | Human-readable description | | `ai_context` | string/object | No | Additional context for AI tools (e.g., synonyms, common terms) | | `fields` | array | No | Row-level attributes for grouping, filtering, and metric expressions | @@ -215,7 +216,17 @@ datasets: synonyms: - "purchases" - "sales" - fields: [] + fields: + - name: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id + - name: order_number + expression: + dialects: + - dialect: ANSI_SQL + expression: order_number custom_extensions: - vendor_name: DBT data: '{"materialized": "table"}' @@ -234,17 +245,18 @@ Relationships define how logical datasets are connected through foreign key cons | `name` | string | Yes | Unique identifier for the relationship | | `from` | string | Yes | The logical dataset on the many side of the relationship | | `to` | string | Yes | The logical dataset on the one side of the relationship | -| `from_columns` | array | Yes | Array of column names in the "from" dataset (foreign key columns) | -| `to_columns` | array | Yes | Array of column names in the "to" dataset (primary or unique key columns) | +| `from_columns` | array | Yes | Array of field names in the "from" dataset (foreign key fields) | +| `to_columns` | array | Yes | Array of field names in the "to" dataset (primary or unique key fields) | | `ai_context` | string/object | No | Additional context for AI tools | | `custom_extensions` | array | No | Vendor-specific attributes | ### Important Notes -- The order of columns in `from_columns` must correspond to the order in `to_columns` -- Both arrays must have the same number of columns -- For simple relationships, use a single column: `[column1]` -- For composite relationships, use multiple columns: `[column1, column2]` +- Despite their names, `from_columns` and `to_columns` contain names of fields declared in the `from` and `to` datasets, not columns of those datasets' sources. See [Name Resolution](#name-resolution). +- The order of fields in `from_columns` must correspond to the order in `to_columns` +- Both arrays must have the same number of fields +- For simple relationships, use a single field: `[field1]` +- For composite relationships, use multiple fields: `[field1, field2]` ### Examples @@ -307,6 +319,7 @@ expression: - Use scalar SQL expressions (no aggregations) - Can be simple column references (e.g., `customer_id`) or computed expressions (e.g., `first_name || ' ' || last_name`) +- Identifiers refer to columns of the dataset's `source`, not to other fields. See [Name Resolution](#name-resolution). - Multiple dialect versions can be provided for the same field ### Dimension Object @@ -416,7 +429,7 @@ There are two types of metrics: - **Dataset-scoped** (`datasets[].metrics`): aggregations that belong to a single dataset. The expression references that dataset's declared fields by single-part name and resolves entirely within it. - **Model-scoped** (the model-root `metrics`): aggregations that span datasets across the model. The expression references fields by multi-part `dataset.field` name and may combine fields across datasets. -See [Metric Scoping](#metric-scoping) for the rules governing each. +Metric expressions refer to dataset fields. They cannot refer to columns of a dataset's `source` or to other metrics. See [Metric Scoping](#metric-scoping) for the rules governing each and [Name Resolution](#name-resolution) for how names resolve. ### Schema @@ -643,6 +656,106 @@ The following are well-known examples: --- +## Name Resolution + +An Ossie model has three levels of names: + +1. **Source columns.** The columns of the table, view, or query named by a + dataset's `source`. The Ossie document does not declare them. +2. **Dataset fields.** The fields declared in a dataset's `fields`. Each field + is defined in terms of that dataset's source columns, just as a view's + `SELECT` list can only reference columns from its `FROM` clause. A dataset's + fields are its public interface. +3. **Metrics.** Dataset-scoped metrics (`datasets[].metrics`) and model-scoped + metrics (the model-root `metrics`), defined in terms of dataset fields. + +Metrics and relationships do not reach through a dataset to its source +columns, so a dataset that declares no fields exposes nothing to them. + +| Where the name appears | What it refers to | +|------------------------|-------------------| +| Field `expression` | A column of the same dataset's `source` | +| `datasets[].primary_key`, `datasets[].unique_keys` | A field of the same dataset | +| `relationships[].from_columns` | A field of the `from` dataset | +| `relationships[].to_columns` | A field of the `to` dataset | +| Dataset-scoped metric `expression` (`datasets[].metrics`) | A field of the same dataset, unqualified | +| Model-scoped metric `expression` (model-root `metrics`) | A field of the named dataset, qualified as `dataset.field` | + +For example, a dataset can expose the source column `o_totalprice` as the +field `total_amount`, and derive `total_usd` from `total_cents`: + +```yaml +datasets: + - name: orders + source: sales.public.orders # columns: o_orderkey, o_custkey, o_totalprice, total_cents + primary_key: [order_id] # field name, not o_orderkey + fields: + - name: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: o_orderkey # source column + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: o_custkey + - name: total_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: o_totalprice + - name: total_usd + expression: + dialects: + - dialect: ANSI_SQL + expression: total_cents / 100.0 + +relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: [customer_id] # field of orders, not o_custkey + to_columns: [id] # field of customers (dataset not shown) + +metrics: + - name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.total_usd) # field, not a source column +``` + +In this model, these references are invalid: + +- `SUM(orders.o_totalprice)` and `from_columns: [o_custkey]`, because + `o_totalprice` and `o_custkey` are source columns, not fields of `orders`. +- `SUM(total_usd)` in a model-scoped metric, because model-scoped metric + expressions must qualify field names with the dataset name, even when only + one dataset has a field with that name. +- `SUM(o_totalprice)` in a metric declared on `orders`, because dataset-scoped + metrics reach only the dataset's declared fields, not its source columns. + +If a field and a source column share a name (as in a field `amount` defined as +`amount`), the name refers to the source column inside that dataset's field +expressions and to the field everywhere else. + +Consumers that generate SQL against the source tables, rather than against a +layer that exposes the fields, must replace each field reference with the +field's expression. For example, `SUM(orders.total_amount)` compiles to +`SUM(o_totalprice)` over `sales.public.orders`. + +This version of the specification does not support: + +- a field expression that refers to another field, in the same or another + dataset; +- a metric expression that refers to another metric; +- a dataset that uses another dataset as its `source`. + +Future versions may add these. + +--- + ## Complete Example Here's a complete semantic model example showing all components working together: @@ -796,6 +909,7 @@ ai_context: - **0.2.0.dev0** (Unreleased): In-development next minor release. Schema is mutable; do not depend on this version in production. - Breaking: each standalone document contains one model directly at the root; the `semantic_model` array is removed. + - Breaking: metric expressions, `primary_key`, `unique_keys`, and relationship `from_columns`/`to_columns` refer to dataset fields, not columns of the dataset's `source`. Model-scoped metric expressions must use qualified `dataset.field` references; dataset-scoped metric expressions use unqualified field names. Earlier versions did not specify this, and some converters read these names as source columns; those converters must now resolve them as fields. See [Name Resolution](#name-resolution). - **0.1.1** (2025-12-11): Initial release - Core semantic model structure - Support for datasets, relationships, fields, and metrics diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index a32bd035..d56629c5 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -117,28 +117,30 @@ datasets: # Format should be either database_name.schema_name.table_name or query source: string - # Optional: Primary key definition that uniquely identifies rows in this dataset - # Can be a single column or a composite of multiple columns + # Optional: Dataset fields that uniquely identify rows + # These are fields declared in the dataset, not columns of its source + # Can be a single field or a composite of multiple fields # This is the preferred unique identifier for this dataset and is used in relationships to determine many-to-one or one-to-one. # Examples: # primary_key: [customer_id] # Simple primary key # # primary_key: [order_id, line_number] # Composite primary key - primary_key: [] # Array of column names (single or composite) + primary_key: [] # Array of field names - # Optional: Array of unique key definitions that uniquely identify rows in this dataset - # Each unique key can be a single column or a composite of multiple columns + # Optional: Array of unique key definitions, each a list of dataset fields + # These are fields declared in the dataset, not columns of its source + # Each unique key can be a single field or a composite of multiple fields # Used for determining relationship type of either many-to-one or one-to-one # Examples: # unique_keys: - # - [column1] - # - [column2, column3] + # - [field1] + # - [field2, field3] # # unique_keys: - # - [column1, column2] - # - [column3, column4] + # - [field1, field2] + # - [field3, field4] unique_keys: - - [] # Array of column names (single or composite) + - [] # Array of field names # Optional: Human-readable description of the logical dataset description: string @@ -189,21 +191,23 @@ relationships: # References a logical dataset name to: string - # Required: Array of column names in the "from" dataset (foreign key columns) - # For simple relationships, use a single column: [column1] - # For composite relationships, use multiple columns: [column1, column2] - # The order of columns must correspond to the order in to_columns + # Required: Array of field names in the "from" dataset (foreign key fields) + # These are fields declared in the dataset, not columns of its source + # For simple relationships, use a single field: [field1] + # For composite relationships, use multiple fields: [field1, field2] + # The order of fields must correspond to the order in to_columns # Examples: # - [customer_id] # Simple foreign key # - [order_id, line_number] # Composite foreign key - from_columns: [] # Array of column names + from_columns: [] # Array of field names - # Required: Array of column names in the "to" dataset (primary or unique key columns) - # Must have the same number of columns as from_columns in corresponding order + # Required: Array of field names in the "to" dataset (primary or unique key fields) + # These are fields declared in the dataset, not columns of its source + # Must have the same number of fields as from_columns in corresponding order # Examples: # - [id] # Simple key # - [order_id, line_number] # Composite key - to_columns: [] # Array of column names + to_columns: [] # Array of field names # Optional: Additional context for AI tools (e.g., synonyms, business context) # Helps LLMs understand when and why the datasets should be joined @@ -225,6 +229,7 @@ fields: # Supports multiple SQL dialects for cross-platform compatibility # Each field can have expressions in different dialects for portability # Can be a simple column reference or a complex scalar expression + # Identifiers refer to columns of the dataset's source, not to other fields expression: dialects: - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" @@ -284,6 +289,10 @@ metrics: # Required: Expression definition with dialect support # Supports multiple SQL dialects for cross-platform compatibility # Each metric can have expressions in different dialects for portability + # References are dataset fields, not columns of the dataset's source + # Model-scoped metrics qualify them as dataset.field; dataset-scoped + # metrics use unqualified field names + # Metrics cannot reference other metrics expression: dialects: - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index 0aa290c7..7837032f 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -71,6 +71,16 @@ datasets: - "product" - "item" + - name: ss_ticket_number + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ticket_number + description: Ticket number identifying the sales transaction + datatype: Integer + dimension: + is_time: false + - name: ss_customer_sk expression: dialects: