From f9b11628961d7c4d42595c8b4618ab4b7f88bd35 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:13:51 +0200 Subject: [PATCH 1/5] test(compliance): add portability rung M6, date and time scalars One dataset with YEAR and DATE_TRUNC fields and an AVG(DATEDIFF(...)) metric, using the part-first DATEDIFF(part, start, end) order from expression_language.md. Adds one construct on top of the baseline. Co-authored-by: Isaac --- .../portability/m6/m6_date_scalars.ossie.yaml | 59 +++++++++++++++++++ compliance/portability/manifest.yaml | 10 ++++ 2 files changed, 69 insertions(+) create mode 100644 compliance/portability/m6/m6_date_scalars.ossie.yaml diff --git a/compliance/portability/m6/m6_date_scalars.ossie.yaml b/compliance/portability/m6/m6_date_scalars.ossie.yaml new file mode 100644 index 00000000..50e73736 --- /dev/null +++ b/compliance/portability/m6/m6_date_scalars.ossie.yaml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M6 ladder rung: date and time scalars from the portable core (YEAR, +# DATE_TRUNC, DATEDIFF). DATEDIFF uses the spec's part-first argument order +# DATEDIFF(part, start, end). The date-part fields resolve inside their own +# dataset (sibling columns), and the DATEDIFF metric uses the qualified form. +version: "0.2.0.dev0" +name: m6_sales +description: "M6: date and time scalar functions from the portable core." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: order_date + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date + datatype: Date + - name: ship_date + expression: + dialects: + - dialect: ANSI_SQL + expression: ship_date + datatype: Date + - name: order_year + expression: + dialects: + - dialect: ANSI_SQL + expression: YEAR(order_date) + datatype: Integer + - name: order_month_start + expression: + dialects: + - dialect: ANSI_SQL + expression: DATE_TRUNC('month', order_date) + datatype: Date +metrics: + - name: avg_fulfillment_days + expression: + dialects: + - dialect: ANSI_SQL + expression: AVG(DATEDIFF('day', sales.order_date, sales.ship_date)) + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 0382f779..21a9cd35 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -103,3 +103,13 @@ rungs: variants: - {id: M3.filtered_agg.qualified, file: m3/m3_filtered_agg_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - {id: M3.filtered_agg.bare, file: m3/m3_filtered_agg_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} + + - id: M6 + description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) + element_under_test: + kind: fields and metric + name: order_year, order_month_start, avg_fulfillment_days + construct: date scalar functions (DATEDIFF(part, start, end) per expression_language.md) + reference_form_primary: qualified # the fields reference sibling columns intra-dataset + variants: + - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} From 102d2d83c8cff0e356e6e8151624a9b19b84cdc6 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:14:20 +0200 Subject: [PATCH 2/5] test(compliance): add the FILTER (WHERE ...) form to portability rung M3 expression_language.md makes the aggregate FILTER (WHERE ...) clause REQUIRED. Two variants (qualified and bare) express the same filtered aggregate as the existing CASE WHEN fixtures, so a difference between the two forms isolates handling of the clause itself. Co-authored-by: Isaac --- .../m3/m3_filter_clause_bare.ossie.yaml | 47 +++++++++++++++++++ .../m3/m3_filter_clause_qualified.ossie.yaml | 47 +++++++++++++++++++ compliance/portability/manifest.yaml | 4 +- 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 compliance/portability/m3/m3_filter_clause_bare.ossie.yaml create mode 100644 compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml diff --git a/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml b/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml new file mode 100644 index 00000000..795743af --- /dev/null +++ b/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M3 ladder rung, filter-clause form: the same filtered aggregate as the CASE +# WHEN fixtures, written with the aggregate FILTER (WHERE ...) clause that +# expression_language.md makes REQUIRED. Comparing the two forms shows whether a +# converter handles the clause itself. Variant: BARE reference form, kept as a sibling variant. +version: "0.2.0.dev0" +name: m3_sales +description: "M3: conditional aggregation with the FILTER (WHERE ...) clause, bare reference form." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + datatype: Decimal + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + datatype: String +metrics: + - name: completed_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(amount) FILTER (WHERE status = 'completed') + datatype: Decimal diff --git a/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml b/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml new file mode 100644 index 00000000..49af6d34 --- /dev/null +++ b/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M3 ladder rung, filter-clause form: the same filtered aggregate as the CASE +# WHEN fixtures, written with the aggregate FILTER (WHERE ...) clause that +# expression_language.md makes REQUIRED. Comparing the two forms shows whether a +# converter handles the clause itself. Variant: QUALIFIED reference form, carried as primary. +version: "0.2.0.dev0" +name: m3_sales +description: "M3: conditional aggregation with the FILTER (WHERE ...) clause, qualified reference form." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + datatype: Decimal + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + datatype: String +metrics: + - name: completed_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(sales.amount) FILTER (WHERE sales.status = 'completed') + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 21a9cd35..965b0cd0 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -94,7 +94,7 @@ rungs: - {id: M2.aggregates.bare, file: m2/m2_aggregates_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} - id: M3 - description: a filtered aggregate in the portable form SUM(CASE WHEN ... THEN x ELSE 0 END) + description: a filtered aggregate, in the portable form SUM(CASE WHEN ... THEN x ELSE 0 END) and with the REQUIRED FILTER (WHERE ...) clause element_under_test: kind: metric name: completed_amount @@ -103,6 +103,8 @@ rungs: variants: - {id: M3.filtered_agg.qualified, file: m3/m3_filtered_agg_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - {id: M3.filtered_agg.bare, file: m3/m3_filtered_agg_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} + - {id: M3.filter_clause.qualified, file: m3/m3_filter_clause_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: filter-clause-form} + - {id: M3.filter_clause.bare, file: m3/m3_filter_clause_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: filter-clause-form} - id: M6 description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) From bc64a5c529e91f7e4c621545a7bc9ce98495c21f Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:14:42 +0200 Subject: [PATCH 3/5] test(compliance): add portability rung M4, a cross-dataset metric Two datasets joined by a declared relationship and one model-scoped metric that aggregates across the join (SUM over one dataset divided by a COUNT DISTINCT over the other), referencing fields as dataset.field. Co-authored-by: Isaac --- .../m4/m4_cross_dataset.ossie.yaml | 63 +++++++++++++++++++ compliance/portability/manifest.yaml | 10 +++ 2 files changed, 73 insertions(+) create mode 100644 compliance/portability/m4/m4_cross_dataset.ossie.yaml diff --git a/compliance/portability/m4/m4_cross_dataset.ossie.yaml b/compliance/portability/m4/m4_cross_dataset.ossie.yaml new file mode 100644 index 00000000..768eb790 --- /dev/null +++ b/compliance/portability/m4/m4_cross_dataset.ossie.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M4 ladder rung: two datasets and a relationship, with a metric that +# aggregates across the join. A cross-dataset metric must qualify its columns +# by dataset to be unambiguous, so this rung has a single qualified fixture +# (no bare variant: a bare column would not say which dataset it belongs to). +version: "0.2.0.dev0" +name: m4_sales +description: "M4: two datasets, a relationship, and a cross-dataset metric." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + datatype: Decimal + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + datatype: Integer + - name: customers + source: samples.sales.customers + primary_key: [customer_id] + fields: + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + datatype: Integer +relationships: + - name: sales_to_customer + from: sales + to: customers + from_columns: [customer_id] + to_columns: [customer_id] +metrics: + - name: revenue_per_customer + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(sales.amount) / COUNT(DISTINCT customers.customer_id) + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 965b0cd0..dad7a083 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -115,3 +115,13 @@ rungs: reference_form_primary: qualified # the fields reference sibling columns intra-dataset variants: - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} + + - id: M4 + description: two datasets joined by a relationship, with a model-scoped metric across the join + element_under_test: + kind: metric + name: revenue_per_customer + construct: cross-dataset (model-scoped) metric over a declared relationship + reference_form_primary: qualified # model-scoped metrics reference dataset.field + variants: + - {id: M4.cross_dataset, file: m4/m4_cross_dataset.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} From 44587329b3e2fd81c9e4c459b3d4f81c848ea861 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:15:07 +0200 Subject: [PATCH 4/5] test(compliance): add portability rung M5, metric-to-metric composition A model-scoped metric (avg_order_value) composed from two other model-scoped metrics referenced by name, per Metric Scoping in core-spec/spec.md. Co-authored-by: Isaac --- .../portability/m5/m5_metric_ref.ossie.yaml | 59 +++++++++++++++++++ compliance/portability/manifest.yaml | 10 ++++ 2 files changed, 69 insertions(+) create mode 100644 compliance/portability/m5/m5_metric_ref.ossie.yaml diff --git a/compliance/portability/m5/m5_metric_ref.ossie.yaml b/compliance/portability/m5/m5_metric_ref.ossie.yaml new file mode 100644 index 00000000..29eda689 --- /dev/null +++ b/compliance/portability/m5/m5_metric_ref.ossie.yaml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M5 ladder rung: metric composition, a model-scoped metric that references +# other model-scoped metrics by name (avg_order_value refers to total_amount and +# order_count), as described under Metric Scoping in core-spec/spec.md, where a +# model-scoped metric is referenced as metric_name. Single fixture. +version: "0.2.0.dev0" +name: m5_sales +description: "M5: a metric that references other metrics (composition)." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + datatype: Decimal + - name: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id + datatype: Integer +metrics: + - name: total_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(sales.amount) + datatype: Decimal + - name: order_count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(sales.order_id) + datatype: Integer + - name: avg_order_value + expression: + dialects: + - dialect: ANSI_SQL + expression: total_amount / order_count + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index dad7a083..a4965ecc 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -125,3 +125,13 @@ rungs: reference_form_primary: qualified # model-scoped metrics reference dataset.field variants: - {id: M4.cross_dataset, file: m4/m4_cross_dataset.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} + + - id: M5 + description: a model-scoped metric composed from other model-scoped metrics, referenced by name + element_under_test: + kind: metric + name: avg_order_value + construct: metric-to-metric reference (total_amount / order_count) + reference_form_primary: qualified # base metrics qualify dataset.field; the composite references metric names + variants: + - {id: M5.metric_ref, file: m5/m5_metric_ref.ossie.yaml, tag: ANSI_SQL, reference_form: metric-name, role: primary} From 9e425fef60190b468cbbf24d5d7633754c7dc423 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:15:42 +0200 Subject: [PATCH 5/5] docs(compliance): describe portability rungs M0-M6 Lists the M3 FILTER (WHERE ...) form and rungs M4-M6 in the README table, notes that M7 and M8 wait on the window ORDER BY NULL-placement default, and records that the fixtures live in this repository. Co-authored-by: Isaac --- compliance/portability/README.md | 19 +++++++++++-------- compliance/portability/manifest.yaml | 25 ++++++++++++------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/compliance/portability/README.md b/compliance/portability/README.md index 9039231a..7a87927f 100644 --- a/compliance/portability/README.md +++ b/compliance/portability/README.md @@ -17,7 +17,7 @@ under the License. --> -# Portability measurement ladder (M0-M3) +# Portability measurement ladder (M0-M6) A small set of portable fixtures, and a grader, for measuring how faithfully a model survives a round-trip through each converter. This is issue #576 @@ -37,7 +37,10 @@ difference in a converter's output localizes to a single construct: | M0 | baseline | one dataset, one field, one `SUM` metric | | M1 | a scalar field, no aggregation | `amount * 1.1` calculated column | | M2 | the core aggregate set | `SUM`, `COUNT`, `COUNT(DISTINCT)`, `AVG`, `MIN`, `MAX` | -| M3 | a filtered aggregate | `SUM(CASE WHEN ... THEN x ELSE 0 END)` | +| M3 | a filtered aggregate | `SUM(CASE WHEN ... THEN x ELSE 0 END)`, and the same with `FILTER (WHERE ...)` | +| M4 | a cross-dataset metric | two datasets, a relationship, a model-scoped metric across the join | +| M5 | metric composition | a model-scoped metric built from two other metrics by name | +| M6 | date and time scalars | `YEAR`, `DATE_TRUNC`, `DATEDIFF(part, start, end)` | `manifest.yaml` lists the fixtures, the element each rung exercises, and the per-element outcome vocabulary the grader assigns (PORTABLE / TAG_PRESERVED, @@ -61,10 +64,10 @@ grades the real fixtures all-PORTABLE. See `grader/README.md`. ## Scope and placement -Rungs M4-M8 (relationships and cross-dataset metrics, metric-to-metric -composition, date and time scalars, window and offset functions) are held -pending #354, #591 and #587, and are not in this set. +Rungs M7 and M8 (window and offset functions: time intelligence and +ratio-to-total) are held until the default NULL placement in window `ORDER BY` +is settled (#587), and are not in this set. -Location is a starting point. If these should sit with the interoperability -harness from #348 or elsewhere once that lands, they can move; the fixtures are -self-contained and the manifest paths are relative. +The fixtures measure the specification rather than any one converter, so they +live in this repository; the fixtures are self-contained and the manifest paths +are relative. diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index a4965ecc..3d8cd12b 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -35,9 +35,8 @@ # expression, so it carries a single fixture. # # All fixtures validate.py-pass on apache/ossie main (9531429) with sqlglot -# 29.0.1. Rungs M4-M8 (relationships, metric composition, date scalars, window -# and offset functions) are held pending #354, #591 and #587; they are not in -# this set. +# 29.0.1. Rungs M7 and M8 (window and offset functions) are held pending #587; +# they are not in this set. outcomes: # per-element, assigned by the grader after diffing input vs returned output @@ -106,16 +105,6 @@ rungs: - {id: M3.filter_clause.qualified, file: m3/m3_filter_clause_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: filter-clause-form} - {id: M3.filter_clause.bare, file: m3/m3_filter_clause_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: filter-clause-form} - - id: M6 - description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) - element_under_test: - kind: fields and metric - name: order_year, order_month_start, avg_fulfillment_days - construct: date scalar functions (DATEDIFF(part, start, end) per expression_language.md) - reference_form_primary: qualified # the fields reference sibling columns intra-dataset - variants: - - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - - id: M4 description: two datasets joined by a relationship, with a model-scoped metric across the join element_under_test: @@ -135,3 +124,13 @@ rungs: reference_form_primary: qualified # base metrics qualify dataset.field; the composite references metric names variants: - {id: M5.metric_ref, file: m5/m5_metric_ref.ossie.yaml, tag: ANSI_SQL, reference_form: metric-name, role: primary} + + - id: M6 + description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) + element_under_test: + kind: fields and metric + name: order_year, order_month_start, avg_fulfillment_days + construct: date scalar functions (DATEDIFF(part, start, end) per expression_language.md) + reference_form_primary: qualified # the fields reference sibling columns intra-dataset + variants: + - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary}