refactor: dedup regex/validation/provider code (file-search-on); fix JSON in predicate

.gitignore

@@ -18,6 +18,16 @@
 examples/*/main
 examples/*/main.exe
 
+# Example binaries built without -o (named after the directory)
+examples/basic/basic
+examples/comprehensions/comprehensions
+examples/context/context
+examples/index_analysis/index_analysis
+examples/load_table_schema/load_table_schema
+examples/logging/logging
+examples/parameterized/parameterized
+examples/string_extensions/string_extensions
+
 # Claude Code settings
 .claude/settings.local.json
 

CHANGELOG.md

@@ -1,6 +1,22 @@
 # Changelog
 
 ## [Unreleased]
+### Fixed
+- **JSON array membership (`in`) now generates a correct boolean predicate on
+  every dialect.** Each dialect now owns the full predicate, emitting both the
+  element and the array expression, instead of relying on the caller to prepend
+  `elem = `. This resolves semantically wrong SQL on several dialects:
+  - **MySQL**: switched from `JSON_CONTAINS(arr, CAST(? AS JSON))` (which
+    emitted a stray `?` and ignored the element) to
+    `JSON_OVERLAPS(JSON_ARRAY(elem), arr)`.
+  - **SQLite/DuckDB**: switched from a bare `(SELECT value FROM json_each(arr))`
+    scalar subquery to `EXISTS (SELECT 1 FROM json_each(arr) WHERE value = elem)`.
+  - **BigQuery**: switched from the invalid `= UNNEST(...)` form to
+    `elem IN UNNEST(JSON_VALUE_ARRAY(arr))`.
+  - **PostgreSQL**: unchanged semantics (`elem = ANY(ARRAY(SELECT jsonFunc(arr)))`).
+  - **Spark**: `array_contains(from_json(arr, 'ARRAY<STRING>'), elem)`.
+
+  Ported from cel2sql4j ([SPANDigital/cel2sql4j@1835215](https://github.com/SPANDigital/cel2sql4j/commit/1835215bb1244b3b15c82315f264354566cfa499)).
 
 ## [3.8.4] - 2026-06-08
 ### Changed

bigquery/provider.go

@@ -10,9 +10,9 @@ import (
 	bq "cloud.google.com/go/bigquery"
 	"github.com/google/cel-go/checker/decls"
 	"github.com/google/cel-go/common/types"
-	"github.com/google/cel-go/common/types/ref"
 	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
 
+	"github.com/spandigital/cel2sql/v3/internal/celprovider"
 	"github.com/spandigital/cel2sql/v3/schema"
 )
 
@@ -40,14 +40,14 @@ type TypeProvider interface {
 }
 
 type typeProvider struct {
-	schemas   map[string]Schema
+	celprovider.Base
 	client    *bq.Client
 	datasetID string
 }
 
 // NewTypeProvider creates a new BigQuery type provider with pre-defined schemas.
 func NewTypeProvider(schemas map[string]Schema) TypeProvider {
-	return &typeProvider{schemas: schemas}
+	return &typeProvider{Base: celprovider.Base{Schemas: schemas, Mapper: bigqueryTypeToCELExprType}}
 }
 
 // NewTypeProviderWithClient creates a new BigQuery type provider that can introspect database schemas.
@@ -61,7 +61,7 @@ func NewTypeProviderWithClient(_ context.Context, client *bq.Client, datasetID s
 	}
 
 	return &typeProvider{
-		schemas:   make(map[string]Schema),
+		Base:      celprovider.Base{Schemas: make(map[string]Schema), Mapper: bigqueryTypeToCELExprType},
 		client:    client,
 		datasetID: datasetID,
 	}, nil
@@ -83,7 +83,7 @@ func (tp *typeProvider) LoadTableSchema(ctx context.Context, tableName string) e
 		return fmt.Errorf("%w: table %q has no columns", ErrInvalidSchema, tableName)
 	}
 
-	tp.schemas[tableName] = NewSchema(fields)
+	tp.Schemas[tableName] = NewSchema(fields)
 	return nil
 }
 
@@ -127,75 +127,6 @@ func bigqueryFieldTypeToString(ft bq.FieldType) string {
 	return strings.ToLower(string(ft))
 }
 
-// Close is a no-op since we don't own the *bigquery.Client.
-func (tp *typeProvider) Close() {
-	// No-op: caller owns the *bigquery.Client connection
-}
-
-// GetSchemas returns the schemas known to this type provider.
-func (tp *typeProvider) GetSchemas() map[string]Schema {
-	return tp.schemas
-}
-
-// EnumValue implements types.Provider.
-func (tp *typeProvider) EnumValue(_ string) ref.Val {
-	return types.NewErr("unknown enum value")
-}
-
-// FindIdent implements types.Provider.
-func (tp *typeProvider) FindIdent(_ string) (ref.Val, bool) {
-	return nil, false
-}
-
-// FindStructType implements types.Provider.
-func (tp *typeProvider) FindStructType(structType string) (*types.Type, bool) {
-	if _, ok := tp.schemas[structType]; ok {
-		return types.NewObjectType(structType), true
-	}
-	return nil, false
-}
-
-// FindStructFieldNames implements types.Provider.
-func (tp *typeProvider) FindStructFieldNames(structType string) ([]string, bool) {
-	s, ok := tp.schemas[structType]
-	if !ok {
-		return nil, false
-	}
-	fields := s.Fields()
-	names := make([]string, len(fields))
-	for i, f := range fields {
-		names[i] = f.Name
-	}
-	return names, true
-}
-
-// FindStructFieldType implements types.Provider.
-func (tp *typeProvider) FindStructFieldType(structType, fieldName string) (*types.FieldType, bool) {
-	s, ok := tp.schemas[structType]
-	if !ok {
-		return nil, false
-	}
-	field, found := s.FindField(fieldName)
-	if !found {
-		return nil, false
-	}
-
-	exprType := bigqueryTypeToCELExprType(field)
-	celType, err := types.ExprTypeToType(exprType)
-	if err != nil {
-		return nil, false
-	}
-
-	return &types.FieldType{
-		Type: celType,
-	}, true
-}
-
-// NewValue implements types.Provider.
-func (tp *typeProvider) NewValue(_ string, _ map[string]ref.Val) ref.Val {
-	return types.NewErr("unknown type in schema")
-}
-
 // bigqueryTypeToCELExprType converts a BigQuery field schema to a CEL expression type.
 func bigqueryTypeToCELExprType(field *schema.FieldSchema) *exprpb.Type {
 	baseType := bigqueryBaseTypeToCEL(field.Type)

cel2sql.go

@@ -702,12 +702,25 @@ func (con *converter) visitCallBinary(expr *exprpb.Expr) error {
 		)
 	}
 
-	// Handle array membership (IN operator with list) via dialect before writing LHS.
-	// This allows dialects like SQLite to use a fundamentally different pattern
-	// (e.g., "elem IN (SELECT value FROM json_each(array))") instead of "elem = ANY(array)".
-	if fun == operators.In && isListType(rhsType) {
-		// Non-JSON list membership
-		if !isFieldAccessExpression(rhs) || !con.isJSONArrayField(rhs) {
+	// Handle array membership (IN operator) via dialect before writing LHS.
+	// This allows each dialect to own the complete boolean predicate, using a
+	// fundamentally different pattern (e.g., SQLite's
+	// "EXISTS (SELECT 1 FROM json_each(array) WHERE value = elem)") instead of
+	// the caller emitting "elem = ANY(array)".
+	if fun == operators.In && (isListType(rhsType) || isFieldAccessExpression(rhs)) {
+		// JSON array membership: the dialect emits both the element and array.
+		if isFieldAccessExpression(rhs) && con.isJSONArrayField(rhs) {
+			writeElem := func() error { return con.visitMaybeNested(lhs, lhsParen) }
+			if con.isNestedJSONAccess(rhs) {
+				return con.dialect.WriteNestedJSONArrayMembership(&con.str, writeElem,
+					func() error { return con.visitNestedJSONForArray(rhs) })
+			}
+			jsonFunc := con.getJSONArrayFunction(rhs)
+			return con.dialect.WriteJSONArrayMembership(&con.str, jsonFunc, writeElem,
+				func() error { return con.visitMaybeNested(rhs, rhsParen) })
+		}
+		// Non-JSON list membership.
+		if isListType(rhsType) {
 			return con.dialect.WriteArrayMembership(&con.str,
 				func() error { return con.visitMaybeNested(lhs, lhsParen) },
 				func() error { return con.visitMaybeNested(rhs, rhsParen) },
@@ -769,40 +782,18 @@ func (con *converter) visitCallBinary(expr *exprpb.Expr) error {
 	con.str.WriteString(" ")
 	con.str.WriteString(operator)
 	con.str.WriteString(" ")
-	if fun == operators.In && (isListType(rhsType) || isFieldAccessExpression(rhs)) {
-		// Check if we're dealing with a JSON array
-		if isFieldAccessExpression(rhs) && con.isJSONArrayField(rhs) {
-			// For JSON arrays, use dialect-specific JSON array membership
-			jsonFunc := con.getJSONArrayFunction(rhs)
-
-			// For nested JSON access like settings.permissions, we need to handle differently
-			if con.isNestedJSONAccess(rhs) {
-				// Use dialect-specific nested JSON array membership
-				if err := con.dialect.WriteNestedJSONArrayMembership(&con.str, func() error {
-					return con.visitNestedJSONForArray(rhs)
-				}); err != nil {
-					return err
-				}
-				return nil
-			}
-			// For direct JSON array access
-			if err := con.dialect.WriteJSONArrayMembership(&con.str, jsonFunc, func() error {
-				return con.visitMaybeNested(rhs, rhsParen)
-			}); err != nil {
-				return err
-			}
-			return nil
-		}
+	// Remaining membership case: field access on a non-JSON, non-list-typed
+	// column (e.g. a Dyn-typed array column) wraps the RHS in ANY().
+	// JSON arrays and list literals are handled by the dialect before the LHS
+	// is written.
+	if fun == operators.In && isFieldAccessExpression(rhs) {
 		con.str.WriteString("ANY(")
 	}
 	if err := con.visitMaybeNested(rhs, rhsParen); err != nil {
 		return err
 	}
-	if fun == operators.In && (isListType(rhsType) || isFieldAccessExpression(rhs)) {
-		// Check if we're dealing with a JSON array - already handled above for JSON arrays
-		if !isFieldAccessExpression(rhs) || !con.isJSONArrayField(rhs) {
-			con.str.WriteString(")")
-		}
+	if fun == operators.In && isFieldAccessExpression(rhs) {
+		con.str.WriteString(")")
 	}
 	return nil
 }

dialect/bigquery/dialect.go

@@ -261,20 +261,28 @@ func (d *Dialect) WriteJSONExtractPath(w *strings.Builder, pathSegments []string
 	return nil
 }
 
-// WriteJSONArrayMembership writes BigQuery JSON array membership.
-func (d *Dialect) WriteJSONArrayMembership(w *strings.Builder, _ string, writeExpr func() error) error {
-	w.WriteString("UNNEST(JSON_VALUE_ARRAY(")
-	if err := writeExpr(); err != nil {
+// WriteJSONArrayMembership writes BigQuery JSON array membership using
+// elem IN UNNEST(JSON_VALUE_ARRAY(arr)).
+func (d *Dialect) WriteJSONArrayMembership(w *strings.Builder, _ string, writeElem func() error, writeArray func() error) error {
+	if err := writeElem(); err != nil {
+		return err
+	}
+	w.WriteString(" IN UNNEST(JSON_VALUE_ARRAY(")
+	if err := writeArray(); err != nil {
 		return err
 	}
 	w.WriteString("))")
 	return nil
 }
 
-// WriteNestedJSONArrayMembership writes BigQuery nested JSON array membership.
-func (d *Dialect) WriteNestedJSONArrayMembership(w *strings.Builder, writeExpr func() error) error {
-	w.WriteString("UNNEST(JSON_VALUE_ARRAY(")
-	if err := writeExpr(); err != nil {
+// WriteNestedJSONArrayMembership writes BigQuery nested JSON array membership using
+// elem IN UNNEST(JSON_VALUE_ARRAY(arr)).
+func (d *Dialect) WriteNestedJSONArrayMembership(w *strings.Builder, writeElem func() error, writeArray func() error) error {
+	if err := writeElem(); err != nil {
+		return err
+	}
+	w.WriteString(" IN UNNEST(JSON_VALUE_ARRAY(")
+	if err := writeArray(); err != nil {
 		return err
 	}
 	w.WriteString("))")