feat: AST SQL rewriting

docs-mintlify/reference/core-data-apis/rest-api/reference.mdx

@@ -339,6 +339,129 @@ Successful response:
 }
 ```
 
+## `{base_path}/v1/sql-filters`
+
+Reads and edits the filters of a [SQL API](/reference/core-data-apis/sql-api) query,
+in the [REST (JSON) API filter format](/reference/core-data-apis/rest-api/query-format#filters-format).
+
+Reading is plan-wide: `GET` reports the filters of the whole query, CTEs and
+subqueries included. Rewriting touches the outermost `SELECT` only, and only
+members available in it can be filtered, so a reported filter that lives only
+in a CTE or a subquery cannot be deleted or replaced, and `set` leaves it in
+place.
+
+### Reading filters
+
+`GET` returns the filters of a query, as the query planner resolves them.
+
+| Parameter, type | Description | Required |
+| --- | --- | --- |
+| `query`, `string` | SQL API query to read the filters of | ✅ Yes |
+
+```bash
+curl \
+  -H "Authorization: TOKEN" \
+  -G --data-urlencode "query=SELECT MEASURE(total_amount) FROM orders WHERE status = 'shipped'" \
+  http://localhost:4000/cubejs-api/v1/sql-filters
+```
+
+```json
+{
+  "status": "ok",
+  "filters": [
+    {
+      "member": "orders.status",
+      "operator": "equals",
+      "values": ["shipped"]
+    }
+  ]
+}
+```
+
+What comes back is what the planner made of the query rather than what the query
+spells out: two bounds on a time dimension are reported as a single
+`inDateRange` filter, and a relative date such as `CURRENT_DATE - INTERVAL '7 days'`
+is reported as the date it was worked out to be.
+
+### Editing filters
+
+`POST` rewrites the query and returns the result. Exactly one of `add`, `set`,
+`delete` or `replace` must be present.
+
+| Parameter, type | Description | Required |
+| --- | --- | --- |
+| `query`, `string` | SQL API query to rewrite | ✅ Yes |
+| `add`, `array` | Filters to add to the outermost `SELECT` | One of the four |
+| `set`, `array` | Filters to replace the query's reported filters with: the filters `GET` reports are removed where the outermost `SELECT` spells them out (see below), then these are added | One of the four |
+| `delete`, `array` | Filters to remove; every occurrence of an equal filter is removed | One of the four |
+| `replace`, `object` | `{ "old": [...], "new": [...] }`; every occurrence of an equal filter in `old` is replaced | One of the four |
+
+A filter is either a single filter or an `and`/`or` filter group, exactly as in
+the [REST (JSON) API query format](/reference/core-data-apis/rest-api/query-format#filters-format).
+A request carries at most 500 filters, counting the ones nested inside a group
+rather than the entries of the array, and at most 4 MiB and 4,000 operators of
+SQL. A number or a boolean in `values` is
+accepted and carried as its string form, as `/v1/load` takes it; the SQL
+written follows the member's type. Unlike `/v1/load`, `null` is not accepted.
+
+The response carries the rewritten query and the filters it reports afterwards:
+
+| Property, type | Description |
+| --- | --- |
+| `status`, `string` | `ok`, or `error` for a failure the planner reports; a request the gateway rejects (`400`, `403`) or a server fault carries `error` alone |
+| `sql`, `string` | Rewritten query (only present when `status` is `ok`) |
+| `filters`, `array` | Filters of the rewritten query (only present when `status` is `ok`) |
+| `error`, `string` | Error message (only present when `status` is `error`) |
+
+A filter this endpoint writes is spelled qualified and quoted, as in
+`orders."status" = 'completed'`, whatever spelling the query used for the
+filters it already held.
+
+A reported filter the outermost `SELECT` does not spell out the way this
+endpoint writes it is matched by its column instead: when it is the only
+reported filter on its member, every literal predicate on that column is
+removed, so `delete` and `set` may drop predicates that were not asked about;
+when it shares its member with another reported filter, it may be kept.
+Compare the returned `filters` with what was asked for.
+
+A query that cannot be planned, a member that is not available in the outermost
+`SELECT`, and a filter to `replace` that is not present are all answered with
+`400` and an `error`. Deleting a filter that is not there is not an error: the
+query comes back unchanged.
+
+### Example
+
+Replacing a status filter:
+
+```bash
+curl \
+  -X POST \
+  -H "Authorization: TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "query": "SELECT MEASURE(total_amount) FROM orders WHERE status = '\''shipped'\''",
+    "replace": {
+      "old": [{"member": "orders.status", "operator": "equals", "values": ["shipped"]}],
+      "new": [{"member": "orders.status", "operator": "equals", "values": ["completed"]}]
+    }
+  }' \
+  http://localhost:4000/cubejs-api/v1/sql-filters
+```
+
+```json
+{
+  "status": "ok",
+  "sql": "SELECT MEASURE(total_amount) FROM orders WHERE orders.\"status\" = 'completed'",
+  "filters": [
+    {
+      "member": "orders.status",
+      "operator": "equals",
+      "values": ["completed"]
+    }
+  ]
+}
+```
+
 ## `{base_path}/v1/meta`
 
 <Info>

packages/cubejs-api-gateway/src/gateway.ts

@@ -22,6 +22,7 @@
   redactSqlLiterals,
   rowsToColumnar,
 } from '@cubejs-backend/native';
+import type { SqlFilterItem, SqlFiltersResponse } from '@cubejs-backend/native';
 import type {
   Application as ExpressApplication,
   ErrorRequestHandler,
@@ -484,6 +485,26 @@
       });
     }));
 
+    app.get(`${this.basePath}/v1/sql-filters`, userMiddlewares, userAsyncHandler(async (req: any, res) => {
+      await this.getSqlFilters({
+        query: req.query.query,
+        context: req.context,
+        res: this.resToResultFn(res)
+      });
+    }));
+
+    app.post(`${this.basePath}/v1/sql-filters`, jsonParser, userMiddlewares, userAsyncHandler(async (req, res) => {
+      await this.modifySqlFilters({
+        query: req.body.query,
+        add: req.body.add,
+        set: req.body.set,
+        delete: req.body.delete,
+        replace: req.body.replace,
+        context: req.context,
+        res: this.resToResultFn(res)
+      });
+    }));
+
     app.get(`${this.basePath}/v1/dry-run`, userMiddlewares, userAsyncHandler(async (req: any, res) => {
       await this.dryRun({
         query: req.query.query,
@@ -1553,6 +1574,182 @@
     }
   }
 
+  /**
+   * Responds with the result of a SQL filters operation: an in-band
+   * `{ status: 'error', error }` is a 400, logged like every other one. Which
+   * failures arrive in-band and which are thrown is decided by
+   * `in_band_or_thrown` in the native layer.
+   */
+  protected async resSqlFilters(
+    result: SqlFiltersResponse,
+    res: ResponseResultFn,
+    { query, context, requestStarted }: { query: string, context: any, requestStarted: Date },
+  ) {
+    if (result.status === 'error') {
+      this.log({
+        type: 'User Error',
+        query: { sql: query },
+        redactedQuery: this.redactedSqlForLog(query),
+        error: result.error,
+        duration: this.duration(requestStarted),
+      }, context);
+      const requestId = getEnv('devMode') || context?.signedWithPlaygroundAuthSecret
+        ? context?.requestId
+        : undefined;
+      await res({ ...result, requestId }, { status: 400 });
+      return;
+    }
+
+    await res(result);
+  }
+
+  /**
+   * Returns the list of Cube filters of a SQL query in Cube query format,
+   * extracted from the logical plan of the query.
+   */
+  protected async getSqlFilters({
+    query,
+    context,
+    res,
+  }: { query: string } & BaseRequest) {
+    const requestStarted = new Date();
+
+    try {
+      await this.assertApiScope('sql', context.securityContext);
+
+      if (typeof query !== 'string' || !query.trim()) {
+        throw new UserError('query parameter must be a non-empty string');
+      }
+
+      const result = await this.sqlServer.getSqlFilters(query, context.securityContext);
+
+      await this.resSqlFilters(result, res, { query, context, requestStarted });
+    } catch (e: any) {
+      this.handleError({
+        e,
+        context,
+        query: { sql: query },
+        redactedQuery: this.redactedSqlForLog(query),
+        res,
+        requestStarted,
+      });
+    }
+  }
+
+  /**
+   * Rewrites the filters of a SQL API query with exactly one of `add`, `set`,
+   * `delete` or `replace`. The semantics live in cubesql's `ast_conv` and are
+   * documented at `reference/core-data-apis/rest-api/reference.mdx`.
+   */
+  protected async modifySqlFilters({
+    query,
+    add,
+    set,
+    delete: deleteFilters,
+    replace,
+    context,
+    res,
+  }: { query: string, add?: unknown, set?: unknown, delete?: unknown, replace?: unknown } & BaseRequest) {
+    const requestStarted = new Date();
+
+    try {
+      await this.assertApiScope('sql', context.securityContext);
+
+      if (typeof query !== 'string' || !query.trim()) {
+        throw new UserError('query parameter must be a non-empty string');
+      }
+
+      const requestedOps = [add, set, deleteFilters, replace].filter((op) => op !== undefined);
+      if (requestedOps.length !== 1) {
+        throw new UserError('Exactly one of add, set, delete or replace parameters is required');
+      }
+
+      // The shape alone is checked here; the filter count is bounded by the
+      // native layer, which answers in-band and lands on the same 400. A number
+      // or a boolean in `values` is carried as its string form, as /v1/load
+      // does; a null has no place in the native filter type and is refused.
+      const normalizeValue = (value: unknown): string => {
+        if (typeof value === 'string') {
+          return value;
+        }
+        if (typeof value === 'number' || typeof value === 'boolean') {
+          return String(value);
+        }
+        throw new UserError('filter values must be strings, numbers or booleans');
+      };
+      const normalizeFilter = (source: any): SqlFilterItem => {
+        if (typeof source !== 'object' || source === null) {
+          return source;
+        }
+
+        const filter: SqlFilterItem = { ...source };
+
+        if (Array.isArray(source.values)) {
+          filter.values = source.values.map(normalizeValue);
+        }
+
+        for (const group of ['and', 'or'] as const) {
+          if (Array.isArray(source[group])) {
+            filter[group] = source[group].map(normalizeFilter);
+          }
+        }
+
+        return filter;
+      };
+      const assertFilterArray = (filters: unknown, name: string): SqlFilterItem[] => {
+        if (!Array.isArray(filters)) {
+          throw new UserError(`${name} parameter must be an array of filters`);
+        }
+
+        return filters.map(normalizeFilter);
+      };
+
+      if (add !== undefined) {
+        const result = await this.sqlServer.addSqlFilters(query, assertFilterArray(add, 'add'), context.securityContext);
+
+        await this.resSqlFilters(result, res, { query, context, requestStarted });
+        return;
+      }
+
+      if (set !== undefined) {
+        const result = await this.sqlServer.setSqlFilters(query, assertFilterArray(set, 'set'), context.securityContext);
+
+        await this.resSqlFilters(result, res, { query, context, requestStarted });
+        return;
+      }
+
+      if (deleteFilters !== undefined) {
+        const result = await this.sqlServer.deleteSqlFilters(query, assertFilterArray(deleteFilters, 'delete'), context.securityContext);
+
+        await this.resSqlFilters(result, res, { query, context, requestStarted });
+        return;
+      }
+
+      if (typeof replace !== 'object' || replace === null || Array.isArray(replace)) {
+        throw new UserError('replace parameter must be an object with old and new filter arrays');
+      }
+
+      const { old: oldFilters, new: newFilters } = replace as Record<string, unknown>;
+      const result = await this.sqlServer.replaceSqlFilters(
+        query,
+        assertFilterArray(oldFilters, 'replace.old'),
+        assertFilterArray(newFilters, 'replace.new'),
+        context.securityContext,
+      );
+
+      await this.resSqlFilters(result, res, { query, context, requestStarted });
+    } catch (e: any) {
+      this.handleError({
+        e,
+        context,
+        query: { sql: query },
+        redactedQuery: this.redactedSqlForLog(query),
+        res,
+        requestStarted,
+      });
+    }
+  }
+
   public async sql({
     query,
     context,

packages/cubejs-api-gateway/src/sql-server.ts

@@ -5,11 +5,18 @@ import {
   execSql,
   sql4sql,
   rest4sql,
+  getSqlFilters,
+  addSqlFilters,
+  setSqlFilters,
+  deleteSqlFilters,
+  replaceSqlFilters,
   SqlInterfaceInstance,
   Request as NativeRequest,
   LoadRequestMeta,
   Sql4SqlResponse,
   QueryConvertResponse,
+  SqlFilterItem,
+  SqlFiltersResponse,
 } from '@cubejs-backend/native';
 import type { ShutdownMode } from '@cubejs-backend/native';
 import { displayCLIWarning, getEnv, CacheMode } from '@cubejs-backend/shared';
@@ -88,6 +95,26 @@ export class SQLServer {
     return rest4sql(this.getSqlInterfaceInstance(), sqlQuery, securityContext);
   }
 
+  public async getSqlFilters(sqlQuery: string, securityContext?: unknown): Promise<SqlFiltersResponse> {
+    return getSqlFilters(this.getSqlInterfaceInstance(), sqlQuery, securityContext);
+  }
+
+  public async addSqlFilters(sqlQuery: string, filters: SqlFilterItem[], securityContext?: unknown): Promise<SqlFiltersResponse> {
+    return addSqlFilters(this.getSqlInterfaceInstance(), sqlQuery, filters, securityContext);
+  }
+
+  public async setSqlFilters(sqlQuery: string, filters: SqlFilterItem[], securityContext?: unknown): Promise<SqlFiltersResponse> {
+    return setSqlFilters(this.getSqlInterfaceInstance(), sqlQuery, filters, securityContext);
+  }
+
+  public async deleteSqlFilters(sqlQuery: string, filters: SqlFilterItem[], securityContext?: unknown): Promise<SqlFiltersResponse> {
+    return deleteSqlFilters(this.getSqlInterfaceInstance(), sqlQuery, filters, securityContext);
+  }
+
+  public async replaceSqlFilters(sqlQuery: string, oldFilters: SqlFilterItem[], newFilters: SqlFilterItem[], securityContext?: unknown): Promise<SqlFiltersResponse> {
+    return replaceSqlFilters(this.getSqlInterfaceInstance(), sqlQuery, oldFilters, newFilters, securityContext);
+  }
+
   protected buildCheckSqlAuth(options: SQLServerOptions): CheckSQLAuthFn {
     return (options.checkSqlAuth && this.wrapCheckSqlAuthFn(options.checkSqlAuth))
       || this.createDefaultCheckSqlAuthFn(options);