feat(pkg-r): route ggsql viz execution through ggsql::caching_reader()

pkg-r/DESCRIPTION

@@ -61,6 +61,8 @@ Suggests:
     withr
 VignetteBuilder:
     knitr
+Remotes: 
+    ggsql=posit-dev/ggsql-r#18
 Config/roxygen2/version: 8.1.1
 Config/testthat/edition: 3
 Config/testthat/parallel: true

pkg-r/NEWS.md

@@ -1,5 +1,9 @@
 # querychat (development version)
 
+## New features
+
+* The `"visualize"` tool now executes ggsql queries end to end through a caching reader (`ggsql::caching_reader()`): identical base reads across chart refinements are memoized (a session of refinements over one shared base read costs 1 source read instead of N), and queries with layer-level sources (`DRAW ... FROM <table>`) are now supported instead of rejected. Requires ggsql >= 0.5.2.9000; see the caching section of `vignette("visualize")` for tuning knobs and the staleness caveat.
+
 ## Bug fixes
 
 * JSON pins now fall back to `pins::pin_read()` when DuckDB's JSON extension cannot be installed or loaded, including on builds that disable extension loading. Native DuckDB JSON reading is retained when the extension is available.

pkg-r/R/QueryChat.R

@@ -1654,13 +1654,13 @@ check_viz_deps <- function(tools) {
   if (
     is.null(tools) ||
       !"visualize" %in% tools ||
-      rlang::is_installed("ggsql", version = "0.5.2")
+      rlang::is_installed("ggsql", version = "0.5.2.9000")
   ) {
     return(tools)
   }
   rlang::warn(
     c(
-      'The "visualize" tool requires {.pkg ggsql} 0.5.2 or later.',
+      'The "visualize" tool requires {.pkg ggsql} 0.5.2.9000 or later.',
       "i" = 'Install it with `install.packages("ggsql")`; continuing without the "visualize" tool.'
     ),
     .frequency = "once",

pkg-r/R/querychat_module.R

@@ -354,20 +354,22 @@ restore_record_list <- function(x) {
 }
 
 restore_viz_widgets <- function(executor, saved_widgets, session) {
-  if (!rlang::is_installed("ggsql")) {
+  if (!rlang::is_installed("ggsql", version = "0.5.2.9000")) {
     warning(
-      "ggsql is not installed; skipping restoration of visualization widgets.",
+      "ggsql >= 0.5.2.9000 is not installed; skipping restoration of visualization widgets.",
       call. = FALSE
     )
     return(list())
   }
 
   restored <- list()
+  # One caching reader shared across restored widgets: identical base reads
+  # across widgets are served from the reader's memoized cache.
+  reader <- viz_caching_reader(executor)
   for (entry in saved_widgets) {
     tryCatch(
       {
-        validated <- ggsql::ggsql_validate(entry$ggsql)
-        spec <- execute_ggsql(executor, validated)
+        spec <- ggsql::ggsql_execute(reader, entry$ggsql)
         session$output[[entry$widget_id]] <- ggsql::renderGgsql(spec)
         restored <- c(restored, list(entry))
       },

pkg-r/R/querychat_viz.R

@@ -38,11 +38,45 @@ tool_visualize_impl <- function(executor, session, update_fn) {
   force(session)
   force(update_fn)
 
+  # One caching reader per chat client (this closure is created once per
+  # client): memoized base reads are shared across viz calls in the session,
+  # and the reader's cache is freed when the client is dropped. Created
+  # lazily so clients that never visualize don't pay for an idle cache.
+  reader <- NULL
+
   function(ggsql, title) {
-    visualize_result(executor, session, update_fn, ggsql, title)
+    if (is.null(reader)) {
+      reader <<- viz_caching_reader(executor)
+    }
+    visualize_result(reader, session, update_fn, ggsql, title)
   }
 }
 
+# A ggsql caching reader serving base reads through a querychat executor.
+# Writes are never forwarded to the executor: ggsql's CachingReader routes
+# them to its own in-memory cache, so only `execute_sql` is bridged.
+viz_caching_reader <- function(executor) {
+  rlang::check_installed(
+    "ggsql",
+    version = "0.5.2.9000",
+    reason = "for visualization support (ggsql::caching_reader)."
+  )
+  # Indirect lookup: caching_reader() doesn't exist in ggsql <= 0.5.2, so a
+  # static ggsql::caching_reader reference trips R CMD check's "missing or
+  # unexported object" warning when an older ggsql is installed.
+  caching_reader <- getExportedValue("ggsql", "caching_reader")
+  caching_reader(
+    primary = function(sql) {
+      df <- executor$execute_query(sql)
+      if (inherits(df, "tbl_sql")) {
+        # Materialize the query for ggsql, {dplyr} guaranteed by TblSqlSource
+        df <- dplyr::collect(df)
+      }
+      df
+    }
+  )
+}
+
 random_hex <- function(n_bytes = 8) {
   paste0(
     format(as.hexmode(sample(256, n_bytes, replace = TRUE) - 1L), width = 2),
@@ -51,13 +85,17 @@ random_hex <- function(n_bytes = 8) {
 }
 
 visualize_result <- function(
-  executor,
+  reader,
   session,
   update_fn,
   ggsql_str,
   title
 ) {
-  rlang::check_installed("ggsql", reason = "for visualization support.")
+  rlang::check_installed(
+    "ggsql",
+    version = "0.5.2.9000",
+    reason = "for visualization support (ggsql::caching_reader)."
+  )
 
   validated <- ggsql::ggsql_validate(ggsql_str)
   has_visual <- ggsql::ggsql_has_visual(validated)
@@ -72,7 +110,7 @@ visualize_result <- function(
     rlang::abort(collapse_validation_errors(validated))
   }
 
-  spec <- execute_ggsql(executor, validated)
+  spec <- ggsql::ggsql_execute(reader, ggsql_str)
 
   widget_id <- paste0("querychat_viz_", random_hex())
 
@@ -311,117 +349,3 @@ render_viz_tool_description <- function(db_type, has_tool_query = FALSE) {
     )
   )
 }
-
-#' Execute a pre-validated ggsql query against an executor
-#'
-#' Executes the SQL portion through an executor (preserving database pushdown),
-#' then feeds the result into a ggsql DuckDB reader to produce a Spec.
-#'
-#' @param executor A querychat QueryExecutor R6 object.
-#' @param validated A pre-validated ggsql query (from `ggsql::ggsql_validate()`).
-#'   Must be a list with `$sql` and `$visual` fields.
-#'
-#' @return A `ggsql::Spec` R6 object (the writer-independent plot specification).
-#'
-#' @keywords internal
-execute_ggsql <- function(executor, validated) {
-  rlang::check_installed("ggsql", reason = "for visualization support.")
-
-  visual <- validated$visual
-
-  if (has_layer_level_source(visual)) {
-    cli::cli_abort(
-      "Layer-specific sources are not currently supported in querychat visual queries. Rewrite the query so that all layers come from the final SQL result."
-    )
-  }
-
-  df <- executor$execute_query(validated$sql)
-
-  if (inherits(df, "tbl_sql")) {
-    # Materialize the query for ggsql, {dplyr} guaranteed by TblSqlSource
-    df <- dplyr::collect(df)
-  }
-
-  reader <- ggsql::duckdb_reader()
-  table <- extract_visualise_table(visual)
-
-  if (!is.null(table)) {
-    # VISUALISE [mappings] FROM <table> — register data under the
-    # referenced table name and execute the visual part directly.
-    name <- if (startsWith(table, '"') && endsWith(table, '"')) {
-      substr(table, 2, nchar(table) - 1)
-    } else {
-      table
-    }
-    ggsql::ggsql_register(reader, df, name)
-    ggsql::ggsql_execute(reader, visual)
-  } else {
-    # SELECT ... VISUALISE — no FROM in VISUALISE clause, so register
-    # under a synthetic name and prepend a SELECT.
-    ggsql::ggsql_register(reader, df, "_data")
-    ggsql::ggsql_execute(reader, paste("SELECT * FROM _data", visual))
-  }
-}
-
-#' Extract the table name from a VISUALISE clause's FROM, if present
-#'
-#' Looks only in the portion of the visual string before the first DRAW keyword,
-#' so FROM clauses inside DRAW (e.g., MAPPING x FROM other) are ignored.
-#'
-#' @param visual A ggsql VISUALISE string.
-#' @return The table name string (possibly quoted), or `NULL` if not present.
-#'
-#' @keywords internal
-extract_visualise_table <- function(visual) {
-  draw_pos <- regexpr("\\bDRAW\\b", visual, ignore.case = TRUE, perl = TRUE)
-  vis_clause <- if (draw_pos > 0) substr(visual, 1, draw_pos - 1L) else visual
-  m <- regmatches(
-    vis_clause,
-    regexpr(
-      '\\bFROM\\s+("[^"]+?"|\\S+)',
-      vis_clause,
-      ignore.case = TRUE,
-      perl = TRUE
-    )
-  )
-  if (length(m) == 0 || !nzchar(m)) {
-    return(NULL)
-  }
-  sub("^(?i)FROM\\s+", "", m, perl = TRUE)
-}
-
-#' Detect whether a VISUALISE string has a layer-level FROM source
-#'
-#' Returns `TRUE` when a DRAW clause defines its own `FROM <source>` via a
-#' MAPPING sub-clause. Querychat replays VISUALISE against a single local
-#' relation, so layer-specific sources cannot be preserved reliably.
-#'
-#' @param visual A ggsql VISUALISE string.
-#' @return `TRUE` if any DRAW clause contains a MAPPING ... FROM source.
-#'
-#' @keywords internal
-has_layer_level_source <- function(visual) {
-  # Split at clause boundaries (DRAW, SCALE, etc.) using a lookbehind for
-  # whitespace rather than \b, which can split mid-word in R's PCRE engine.
-  clauses <- strsplit(
-    visual,
-    "(?i)(?<=\\s)(?=DRAW|SCALE|PROJECT|FACET|PLACE|LABEL|THEME)",
-    perl = TRUE
-  )[[1]]
-  for (clause in clauses) {
-    if (!grepl("^\\s*DRAW\\b", clause, ignore.case = TRUE, perl = TRUE)) {
-      next
-    }
-    if (
-      grepl(
-        "\\bMAPPING\\b[\\s\\S]*?\\bFROM\\s+(\"[^\"]+?\"|\\S+)",
-        clause,
-        ignore.case = TRUE,
-        perl = TRUE
-      )
-    ) {
-      return(TRUE)
-    }
-  }
-  FALSE
-}

pkg-r/inst/prompts/ggsql-syntax.md

@@ -513,34 +513,15 @@ PROJECT TO polar SETTING inner => 0.5
     VISUALISE FROM titanic
     DRAW bar MAPPING class AS x
     ```
-3. **In querychat, all layers must come from the final SQL result**: Do not use layer-specific `FROM source` inside `DRAW ... MAPPING ...` clauses. If you need raw data and a summary in one chart, put both into one final relation and distinguish layers with a column such as `layer_type`:
+3. **Layers can draw from other relations in the query**: Attach `FROM <source>` to a layer's `MAPPING` to combine different granularities in one chart — e.g. raw points from the final SELECT with a summary line from a CTE. The `FROM` belongs to `MAPPING` (a bare `DRAW line FROM t` is a parse error). Explicit layer mappings resolve against the layer's source; aesthetics the layer doesn't map are inherited from VISUALISE:
     ```sql
-    WITH raw AS (
-      SELECT
-        date,
-        amount,
-        region,
-        'raw' AS layer_type
-      FROM sales
-    ),
-    summary AS (
-      SELECT
-        date,
-        AVG(amount) AS amount,
-        region,
-        'summary' AS layer_type
-      FROM sales
-      GROUP BY date, region
-    ),
-    combined AS (
-      SELECT * FROM raw
-      UNION ALL
-      SELECT * FROM summary
+    WITH summary AS (
+      SELECT date, AVG(amount) AS amount FROM sales GROUP BY date
     )
-    SELECT * FROM combined
+    SELECT date, amount FROM sales
     VISUALISE date AS x, amount AS y
-    DRAW point MAPPING region AS color FILTER layer_type = 'raw'
-    DRAW line MAPPING region AS color FILTER layer_type = 'summary'
+    DRAW point SETTING opacity => 0.3
+    DRAW line MAPPING date AS x, amount AS y FROM summary
     ```
 4. **String values use single quotes**: In SETTING, LABEL, and RENAMING clauses, always use single quotes for string values. Double quotes cause parse errors.
 5. **Statistical layers**: When using `histogram`, `bar` (without y), `density`, `smooth`, `violin`, or `boxplot`, the layer computes statistics. Use REMAPPING to access `density`, `intensity`, `proportion`, etc.

pkg-r/tests/testthat/test-QueryChat.R

@@ -441,7 +441,7 @@ describe("QueryChat$client()", {
   })
 
   it("registers visualize tool when tools include 'visualize'", {
-    skip_if_not_installed("ggsql")
+    skip_if_not_installed("ggsql", minimum_version = "0.5.2.9000")
     session <- structure(
       list(
         output = list(),
@@ -467,7 +467,7 @@ describe("QueryChat$client()", {
   })
 
   it("registers visualize tool without a session", {
-    skip_if_not_installed("ggsql")
+    skip_if_not_installed("ggsql", minimum_version = "0.5.2.9000")
 
     qc <- QueryChat$new(
       new_test_df(),
@@ -482,7 +482,7 @@ describe("QueryChat$client()", {
   })
 
   it("registers only visualize tool when tools = 'visualize'", {
-    skip_if_not_installed("ggsql")
+    skip_if_not_installed("ggsql", minimum_version = "0.5.2.9000")
     session <- structure(
       list(
         output = list(),