From 9dba2e53d443aeacaf7be176fa7fdd8b5b4f4594 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Mon, 4 May 2026 15:55:50 -0700 Subject: [PATCH 1/6] Start query RP topics doc --- modules/sql/pages/query/index.adoc | 3 + .../pages/query/query-redpanda-topics.adoc | 66 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 modules/sql/pages/query/index.adoc create mode 100644 modules/sql/pages/query/query-redpanda-topics.adoc diff --git a/modules/sql/pages/query/index.adoc b/modules/sql/pages/query/index.adoc new file mode 100644 index 00000000..a3ae5662 --- /dev/null +++ b/modules/sql/pages/query/index.adoc @@ -0,0 +1,3 @@ += Query data +:description: Query streaming data in Redpanda topics and lakehouse data in Iceberg tables using standard PostgreSQL syntax. +:page-layout: index diff --git a/modules/sql/pages/query/query-redpanda-topics.adoc b/modules/sql/pages/query/query-redpanda-topics.adoc new file mode 100644 index 00000000..5173f96d --- /dev/null +++ b/modules/sql/pages/query/query-redpanda-topics.adoc @@ -0,0 +1,66 @@ += Query Redpanda topics +:description: Map a Redpanda topic to a SQL table and run analytical queries directly against streaming data. +:page-topic-type: how-to +:personas: app_developer, data_engineer +:learning-objective-1: Map a Redpanda topic to a SQL table using a Redpanda catalog +:learning-objective-2: Run analytical SQL queries against Redpanda topic data + +Map a Redpanda topic to a SQL table to run analytical queries directly against streaming data without building ETL pipelines. Redpanda SQL reads each record's fields from the topic's Protobuf schema in Schema Registry. + +After completing these steps, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} + +== Prerequisites + +Before you query a topic with SQL: + +* Enable the Redpanda SQL engine on your Redpanda Cloud Bring Your Own Cloud (BYOC) cluster. See xref:sql:get-started/sql-quickstart.adoc[Quickstart]. +* Connect to Redpanda SQL with `psql` or another PostgreSQL client. See xref:sql:connect-to-sql/index.adoc[Connect to Redpanda SQL]. +* Confirm that the Redpanda topic you want to query has a Protobuf schema registered in Schema Registry. + +== Map the topic to a SQL table + +Each Redpanda topic appears as a SQL table inside a Redpanda catalog. When you enable the SQL engine, Redpanda SQL automatically creates a catalog named `default_redpanda_connection` that points at your cluster. + +Define a table against the topic with `CREATE TABLE`: + +[source,sql] +---- +CREATE TABLE default_redpanda_connection=>orders WITH ( + topic = 'orders', + schema_subject = 'orders-value' +); +---- + +Replace `orders` with your topic name and `orders-value` with the Schema Registry subject that holds the topic's value schema. + +The table inherits its column definitions from the registered Protobuf schema. Each top-level Protobuf field becomes a SQL column. + +== Run queries + +Query the table with standard `SELECT` syntax. The following query returns the first 10 records: + +[source,sql] +---- +SELECT * FROM default_redpanda_connection=>orders LIMIT 10; +---- + +Aggregate and filter records using familiar PostgreSQL constructs: + +[source,sql] +---- +SELECT customer_id, SUM(amount) AS total +FROM default_redpanda_connection=>orders +WHERE status = 'completed' +GROUP BY customer_id +ORDER BY total DESC +LIMIT 10; +---- + +== Next steps + +* xref:reference:sql/sql-statements/create-table.adoc[CREATE TABLE]: full reference for the table-against-topic syntax, including all options. +* xref:reference:sql/sql-statements/create-redpanda-catalog.adoc[CREATE REDPANDA CATALOG]: define additional Redpanda catalogs with custom connection settings. +* xref:reference:sql/index.adoc[Redpanda SQL Reference]: supported SQL statements, clauses, data types, and functions. From ffb1285fc379795cefe207f833cdd670dc96d000 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Thu, 7 May 2026 19:27:48 -0700 Subject: [PATCH 2/6] DOC-1990: Move query directory and rewrite streaming-topics how-to Renames modules/sql/pages/query/ to modules/sql/pages/query-data/ and renames the streaming-topic how-to from query-redpanda-topics.adoc to query-streaming-topics.adoc to match the SQL GA IA. Retitles the page "Query streaming topics" and reframes the description and learning objectives around live streaming data; bridge-query and Iceberg content stays out of this page (DOC-2006 owns the Iceberg-topics how-to). Adds a pointer to the Iceberg topics how-to under the intro and lists it under Next steps. Updates the enable-prereq xref to point to the Enable Redpanda SQL page. Drops the CREATE REDPANDA CATALOG link from Next steps to align with the v1 framing that users do not typically create their own Redpanda catalog. Reframes the Query data index page description for v1 Iceberg scope (live and historical data in Redpanda topics; no external Iceberg lakehouse). Co-Authored-By: Claude Opus 4.7 (1M context) --- modules/sql/pages/query-data/index.adoc | 3 +++ .../query-streaming-topics.adoc} | 16 +++++++++------- modules/sql/pages/query/index.adoc | 3 --- 3 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 modules/sql/pages/query-data/index.adoc rename modules/sql/pages/{query/query-redpanda-topics.adoc => query-data/query-streaming-topics.adoc} (71%) delete mode 100644 modules/sql/pages/query/index.adoc diff --git a/modules/sql/pages/query-data/index.adoc b/modules/sql/pages/query-data/index.adoc new file mode 100644 index 00000000..c9e39d9e --- /dev/null +++ b/modules/sql/pages/query-data/index.adoc @@ -0,0 +1,3 @@ += Query data +:description: Query live and historical data in your Redpanda topics using standard PostgreSQL syntax. +:page-layout: index diff --git a/modules/sql/pages/query/query-redpanda-topics.adoc b/modules/sql/pages/query-data/query-streaming-topics.adoc similarity index 71% rename from modules/sql/pages/query/query-redpanda-topics.adoc rename to modules/sql/pages/query-data/query-streaming-topics.adoc index 5173f96d..8f6115f1 100644 --- a/modules/sql/pages/query/query-redpanda-topics.adoc +++ b/modules/sql/pages/query-data/query-streaming-topics.adoc @@ -1,11 +1,13 @@ -= Query Redpanda topics -:description: Map a Redpanda topic to a SQL table and run analytical queries directly against streaming data. += Query streaming topics +:description: Map a Redpanda topic to a SQL table and run analytical queries directly against live streaming data. :page-topic-type: how-to :personas: app_developer, data_engineer -:learning-objective-1: Map a Redpanda topic to a SQL table using a Redpanda catalog -:learning-objective-2: Run analytical SQL queries against Redpanda topic data +:learning-objective-1: Map a streaming Redpanda topic to a SQL table using the default Redpanda catalog +:learning-objective-2: Run analytical SQL queries against live topic data -Map a Redpanda topic to a SQL table to run analytical queries directly against streaming data without building ETL pipelines. Redpanda SQL reads each record's fields from the topic's Protobuf schema in Schema Registry. +Map a Redpanda topic to a SQL table to run analytical queries directly against live streaming data without building ETL pipelines. Redpanda SQL reads each record's fields from the topic's Protobuf schema in Schema Registry. + +To query the Iceberg-translated history of a Redpanda topic, see xref:sql:query-data/query-iceberg-topics.adoc[]. After completing these steps, you will be able to: @@ -16,7 +18,7 @@ After completing these steps, you will be able to: Before you query a topic with SQL: -* Enable the Redpanda SQL engine on your Redpanda Cloud Bring Your Own Cloud (BYOC) cluster. See xref:sql:get-started/sql-quickstart.adoc[Quickstart]. +* Enable the Redpanda SQL engine on your Redpanda Cloud Bring Your Own Cloud (BYOC) cluster. See xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL]. * Connect to Redpanda SQL with `psql` or another PostgreSQL client. See xref:sql:connect-to-sql/index.adoc[Connect to Redpanda SQL]. * Confirm that the Redpanda topic you want to query has a Protobuf schema registered in Schema Registry. @@ -61,6 +63,6 @@ LIMIT 10; == Next steps +* xref:sql:query-data/query-iceberg-topics.adoc[Query Iceberg topics]: query the Iceberg-translated history of an Iceberg-enabled Redpanda topic, and run a single query that spans live and historical records. * xref:reference:sql/sql-statements/create-table.adoc[CREATE TABLE]: full reference for the table-against-topic syntax, including all options. -* xref:reference:sql/sql-statements/create-redpanda-catalog.adoc[CREATE REDPANDA CATALOG]: define additional Redpanda catalogs with custom connection settings. * xref:reference:sql/index.adoc[Redpanda SQL Reference]: supported SQL statements, clauses, data types, and functions. diff --git a/modules/sql/pages/query/index.adoc b/modules/sql/pages/query/index.adoc deleted file mode 100644 index a3ae5662..00000000 --- a/modules/sql/pages/query/index.adoc +++ /dev/null @@ -1,3 +0,0 @@ -= Query data -:description: Query streaming data in Redpanda topics and lakehouse data in Iceberg tables using standard PostgreSQL syntax. -:page-layout: index From 522ad59cb0287f49d4f1c43994ed3a12b8e894df Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 13 May 2026 18:59:41 -0700 Subject: [PATCH 3/6] redpanda-catalogs work done in DOC-2049 # Conflicts: # modules/sql/pages/query-data/redpanda-catalogs.adoc --- .../pages/query-data/redpanda-catalogs.adoc | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 modules/sql/pages/query-data/redpanda-catalogs.adoc diff --git a/modules/sql/pages/query-data/redpanda-catalogs.adoc b/modules/sql/pages/query-data/redpanda-catalogs.adoc deleted file mode 100644 index bffbd247..00000000 --- a/modules/sql/pages/query-data/redpanda-catalogs.adoc +++ /dev/null @@ -1,81 +0,0 @@ -= Redpanda Catalogs -:description: Redpanda catalogs are named connections that map Redpanda topics to queryable SQL tables. -:page-topic-type: reference - -Redpanda catalogs are named connections that let you query Redpanda topics using standard SQL. The catalog model consists of three core concepts: - -* Catalogs: Named connections to a Redpanda cluster, created with xref:reference:sql/sql-statements/create-redpanda-catalog.adoc[CREATE REDPANDA CATALOG]. -* Tables: Redpanda topics mapped as queryable SQL tables using the `catalog_name\=>table_name` syntax, created with xref:reference:sql/sql-statements/create-table.adoc[CREATE TABLE]. -* Storage connections: Named connections to external object storage such as Amazon S3, created with xref:reference:sql/sql-statements/create-storage.adoc[CREATE STORAGE]. - -NOTE: Redpanda SQL operates in read-only mode. Data mutation operations such as `INSERT`, `UPDATE`, and `DELETE` are not available. Data is ingested into Redpanda topics and made queryable through catalog mappings. - -== Typical workflow - -To query Redpanda topic data with SQL: - -. Create a catalog connection: -+ -[source,sql] ----- -CREATE REDPANDA CATALOG production_redpanda -WITH ( - initial_brokers = 'broker1:9092', - schema_registry_url = 'http://schema-registry:8081' -); ----- - -. Map a topic as a table: -+ -[source,sql] ----- -CREATE TABLE production_redpanda=>user_events -WITH (topic = 'user-events'); ----- - -. Query the data: -+ -[source,sql] ----- -SELECT * FROM production_redpanda=>user_events LIMIT 10; ----- - -== Related statements - -[cols="<40%,<60%",options="header"] -|=== -|Statement |Description - -|xref:reference:sql/sql-statements/create-redpanda-catalog.adoc[CREATE REDPANDA CATALOG] -|Create a catalog connection to a Redpanda cluster. - -|xref:reference:sql/sql-statements/alter-redpanda-catalog.adoc[ALTER REDPANDA CATALOG] -|Modify connection properties of an existing catalog. - -|xref:reference:sql/sql-statements/create-table.adoc[CREATE TABLE] -|Map a Redpanda topic to a SQL table through a catalog. - -|xref:reference:sql/sql-statements/alter-table.adoc[ALTER TABLE] -|Modify options of an existing catalog table. - -|xref:reference:sql/sql-statements/drop-table.adoc[DROP TABLE] -|Remove a catalog table mapping. - -|xref:reference:sql/sql-statements/drop-redpanda-catalog.adoc[DROP REDPANDA CATALOG] -|Remove a Redpanda catalog connection. - -|xref:reference:sql/sql-statements/drop-storage.adoc[DROP STORAGE] -|Remove a named storage definition. - -|xref:reference:sql/sql-statements/show-tables.adoc[SHOW TABLES] -|List tables within a catalog. - -|xref:reference:sql/sql-statements/describe.adoc[DESCRIBE] -|Show details about a catalog or catalog table. - -|xref:reference:sql/sql-statements/create-storage.adoc[CREATE STORAGE] -|Create a connection to external object storage. - -|xref:reference:sql/sql-statements/alter-storage.adoc[ALTER STORAGE] -|Modify an existing storage connection. -|=== From ddefdadce7eb5d477c3db8196fc0b09b63841487 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 13 May 2026 19:00:19 -0700 Subject: [PATCH 4/6] Change rp connection to catalog --- .../query-data/query-streaming-topics.adoc | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/modules/sql/pages/query-data/query-streaming-topics.adoc b/modules/sql/pages/query-data/query-streaming-topics.adoc index 8f6115f1..262632f1 100644 --- a/modules/sql/pages/query-data/query-streaming-topics.adoc +++ b/modules/sql/pages/query-data/query-streaming-topics.adoc @@ -5,7 +5,7 @@ :learning-objective-1: Map a streaming Redpanda topic to a SQL table using the default Redpanda catalog :learning-objective-2: Run analytical SQL queries against live topic data -Map a Redpanda topic to a SQL table to run analytical queries directly against live streaming data without building ETL pipelines. Redpanda SQL reads each record's fields from the topic's Protobuf schema in Schema Registry. +Map a Redpanda topic to a SQL table to run analytical queries directly against live streaming data without building ETL pipelines. Redpanda SQL reads each record's fields from the topic's schema in Schema Registry. To query the Iceberg-translated history of a Redpanda topic, see xref:sql:query-data/query-iceberg-topics.adoc[]. @@ -18,9 +18,14 @@ After completing these steps, you will be able to: Before you query a topic with SQL: -* Enable the Redpanda SQL engine on your Redpanda Cloud Bring Your Own Cloud (BYOC) cluster. See xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL]. +* Enable the Redpanda SQL engine on your Redpanda Bring Your Own Cloud (BYOC) cluster. See xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL]. * Connect to Redpanda SQL with `psql` or another PostgreSQL client. See xref:sql:connect-to-sql/index.adoc[Connect to Redpanda SQL]. -* Confirm that the Redpanda topic you want to query has a Protobuf schema registered in Schema Registry. +* Confirm that the Redpanda topic you want to query has a schema registered in Schema Registry. + +// TODO: Confirm permissions/roles/ACLs required +// Is it possible to use a topic without a registered schema? +// Any specific limitations on Protobuf vs JSON vs Avro formats? +// Any requirements related to wire format and subject naming strategy? == Map the topic to a SQL table @@ -30,7 +35,7 @@ Define a table against the topic with `CREATE TABLE`: [source,sql] ---- -CREATE TABLE default_redpanda_connection=>orders WITH ( +CREATE TABLE default_redpanda_catalog=>orders WITH ( topic = 'orders', schema_subject = 'orders-value' ); @@ -38,7 +43,8 @@ CREATE TABLE default_redpanda_connection=>orders WITH ( Replace `orders` with your topic name and `orders-value` with the Schema Registry subject that holds the topic's value schema. -The table inherits its column definitions from the registered Protobuf schema. Each top-level Protobuf field becomes a SQL column. +// TODO: Nested fields? +The table inherits its column definitions from the registered schema. For Protobuf schemas, Redpanda SQL maps each top-level field to a SQL column. == Run queries @@ -46,7 +52,7 @@ Query the table with standard `SELECT` syntax. The following query returns the f [source,sql] ---- -SELECT * FROM default_redpanda_connection=>orders LIMIT 10; +SELECT * FROM default_redpanda_catalog=>orders LIMIT 10; ---- Aggregate and filter records using familiar PostgreSQL constructs: @@ -54,7 +60,7 @@ Aggregate and filter records using familiar PostgreSQL constructs: [source,sql] ---- SELECT customer_id, SUM(amount) AS total -FROM default_redpanda_connection=>orders +FROM default_redpanda_catalog=>orders WHERE status = 'completed' GROUP BY customer_id ORDER BY total DESC From f97c053d4775da4c6ebfeb61b4601d6a2cda102f Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 13 May 2026 19:06:23 -0700 Subject: [PATCH 5/6] Revert "redpanda-catalogs work done in DOC-2049" This reverts commit 522ad59cb0287f49d4f1c43994ed3a12b8e894df. --- modules/sql/pages/query-data/redpanda-catalogs.adoc | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 modules/sql/pages/query-data/redpanda-catalogs.adoc diff --git a/modules/sql/pages/query-data/redpanda-catalogs.adoc b/modules/sql/pages/query-data/redpanda-catalogs.adoc new file mode 100644 index 00000000..e69de29b From ec651fdd52639d00432edb713dc8e040bff04f0c Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 13 May 2026 19:07:07 -0700 Subject: [PATCH 6/6] Stub file --- modules/sql/pages/query-data/redpanda-catalogs.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/sql/pages/query-data/redpanda-catalogs.adoc b/modules/sql/pages/query-data/redpanda-catalogs.adoc index e69de29b..c54df0b2 100644 --- a/modules/sql/pages/query-data/redpanda-catalogs.adoc +++ b/modules/sql/pages/query-data/redpanda-catalogs.adoc @@ -0,0 +1,2 @@ += Redpanda Catalogs +// stub \ No newline at end of file