From fa4831d5b3b66af6e7dc5ad453ec9ca63cde5076 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Mon, 20 Jul 2026 17:27:22 -0700 Subject: [PATCH 1/3] Draft SQL BYOVPC on GCP --- modules/ROOT/nav.adoc | 4 +- .../get-started/enable-sql-byovpc-gcp.adoc | 370 ++++++++++++++++++ 2 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 78d6a9346..d028481c2 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -333,7 +333,9 @@ * xref:sql:index.adoc[Redpanda SQL] ** xref:sql:get-started/index.adoc[Get Started] *** xref:sql:get-started/sql-quickstart.adoc[Quickstart] -*** xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL] +*** xref:sql:get-started/deploy-sql-cluster.adoc[Enable Redpanda SQL on BYOC] +*** xref:sql:get-started/enable-sql-byovpc.adoc[Enable Redpanda SQL on BYOVPC] +**** xref:sql:get-started/enable-sql-byovpc-gcp.adoc[GCP] *** xref:sql:get-started/overview.adoc[] **** xref:sql:get-started/oltp-vs-olap.adoc[] **** xref:sql:get-started/redpanda-sql-vs-postgresql.adoc[] diff --git a/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc b/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc new file mode 100644 index 000000000..b74c9e69c --- /dev/null +++ b/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc @@ -0,0 +1,370 @@ += Enable Redpanda SQL on a BYOVPC Cluster on GCP +:description: Provision SQL-specific GCP resources with Terraform and enable the Redpanda SQL engine on a new or existing BYOVPC cluster on GCP. +:page-topic-type: how-to +:personas: platform_admin +:learning-objective-1: Provision SQL-specific GCP resources using the Redpanda BYOVPC Terraform module +:learning-objective-2: Enable Redpanda SQL on a new or existing BYOVPC cluster by supplying customer-managed resources +:learning-objective-3: Verify that the SQL engine is running and ready to accept connections + +include::shared:partial$feature-flag-rpcn.adoc[] + +Enable Redpanda SQL on a Bring Your Own Virtual Private Cloud (BYOVPC) cluster on GCP so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. You can enable SQL when creating a new BYOVPC cluster or on an existing one. + +Unlike standard BYOC clusters, Redpanda does not create resources in your GCP project for BYOVPC clusters. You must provision the SQL-specific GCP resources yourself and supply them to the cluster as customer-managed resources (CMRs) before enabling the SQL engine. + +After reading this page, you will be able to: + +* [ ] {learning-objective-1} +* [ ] {learning-objective-2} +* [ ] {learning-objective-3} + +== Prerequisites + +* For a new cluster: a BYOVPC network on GCP. For an existing cluster: a deployed xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[BYOVPC cluster on GCP]. +* The https://registry.terraform.io/modules/redpanda-data/redpanda-byovpc/gcp/latest[Redpanda BYOVPC Terraform module^] and https://developer.hashicorp.com/terraform[Terraform^] version 1.8.5 or later. +* Admin permissions in your Redpanda Cloud organization on xref:billing:billing.adoc[usage-based billing]. +* If using the Cloud API, a valid glossterm:bearer token[]. See link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. + +== Provision SQL resources with Terraform + +In your `terraform-gcp-redpanda-byovpc` module configuration, set `enable_redpanda_sql = true`: + +[,hcl] +---- +module "redpanda_byovpc" { + source = "redpanda-data/redpanda-byovpc/gcp" + # ... existing configuration ... + enable_redpanda_sql = true +} +---- + +Apply the updated configuration: + +[,bash] +---- +terraform apply +---- + +This creates SQL-specific resources in your GCP project: + +* A service account for the Redpanda SQL API +* A service account for the Redpanda SQL compute nodes +* A GCS bucket for SQL data storage + +It also derives a Secret Manager prefix and grants the API service account access to any secrets created under it, reserving the prefix for future Redpanda SQL Iceberg catalog credentials. + +After applying, Terraform outputs the following values. Supply these to the cluster when you enable SQL. + +[cols="1,3",options="header"] +|=== +| Module output | Description +| `redpanda_sql_api_service_account_email` | Email of the service account for the Redpanda SQL API +| `redpanda_sql_service_account_email` | Email of the service account for Redpanda SQL compute nodes +| `redpanda_sql_storage_bucket_name` | Name of the GCS bucket for SQL data storage +| `redpanda_sql_secret_manager_prefix` | Secret Manager prefix reserved for Redpanda SQL Iceberg catalog credentials +|=== + +== Enable Redpanda SQL + +Use the Terraform provider or the Cloud API to enable the SQL engine. The steps differ depending on whether you are creating a new cluster or updating an existing one. + +[NOTE] +==== +The Cloud Console does not yet support enabling Redpanda SQL on a GCP BYOVPC cluster. Use Terraform or the Cloud API. +==== + +=== On a new cluster + +[NOTE] +==== +To enable SQL when creating a new BYOVPC cluster, use the Terraform provider or the Cloud API. + +If your cluster already exists, skip to <>. +==== + +[IMPORTANT] +==== +The SQL customer-managed resources `rpsql_api_service_account`, `rpsql_service_account`, and `rpsql_cloud_storage_bucket` are immutable while SQL is enabled. To change any of these resources, disable SQL first. + +`rpsql_secret_manager_prefix` can be changed without disabling SQL, but any secrets previously mapped under the old prefix become invalid. You must manually move them to the new prefix. +==== + +[tabs] +===== +Terraform:: ++ +-- +Add the SQL customer-managed resource fields and the `rpsql` block to the `redpanda_cluster` resource in your BYOVPC Terraform configuration. For the base `redpanda_cluster` configuration, see xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[Create a BYOVPC Cluster on GCP]. + +[,hcl] +---- +resource "redpanda_cluster" "cluster" { + # ... existing BYOVPC cluster configuration ... + customer_managed_resources = { + gcp = { + # ... existing BYOVPC customer-managed resources ... + rpsql_api_service_account = { + email = module.redpanda_byovpc.redpanda_sql_api_service_account_email + } + rpsql_service_account = { + email = module.redpanda_byovpc.redpanda_sql_service_account_email + } + rpsql_cloud_storage_bucket = { + name = module.redpanda_byovpc.redpanda_sql_storage_bucket_name + } + rpsql_secret_manager_prefix = module.redpanda_byovpc.redpanda_sql_secret_manager_prefix + } + } + rpsql = { + enabled = true + replicas = 1 + zones = [""] + } +} +---- + +Replace the placeholders with your own values: + +* `replicas`: Initial number of SQL compute nodes (minimum 1, maximum 9). +* `` (optional): One of the cluster's glossterm:availability zones[]. Even if the Redpanda cluster spans multiple AZs, the SQL engine always deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. + +Apply the configuration: + +[,bash] +---- +terraform apply +---- +-- + +Cloud API:: ++ +-- +When creating a new BYOVPC cluster via the Cloud API, include the SQL customer-managed resource fields and the `rpsql` configuration in the link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] payload alongside your existing BYOVPC customer-managed resources. + +. link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. +. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] request. Include your existing BYOVPC customer-managed resource fields (see xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[Create a BYOVPC Cluster on GCP]) and add the SQL customer-managed resource fields and `rpsql` block: ++ +[,bash] +---- +curl -X POST "https://api.redpanda.com/v1/clusters" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "cluster": { + "name": "", + "type": "TYPE_BYOC", + "cloud_provider": "CLOUD_PROVIDER_GCP", + "region": "", + "zones": ["", "", ""], + "network_id": "", + "throughput_tier": "", + "resource_group_id": "", + "customer_managed_resources": { + "gcp": { + "rpsql_api_service_account": {"email": ""}, + "rpsql_service_account": {"email": ""}, + "rpsql_cloud_storage_bucket": {"name": ""}, + "rpsql_secret_manager_prefix": "" + } + }, + "rpsql": { + "enabled": true, + "replicas": , + "zones": [""] + } + } + }' +---- ++ +Replace the placeholders with your own values: ++ +* ``, ``, ``, ``: The corresponding outputs from the Terraform module. +* ``: Initial number of SQL compute nodes (minimum 1, maximum 9). +* `` (optional): One of the cluster's glossterm:availability zones[]. Even if the Redpanda cluster spans multiple AZs, the SQL engine always deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. ++ +. The request returns the ID of a long-running operation. Poll the link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] endpoint until the operation completes: ++ +[,bash] +---- +curl -X GET "https://api.redpanda.com/v1/operations/{operation.id}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" +---- ++ +When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. +-- +===== + +=== On an existing cluster + +[tabs] +===== +Terraform:: ++ +-- +Add the SQL customer-managed resource fields and the `rpsql` block to the `redpanda_cluster` resource in your BYOVPC Terraform configuration: + +[,hcl] +---- +resource "redpanda_cluster" "cluster" { + # ... existing BYOVPC cluster configuration ... + customer_managed_resources = { + gcp = { + # ... existing BYOVPC customer-managed resources ... + rpsql_api_service_account = { + email = module.redpanda_byovpc.redpanda_sql_api_service_account_email + } + rpsql_service_account = { + email = module.redpanda_byovpc.redpanda_sql_service_account_email + } + rpsql_cloud_storage_bucket = { + name = module.redpanda_byovpc.redpanda_sql_storage_bucket_name + } + rpsql_secret_manager_prefix = module.redpanda_byovpc.redpanda_sql_secret_manager_prefix + } + } + rpsql = { + enabled = true + replicas = 1 + zones = [""] + } +} +---- + +Replace the placeholders with your own values: + +* `replicas`: Initial number of SQL compute nodes (minimum 1, maximum 9). +* `` (optional): One of the cluster's glossterm:availability zones[]. Even if the Redpanda cluster spans multiple AZs, the SQL engine always deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. + +Apply the configuration: + +[,bash] +---- +terraform apply +---- +-- + +Cloud API:: ++ +-- +. link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. +. Locate the cluster ID in the *Details* section of the cluster overview in the Cloud Console. +. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecluster[`PATCH /v1/clusters/{cluster.id}`] request with the SQL customer-managed resources and `rpsql.enabled` set to `true`. Replace `{cluster.id}` with your cluster ID: ++ +[,bash] +---- +curl -X PATCH "https://api.redpanda.com/v1/clusters/{cluster.id}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "customer_managed_resources": { + "gcp": { + "rpsql_api_service_account": {"email": ""}, + "rpsql_service_account": {"email": ""}, + "rpsql_cloud_storage_bucket": {"name": ""}, + "rpsql_secret_manager_prefix": "" + } + }, + "rpsql": { + "enabled": true, + "replicas": , + "zones": [""] + } + }' +---- ++ +Replace the placeholders with your own values: ++ +* ``, ``, ``, ``: The corresponding outputs from the Terraform module. +* ``: Initial number of SQL compute nodes (minimum 1, maximum 9). +* `` (optional): One of the cluster's glossterm:availability zones[]. Even if the Redpanda cluster spans multiple AZs, the SQL engine always deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. ++ +. The request returns the ID of a long-running operation. Poll the link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] endpoint until the operation completes: ++ +[,bash] +---- +curl -X GET "https://api.redpanda.com/v1/operations/{operation.id}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" +---- ++ +When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. +-- +===== + +== Verify the SQL engine is running + +After you enable Redpanda SQL, a *Redpanda SQL* tile appears in the cluster overview under *Products*. The tile shows the engine status, active sessions, queries per minute, and node count. When the *Nodes* indicator shows all nodes as ready (for example, *1/1*), the engine is provisioned and accepting connections. Provisioning can take up to 30 minutes. + +For the API flow, poll the long-running operation until it returns `STATE_COMPLETED`. + +To verify the SQL engine is running, connect with `psql` version 16 or later, or another PostgreSQL client, using the connection details on the *SQL* tab under *Connection details*. You can also query data directly using the xref:sql:query-data/sql-editor.adoc[SQL editor] in the navigation menu. + +// TODO: Confirm with SME whether OIDC bearer-token authentication works on GCP before publishing. +// A 2026-07-08 testing note on an unrelated cloudv2 PR (BigLake/Iceberg validation) reported that +// Oxla's OIDC client couldn't fetch the Auth0 discovery document on GCP (a certificate issue), but +// that finding wasn't isolated to end-user OIDC auth specifically and may already be fixed. If still +// broken, add a NOTE here scoping authentication to SCRAM (or OIDC client credentials, if unaffected) +// and cross-check xref:sql:connect-to-sql/authenticate.adoc[] for the same caveat. + +== Disable Redpanda SQL + +[WARNING] +==== +Disabling Redpanda SQL tears down the SQL compute engine and clears its catalog state (catalog metadata, table mappings, and role/grant data). In-flight queries fail when SQL is disabled. +==== + +If you disable Redpanda SQL, Redpanda topic data, Schema Registry subjects, and any Iceberg-committed history for Iceberg-enabled topics are not affected. The Redpanda cluster itself continues to run normally and only the SQL engine and its associated state are removed. + +Re-enabling SQL on the same cluster provisions a fresh engine. Redpanda does not restore prior catalog state, table mappings, or grants. You must re-create catalogs, tables, and grants after re-enabling. + +[NOTE] +==== +The SQL-specific GCP resources (service accounts and GCS bucket) are not deleted when you disable SQL. They remain in your project and can be reused if you re-enable SQL. To remove them, set `enable_redpanda_sql = false` in the Terraform module and run `terraform apply`, or run `terraform destroy` to remove all BYOVPC resources. +==== + +[tabs] +===== +Terraform:: ++ +-- +Set `rpsql.enabled` to `false` in the `redpanda_cluster` resource and apply: + +[,hcl] +---- +resource "redpanda_cluster" "cluster" { + # ... existing BYOVPC cluster configuration ... + rpsql = { + enabled = false + } +} +---- + +[,bash] +---- +terraform apply +---- +-- + +Cloud API:: ++ +-- +Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecluster[`PATCH /v1/clusters/{cluster.id}`] request with `rpsql.enabled` set to `false`. Replace `{cluster.id}` with your cluster ID: + +[,bash] +---- +curl -X PATCH "https://api.redpanda.com/v1/clusters/{cluster.id}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"rpsql":{"enabled":false}}' +---- + +The request returns the ID of a long-running operation. Poll link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] until the operation completes. +-- +===== + +== Next steps + +* xref:sql:get-started/deploy-sql-cluster.adoc[Scale the SQL engine]: Adjust compute size after enabling SQL. +* xref:sql:get-started/sql-quickstart.adoc[Quickstart]: Connect to Redpanda SQL with `psql` and run your first query. +* xref:sql:connect-to-sql/authenticate.adoc[Authenticate to Redpanda SQL]: Connect with OIDC client credentials or a SCRAM password. From 0f86bfdd6a0fe9a1c8c6ee18fe163a45c854286a Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 22 Jul 2026 18:55:44 -0700 Subject: [PATCH 2/3] Update draft with correct GCP BYOVPC customer workflow --- .../get-started/enable-sql-byovpc-gcp.adoc | 363 +++++------------- 1 file changed, 91 insertions(+), 272 deletions(-) diff --git a/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc b/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc index b74c9e69c..e391e4cab 100644 --- a/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc +++ b/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc @@ -1,16 +1,16 @@ = Enable Redpanda SQL on a BYOVPC Cluster on GCP -:description: Provision SQL-specific GCP resources with Terraform and enable the Redpanda SQL engine on a new or existing BYOVPC cluster on GCP. +:description: Provision SQL-specific GCP resources and enable the Redpanda SQL engine on a running BYOVPC cluster on GCP using the Cloud API. :page-topic-type: how-to :personas: platform_admin -:learning-objective-1: Provision SQL-specific GCP resources using the Redpanda BYOVPC Terraform module -:learning-objective-2: Enable Redpanda SQL on a new or existing BYOVPC cluster by supplying customer-managed resources +:learning-objective-1: Provision the SQL-specific GCP customer-managed resources +:learning-objective-2: Enable Redpanda SQL on a running BYOVPC cluster with the Cloud API :learning-objective-3: Verify that the SQL engine is running and ready to accept connections include::shared:partial$feature-flag-rpcn.adoc[] -Enable Redpanda SQL on a Bring Your Own Virtual Private Cloud (BYOVPC) cluster on GCP so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. You can enable SQL when creating a new BYOVPC cluster or on an existing one. +Enable Redpanda SQL on a running Bring Your Own Virtual Private Cloud (BYOVPC) cluster on GCP so you can query streaming data in Redpanda topics using standard PostgreSQL syntax. -Unlike standard BYOC clusters, Redpanda does not create resources in your GCP project for BYOVPC clusters. You must provision the SQL-specific GCP resources yourself and supply them to the cluster as customer-managed resources (CMRs) before enabling the SQL engine. +Unlike standard BYOC clusters, Redpanda does not create resources in your GCP project for BYOVPC clusters. You must provision the SQL-specific GCP resources yourself and supply them to the cluster as customer-managed resources (CMRs). On GCP, you enable Redpanda SQL as a day-2 operation with the Cloud API: the Cloud Console cannot supply the SQL customer-managed resources for a BYOVPC cluster. After reading this page, you will be able to: @@ -20,157 +20,82 @@ After reading this page, you will be able to: == Prerequisites -* For a new cluster: a BYOVPC network on GCP. For an existing cluster: a deployed xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[BYOVPC cluster on GCP]. -* The https://registry.terraform.io/modules/redpanda-data/redpanda-byovpc/gcp/latest[Redpanda BYOVPC Terraform module^] and https://developer.hashicorp.com/terraform[Terraform^] version 1.8.5 or later. -* Admin permissions in your Redpanda Cloud organization on xref:billing:billing.adoc[usage-based billing]. -* If using the Cloud API, a valid glossterm:bearer token[]. See link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. +* A running xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[BYOVPC cluster on GCP], with its base customer-managed resources already provisioned. +* Admin permissions in your Redpanda Cloud organization on xref:billing:billing.adoc[usage-based billing], and a Redpanda Cloud service account (client ID and secret) with permission to update clusters. +* The https://cloud.google.com/sdk/docs/install[Google Cloud CLI^] (`gcloud`) with the beta component, authenticated with Application Default Credentials (`gcloud auth application-default login`) and your service project selected. +* To verify the engine after enabling it: `psql` version 16 or later on a host inside the cluster's VPC. A BYOVPC cluster is private, so the SQL endpoint is not reachable from outside the VPC. -== Provision SQL resources with Terraform +== Provision the SQL customer-managed resources -In your `terraform-gcp-redpanda-byovpc` module configuration, set `enable_redpanda_sql = true`: +Redpanda SQL on a GCP BYOVPC cluster requires four customer-managed resources that the base BYOVPC provisioning does not create: -[,hcl] ----- -module "redpanda_byovpc" { - source = "redpanda-data/redpanda-byovpc/gcp" - # ... existing configuration ... - enable_redpanda_sql = true -} ----- +[cols="1,2",options="header"] +|=== +| Resource | Purpose +| A GCP service account for the SQL compute nodes | Reads and writes the SQL data-storage bucket. +| A GCP service account for the SQL API | Accesses Secret Manager secrets under the SQL prefix. +| A GCS bucket for SQL data storage | Stores the SQL engine's data. +| A Secret Manager prefix | A string namespace (not a resource) that reserves where Redpanda SQL Iceberg catalog credentials are stored. +|=== -Apply the updated configuration: +Create the two service accounts and the storage bucket, and choose a Secret Manager prefix. Replace `` and `` with your own values, and set the bucket location to your cluster's region. [,bash] ---- -terraform apply ----- - -This creates SQL-specific resources in your GCP project: +# Service accounts +gcloud iam service-accounts create "redpanda-rpsql-" \ + --project "" --display-name "Redpanda SQL Service Account" +gcloud iam service-accounts create "redpanda-rpsql-api-" \ + --project "" --display-name "Redpanda SQL API Service Account" -* A service account for the Redpanda SQL API -* A service account for the Redpanda SQL compute nodes -* A GCS bucket for SQL data storage +# SQL data-storage bucket +gcloud storage buckets create "gs://redpanda-rpsql-storage-" \ + --project "" --location "" --uniform-bucket-level-access +---- -It also derives a Secret Manager prefix and grants the API service account access to any secrets created under it, reserving the prefix for future Redpanda SQL Iceberg catalog credentials. +// TODO: SME/source confirmation needed on the exact IAM role bindings for the two SQL service +// accounts and whether a roles/iam.workloadIdentityUser binding to the Oxla Kubernetes service +// account is required. Marat's verified run used broad predefined roles (roles/storage.objectAdmin +// on the bucket for the compute SA; roles/secretmanager.admin on the project for both SAs) and +// flagged the workload-identity binding as possibly needed. The terraform-gcp-redpanda-byovpc +// module (iam_rpsql.tf) instead grants narrower custom roles plus workload-identity bindings. +// Confirm the canonical customer-facing IAM before publishing this section. -After applying, Terraform outputs the following values. Supply these to the cluster when you enable SQL. +Grant each service account the permissions it needs. Confirm the exact roles with your Redpanda account team before you rely on this in production. -[cols="1,3",options="header"] -|=== -| Module output | Description -| `redpanda_sql_api_service_account_email` | Email of the service account for the Redpanda SQL API -| `redpanda_sql_service_account_email` | Email of the service account for Redpanda SQL compute nodes -| `redpanda_sql_storage_bucket_name` | Name of the GCS bucket for SQL data storage -| `redpanda_sql_secret_manager_prefix` | Secret Manager prefix reserved for Redpanda SQL Iceberg catalog credentials -|=== +The Secret Manager prefix is a string you choose (for example, `redpanda-rpsql-`); it is not a GCP resource you create. == Enable Redpanda SQL -Use the Terraform provider or the Cloud API to enable the SQL engine. The steps differ depending on whether you are creating a new cluster or updating an existing one. - -[NOTE] -==== -The Cloud Console does not yet support enabling Redpanda SQL on a GCP BYOVPC cluster. Use Terraform or the Cloud API. -==== - -=== On a new cluster - -[NOTE] -==== -To enable SQL when creating a new BYOVPC cluster, use the Terraform provider or the Cloud API. - -If your cluster already exists, skip to <>. -==== +Enable the SQL engine with a Cloud API `PATCH` request. The Cloud Console does not support supplying the SQL customer-managed resources for a BYOVPC cluster. [IMPORTANT] ==== -The SQL customer-managed resources `rpsql_api_service_account`, `rpsql_service_account`, and `rpsql_cloud_storage_bucket` are immutable while SQL is enabled. To change any of these resources, disable SQL first. - -`rpsql_secret_manager_prefix` can be changed without disabling SQL, but any secrets previously mapped under the old prefix become invalid. You must manually move them to the new prefix. +The `PATCH` request must use an explicit leaf-path `update_mask` that lists each field you set, including each SQL customer-managed resource. A grouped mask (for example, `customer_managed_resources`) is silently dropped: the operation returns `STATE_COMPLETED`, but SQL stays disabled. ==== -[tabs] -===== -Terraform:: -+ --- -Add the SQL customer-managed resource fields and the `rpsql` block to the `redpanda_cluster` resource in your BYOVPC Terraform configuration. For the base `redpanda_cluster` configuration, see xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[Create a BYOVPC Cluster on GCP]. - -[,hcl] ----- -resource "redpanda_cluster" "cluster" { - # ... existing BYOVPC cluster configuration ... - customer_managed_resources = { - gcp = { - # ... existing BYOVPC customer-managed resources ... - rpsql_api_service_account = { - email = module.redpanda_byovpc.redpanda_sql_api_service_account_email - } - rpsql_service_account = { - email = module.redpanda_byovpc.redpanda_sql_service_account_email - } - rpsql_cloud_storage_bucket = { - name = module.redpanda_byovpc.redpanda_sql_storage_bucket_name - } - rpsql_secret_manager_prefix = module.redpanda_byovpc.redpanda_sql_secret_manager_prefix - } - } - rpsql = { - enabled = true - replicas = 1 - zones = [""] - } -} ----- +The SQL customer-managed resources `rpsql_service_account`, `rpsql_api_service_account`, and `rpsql_cloud_storage_bucket` are immutable while SQL is enabled. To change any of them, disable SQL first. You can change `rpsql_secret_manager_prefix` without disabling SQL, but any secrets mapped under the old prefix become invalid, and you must move them to the new prefix manually. -Replace the placeholders with your own values: - -* `replicas`: Initial number of SQL compute nodes (minimum 1, maximum 9). -* `` (optional): One of the cluster's glossterm:availability zones[]. Even if the Redpanda cluster spans multiple AZs, the SQL engine always deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. - -Apply the configuration: - -[,bash] ----- -terraform apply ----- --- - -Cloud API:: -+ --- -When creating a new BYOVPC cluster via the Cloud API, include the SQL customer-managed resource fields and the `rpsql` configuration in the link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] payload alongside your existing BYOVPC customer-managed resources. - -. link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. -. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_createcluster[`POST /v1/clusters`] request. Include your existing BYOVPC customer-managed resource fields (see xref:get-started:cluster-types/byoc/gcp/vpc-byo-gcp.adoc[Create a BYOVPC Cluster on GCP]) and add the SQL customer-managed resource fields and `rpsql` block: +. link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API] and obtain a bearer token. +. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecluster[`PATCH /v1/clusters/{cluster.id}`] request. Set the `update_mask` query parameter to the explicit leaf paths, and provide the `rpsql` block and the four SQL customer-managed resources in the body. Replace `{cluster-id}` and the placeholder values with your own: + [,bash] ---- -curl -X POST "https://api.redpanda.com/v1/clusters" \ +curl -sX PATCH "https://api.redpanda.com/v1/clusters/{cluster-id}?update_mask=rpsql.enabled,rpsql.replicas,rpsql.zones,customer_managed_resources.gcp.rpsql_service_account.email,customer_managed_resources.gcp.rpsql_api_service_account.email,customer_managed_resources.gcp.rpsql_cloud_storage_bucket.name,customer_managed_resources.gcp.rpsql_secret_manager_prefix" \ -H "Authorization: Bearer $AUTH_TOKEN" \ -H "Content-Type: application/json" \ -d '{ - "cluster": { - "name": "", - "type": "TYPE_BYOC", - "cloud_provider": "CLOUD_PROVIDER_GCP", - "region": "", - "zones": ["", "", ""], - "network_id": "", - "throughput_tier": "", - "resource_group_id": "", - "customer_managed_resources": { - "gcp": { - "rpsql_api_service_account": {"email": ""}, - "rpsql_service_account": {"email": ""}, - "rpsql_cloud_storage_bucket": {"name": ""}, - "rpsql_secret_manager_prefix": "" - } - }, - "rpsql": { - "enabled": true, - "replicas": , - "zones": [""] + "rpsql": { + "enabled": true, + "replicas": , + "zones": [""] + }, + "customer_managed_resources": { + "gcp": { + "rpsql_service_account": {"email": ""}, + "rpsql_api_service_account": {"email": ""}, + "rpsql_cloud_storage_bucket": {"name": ""}, + "rpsql_secret_manager_prefix": "" } } }' @@ -178,134 +103,60 @@ curl -X POST "https://api.redpanda.com/v1/clusters" \ + Replace the placeholders with your own values: + -* ``, ``, ``, ``: The corresponding outputs from the Terraform module. * ``: Initial number of SQL compute nodes (minimum 1, maximum 9). * `` (optional): One of the cluster's glossterm:availability zones[]. Even if the Redpanda cluster spans multiple AZs, the SQL engine always deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. +* The four `rpsql_*` values: the service account emails, bucket name, and Secret Manager prefix from <>. + -. The request returns the ID of a long-running operation. Poll the link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] endpoint until the operation completes: +A genuine enable returns a long-running operation that stays `STATE_IN_PROGRESS` for roughly 15 to 30 minutes while the SQL node group provisions. ++ +. Poll the link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] endpoint until the operation completes: + [,bash] ---- -curl -X GET "https://api.redpanda.com/v1/operations/{operation.id}" \ +curl -s "https://api.redpanda.com/v1/operations/{operation-id}" \ -H "Authorization: Bearer $AUTH_TOKEN" \ -H "Content-Type: application/json" ---- -+ -When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. --- -===== -=== On an existing cluster +== Verify the SQL engine is running -[tabs] -===== -Terraform:: -+ --- -Add the SQL customer-managed resource fields and the `rpsql` block to the `redpanda_cluster` resource in your BYOVPC Terraform configuration: +A completed operation does not guarantee that SQL was enabled. Confirm the result by fetching the cluster and checking that the `rpsql` block and all four SQL customer-managed resources are populated: -[,hcl] +[,bash] ---- -resource "redpanda_cluster" "cluster" { - # ... existing BYOVPC cluster configuration ... - customer_managed_resources = { - gcp = { - # ... existing BYOVPC customer-managed resources ... - rpsql_api_service_account = { - email = module.redpanda_byovpc.redpanda_sql_api_service_account_email - } - rpsql_service_account = { - email = module.redpanda_byovpc.redpanda_sql_service_account_email - } - rpsql_cloud_storage_bucket = { - name = module.redpanda_byovpc.redpanda_sql_storage_bucket_name - } - rpsql_secret_manager_prefix = module.redpanda_byovpc.redpanda_sql_secret_manager_prefix - } - } - rpsql = { - enabled = true - replicas = 1 - zones = [""] - } -} +curl -s "https://api.redpanda.com/v1/clusters/{cluster-id}" \ + -H "Authorization: Bearer $AUTH_TOKEN" ---- -Replace the placeholders with your own values: - -* `replicas`: Initial number of SQL compute nodes (minimum 1, maximum 9). -* `` (optional): One of the cluster's glossterm:availability zones[]. Even if the Redpanda cluster spans multiple AZs, the SQL engine always deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. +SQL is enabled when the response shows `rpsql.enabled` set to `true` with a non-empty `rpsql.url`, and the four `customer_managed_resources.gcp.rpsql_*` fields are populated (not null). For example: -Apply the configuration: - -[,bash] +[,json,role="no-copy"] ---- -terraform apply +"rpsql": { + "enabled": true, + "replicas": 3, + "url": "oxla-..byoc..cloud.redpanda.com:5432", + "zones": [""] +} ---- --- -Cloud API:: -+ --- -. link:/api/doc/cloud-controlplane/authentication[Authenticate to the Cloud API]. -. Locate the cluster ID in the *Details* section of the cluster overview in the Cloud Console. -. Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecluster[`PATCH /v1/clusters/{cluster.id}`] request with the SQL customer-managed resources and `rpsql.enabled` set to `true`. Replace `{cluster.id}` with your cluster ID: -+ -[,bash] ----- -curl -X PATCH "https://api.redpanda.com/v1/clusters/{cluster.id}" \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "customer_managed_resources": { - "gcp": { - "rpsql_api_service_account": {"email": ""}, - "rpsql_service_account": {"email": ""}, - "rpsql_cloud_storage_bucket": {"name": ""}, - "rpsql_secret_manager_prefix": "" - } - }, - "rpsql": { - "enabled": true, - "replicas": , - "zones": [""] - } - }' ----- -+ -Replace the placeholders with your own values: -+ -* ``, ``, ``, ``: The corresponding outputs from the Terraform module. -* ``: Initial number of SQL compute nodes (minimum 1, maximum 9). -* `` (optional): One of the cluster's glossterm:availability zones[]. Even if the Redpanda cluster spans multiple AZs, the SQL engine always deploys to a single AZ. If omitted, Redpanda selects a zone automatically. The AZ is locked when SQL is first enabled and cannot be changed afterward. -+ -. The request returns the ID of a long-running operation. Poll the link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] endpoint until the operation completes: -+ +A *Redpanda SQL* tile also appears in the cluster overview in the Cloud Console, under *Products*, showing engine status and node count. Provisioning can take up to 30 minutes. + +To connect from a host inside the cluster's VPC, use the SQL endpoint URL with `psql` version 16 or later and a bearer token: + [,bash] ---- -curl -X GET "https://api.redpanda.com/v1/operations/{operation.id}" \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" +PGPASSWORD="$AUTH_TOKEN" psql \ + "host= port=5432 dbname=oxla user=ignored options='-c auth_method=bearer' sslmode=require" \ + -c 'SHOW NODES;' ---- -+ -When the operation is complete, the response shows `"state": "STATE_COMPLETED"`. --- -===== - -== Verify the SQL engine is running - -After you enable Redpanda SQL, a *Redpanda SQL* tile appears in the cluster overview under *Products*. The tile shows the engine status, active sessions, queries per minute, and node count. When the *Nodes* indicator shows all nodes as ready (for example, *1/1*), the engine is provisioned and accepting connections. Provisioning can take up to 30 minutes. - -For the API flow, poll the long-running operation until it returns `STATE_COMPLETED`. -To verify the SQL engine is running, connect with `psql` version 16 or later, or another PostgreSQL client, using the connection details on the *SQL* tab under *Connection details*. You can also query data directly using the xref:sql:query-data/sql-editor.adoc[SQL editor] in the navigation menu. +This example uses bearer-token authentication (`auth_method=bearer`), so Redpanda SQL takes the session identity from the token and ignores the `user` field. Redpanda SQL always connects to the `oxla` database regardless of the `dbname` value. For client-credentials and SCRAM authentication, where the `user` field carries the identity, see xref:sql:connect-to-sql/authenticate.adoc[Authenticate to Redpanda SQL]. -// TODO: Confirm with SME whether OIDC bearer-token authentication works on GCP before publishing. -// A 2026-07-08 testing note on an unrelated cloudv2 PR (BigLake/Iceberg validation) reported that -// Oxla's OIDC client couldn't fetch the Auth0 discovery document on GCP (a certificate issue), but -// that finding wasn't isolated to end-user OIDC auth specifically and may already be fixed. If still -// broken, add a NOTE here scoping authentication to SCRAM (or OIDC client credentials, if unaffected) -// and cross-check xref:sql:connect-to-sql/authenticate.adoc[] for the same caveat. +// TODO: Confirm with SME whether OIDC bearer-token authentication works on GCP for end users. +// Marat's 2026-07-21 run connected successfully with a Cloud bearer token (auth_method=bearer), +// which suggests the earlier "OIDC broken on GCP" concern may be resolved. Cross-check +// xref:sql:connect-to-sql/authenticate.adoc[] before publishing. == Disable Redpanda SQL @@ -314,57 +165,25 @@ To verify the SQL engine is running, connect with `psql` version 16 or later, or Disabling Redpanda SQL tears down the SQL compute engine and clears its catalog state (catalog metadata, table mappings, and role/grant data). In-flight queries fail when SQL is disabled. ==== -If you disable Redpanda SQL, Redpanda topic data, Schema Registry subjects, and any Iceberg-committed history for Iceberg-enabled topics are not affected. The Redpanda cluster itself continues to run normally and only the SQL engine and its associated state are removed. +Redpanda topic data, Schema Registry subjects, and any Iceberg-committed history for Iceberg-enabled topics are not affected. The Redpanda cluster itself continues to run normally, and only the SQL engine and its associated state are removed. Re-enabling SQL on the same cluster provisions a fresh engine. Redpanda does not restore prior catalog state, table mappings, or grants. You must re-create catalogs, tables, and grants after re-enabling. -[NOTE] -==== -The SQL-specific GCP resources (service accounts and GCS bucket) are not deleted when you disable SQL. They remain in your project and can be reused if you re-enable SQL. To remove them, set `enable_redpanda_sql = false` in the Terraform module and run `terraform apply`, or run `terraform destroy` to remove all BYOVPC resources. -==== - -[tabs] -===== -Terraform:: -+ --- -Set `rpsql.enabled` to `false` in the `redpanda_cluster` resource and apply: +The SQL-specific GCP resources (service accounts and bucket) are not deleted when you disable SQL. They remain in your project and can be reused if you re-enable SQL. -[,hcl] ----- -resource "redpanda_cluster" "cluster" { - # ... existing BYOVPC cluster configuration ... - rpsql = { - enabled = false - } -} ----- - -[,bash] ----- -terraform apply ----- --- - -Cloud API:: -+ --- -Make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecluster[`PATCH /v1/clusters/{cluster.id}`] request with `rpsql.enabled` set to `false`. Replace `{cluster.id}` with your cluster ID: +To disable SQL, make a link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecluster[`PATCH /v1/clusters/{cluster.id}`] request with `rpsql.enabled` set to `false`: [,bash] ---- -curl -X PATCH "https://api.redpanda.com/v1/clusters/{cluster.id}" \ +curl -sX PATCH "https://api.redpanda.com/v1/clusters/{cluster-id}?update_mask=rpsql.enabled" \ -H "Authorization: Bearer $AUTH_TOKEN" \ -H "Content-Type: application/json" \ - -d '{"rpsql":{"enabled":false}}' + -d '{"rpsql": {"enabled": false}}' ---- -The request returns the ID of a long-running operation. Poll link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] until the operation completes. --- -===== +Poll the link:/api/doc/cloud-controlplane/operation/operation-operationservice_getoperation[`GET /v1/operations/{operation.id}`] endpoint until the operation completes. == Next steps -* xref:sql:get-started/deploy-sql-cluster.adoc[Scale the SQL engine]: Adjust compute size after enabling SQL. * xref:sql:get-started/sql-quickstart.adoc[Quickstart]: Connect to Redpanda SQL with `psql` and run your first query. * xref:sql:connect-to-sql/authenticate.adoc[Authenticate to Redpanda SQL]: Connect with OIDC client credentials or a SCRAM password. From 957be96f626c51479eabb0470814dc91d13d30d8 Mon Sep 17 00:00:00 2001 From: Kat Batuigas Date: Wed, 22 Jul 2026 19:02:31 -0700 Subject: [PATCH 3/3] Update TODOs --- .../pages/get-started/enable-sql-byovpc-gcp.adoc | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc b/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc index e391e4cab..c8f7a49b8 100644 --- a/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc +++ b/modules/sql/pages/get-started/enable-sql-byovpc-gcp.adoc @@ -53,13 +53,7 @@ gcloud storage buckets create "gs://redpanda-rpsql-storage-" \ --project "" --location "" --uniform-bucket-level-access ---- -// TODO: SME/source confirmation needed on the exact IAM role bindings for the two SQL service -// accounts and whether a roles/iam.workloadIdentityUser binding to the Oxla Kubernetes service -// account is required. Marat's verified run used broad predefined roles (roles/storage.objectAdmin -// on the bucket for the compute SA; roles/secretmanager.admin on the project for both SAs) and -// flagged the workload-identity binding as possibly needed. The terraform-gcp-redpanda-byovpc -// module (iam_rpsql.tf) instead grants narrower custom roles plus workload-identity bindings. -// Confirm the canonical customer-facing IAM before publishing this section. +// TODO: Document the IAM roles the two SQL service accounts require. Pending SME confirmation. Grant each service account the permissions it needs. Confirm the exact roles with your Redpanda account team before you rely on this in production. @@ -153,10 +147,7 @@ PGPASSWORD="$AUTH_TOKEN" psql \ This example uses bearer-token authentication (`auth_method=bearer`), so Redpanda SQL takes the session identity from the token and ignores the `user` field. Redpanda SQL always connects to the `oxla` database regardless of the `dbname` value. For client-credentials and SCRAM authentication, where the `user` field carries the identity, see xref:sql:connect-to-sql/authenticate.adoc[Authenticate to Redpanda SQL]. -// TODO: Confirm with SME whether OIDC bearer-token authentication works on GCP for end users. -// Marat's 2026-07-21 run connected successfully with a Cloud bearer token (auth_method=bearer), -// which suggests the earlier "OIDC broken on GCP" concern may be resolved. Cross-check -// xref:sql:connect-to-sql/authenticate.adoc[] before publishing. +// TODO: Confirm OIDC bearer-token authentication on GCP before publishing; cross-check authenticate.adoc. == Disable Redpanda SQL