diff --git a/knowledge_base/lakebase_synced_tables/.gitignore b/knowledge_base/lakebase_synced_tables/.gitignore new file mode 100644 index 0000000..15bcc6d --- /dev/null +++ b/knowledge_base/lakebase_synced_tables/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/knowledge_base/lakebase_synced_tables/README.md b/knowledge_base/lakebase_synced_tables/README.md new file mode 100644 index 0000000..80f8f35 --- /dev/null +++ b/knowledge_base/lakebase_synced_tables/README.md @@ -0,0 +1,59 @@ +# Lakebase synced tables + +This Declarative Automation Bundles example syncs a Unity Catalog Delta table to Lakebase Postgres. Each target creates its own project, branch, endpoint, and synced table. + +See the [Lakebase Autoscaling documentation](https://docs.databricks.com/aws/en/oltp/) for more information. + +## Prerequisites + +- Databricks CLI v1.20.0 or above. +- A workspace with Lakebase Autoscaling and serverless pipelines. +- A source Delta table with change data feed enabled and an `id` primary key column. +- A standard catalog and schema for pipeline storage. +- A new project ID. The bundle creates and purges its project during deployment and cleanup. +- The `psql` client, for the row count in step 5. + +## Usage + +1. Set a unique project ID and your Unity Catalog object names. + + ```sh + export BUNDLE_VAR_project_id=my-synced-tables + export BUNDLE_VAR_source_table=main.example.source_table + export BUNDLE_VAR_destination_table=main.example.synced_source_table + export BUNDLE_VAR_pipeline_storage_catalog=main + export BUNDLE_VAR_pipeline_storage_schema=example + ``` + +2. Validate the bundle. + + ```sh + databricks bundle validate + ``` + +3. Deploy the development project and synced table. + + ```sh + databricks bundle deploy + ``` + +4. Deploy the production project and synced table. + + ```sh + databricks bundle deploy -t prod + ``` + +5. Verify the development and production sync status. + + ```sh + databricks postgres get-synced-table "synced_tables/${BUNDLE_VAR_destination_table}_dev" + databricks postgres get-synced-table "synced_tables/${BUNDLE_VAR_destination_table}_prod" + databricks psql --project "${BUNDLE_VAR_project_id}-dev" --branch production --endpoint primary -- -d databricks_postgres -c "SELECT count(*) FROM ${BUNDLE_VAR_destination_table#*.}_dev;" + ``` + +6. Remove the resources for each deployed target. + + ```sh + databricks bundle destroy --auto-approve + databricks bundle destroy -t prod --auto-approve + ``` diff --git a/knowledge_base/lakebase_synced_tables/databricks.yml b/knowledge_base/lakebase_synced_tables/databricks.yml new file mode 100644 index 0000000..c9985d3 --- /dev/null +++ b/knowledge_base/lakebase_synced_tables/databricks.yml @@ -0,0 +1,28 @@ +bundle: + name: lakebase-synced-tables + databricks_cli_version: ">= 1.20.0" + +include: + - resources/*.yml + +variables: + project_id: + description: Unique Lakebase project ID + source_table: + description: Three-part name of the source Delta table + destination_table: + description: Three-part name of the destination synced table + pipeline_storage_catalog: + description: Standard catalog for pipeline storage + pipeline_storage_schema: + description: Schema for pipeline storage +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} diff --git a/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml b/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml new file mode 100644 index 0000000..8d0a50b --- /dev/null +++ b/knowledge_base/lakebase_synced_tables/resources/lakebase.postgres.yml @@ -0,0 +1,46 @@ +resources: + postgres_projects: + synced_tables: + project_id: ${var.project_id}-${bundle.target} + display_name: Lakebase synced tables example + pg_version: 17 + purge_on_delete: true + + postgres_branches: + target_branch: + parent: ${resources.postgres_projects.synced_tables.id} + branch_id: production + no_expiry: true + replace_existing: true + purge_on_delete: true + + postgres_endpoints: + target_endpoint: + parent: ${resources.postgres_branches.target_branch.id} + endpoint_id: primary + endpoint_type: ENDPOINT_TYPE_READ_WRITE + replace_existing: true + autoscaling_limit_min_cu: 0.5 + autoscaling_limit_max_cu: 1 + suspend_timeout_duration: 60s + + postgres_snapshot_schedules: + daily_snapshots: + branch: ${resources.postgres_branches.target_branch.id} + schedule: + - daily_schedule: + hour: 0 + retention: 3600s + + postgres_synced_tables: + example: + synced_table_id: ${var.destination_table}_${bundle.target} + source_table_full_name: ${var.source_table} + branch: ${resources.postgres_branches.target_branch.id} + postgres_database: databricks_postgres + primary_key_columns: + - id + scheduling_policy: TRIGGERED + new_pipeline_spec: + storage_catalog: ${var.pipeline_storage_catalog} + storage_schema: ${var.pipeline_storage_schema}