From afea16f90e97e66fce701c44b601a0c5d3af3a65 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:12:32 +0000 Subject: [PATCH 1/3] Add knowledge_base example for complex variables and lookups Co-authored-by: omnigent --- .../complex_variables_and_lookups/.gitignore | 1 + .../complex_variables_and_lookups/README.md | 45 +++++++++++++ .../databricks.yml | 67 +++++++++++++++++++ .../resources/lookup_demo.job.yml | 41 ++++++++++++ .../complex_variables_and_lookups/src/main.py | 22 ++++++ 5 files changed, 176 insertions(+) create mode 100644 knowledge_base/complex_variables_and_lookups/.gitignore create mode 100644 knowledge_base/complex_variables_and_lookups/README.md create mode 100644 knowledge_base/complex_variables_and_lookups/databricks.yml create mode 100644 knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml create mode 100644 knowledge_base/complex_variables_and_lookups/src/main.py diff --git a/knowledge_base/complex_variables_and_lookups/.gitignore b/knowledge_base/complex_variables_and_lookups/.gitignore new file mode 100644 index 00000000..15bcc6dd --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/knowledge_base/complex_variables_and_lookups/README.md b/knowledge_base/complex_variables_and_lookups/README.md new file mode 100644 index 00000000..f1f85c3e --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/README.md @@ -0,0 +1,45 @@ +# Complex variables and lookups + +This Declarative Automation Bundles example shows `type: complex` variables and `lookup` variables. +It resolves warehouse, cluster policy, service principal, cluster, job, and metastore IDs by name. +A small serverless job prints the resolved IDs. + +For more information, see the [variables documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/variables). + +## Prerequisites + +* Databricks CLI v1.20.0 or above +* A SQL warehouse, cluster policy, service principal, cluster, and job in the workspace +* Update each `lookup` name in `databricks.yml` to match those resources +* Metastore lookup needs account admin, or set `BUNDLE_VAR_metastore_id` + +## Usage + +1. Set lookup names in `databricks.yml` to resources in your workspace. + +2. Optional: override the complex `environment_spec` with a file. + + Precedence is CLI `--var` > `BUNDLE_VAR_*` > `.databricks/bundle//variable-overrides.json` > defaults. + Complex variables use the overrides file or target defaults. They do not use `BUNDLE_VAR_*`. + + ```bash + mkdir -p .databricks/bundle/dev + printf '%s\n' '{"environment_spec":{"environment_version":"4"}}' > .databricks/bundle/dev/variable-overrides.json + ``` + +3. Optional: override a lookup ID: `export BUNDLE_VAR_warehouse_id=` + +4. Validate and deploy: `databricks bundle validate -t dev` then `databricks bundle deploy -t dev` + +5. Run the job: `databricks bundle run -t dev lookup_demo` + +6. Deploy and run production: `databricks bundle deploy -t prod` then `databricks bundle run -t prod lookup_demo` + +7. Remove resources: `databricks bundle destroy -t dev --auto-approve` and `databricks bundle destroy -t prod --auto-approve` + +| Override source | Example | +| --- | --- | +| CLI flag | `--var="warehouse_id=..."` | +| Environment | `BUNDLE_VAR_warehouse_id=` | +| Overrides file | `.databricks/bundle/dev/variable-overrides.json` | +| Default / target | `variables` in `databricks.yml` | diff --git a/knowledge_base/complex_variables_and_lookups/databricks.yml b/knowledge_base/complex_variables_and_lookups/databricks.yml new file mode 100644 index 00000000..30cae794 --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/databricks.yml @@ -0,0 +1,67 @@ +# Complex variables and lookups. +# Replace each lookup name with a resource that exists in your workspace. +# See https://docs.databricks.com/dev-tools/bundles/variables.html +bundle: + name: complex_variables_and_lookups + +include: + - resources/*.yml + +variables: + warehouse_id: + description: SQL warehouse ID resolved by name + lookup: + warehouse: Serverless Starter Warehouse + + cluster_policy_id: + description: Cluster policy ID resolved by name + lookup: + cluster_policy: kb6-example-cluster-policy + + service_principal_id: + description: Service principal application ID resolved by name + lookup: + service_principal: kb6-example-sp + + cluster_id: + description: All-purpose cluster ID resolved by name + lookup: + cluster: kb6-example-cluster + + job_id: + description: Job ID resolved by name + lookup: + job: kb6-example-job + + metastore_id: + description: >- + Metastore ID resolved by name. Lookup needs account admin. + Otherwise set BUNDLE_VAR_metastore_id to the metastore UUID. + lookup: + metastore: your-metastore-name + + # Complex map used as the serverless task environment. Override per target + # or with .databricks/bundle//variable-overrides.json (not BUNDLE_VAR_*). + environment_spec: + description: Serverless environment spec for the Python task + type: complex + default: + environment_version: "4" + +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + variables: + environment_spec: + environment_version: "3" + + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + variables: + environment_spec: + environment_version: "4" diff --git a/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml b/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml new file mode 100644 index 00000000..e740da54 --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml @@ -0,0 +1,41 @@ +resources: + jobs: + lookup_demo: + name: Complex variables and lookups demo + + parameters: + - name: warehouse_id + default: ${var.warehouse_id} + - name: cluster_policy_id + default: ${var.cluster_policy_id} + - name: service_principal_id + default: ${var.service_principal_id} + - name: cluster_id + default: ${var.cluster_id} + - name: job_id + default: ${var.job_id} + - name: metastore_id + default: ${var.metastore_id} + + tasks: + - task_key: print_lookups + spark_python_task: + python_file: ../src/main.py + parameters: + - --warehouse-id + - "{{job.parameters.warehouse_id}}" + - --cluster-policy-id + - "{{job.parameters.cluster_policy_id}}" + - --service-principal-id + - "{{job.parameters.service_principal_id}}" + - --cluster-id + - "{{job.parameters.cluster_id}}" + - --job-id + - "{{job.parameters.job_id}}" + - --metastore-id + - "{{job.parameters.metastore_id}}" + environment_key: default + + environments: + - environment_key: default + spec: ${var.environment_spec} diff --git a/knowledge_base/complex_variables_and_lookups/src/main.py b/knowledge_base/complex_variables_and_lookups/src/main.py new file mode 100644 index 00000000..5618211b --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/src/main.py @@ -0,0 +1,22 @@ +import argparse +import json + + +parser = argparse.ArgumentParser() +parser.add_argument("--warehouse-id", default="") +parser.add_argument("--cluster-policy-id", default="") +parser.add_argument("--service-principal-id", default="") +parser.add_argument("--cluster-id", default="") +parser.add_argument("--job-id", default="") +parser.add_argument("--metastore-id", default="") +args = parser.parse_args() + +resolved = { + "warehouse_id": args.warehouse_id, + "cluster_policy_id": args.cluster_policy_id, + "service_principal_id": args.service_principal_id, + "cluster_id": args.cluster_id, + "job_id": args.job_id, + "metastore_id": args.metastore_id, +} +print("resolved_lookups=" + json.dumps(resolved, sort_keys=True)) From d097229caa891b450ea3c340c0b4826079b4c909 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:18:20 +0000 Subject: [PATCH 2/3] Drop metastore lookup and neutralize placeholder names Co-authored-by: omnigent --- .../complex_variables_and_lookups/README.md | 7 +------ .../databricks.yml | 17 +++++------------ .../resources/lookup_demo.job.yml | 4 ---- .../complex_variables_and_lookups/src/main.py | 2 -- 4 files changed, 6 insertions(+), 24 deletions(-) diff --git a/knowledge_base/complex_variables_and_lookups/README.md b/knowledge_base/complex_variables_and_lookups/README.md index f1f85c3e..da614087 100644 --- a/knowledge_base/complex_variables_and_lookups/README.md +++ b/knowledge_base/complex_variables_and_lookups/README.md @@ -1,7 +1,7 @@ # Complex variables and lookups This Declarative Automation Bundles example shows `type: complex` variables and `lookup` variables. -It resolves warehouse, cluster policy, service principal, cluster, job, and metastore IDs by name. +It resolves warehouse, cluster policy, service principal, cluster, and job IDs by name. A small serverless job prints the resolved IDs. For more information, see the [variables documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/variables). @@ -10,8 +10,6 @@ For more information, see the [variables documentation](https://docs.databricks. * Databricks CLI v1.20.0 or above * A SQL warehouse, cluster policy, service principal, cluster, and job in the workspace -* Update each `lookup` name in `databricks.yml` to match those resources -* Metastore lookup needs account admin, or set `BUNDLE_VAR_metastore_id` ## Usage @@ -19,9 +17,6 @@ For more information, see the [variables documentation](https://docs.databricks. 2. Optional: override the complex `environment_spec` with a file. - Precedence is CLI `--var` > `BUNDLE_VAR_*` > `.databricks/bundle//variable-overrides.json` > defaults. - Complex variables use the overrides file or target defaults. They do not use `BUNDLE_VAR_*`. - ```bash mkdir -p .databricks/bundle/dev printf '%s\n' '{"environment_spec":{"environment_version":"4"}}' > .databricks/bundle/dev/variable-overrides.json diff --git a/knowledge_base/complex_variables_and_lookups/databricks.yml b/knowledge_base/complex_variables_and_lookups/databricks.yml index 30cae794..db6f0946 100644 --- a/knowledge_base/complex_variables_and_lookups/databricks.yml +++ b/knowledge_base/complex_variables_and_lookups/databricks.yml @@ -16,32 +16,25 @@ variables: cluster_policy_id: description: Cluster policy ID resolved by name lookup: - cluster_policy: kb6-example-cluster-policy + cluster_policy: my-cluster-policy service_principal_id: description: Service principal application ID resolved by name lookup: - service_principal: kb6-example-sp + service_principal: my-service-principal cluster_id: description: All-purpose cluster ID resolved by name lookup: - cluster: kb6-example-cluster + cluster: my-cluster job_id: description: Job ID resolved by name lookup: - job: kb6-example-job - - metastore_id: - description: >- - Metastore ID resolved by name. Lookup needs account admin. - Otherwise set BUNDLE_VAR_metastore_id to the metastore UUID. - lookup: - metastore: your-metastore-name + job: my-job # Complex map used as the serverless task environment. Override per target - # or with .databricks/bundle//variable-overrides.json (not BUNDLE_VAR_*). + # or with .databricks/bundle//variable-overrides.json. environment_spec: description: Serverless environment spec for the Python task type: complex diff --git a/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml b/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml index e740da54..f490208c 100644 --- a/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml +++ b/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml @@ -14,8 +14,6 @@ resources: default: ${var.cluster_id} - name: job_id default: ${var.job_id} - - name: metastore_id - default: ${var.metastore_id} tasks: - task_key: print_lookups @@ -32,8 +30,6 @@ resources: - "{{job.parameters.cluster_id}}" - --job-id - "{{job.parameters.job_id}}" - - --metastore-id - - "{{job.parameters.metastore_id}}" environment_key: default environments: diff --git a/knowledge_base/complex_variables_and_lookups/src/main.py b/knowledge_base/complex_variables_and_lookups/src/main.py index 5618211b..b23705db 100644 --- a/knowledge_base/complex_variables_and_lookups/src/main.py +++ b/knowledge_base/complex_variables_and_lookups/src/main.py @@ -8,7 +8,6 @@ parser.add_argument("--service-principal-id", default="") parser.add_argument("--cluster-id", default="") parser.add_argument("--job-id", default="") -parser.add_argument("--metastore-id", default="") args = parser.parse_args() resolved = { @@ -17,6 +16,5 @@ "service_principal_id": args.service_principal_id, "cluster_id": args.cluster_id, "job_id": args.job_id, - "metastore_id": args.metastore_id, } print("resolved_lookups=" + json.dumps(resolved, sort_keys=True)) From 15f80e57fb67e762e0d4be917f0f573383ff6365 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 02:41:26 +0000 Subject: [PATCH 3/3] Bump complex variables example environments to 5 and 6. Co-authored-by: omnigent --- knowledge_base/complex_variables_and_lookups/README.md | 2 +- knowledge_base/complex_variables_and_lookups/databricks.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/knowledge_base/complex_variables_and_lookups/README.md b/knowledge_base/complex_variables_and_lookups/README.md index da614087..e327dc87 100644 --- a/knowledge_base/complex_variables_and_lookups/README.md +++ b/knowledge_base/complex_variables_and_lookups/README.md @@ -19,7 +19,7 @@ For more information, see the [variables documentation](https://docs.databricks. ```bash mkdir -p .databricks/bundle/dev - printf '%s\n' '{"environment_spec":{"environment_version":"4"}}' > .databricks/bundle/dev/variable-overrides.json + printf '%s\n' '{"environment_spec":{"environment_version":"6"}}' > .databricks/bundle/dev/variable-overrides.json ``` 3. Optional: override a lookup ID: `export BUNDLE_VAR_warehouse_id=` diff --git a/knowledge_base/complex_variables_and_lookups/databricks.yml b/knowledge_base/complex_variables_and_lookups/databricks.yml index db6f0946..aee9a660 100644 --- a/knowledge_base/complex_variables_and_lookups/databricks.yml +++ b/knowledge_base/complex_variables_and_lookups/databricks.yml @@ -39,7 +39,7 @@ variables: description: Serverless environment spec for the Python task type: complex default: - environment_version: "4" + environment_version: "6" targets: dev: @@ -49,7 +49,7 @@ targets: root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} variables: environment_spec: - environment_version: "3" + environment_version: "5" prod: mode: production @@ -57,4 +57,4 @@ targets: root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} variables: environment_spec: - environment_version: "4" + environment_version: "6"