diff --git a/knowledge_base/complex_variables_and_lookups/.gitignore b/knowledge_base/complex_variables_and_lookups/.gitignore new file mode 100644 index 0000000..15bcc6d --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/knowledge_base/complex_variables_and_lookups/README.md b/knowledge_base/complex_variables_and_lookups/README.md new file mode 100644 index 0000000..e327dc8 --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/README.md @@ -0,0 +1,40 @@ +# Complex variables and lookups + +This Declarative Automation Bundles example shows `type: complex` variables and `lookup` variables. +It resolves warehouse, cluster policy, service principal, cluster, and job IDs by name. +A small serverless job prints the resolved IDs. + +For more information, see the [variables documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/variables). + +## Prerequisites + +* Databricks CLI v1.20.0 or above +* A SQL warehouse, cluster policy, service principal, cluster, and job in the workspace + +## Usage + +1. Set lookup names in `databricks.yml` to resources in your workspace. + +2. Optional: override the complex `environment_spec` with a file. + + ```bash + mkdir -p .databricks/bundle/dev + printf '%s\n' '{"environment_spec":{"environment_version":"6"}}' > .databricks/bundle/dev/variable-overrides.json + ``` + +3. Optional: override a lookup ID: `export BUNDLE_VAR_warehouse_id=` + +4. Validate and deploy: `databricks bundle validate -t dev` then `databricks bundle deploy -t dev` + +5. Run the job: `databricks bundle run -t dev lookup_demo` + +6. Deploy and run production: `databricks bundle deploy -t prod` then `databricks bundle run -t prod lookup_demo` + +7. Remove resources: `databricks bundle destroy -t dev --auto-approve` and `databricks bundle destroy -t prod --auto-approve` + +| Override source | Example | +| --- | --- | +| CLI flag | `--var="warehouse_id=..."` | +| Environment | `BUNDLE_VAR_warehouse_id=` | +| Overrides file | `.databricks/bundle/dev/variable-overrides.json` | +| Default / target | `variables` in `databricks.yml` | diff --git a/knowledge_base/complex_variables_and_lookups/databricks.yml b/knowledge_base/complex_variables_and_lookups/databricks.yml new file mode 100644 index 0000000..aee9a66 --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/databricks.yml @@ -0,0 +1,60 @@ +# Complex variables and lookups. +# Replace each lookup name with a resource that exists in your workspace. +# See https://docs.databricks.com/dev-tools/bundles/variables.html +bundle: + name: complex_variables_and_lookups + +include: + - resources/*.yml + +variables: + warehouse_id: + description: SQL warehouse ID resolved by name + lookup: + warehouse: Serverless Starter Warehouse + + cluster_policy_id: + description: Cluster policy ID resolved by name + lookup: + cluster_policy: my-cluster-policy + + service_principal_id: + description: Service principal application ID resolved by name + lookup: + service_principal: my-service-principal + + cluster_id: + description: All-purpose cluster ID resolved by name + lookup: + cluster: my-cluster + + job_id: + description: Job ID resolved by name + lookup: + job: my-job + + # Complex map used as the serverless task environment. Override per target + # or with .databricks/bundle//variable-overrides.json. + environment_spec: + description: Serverless environment spec for the Python task + type: complex + default: + environment_version: "6" + +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + variables: + environment_spec: + environment_version: "5" + + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + variables: + environment_spec: + environment_version: "6" diff --git a/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml b/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml new file mode 100644 index 0000000..f490208 --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml @@ -0,0 +1,37 @@ +resources: + jobs: + lookup_demo: + name: Complex variables and lookups demo + + parameters: + - name: warehouse_id + default: ${var.warehouse_id} + - name: cluster_policy_id + default: ${var.cluster_policy_id} + - name: service_principal_id + default: ${var.service_principal_id} + - name: cluster_id + default: ${var.cluster_id} + - name: job_id + default: ${var.job_id} + + tasks: + - task_key: print_lookups + spark_python_task: + python_file: ../src/main.py + parameters: + - --warehouse-id + - "{{job.parameters.warehouse_id}}" + - --cluster-policy-id + - "{{job.parameters.cluster_policy_id}}" + - --service-principal-id + - "{{job.parameters.service_principal_id}}" + - --cluster-id + - "{{job.parameters.cluster_id}}" + - --job-id + - "{{job.parameters.job_id}}" + environment_key: default + + environments: + - environment_key: default + spec: ${var.environment_spec} diff --git a/knowledge_base/complex_variables_and_lookups/src/main.py b/knowledge_base/complex_variables_and_lookups/src/main.py new file mode 100644 index 0000000..b23705d --- /dev/null +++ b/knowledge_base/complex_variables_and_lookups/src/main.py @@ -0,0 +1,20 @@ +import argparse +import json + + +parser = argparse.ArgumentParser() +parser.add_argument("--warehouse-id", default="") +parser.add_argument("--cluster-policy-id", default="") +parser.add_argument("--service-principal-id", default="") +parser.add_argument("--cluster-id", default="") +parser.add_argument("--job-id", default="") +args = parser.parse_args() + +resolved = { + "warehouse_id": args.warehouse_id, + "cluster_policy_id": args.cluster_policy_id, + "service_principal_id": args.service_principal_id, + "cluster_id": args.cluster_id, + "job_id": args.job_id, +} +print("resolved_lookups=" + json.dumps(resolved, sort_keys=True))