knowledge_base: add complex_variables_and_lookups example

knowledge_base/complex_variables_and_lookups/.gitignore

@@ -0,0 +1 @@
+.databricks

knowledge_base/complex_variables_and_lookups/README.md

@@ -0,0 +1,40 @@
+# Complex variables and lookups
+
+This Declarative Automation Bundles example shows `type: complex` variables and `lookup` variables.
+It resolves warehouse, cluster policy, service principal, cluster, and job IDs by name.
+A small serverless job prints the resolved IDs.
+
+For more information, see the [variables documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/variables).
+
+## Prerequisites
+
+* Databricks CLI v1.20.0 or above
+* A SQL warehouse, cluster policy, service principal, cluster, and job in the workspace
+
+## Usage
+
+1. Set lookup names in `databricks.yml` to resources in your workspace.
+
+2. Optional: override the complex `environment_spec` with a file.
+
+   ```bash
+   mkdir -p .databricks/bundle/dev
+   printf '%s\n' '{"environment_spec":{"environment_version":"6"}}' > .databricks/bundle/dev/variable-overrides.json
+   ```
+
+3. Optional: override a lookup ID: `export BUNDLE_VAR_warehouse_id=<warehouse-id>`
+
+4. Validate and deploy: `databricks bundle validate -t dev` then `databricks bundle deploy -t dev`
+
+5. Run the job: `databricks bundle run -t dev lookup_demo`
+
+6. Deploy and run production: `databricks bundle deploy -t prod` then `databricks bundle run -t prod lookup_demo`
+
+7. Remove resources: `databricks bundle destroy -t dev --auto-approve` and `databricks bundle destroy -t prod --auto-approve`
+
+| Override source | Example |
+| --- | --- |
+| CLI flag | `--var="warehouse_id=..."` |
+| Environment | `BUNDLE_VAR_warehouse_id=<id>` |
+| Overrides file | `.databricks/bundle/dev/variable-overrides.json` |
+| Default / target | `variables` in `databricks.yml` |

knowledge_base/complex_variables_and_lookups/databricks.yml

@@ -0,0 +1,60 @@
+# Complex variables and lookups.
+# Replace each lookup name with a resource that exists in your workspace.
+# See https://docs.databricks.com/dev-tools/bundles/variables.html
+bundle:
+  name: complex_variables_and_lookups
+
+include:
+  - resources/*.yml
+
+variables:
+  warehouse_id:
+    description: SQL warehouse ID resolved by name
+    lookup:
+      warehouse: Serverless Starter Warehouse
+
+  cluster_policy_id:
+    description: Cluster policy ID resolved by name
+    lookup:
+      cluster_policy: my-cluster-policy
+
+  service_principal_id:
+    description: Service principal application ID resolved by name
+    lookup:
+      service_principal: my-service-principal
+
+  cluster_id:
+    description: All-purpose cluster ID resolved by name
+    lookup:
+      cluster: my-cluster
+
+  job_id:
+    description: Job ID resolved by name
+    lookup:
+      job: my-job
+
+  # Complex map used as the serverless task environment. Override per target
+  # or with .databricks/bundle/<target>/variable-overrides.json.
+  environment_spec:
+    description: Serverless environment spec for the Python task
+    type: complex
+    default:
+      environment_version: "6"
+
+targets:
+  dev:
+    default: true
+    mode: development
+    workspace:
+      root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name}
+    variables:
+      environment_spec:
+        environment_version: "5"
+
+  prod:
+    mode: production
+    workspace:
+      root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}
+    variables:
+      environment_spec:
+        environment_version: "6"

knowledge_base/complex_variables_and_lookups/resources/lookup_demo.job.yml

@@ -0,0 +1,37 @@
+resources:
+  jobs:
+    lookup_demo:
+      name: Complex variables and lookups demo
+
+      parameters:
+        - name: warehouse_id
+          default: ${var.warehouse_id}
+        - name: cluster_policy_id
+          default: ${var.cluster_policy_id}
+        - name: service_principal_id
+          default: ${var.service_principal_id}
+        - name: cluster_id
+          default: ${var.cluster_id}
+        - name: job_id
+          default: ${var.job_id}
+
+      tasks:
+        - task_key: print_lookups
+          spark_python_task:
+            python_file: ../src/main.py
+            parameters:
+              - --warehouse-id
+              - "{{job.parameters.warehouse_id}}"
+              - --cluster-policy-id
+              - "{{job.parameters.cluster_policy_id}}"
+              - --service-principal-id
+              - "{{job.parameters.service_principal_id}}"
+              - --cluster-id
+              - "{{job.parameters.cluster_id}}"
+              - --job-id
+              - "{{job.parameters.job_id}}"
+          environment_key: default
+
+      environments:
+        - environment_key: default
+          spec: ${var.environment_spec}

knowledge_base/complex_variables_and_lookups/src/main.py

@@ -0,0 +1,20 @@
+import argparse
+import json
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--warehouse-id", default="")
+parser.add_argument("--cluster-policy-id", default="")
+parser.add_argument("--service-principal-id", default="")
+parser.add_argument("--cluster-id", default="")
+parser.add_argument("--job-id", default="")
+args = parser.parse_args()
+
+resolved = {
+    "warehouse_id": args.warehouse_id,
+    "cluster_policy_id": args.cluster_policy_id,
+    "service_principal_id": args.service_principal_id,
+    "cluster_id": args.cluster_id,
+    "job_id": args.job_id,
+}
+print("resolved_lookups=" + json.dumps(resolved, sort_keys=True))