diff --git a/knowledge_base/presets_and_multi_workspace_targets/.gitignore b/knowledge_base/presets_and_multi_workspace_targets/.gitignore new file mode 100644 index 0000000..15bcc6d --- /dev/null +++ b/knowledge_base/presets_and_multi_workspace_targets/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/knowledge_base/presets_and_multi_workspace_targets/README.md b/knowledge_base/presets_and_multi_workspace_targets/README.md new file mode 100644 index 0000000..cf1aa67 --- /dev/null +++ b/knowledge_base/presets_and_multi_workspace_targets/README.md @@ -0,0 +1,31 @@ +# Presets and multi-workspace targets + +This Declarative Automation Bundles example shows `presets` on `dev`, `staging`, and `prod` targets. +Each target uses a different `workspace.host` placeholder and an optional `profile`. +It also shows target overrides for tasks, environments, and a staging-only job parameter. + +For more information, see the [deployment modes documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/deployment-modes). + +## Prerequisites + +* Databricks CLI v1.20.0 or above +* Three workspaces, or one workspace for a dry run +* Profiles in `~/.databrickscfg` when you set `profile` + +## Usage + +1. Set each target `workspace.host` in `databricks.yml`. Uncomment `profile` if needed. +2. Validate: `databricks bundle validate -t dev` +3. Deploy: `databricks bundle deploy -t dev` +4. Repeat steps 2–3 with `-t staging` and `-t prod`. +5. Check prefixed names: + +```bash +databricks bundle validate -t dev -o json | jq -r '.resources.jobs.presets_demo.name' +``` + +| Target | Name prefix | Pause status | Concurrency | +| --- | --- | --- | --- | +| `dev` | `[dev presets ]` | `PAUSED` | `4` | +| `staging` | `[staging]` | `PAUSED` | `2` | +| `prod` | `[prod]` | `UNPAUSED` | `1` | diff --git a/knowledge_base/presets_and_multi_workspace_targets/databricks.yml b/knowledge_base/presets_and_multi_workspace_targets/databricks.yml new file mode 100644 index 0000000..69045f6 --- /dev/null +++ b/knowledge_base/presets_and_multi_workspace_targets/databricks.yml @@ -0,0 +1,80 @@ +# Presets and multi-workspace targets example. +# Replace each workspace.host (and optional profile) before you deploy. +# Placeholder hosts match other knowledge_base examples (for example development_cluster). +bundle: + name: presets_and_multi_workspace_targets + +include: + - resources/*.yml + +targets: + # Development target. + dev: + default: true + mode: development + workspace: + host: https://dev-workspace.cloud.databricks.com + # Optional. Must match a section name in ~/.databrickscfg. + # profile: my-dev + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + presets: + name_prefix: "[dev presets ${workspace.current_user.short_name}] " + tags: + env: dev + team: examples + trigger_pause_status: PAUSED + jobs_max_concurrent_runs: 4 + # Only affects pipelines. This bundle has none; kept to show the preset key. + pipelines_development: true + # Dev only. Workspace UI can link to source. The CLI warns and deploys file copies. + source_linked_deployment: true + + staging: + workspace: + host: https://staging-workspace.cloud.databricks.com + # profile: my-staging + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + presets: + name_prefix: "[staging] " + tags: + env: staging + team: examples + trigger_pause_status: PAUSED + jobs_max_concurrent_runs: 2 + pipelines_development: false + resources: + jobs: + presets_demo: + # (c) Staging-only job parameter change (parameters merge by name). + parameters: + - name: batch_size + default: "50" + # (a) Override a task by task_key (list merge). + tasks: + - task_key: process + timeout_seconds: 1800 + + prod: + mode: production + workspace: + host: https://prod-workspace.cloud.databricks.com + # profile: my-prod + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + presets: + name_prefix: "[prod] " + tags: + env: prod + team: examples + trigger_pause_status: UNPAUSED + jobs_max_concurrent_runs: 1 + pipelines_development: false + resources: + jobs: + presets_demo: + # (a) Override a task by task_key. + # (b) Point the task at another environment. Environment lists append on + # override; job clusters merge by job_cluster_key instead. + tasks: + - task_key: process + timeout_seconds: 3600 + environment_key: heavy diff --git a/knowledge_base/presets_and_multi_workspace_targets/resources/presets_demo.job.yml b/knowledge_base/presets_and_multi_workspace_targets/resources/presets_demo.job.yml new file mode 100644 index 0000000..d0f7189 --- /dev/null +++ b/knowledge_base/presets_and_multi_workspace_targets/resources/presets_demo.job.yml @@ -0,0 +1,39 @@ +resources: + jobs: + presets_demo: + name: presets_demo + + # Quartz schedule. Presets set pause_status unless the job sets its own value. + schedule: + quartz_cron_expression: "0 0 9 * * ?" + timezone_id: UTC + + parameters: + - name: batch_size + default: "100" + + tasks: + - task_key: prepare + environment_key: default + spark_python_task: + python_file: ../src/prepare.py + parameters: + - "{{job.parameters.batch_size}}" + + - task_key: process + depends_on: + - task_key: prepare + environment_key: default + spark_python_task: + python_file: ../src/process.py + parameters: + - "{{job.parameters.batch_size}}" + + # Serverless environments. Prefer environment_version over deprecated client. + environments: + - environment_key: default + spec: + environment_version: "5" + - environment_key: heavy + spec: + environment_version: "6" diff --git a/knowledge_base/presets_and_multi_workspace_targets/src/prepare.py b/knowledge_base/presets_and_multi_workspace_targets/src/prepare.py new file mode 100644 index 0000000..c3dca5e --- /dev/null +++ b/knowledge_base/presets_and_multi_workspace_targets/src/prepare.py @@ -0,0 +1,12 @@ +"""Prepare step for the presets demo job.""" + +import sys + + +def main() -> None: + batch_size = sys.argv[1] if len(sys.argv) > 1 else "unset" + print(f"prepare: batch_size={batch_size}") + + +if __name__ == "__main__": + main() diff --git a/knowledge_base/presets_and_multi_workspace_targets/src/process.py b/knowledge_base/presets_and_multi_workspace_targets/src/process.py new file mode 100644 index 0000000..07cffc9 --- /dev/null +++ b/knowledge_base/presets_and_multi_workspace_targets/src/process.py @@ -0,0 +1,12 @@ +"""Process step for the presets demo job.""" + +import sys + + +def main() -> None: + batch_size = sys.argv[1] if len(sys.argv) > 1 else "unset" + print(f"process: batch_size={batch_size}") + + +if __name__ == "__main__": + main()