From 4b67373a5f995d4f719d48ab4231017980c064dd Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:07:17 +0000 Subject: [PATCH 1/4] Add sync include/exclude and scripts knowledge_base example. Co-authored-by: omnigent --- .../.gitignore | 2 + .../README.md | 33 ++++++++++++++ .../databricks.yml | 43 +++++++++++++++++++ .../local_only/secret.txt | 1 + .../resources/read_generated.job.yml | 18 ++++++++ .../scripts/generate_payload.py | 19 ++++++++ .../src/read_generated.py | 16 +++++++ 7 files changed, 132 insertions(+) create mode 100644 knowledge_base/sync_include_exclude_and_scripts/.gitignore create mode 100644 knowledge_base/sync_include_exclude_and_scripts/README.md create mode 100644 knowledge_base/sync_include_exclude_and_scripts/databricks.yml create mode 100644 knowledge_base/sync_include_exclude_and_scripts/local_only/secret.txt create mode 100644 knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml create mode 100644 knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py create mode 100644 knowledge_base/sync_include_exclude_and_scripts/src/read_generated.py diff --git a/knowledge_base/sync_include_exclude_and_scripts/.gitignore b/knowledge_base/sync_include_exclude_and_scripts/.gitignore new file mode 100644 index 0000000..28eaa57 --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/.gitignore @@ -0,0 +1,2 @@ +.databricks +generated/ diff --git a/knowledge_base/sync_include_exclude_and_scripts/README.md b/knowledge_base/sync_include_exclude_and_scripts/README.md new file mode 100644 index 0000000..b3a7300 --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/README.md @@ -0,0 +1,33 @@ +# Sync include, exclude, and scripts + +This Declarative Automation Bundles example shows `sync.include`, `sync.exclude`, and deploy scripts. +A `predeploy` hook writes a gitignored file. `sync.include` uploads it. `sync.exclude` keeps a local file out of the workspace. +A serverless job reads the generated file after deploy. + +For more information, see the [sync settings documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/settings#sync). + +## Prerequisites + +* Databricks CLI v1.20.0 or above +* A workspace with serverless jobs + +## Usage + +1. Validate: `databricks bundle validate -t dev --profile ` +2. Deploy: `databricks bundle deploy -t dev --profile ` +3. List files and confirm `generated/payload.txt` is present and `local_only/secret.txt` is absent: + +```bash +databricks workspace list \ + /Workspace/bundle-examples/sync_include_exclude_and_scripts/dev//files \ + --profile +``` + +4. Run the job: `databricks bundle run -t dev read_generated --profile ` +5. Destroy: `databricks bundle destroy -t dev --auto-approve --profile ` +6. Repeat steps 1–5 with `-t prod` (prod root path has no `` segment). + +| Path | Local | Uploaded | +| --- | --- | --- | +| `generated/payload.txt` | gitignored; written by `predeploy` | yes (`sync.include`) | +| `local_only/secret.txt` | committed | no (`sync.exclude`) | diff --git a/knowledge_base/sync_include_exclude_and_scripts/databricks.yml b/knowledge_base/sync_include_exclude_and_scripts/databricks.yml new file mode 100644 index 0000000..897114b --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/databricks.yml @@ -0,0 +1,43 @@ +# sync.include / sync.exclude and lifecycle scripts. +# CLI 1.20.0 schema: lifecycle hooks live under experimental.scripts (string commands). +# Top-level scripts is a map of {content} for `databricks bundle run `. +# Hook names: preinit, postinit, prebuild, postbuild, predeploy, postdeploy. +bundle: + name: sync_include_exclude_and_scripts + databricks_cli_version: ">= 1.20.0" + +include: + - resources/*.yml + +# Named scripts: run with `databricks bundle run generate_payload`. +scripts: + generate_payload: + content: python3 scripts/generate_payload.py + +# Lifecycle hooks run during deploy. A non-zero exit fails the deploy. +experimental: + scripts: + predeploy: python3 scripts/generate_payload.py + +sync: + # Upload this path even though .gitignore ignores generated/. + include: + - generated/payload.txt + # Keep this local file out of the workspace upload. + exclude: + - local_only/secret.txt + +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + permissions: + - user_name: ${workspace.current_user.userName} + level: CAN_MANAGE diff --git a/knowledge_base/sync_include_exclude_and_scripts/local_only/secret.txt b/knowledge_base/sync_include_exclude_and_scripts/local_only/secret.txt new file mode 100644 index 0000000..d3314d1 --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/local_only/secret.txt @@ -0,0 +1 @@ +this-file-must-not-upload diff --git a/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml b/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml new file mode 100644 index 0000000..d277e59 --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml @@ -0,0 +1,18 @@ +resources: + jobs: + read_generated: + name: read_generated + description: Reads the gitignored file that sync.include uploaded. + + tasks: + - task_key: read_payload + environment_key: default + spark_python_task: + python_file: ../src/read_generated.py + parameters: + - ${workspace.file_path}/generated/payload.txt + + environments: + - environment_key: default + spec: + environment_version: "4" diff --git a/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py b/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py new file mode 100644 index 0000000..3193bef --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py @@ -0,0 +1,19 @@ +"""Create generated/payload.txt for sync.include. Exit non-zero on failure.""" + +from pathlib import Path + + +def main() -> None: + out = Path("generated/payload.txt") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("synced-from-predeploy\n", encoding="utf-8") + if ( + not out.is_file() + or out.read_text(encoding="utf-8").strip() != "synced-from-predeploy" + ): + raise SystemExit("failed to write generated/payload.txt") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/knowledge_base/sync_include_exclude_and_scripts/src/read_generated.py b/knowledge_base/sync_include_exclude_and_scripts/src/read_generated.py new file mode 100644 index 0000000..8b58c0d --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/src/read_generated.py @@ -0,0 +1,16 @@ +"""Read the generated payload that sync.include uploaded.""" + +import sys +from pathlib import Path + + +def main() -> None: + # spark_python_task on serverless does not set __file__; pass the path in. + payload = Path(sys.argv[1]) + text = payload.read_text(encoding="utf-8").strip() + print(f"payload: {text}") + assert text == "synced-from-predeploy", f"unexpected payload: {text!r}" + + +if __name__ == "__main__": + main() From d56b8a3d2c5c5d31179028e21348aaf329c25997 Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:11:46 +0000 Subject: [PATCH 2/4] Make predeploy fail on empty PAYLOAD_TEXT; tighten README. Co-authored-by: omnigent --- .../README.md | 22 +++++++++++-------- .../databricks.yml | 2 +- .../local_only/{secret.txt => notes.txt} | 0 .../scripts/generate_payload.py | 20 +++++++++++------ 4 files changed, 27 insertions(+), 17 deletions(-) rename knowledge_base/sync_include_exclude_and_scripts/local_only/{secret.txt => notes.txt} (100%) diff --git a/knowledge_base/sync_include_exclude_and_scripts/README.md b/knowledge_base/sync_include_exclude_and_scripts/README.md index b3a7300..a6c2e7c 100644 --- a/knowledge_base/sync_include_exclude_and_scripts/README.md +++ b/knowledge_base/sync_include_exclude_and_scripts/README.md @@ -1,8 +1,6 @@ # Sync include, exclude, and scripts -This Declarative Automation Bundles example shows `sync.include`, `sync.exclude`, and deploy scripts. -A `predeploy` hook writes a gitignored file. `sync.include` uploads it. `sync.exclude` keeps a local file out of the workspace. -A serverless job reads the generated file after deploy. +This Declarative Automation Bundles example shows `sync.include`, `sync.exclude`, and a `predeploy` hook that writes a gitignored file for upload while excluding a local file. A serverless job reads the generated file after deploy. For more information, see the [sync settings documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/settings#sync). @@ -15,19 +13,25 @@ For more information, see the [sync settings documentation](https://docs.databri 1. Validate: `databricks bundle validate -t dev --profile ` 2. Deploy: `databricks bundle deploy -t dev --profile ` -3. List files and confirm `generated/payload.txt` is present and `local_only/secret.txt` is absent: +3. Confirm the deploy stops when `PAYLOAD_TEXT` is empty: + +```bash +PAYLOAD_TEXT= databricks bundle deploy -t dev --profile +``` + +4. List the generated file and confirm `local_only/notes.txt` is absent: ```bash databricks workspace list \ - /Workspace/bundle-examples/sync_include_exclude_and_scripts/dev//files \ + /Workspace/bundle-examples/sync_include_exclude_and_scripts/dev//files/generated \ --profile ``` -4. Run the job: `databricks bundle run -t dev read_generated --profile ` -5. Destroy: `databricks bundle destroy -t dev --auto-approve --profile ` -6. Repeat steps 1–5 with `-t prod` (prod root path has no `` segment). +5. Run the job: `databricks bundle run -t dev read_generated --profile ` +6. Destroy: `databricks bundle destroy -t dev --auto-approve --profile ` +7. Repeat steps 1–6 with `-t prod` (prod root path has no `` segment). | Path | Local | Uploaded | | --- | --- | --- | | `generated/payload.txt` | gitignored; written by `predeploy` | yes (`sync.include`) | -| `local_only/secret.txt` | committed | no (`sync.exclude`) | +| `local_only/notes.txt` | committed | no (`sync.exclude`) | diff --git a/knowledge_base/sync_include_exclude_and_scripts/databricks.yml b/knowledge_base/sync_include_exclude_and_scripts/databricks.yml index 897114b..1a6fdd6 100644 --- a/knowledge_base/sync_include_exclude_and_scripts/databricks.yml +++ b/knowledge_base/sync_include_exclude_and_scripts/databricks.yml @@ -25,7 +25,7 @@ sync: - generated/payload.txt # Keep this local file out of the workspace upload. exclude: - - local_only/secret.txt + - local_only/notes.txt targets: dev: diff --git a/knowledge_base/sync_include_exclude_and_scripts/local_only/secret.txt b/knowledge_base/sync_include_exclude_and_scripts/local_only/notes.txt similarity index 100% rename from knowledge_base/sync_include_exclude_and_scripts/local_only/secret.txt rename to knowledge_base/sync_include_exclude_and_scripts/local_only/notes.txt diff --git a/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py b/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py index 3193bef..8cac90c 100644 --- a/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py +++ b/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py @@ -1,17 +1,23 @@ """Create generated/payload.txt for sync.include. Exit non-zero on failure.""" +import os from pathlib import Path +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_TEXT = "synced-from-predeploy" + def main() -> None: - out = Path("generated/payload.txt") + # Unset = default text. Empty PAYLOAD_TEXT fails the predeploy hook (and deploy). + if "PAYLOAD_TEXT" in os.environ and os.environ["PAYLOAD_TEXT"] == "": + raise SystemExit("PAYLOAD_TEXT is empty; unset it or set a non-empty value") + + text = os.environ.get("PAYLOAD_TEXT", DEFAULT_TEXT) + out = ROOT / "generated" / "payload.txt" out.parent.mkdir(parents=True, exist_ok=True) - out.write_text("synced-from-predeploy\n", encoding="utf-8") - if ( - not out.is_file() - or out.read_text(encoding="utf-8").strip() != "synced-from-predeploy" - ): - raise SystemExit("failed to write generated/payload.txt") + out.write_text(text + "\n", encoding="utf-8") + if not out.is_file() or out.read_text(encoding="utf-8").strip() != text: + raise SystemExit(f"failed to write {out}") print(f"wrote {out}") From 76f3ee74e934301b963b9df88e9b8033b6caaf0f Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 00:13:40 +0000 Subject: [PATCH 3/4] Verify include and exclude in sync README Co-authored-by: omnigent --- knowledge_base/sync_include_exclude_and_scripts/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/knowledge_base/sync_include_exclude_and_scripts/README.md b/knowledge_base/sync_include_exclude_and_scripts/README.md index a6c2e7c..a3f6ed6 100644 --- a/knowledge_base/sync_include_exclude_and_scripts/README.md +++ b/knowledge_base/sync_include_exclude_and_scripts/README.md @@ -19,12 +19,12 @@ For more information, see the [sync settings documentation](https://docs.databri PAYLOAD_TEXT= databricks bundle deploy -t dev --profile ``` -4. List the generated file and confirm `local_only/notes.txt` is absent: +4. List the uploaded files. Confirm that `generated/payload.txt` is present and `local_only/` is absent: ```bash -databricks workspace list \ - /Workspace/bundle-examples/sync_include_exclude_and_scripts/dev//files/generated \ - --profile +FILES=/Workspace/bundle-examples/sync_include_exclude_and_scripts/dev//files +databricks workspace list $FILES --profile +databricks workspace list $FILES/generated --profile ``` 5. Run the job: `databricks bundle run -t dev read_generated --profile ` From 19cedfae8e6e83400c05ea92c15064e412622b7d Mon Sep 17 00:00:00 2001 From: Andres Garcia Date: Thu, 22 Oct 2026 02:43:18 +0000 Subject: [PATCH 4/4] Bump sync scripts example serverless environment to 6. Co-authored-by: omnigent --- .../resources/read_generated.job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml b/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml index d277e59..dd27e0b 100644 --- a/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml +++ b/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml @@ -15,4 +15,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "6"