diff --git a/knowledge_base/sync_include_exclude_and_scripts/.gitignore b/knowledge_base/sync_include_exclude_and_scripts/.gitignore new file mode 100644 index 0000000..28eaa57 --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/.gitignore @@ -0,0 +1,2 @@ +.databricks +generated/ diff --git a/knowledge_base/sync_include_exclude_and_scripts/README.md b/knowledge_base/sync_include_exclude_and_scripts/README.md new file mode 100644 index 0000000..a3f6ed6 --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/README.md @@ -0,0 +1,37 @@ +# Sync include, exclude, and scripts + +This Declarative Automation Bundles example shows `sync.include`, `sync.exclude`, and a `predeploy` hook that writes a gitignored file for upload while excluding a local file. A serverless job reads the generated file after deploy. + +For more information, see the [sync settings documentation](https://docs.databricks.com/aws/en/dev-tools/bundles/settings#sync). + +## Prerequisites + +* Databricks CLI v1.20.0 or above +* A workspace with serverless jobs + +## Usage + +1. Validate: `databricks bundle validate -t dev --profile ` +2. Deploy: `databricks bundle deploy -t dev --profile ` +3. Confirm the deploy stops when `PAYLOAD_TEXT` is empty: + +```bash +PAYLOAD_TEXT= databricks bundle deploy -t dev --profile +``` + +4. List the uploaded files. Confirm that `generated/payload.txt` is present and `local_only/` is absent: + +```bash +FILES=/Workspace/bundle-examples/sync_include_exclude_and_scripts/dev//files +databricks workspace list $FILES --profile +databricks workspace list $FILES/generated --profile +``` + +5. Run the job: `databricks bundle run -t dev read_generated --profile ` +6. Destroy: `databricks bundle destroy -t dev --auto-approve --profile ` +7. Repeat steps 1–6 with `-t prod` (prod root path has no `` segment). + +| Path | Local | Uploaded | +| --- | --- | --- | +| `generated/payload.txt` | gitignored; written by `predeploy` | yes (`sync.include`) | +| `local_only/notes.txt` | committed | no (`sync.exclude`) | diff --git a/knowledge_base/sync_include_exclude_and_scripts/databricks.yml b/knowledge_base/sync_include_exclude_and_scripts/databricks.yml new file mode 100644 index 0000000..1a6fdd6 --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/databricks.yml @@ -0,0 +1,43 @@ +# sync.include / sync.exclude and lifecycle scripts. +# CLI 1.20.0 schema: lifecycle hooks live under experimental.scripts (string commands). +# Top-level scripts is a map of {content} for `databricks bundle run `. +# Hook names: preinit, postinit, prebuild, postbuild, predeploy, postdeploy. +bundle: + name: sync_include_exclude_and_scripts + databricks_cli_version: ">= 1.20.0" + +include: + - resources/*.yml + +# Named scripts: run with `databricks bundle run generate_payload`. +scripts: + generate_payload: + content: python3 scripts/generate_payload.py + +# Lifecycle hooks run during deploy. A non-zero exit fails the deploy. +experimental: + scripts: + predeploy: python3 scripts/generate_payload.py + +sync: + # Upload this path even though .gitignore ignores generated/. + include: + - generated/payload.txt + # Keep this local file out of the workspace upload. + exclude: + - local_only/notes.txt + +targets: + dev: + default: true + mode: development + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} + + prod: + mode: production + workspace: + root_path: /Workspace/bundle-examples/${bundle.name}/${bundle.target} + permissions: + - user_name: ${workspace.current_user.userName} + level: CAN_MANAGE diff --git a/knowledge_base/sync_include_exclude_and_scripts/local_only/notes.txt b/knowledge_base/sync_include_exclude_and_scripts/local_only/notes.txt new file mode 100644 index 0000000..d3314d1 --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/local_only/notes.txt @@ -0,0 +1 @@ +this-file-must-not-upload diff --git a/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml b/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml new file mode 100644 index 0000000..dd27e0b --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/resources/read_generated.job.yml @@ -0,0 +1,18 @@ +resources: + jobs: + read_generated: + name: read_generated + description: Reads the gitignored file that sync.include uploaded. + + tasks: + - task_key: read_payload + environment_key: default + spark_python_task: + python_file: ../src/read_generated.py + parameters: + - ${workspace.file_path}/generated/payload.txt + + environments: + - environment_key: default + spec: + environment_version: "6" diff --git a/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py b/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py new file mode 100644 index 0000000..8cac90c --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/scripts/generate_payload.py @@ -0,0 +1,25 @@ +"""Create generated/payload.txt for sync.include. Exit non-zero on failure.""" + +import os +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_TEXT = "synced-from-predeploy" + + +def main() -> None: + # Unset = default text. Empty PAYLOAD_TEXT fails the predeploy hook (and deploy). + if "PAYLOAD_TEXT" in os.environ and os.environ["PAYLOAD_TEXT"] == "": + raise SystemExit("PAYLOAD_TEXT is empty; unset it or set a non-empty value") + + text = os.environ.get("PAYLOAD_TEXT", DEFAULT_TEXT) + out = ROOT / "generated" / "payload.txt" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(text + "\n", encoding="utf-8") + if not out.is_file() or out.read_text(encoding="utf-8").strip() != text: + raise SystemExit(f"failed to write {out}") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/knowledge_base/sync_include_exclude_and_scripts/src/read_generated.py b/knowledge_base/sync_include_exclude_and_scripts/src/read_generated.py new file mode 100644 index 0000000..8b58c0d --- /dev/null +++ b/knowledge_base/sync_include_exclude_and_scripts/src/read_generated.py @@ -0,0 +1,16 @@ +"""Read the generated payload that sync.include uploaded.""" + +import sys +from pathlib import Path + + +def main() -> None: + # spark_python_task on serverless does not set __file__; pass the path in. + payload = Path(sys.argv[1]) + text = payload.read_text(encoding="utf-8").strip() + print(f"payload: {text}") + assert text == "synced-from-predeploy", f"unexpected payload: {text!r}" + + +if __name__ == "__main__": + main()