Support selecting resources for deletion with bundle deploy --select

.nextchanges/bundles/select-deletions.md

@@ -0,0 +1 @@
+* `databricks bundle deploy --select` and `bundle plan --select` can now select resources removed from config, so a selective deploy applies their deletion. Deleting a selected resource also deletes its dependents that are also being removed, mirroring `bundle destroy`. Selecting a resource for deletion errors if a live resource still references it, preventing a broken bundle. An unqualified `--select <name>` is ambiguous when a resource of the same name exists in both config and state as different types. ([#7023](https://github.com/databricks/cli/pull/7023))

acceptance/bundle/select/deletion/databricks.yml.after.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: select-del-$UNIQUE_NAME
+
+resources:
+  jobs:
+    keep:
+      name: keep-$UNIQUE_NAME

acceptance/bundle/select/deletion/databricks.yml.tmpl

@@ -0,0 +1,10 @@
+bundle:
+  name: select-del-$UNIQUE_NAME
+
+resources:
+  jobs:
+    keep:
+      name: keep-$UNIQUE_NAME
+
+    gone:
+      name: gone-$UNIQUE_NAME

acceptance/bundle/select/deletion/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/select/deletion/output.txt

@@ -0,0 +1,43 @@
+
+=== Initial deploy
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-del-[UNIQUE_NAME]/default/files...
+Created jobs.gone
+Created jobs.keep
+Files: 3 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] bundle plan --select jobs.gone
+delete jobs.gone
+
+Plan: 0 to add, 0 to change, 1 to delete, 0 unchanged, 1 not selected
+
+=== bundle deploy --select jobs.gone
+
+>>> [CLI] bundle deploy --select jobs.gone
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-del-[UNIQUE_NAME]/default/files...
+Deleted jobs.gone
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 1 deleted, 0 unchanged, 1 not selected
+
+>>> print_requests.py --sort //jobs/delete
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/delete",
+  "body": {
+    "job_id": [GONE_ID]
+  }
+}
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== Destroy
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.keep
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/select-del-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/select/deletion/script

@@ -0,0 +1,31 @@
+envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    title "Destroy"
+    trace $CLI bundle destroy --auto-approve
+    rm -f "$OUT_REQUESTS"
+}
+trap cleanup EXIT
+
+# Deploy both jobs.
+title "Initial deploy\n"
+trace $CLI bundle deploy
+# Register [KEEP_ID]/[GONE_ID] from state while both still exist, so the delete
+# request body below is masked deterministically.
+replace_ids.py
+
+# Remove 'gone' from config. It is now orphaned in state and planned for deletion.
+envsubst '$UNIQUE_NAME' < databricks.yml.after.tmpl > databricks.yml
+
+# A resource removed from config is selectable by name, like any created or updated
+# one. The plan is filtered to just its deletion.
+trace $CLI bundle plan --select jobs.gone
+
+title "bundle deploy --select jobs.gone\n"
+trace $CLI bundle deploy --select jobs.gone
+
+# The delete call targeted 'gone' only.
+trace print_requests.py --sort //jobs/delete
+
+# A full plan now has nothing to delete and 'keep' is unchanged.
+trace $CLI bundle plan

acceptance/bundle/select/deletion/test.toml

@@ -0,0 +1,4 @@
+Cloud = true
+RecordRequests = true
+# databricks.yml is generated at runtime from the committed templates.
+Ignore = [".databricks", ".gitignore", "databricks.yml"]

acceptance/bundle/select/deletion_dependents/databricks.yml.after.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: select-deldep-$UNIQUE_NAME
+
+resources:
+  jobs:
+    keep:
+      name: keep-$UNIQUE_NAME

acceptance/bundle/select/deletion_dependents/databricks.yml.live_consumer.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: select-deldep-$UNIQUE_NAME
+
+resources:
+  jobs:
+    keep:
+      name: keep-$UNIQUE_NAME
+
+    consumer:
+      name: consumer-$UNIQUE_NAME
+      tasks:
+        - task_key: run_producer
+          run_job_task:
+            job_id: ${resources.jobs.producer.id}

acceptance/bundle/select/deletion_dependents/databricks.yml.tmpl

@@ -0,0 +1,17 @@
+bundle:
+  name: select-deldep-$UNIQUE_NAME
+
+resources:
+  jobs:
+    keep:
+      name: keep-$UNIQUE_NAME
+
+    producer:
+      name: producer-$UNIQUE_NAME
+
+    consumer:
+      name: consumer-$UNIQUE_NAME
+      tasks:
+        - task_key: run_producer
+          run_job_task:
+            job_id: ${resources.jobs.producer.id}

acceptance/bundle/select/deletion_dependents/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/select/deletion_dependents/output.txt

@@ -0,0 +1,71 @@
+
+=== Initial deploy
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-deldep-[UNIQUE_NAME]/default/files...
+Created jobs.consumer
+Created jobs.keep
+Created jobs.producer
+Files: 4 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] bundle plan --select jobs.producer
+Error: cannot delete jobs.producer: jobs.consumer still references it; remove the reference or also remove jobs.consumer from the config
+
+
+>>> [CLI] bundle deploy --select jobs.producer
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-deldep-[UNIQUE_NAME]/default/files...
+Error: cannot delete jobs.producer: jobs.consumer still references it; remove the reference or also remove jobs.consumer from the config
+
+Files: 1 uploaded, 0 deleted
+
+>>> print_requests.py --sort //jobs/delete --keep
+
+>>> [CLI] bundle plan
+update jobs.consumer
+delete jobs.producer
+
+Plan: 0 to add, 1 to change, 1 to delete, 1 unchanged
+
+>>> [CLI] bundle plan --select jobs.producer
+delete jobs.consumer
+delete jobs.producer
+
+Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged, 1 not selected
+
+=== bundle deploy --select jobs.producer
+
+>>> [CLI] bundle deploy --select jobs.producer
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-deldep-[UNIQUE_NAME]/default/files...
+Deleted jobs.consumer
+Deleted jobs.producer
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 2 deleted, 0 unchanged, 1 not selected
+
+>>> print_requests.py --sort //jobs/delete
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/delete",
+  "body": {
+    "job_id": [PRODUCER_ID]
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/delete",
+  "body": {
+    "job_id": [CONSUMER_ID]
+  }
+}
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== Destroy
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.keep
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/select-deldep-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/select/deletion_dependents/script

@@ -0,0 +1,48 @@
+envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    title "Destroy"
+    trace $CLI bundle destroy --auto-approve
+    rm -f "$OUT_REQUESTS"
+}
+trap cleanup EXIT
+
+# Deploy three jobs; consumer references producer, so state records consumer's
+# dependency on producer.
+title "Initial deploy\n"
+trace $CLI bundle deploy
+replace_ids.py
+
+# --- error case: consumer is still live and references producer ---
+# Remove only producer from config; consumer stays with its reference intact.
+# Selecting producer for deletion must error: it would leave consumer broken.
+envsubst '$UNIQUE_NAME' < databricks.yml.live_consumer.tmpl > databricks.yml
+
+musterr trace $CLI bundle plan --select jobs.producer
+
+# bundle deploy --select also errors before sending any delete request.
+musterr trace $CLI bundle deploy --select jobs.producer
+
+# No delete request was sent by either command.
+trace print_requests.py --sort //jobs/delete --keep
+
+# A full plan still works and shows producer as a pending delete.
+trace $CLI bundle plan
+
+# --- success case: both producer and consumer are removed from config ---
+# Remove producer and consumer from config; both are now planned for deletion.
+envsubst '$UNIQUE_NAME' < databricks.yml.after.tmpl > databricks.yml
+
+# Selecting the depended-upon producer for deletion also deletes its dependent
+# consumer (which would otherwise be left referencing a deleted job). 'keep' is
+# untouched. The dependency direction is reversed for deletes.
+trace $CLI bundle plan --select jobs.producer
+
+title "bundle deploy --select jobs.producer\n"
+trace $CLI bundle deploy --select jobs.producer
+
+# Both producer and consumer were deleted; keep was not.
+trace print_requests.py --sort //jobs/delete
+
+# A full plan now has nothing to delete and 'keep' is unchanged.
+trace $CLI bundle plan

acceptance/bundle/select/deletion_dependents/test.toml

@@ -0,0 +1,4 @@
+Cloud = true
+RecordRequests = true
+# databricks.yml is generated at runtime from the committed templates.
+Ignore = [".databricks", ".gitignore", "databricks.yml"]

acceptance/bundle/select/deletion_saved_plan/databricks.yml.after.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: select-del-plan-$UNIQUE_NAME
+
+resources:
+  jobs:
+    keep:
+      name: keep-$UNIQUE_NAME

acceptance/bundle/select/deletion_saved_plan/databricks.yml.tmpl

@@ -0,0 +1,10 @@
+bundle:
+  name: select-del-plan-$UNIQUE_NAME
+
+resources:
+  jobs:
+    keep:
+      name: keep-$UNIQUE_NAME
+
+    gone:
+      name: gone-$UNIQUE_NAME

acceptance/bundle/select/deletion_saved_plan/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/select/deletion_saved_plan/output.txt

@@ -0,0 +1,38 @@
+
+=== Initial deploy
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-del-plan-[UNIQUE_NAME]/default/files...
+Created jobs.gone
+Created jobs.keep
+Files: 3 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== bundle deploy --plan (no --select)
+
+>>> [CLI] bundle deploy --plan plan.json
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-del-plan-[UNIQUE_NAME]/default/files...
+Deleted jobs.gone
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 1 deleted, 0 unchanged, 1 not selected
+
+>>> print_requests.py --sort //jobs/delete
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/delete",
+  "body": {
+    "job_id": [GONE_ID]
+  }
+}
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== Destroy
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.keep
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/select-del-plan-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/select/deletion_saved_plan/script

@@ -0,0 +1,34 @@
+envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    title "Destroy"
+    trace $CLI bundle destroy --auto-approve
+    rm -f "$OUT_REQUESTS"
+}
+trap cleanup EXIT
+
+# Deploy both jobs.
+title "Initial deploy\n"
+trace $CLI bundle deploy
+# Register [KEEP_ID]/[GONE_ID] from state while both still exist, so the delete
+# request body below is masked deterministically.
+replace_ids.py
+
+# Remove 'gone' from config. It is now orphaned in state and planned for deletion.
+envsubst '$UNIQUE_NAME' < databricks.yml.after.tmpl > databricks.yml
+
+# Filter the plan to the deletion and serialize it. (Kept out of recorded output:
+# the JSON embeds remote state that differs between local and cloud.)
+$CLI bundle plan --select jobs.gone -o json > plan.json
+
+# Deploy from the plan file WITHOUT re-passing --select: the deletion was selected
+# at plan time and survives serialization, so applying the saved plan deletes 'gone'
+# and reports the ", 1 not selected" suffix from plan.NotSelected, not the flag.
+title "bundle deploy --plan (no --select)\n"
+trace $CLI bundle deploy --plan plan.json
+
+# The delete call targeted 'gone' only.
+trace print_requests.py --sort //jobs/delete
+
+# A full plan now has nothing to delete and 'keep' is unchanged.
+trace $CLI bundle plan

acceptance/bundle/select/deletion_saved_plan/test.toml

@@ -0,0 +1,5 @@
+Cloud = true
+RecordRequests = true
+# databricks.yml is generated at runtime from the committed templates; the serialized
+# plan embeds local/cloud-specific state, so it is kept out of the recorded output.
+Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json"]

acceptance/bundle/select/invalid_no_upload/databricks.yml.tmpl

@@ -0,0 +1,12 @@
+bundle:
+  name: select-invalid-$UNIQUE_NAME
+
+experimental:
+  scripts:
+    # Must NOT run: the invalid selector is rejected before the predeploy script.
+    predeploy: touch predeploy_ran.marker
+
+resources:
+  jobs:
+    real:
+      name: real-$UNIQUE_NAME

acceptance/bundle/select/invalid_no_upload/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/select/invalid_no_upload/output.txt

@@ -0,0 +1,6 @@
+
+>>> [CLI] bundle deploy --select jobs.nonexistent
+Error: no such resource: jobs.nonexistent
+
+
+>>> print_requests.py --dms //workspace //api/2.0/bundle