Allow bind and unbind with shadow deployment history

acceptance/bundle/deployment/bind/alert/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/catalog/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/cluster/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/dashboard/out.test.toml

@@ -1,3 +1,3 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]
 EnvMatrix.READPLAN = ["", "1"]

acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml

@@ -1,3 +1,3 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]
 EnvMatrix.READPLAN = ["", "1"]

acceptance/bundle/deployment/bind/dashboard/recreation/output.txt

@@ -3,7 +3,7 @@
 Successfully bound dashboard with an id '[DASHBOARD_ID]'
 Run 'bundle deploy' to deploy changes to your workspace
 
->>> print_state.py
+>>> print_state.py --no-dms
 [ETAG]
 
 >>> [CLI] bundle plan

acceptance/bundle/deployment/bind/dashboard/recreation/script

@@ -13,7 +13,7 @@ cleanupRemoveDashboard() {
 trap cleanupRemoveDashboard EXIT
 
 trace $CLI bundle deployment bind dashboard1 "${DASHBOARD_ID}" --auto-approve
-trace print_state.py > out.state_after_bind.json
+trace print_state.py --no-dms > out.state_after_bind.json
 ETAG=$(jq '.state["resources.dashboards.dashboard1"].state.etag // .resources[0].instances[0].attributes.etag // "no-etag"' < out.state_after_bind.json)
 echo $ETAG
 add_repl $ETAG ETAG

acceptance/bundle/deployment/bind/database_instance/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/experiment/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/external_location/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/feature/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/genie_space/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/noop-job/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/noop-job/script

@@ -6,4 +6,4 @@ trace $CLI bundle deployment bind job_1 $job_id --auto-approve
 trace $CLI bundle deploy
 
 # there is a difference: in terraform there is also run_as entry
-trace $CLI jobs get $job_id --output json > out.job.json
+trace $CLI jobs get $job_id --output json | nostamp > out.job.json

acceptance/bundle/deployment/bind/job/python-job/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/python-job/script

@@ -7,4 +7,4 @@ trace $UV_RUN $CLI bundle deployment bind job_1 $job_id --auto-approve
 
 trace $UV_RUN $CLI bundle deploy
 
-trace $CLI jobs get $job_id --output json > out.job.json
+trace $CLI jobs get $job_id --output json | nostamp > out.job.json

acceptance/bundle/deployment/bind/job/stale-state/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/job/unreadable-state/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/mcp_service/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/model_provider_service/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/model_service/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/pipelines/recreate/output.txt

@@ -34,4 +34,4 @@ Recreated pipelines.foo
 Files: 0 uploaded, 0 deleted
 Resources: 1 created, 0 changed, 1 deleted, 0 unchanged
 
->>> print_requests.py ^//workspace/ ^//telemetry-ext
+>>> print_requests.py --nostamp ^//workspace/ ^//telemetry-ext

acceptance/bundle/deployment/bind/pipelines/recreate/script

@@ -19,4 +19,4 @@ trace musterr $CLI bundle deploy
 
 rm -f "$OUT_REQUESTS"
 trace $CLI bundle deploy --auto-approve
-trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json
+trace print_requests.py --nostamp '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json

acceptance/bundle/deployment/bind/pipelines/update/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/pipelines/update/output.txt

@@ -19,4 +19,4 @@ Updated pipelines.foo
 Files: 6 uploaded, 0 deleted
 Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
 
->>> print_requests.py ^//workspace/import ^//workspace/delete ^//telemetry-ext
+>>> print_requests.py --nostamp ^//workspace/import ^//workspace/delete ^//telemetry-ext

acceptance/bundle/deployment/bind/pipelines/update/script

@@ -14,4 +14,4 @@ trace $CLI bundle plan
 
 rm -f "$OUT_REQUESTS"
 trace $CLI bundle deploy --auto-approve
-trace print_requests.py '^//workspace/import' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.json
+trace print_requests.py --nostamp '^//workspace/import' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.json

acceptance/bundle/deployment/bind/postgres_database/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/postgres_role/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/postgres_snapshot_schedule/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/quality-monitor/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/registered-model/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/schema/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/secret-scope/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/vector_search_index/out.test.toml

@@ -1,3 +1,3 @@
 Cloud = true
 CloudSlow = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/bind/volume/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/unbind/grants/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/unbind/job/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/unbind/job/script

@@ -6,4 +6,4 @@ trace $CLI bundle deployment unbind job_1 --var "job_name=My Job"
 
 trace $CLI bundle deploy --var "job_name=Updated Job"
 
-trace $CLI jobs get $job_id --output json
+trace $CLI jobs get $job_id --output json | nostamp

acceptance/bundle/deployment/unbind/permissions/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = true
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/unbind/python-job/out.test.toml

@@ -1,2 +1,2 @@
 Cloud = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deployment/unbind/python-job/script

@@ -8,4 +8,4 @@ trace $UV_RUN $CLI bundle deployment unbind job_1 --var "job_name=My Job"
 
 trace $UV_RUN $CLI bundle deploy --var "job_name=Updated Job"
 
-trace $CLI jobs get $job_id --output json
+trace $CLI jobs get $job_id --output json | nostamp

acceptance/bundle/dms/bind-unbind/output.txt

@@ -0,0 +1,63 @@
+
+=== Bind an existing job with deployment history enabled
+>>> [CLI] bundle deployment bind foo [JOB_ID] --auto-approve
+Successfully bound job with an id '[JOB_ID]'
+Run 'bundle deploy' to deploy changes to your workspace
+
+=== Deploy keeps the bound job ID and enables shadow history
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> jq {features, resource_id: .state["resources.jobs.foo"].__id__} .databricks/bundle/default/resources.json
+{
+  "features": {
+    "deployment_history_shadow": {}
+  },
+  "resource_id": "[JOB_ID]"
+}
+
+=== Unbind preserves the shadow history marker and leaves the job intact
+>>> [CLI] bundle deployment unbind foo
+
+>>> jq {features, state} .databricks/bundle/default/resources.json
+{
+  "features": {
+    "deployment_history_shadow": {}
+  },
+  "state": {}
+}
+
+>>> [CLI] jobs get [JOB_ID]
+{
+  "job_id": [JOB_ID]
+}
+
+=== Rebind preserves the existing shadow history marker
+>>> [CLI] bundle deployment bind foo [JOB_ID] --auto-approve
+Successfully bound job with an id '[JOB_ID]'
+Run 'bundle deploy' to deploy changes to your workspace
+
+>>> jq {features, resource_id: .state["resources.jobs.foo"].__id__} .databricks/bundle/default/resources.json
+{
+  "features": {
+    "deployment_history_shadow": {}
+  },
+  "resource_id": "[JOB_ID]"
+}
+
+=== A subsequent deploy continues managing the same job
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files...
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+
+>>> jq {features, resource_id: .state["resources.jobs.foo"].__id__} .databricks/bundle/default/resources.json
+{
+  "features": {
+    "deployment_history_shadow": {}
+  },
+  "resource_id": "[JOB_ID]"
+}

acceptance/bundle/dms/bind-unbind/script

@@ -0,0 +1,23 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+title "Bind an existing job with deployment history enabled"
+job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
+add_repl "$job_id" JOB_ID
+trace $CLI bundle deployment bind foo "$job_id" --auto-approve
+
+title "Deploy keeps the bound job ID and enables shadow history"
+trace $CLI bundle deploy
+trace jq '{features, resource_id: .state["resources.jobs.foo"].__id__}' .databricks/bundle/default/resources.json
+
+title "Unbind preserves the shadow history marker and leaves the job intact"
+trace $CLI bundle deployment unbind foo
+trace jq '{features, state}' .databricks/bundle/default/resources.json
+trace $CLI jobs get "$job_id" | jq '{job_id}'
+
+title "Rebind preserves the existing shadow history marker"
+trace $CLI bundle deployment bind foo "$job_id" --auto-approve
+trace jq '{features, resource_id: .state["resources.jobs.foo"].__id__}' .databricks/bundle/default/resources.json
+
+title "A subsequent deploy continues managing the same job"
+trace $CLI bundle deploy
+trace jq '{features, resource_id: .state["resources.jobs.foo"].__id__}' .databricks/bundle/default/resources.json

acceptance/bundle/dms/bind-unbind/test.toml

@@ -0,0 +1,3 @@
+RecordRequests = false
+# Verify shadow history markers and resource IDs across bind, deploy, unbind, and rebind.
+Cloud = false

acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml

@@ -1,4 +1,4 @@
 Cloud = true
 CloudEnvs.azure = false
 CloudEnvs.gcp = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml

@@ -1,4 +1,4 @@
 Cloud = false
 CloudEnvs.azure = false
 CloudEnvs.gcp = false
-EnvMatrix.DMS = [""]
+EnvMatrix.DMS = ["", "true"]