Support DMS migration during bundle deploy

acceptance/bundle/dms/existing-state/databricks.yml.tmpl

@@ -1,7 +1,16 @@
 bundle:
   name: dms-existing-state-$UNIQUE_NAME
 
+experimental:
+  deployment_history: false
+
 resources:
   jobs:
     one:
       name: one
+    two:
+      name: two
+      tasks:
+        - task_key: run_one
+          run_job_task:
+            job_id: ${resources.jobs.one.id}

acceptance/bundle/dms/existing-state/output.txt

@@ -1,78 +1,120 @@
 
-=== Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS
->>> [CLI] bundle deploy
+=== 1. Create two jobs without history (direct state serial 1)
+Config disables history even though the environment enables it.
+>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
 Created jobs.one
+Created jobs.two
 Files: 2 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
-
->>> [CLI] bundle plan -o json
-{
-  "plan_version": [PLAN_VERSION],
-  "cli_version": "[CLI_VERSION]",
-  "lineage": "[UUID]",
-  "serial": 1
-}
-
->>> [CLI] bundle plan
-Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
-
->>> print_requests.py --dms //api/2.0/bundle --oneline
-
->>> jq {state_version, features} .databricks/bundle/default/resources.json
-{
-  "state_version": 3,
-  "features": null
-}
-
-=== Enabling recording is refused because the existing deployment is not recorded
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
-Error: deployment history setting (true) does not match the existing state (false)
-
-Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over
-
-
->>> print_requests.py --dms //api/2.0/bundle --oneline
-
-=== Wiping the local cache still refuses after pulling the non-DMS state from the workspace
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
-Error: deployment history setting (true) does not match the existing state (false)
-
-Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
 
+=== Save the original resources.json for the backup check
+>>> cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json
 
-=== A leftover local WAL with the wrong lineage is still refused
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
-Error: reading state from [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: WAL recovery failed: WAL lineage ("stale") does not match state lineage ("[UUID]")
-
-
-=== Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve
-The following resources will be deleted:
-  delete resources.jobs.one
+=== 2. Enable history and rename job one
+Expect: version 2 registers old state; version 3 applies the new name; no new jobs.
+>>> update_file.py databricks.yml deployment_history: false deployment_history: true
 
-All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default
+>>> update_file.py databricks.yml name: one name: updated
 
-Destroy: 1 deleted
-
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
+>>> [CLI] bundle deploy
+Enabling deployment history...
+Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
-Created jobs.one
-Files: 2 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
-
->>> print_requests.py --dms //api/2.0/bundle --oneline
-{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}}
-{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
-
+Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3
+Updated jobs.one
+Updated jobs.two
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 2 changed, 0 deleted, 0 unchanged
+
+=== 3. Check local state
+Expect: serial=3; history enabled; no resource state or WAL.
 >>> find.py resources[.]json[.]wal --expect 0
 
->>> jq {state_version, features} .databricks/bundle/default/resources.json
+>>> jq {serial, features, state} .databricks/bundle/default/resources.json
 {
-  "state_version": 3,
+  "serial": 3,
   "features": {
     "deployment_history": {}
+  },
+  "state": {}
+}
+
+=== 4. Check backup
+Expect: original IDs, names and dependencies; exact pre-migration bytes.
+>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.first.json
+
+>>> cat .databricks/resources.backup.first.json
+{
+ "state_version": 3,
+ "cli_version": "[CLI_VERSION]",
+ "lineage": "[UUID]",
+ "serial": 1,
+ "state": {
+  "resources.jobs.one": {
+   "__id__": "[ONE_ID]",
+   "state": {
+    "deployment": {
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "one",
+    "queue": {
+     "enabled": true
+    }
+   }
+  },
+  "resources.jobs.two": {
+   "__id__": "[TWO_ID]",
+   "state": {
+    "deployment": {
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "two",
+    "queue": {
+     "enabled": true
+    },
+    "tasks": [
+     {
+      "run_job_task": {
+       "job_id": [ONE_ID]
+      },
+      "task_key": "run_one"
+     }
+    ]
+   },
+   "depends_on": [
+    {
+     "node": "resources.jobs.one",
+     "label": "${resources.jobs.one.id}"
+    }
+   ]
   }
+ }
 }
+>>> cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json
+
+=== 5. Check DMS records
+Expect: completed versions 2 and 3; original job IDs and dependency preserved.
+=== Version history: migration 2 completes before deployment 3
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json
+[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}]
+
+=== Migration 2: register the original job IDs (INITIAL_REGISTER)
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json
+[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}]
+
+=== Deployment 3: record updates separately
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json
+[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}]
+
+=== Current resources: new name, original IDs, and job two still depends on job one
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
+[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}]

acceptance/bundle/dms/existing-state/script

@@ -1,38 +1,40 @@
 envsubst < databricks.yml.tmpl > databricks.yml
 
-title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS"
-trace $CLI bundle deploy
-dms_plan
-trace print_requests.py --dms //api/2.0/bundle --oneline
-trace jq '{state_version, features}' .databricks/bundle/default/resources.json
+title "1. Create two jobs without history (direct state serial 1)\nConfig disables history even though the environment enables it."
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy 2>&1 | contains.py '!Current Deployment Version:' '!Enabling deployment history'
+replace_ids.py
+title "Save the original resources.json for the backup check"
+trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json
 
-title "Enabling recording is refused because the existing deployment is not recorded"
-musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
-trace print_requests.py --dms //api/2.0/bundle --oneline
+title "2. Enable history and rename job one\nExpect: version 2 registers old state; version 3 applies the new name; no new jobs."
+export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false
+trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true'
+trace update_file.py databricks.yml 'name: one' 'name: updated'
+trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=2' '?version=3' 'Resources: 0 created'
 
-title "Wiping the local cache still refuses after pulling the non-DMS state from the workspace"
-rm -rf .databricks
-musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
+title "3. Check local state\nExpect: serial=3; history enabled; no resource state or WAL."
+trace find.py 'resources[.]json[.]wal' --expect 0
+trace jq '{serial, features, state}' .databricks/bundle/default/resources.json
 
-title "A leftover local WAL with the wrong lineage is still refused"
-mkdir -p .databricks/bundle/default
-printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal
-musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
+title "4. Check backup\nExpect: original IDs, names and dependencies; exact pre-migration bytes."
+trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json
+trace cat .databricks/resources.backup.first.json
+trace cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json
 
-title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone"
-# Clear the stale WAL first: it was refused, not deleted, and destroy runs without recording, so it
-# would otherwise try to replay it and fail on the lineage mismatch.
-rm -f .databricks/bundle/default/resources.json.wal
-# Keep the mismatched setting here to verify that destroy ignores configuration and follows state.
-trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve
-trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
-trace print_requests.py --dms //api/2.0/bundle --oneline
-trace find.py 'resources[.]json[.]wal' --expect 0
-trace jq '{state_version, features}' .databricks/bundle/default/resources.json
-DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py
-deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
+title "5. Check DMS records\nExpect: completed versions 2 and 3; original job IDs and dependency preserved."
+deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
 add_repl "$deployment_id" DEPLOYMENT_ID
+title "Version history: migration 2 completes before deployment 3"
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})'
+title "Migration 2: register the original job IDs (INITIAL_REGISTER)"
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})'
+title "Deployment 3: record updates separately"
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/3" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})'
+title "Current resources: new name, original IDs, and job two still depends on job one"
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -c '
+    sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id,
+        state: (.state | fromjson | {name: .state.name,
+            tasks: [.state.tasks[]? | {task_key, job_id: .run_job_task.job_id}],
+            depends_on: (.depends_on // [])})})'
 
-# The recorded requests are asserted above via print_requests.py; drop whatever the run added after
-# that (telemetry, and the state probes of the non-recording variant) so nothing is left to compare.
 rm -f "$OUT_REQUESTS"

acceptance/bundle/dms/existing-state/test.toml

@@ -1,3 +1,17 @@
-# This test deploys without recording first, then turns it on per-command, so it opts out of
-# the parent's DMS=true and drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY inline instead.
+# This test checks config precedence with conflicting environment settings,
+# so it drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY instead of the parent's matrix.
 EnvMatrix.DMS = [""]
+
+# acceptance/bundle/test.toml removes Current Deployment Version for shared DMS/non-DMS goldens.
+# Inherited Repls append and cannot be disabled here.
+# Mask at Order -1 before the parent's default 0, then restore at +1.
+# This lets these tests assert both migration and deployment links.
+[[Repls]]
+Old = '(?m)^Current Deployment Version: '
+New = '__CURRENT_DEPLOYMENT_VERSION__: '
+Order = -1
+
+[[Repls]]
+Old = '(?m)^__CURRENT_DEPLOYMENT_VERSION__: '
+New = 'Current Deployment Version: '
+Order = 1

acceptance/bundle/dms/migration-before-deploy/databricks.yml.tmpl

@@ -0,0 +1,18 @@
+bundle:
+  name: dms-migration-before-deploy-$UNIQUE_NAME
+
+experimental:
+  deployment_history: false
+
+sync:
+  exclude:
+    - LOG.*
+
+resources:
+  jobs:
+    job:
+      name: migration-before-deploy
+  schemas:
+    schema:
+      name: dms_migration_before_deploy_$UNIQUE_NAME
+      catalog_name: main

acceptance/bundle/dms/migration-before-deploy/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]