From bc860513476e11a3354e572321a60643ef6a2639 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 12:03:08 +0000 Subject: [PATCH 01/25] Support DMS migration during bundle deploy --- .../dms/existing-state/databricks.yml.tmpl | 9 ++ .../bundle/dms/existing-state/output.txt | 149 +++++++++++------- acceptance/bundle/dms/existing-state/script | 56 ++++--- .../bundle/dms/existing-state/test.toml | 18 ++- .../repeated-migration/databricks.yml.tmpl | 14 ++ .../dms/repeated-migration/out.test.toml | 2 + .../bundle/dms/repeated-migration/output.txt | 82 ++++++++++ .../bundle/dms/repeated-migration/script | 59 +++++++ .../bundle/dms/repeated-migration/test.toml | 13 ++ .../migrate/auto/dms-migration/output.txt | 27 ++-- .../bundle/migrate/auto/dms-migration/script | 14 +- bundle/bundle.go | 3 + bundle/config/experimental.go | 6 +- bundle/deploy/dmsmigration/migrate.go | 77 +++++++++ bundle/direct/dstate/dms.go | 15 ++ bundle/phases/deploy.go | 32 +++- bundle/phases/dms.go | 6 +- bundle/statemgmt/state_push.go | 22 +++ cmd/bundle/utils/process.go | 43 ++++- libs/testserver/bundledeployments.go | 22 +++ libs/testserver/handlers.go | 3 + 21 files changed, 549 insertions(+), 123 deletions(-) create mode 100644 acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/repeated-migration/out.test.toml create mode 100644 acceptance/bundle/dms/repeated-migration/output.txt create mode 100644 acceptance/bundle/dms/repeated-migration/script create mode 100644 acceptance/bundle/dms/repeated-migration/test.toml create mode 100644 bundle/deploy/dmsmigration/migrate.go diff --git a/acceptance/bundle/dms/existing-state/databricks.yml.tmpl b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl index 29addd6433e..ff21609cb89 100644 --- a/acceptance/bundle/dms/existing-state/databricks.yml.tmpl +++ b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl @@ -1,7 +1,16 @@ bundle: name: dms-existing-state-$UNIQUE_NAME +experimental: + deployment_history: false + resources: jobs: one: name: one + two: + name: two + tasks: + - task_key: run_one + run_job_task: + job_id: ${resources.jobs.one.id} diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 4057525509b..adf7cb6a09f 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -1,78 +1,111 @@ -=== Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS ->>> [CLI] bundle deploy +=== Config disables recording even when the environment enables it +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Created jobs.one +Created jobs.two Files: 2 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> [CLI] bundle plan -o json -{ - "plan_version": [PLAN_VERSION], - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1 -} +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +>>> cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json ->>> print_requests.py --dms //api/2.0/bundle --oneline - ->>> jq {state_version, features} .databricks/bundle/default/resources.json +>>> jq {state_version, serial, features} .databricks/bundle/default/resources.json { "state_version": 3, + "serial": 1, "features": null } -=== Enabling recording is refused because the existing deployment is not recorded ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: deployment history setting (true) does not match the existing state (false) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over - - ->>> print_requests.py --dms //api/2.0/bundle --oneline - -=== Wiping the local cache still refuses after pulling the non-DMS state from the workspace ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: deployment history setting (true) does not match the existing state (false) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over - - -=== A leftover local WAL with the wrong lineage is still refused ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: reading state from [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: WAL recovery failed: WAL lineage ("stale") does not match state lineage ("[UUID]") - - -=== Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.one - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default - -Destroy: 1 deleted - ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +=== Config enables recording despite the disabled environment: register version 2, then deploy version 3 +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... -Created jobs.one -Files: 2 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 +Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 +Updated jobs.one +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> find.py resources[.]json[.]wal --expect 0 ->>> jq {state_version, features} .databricks/bundle/default/resources.json +>>> jq {features, state} .databricks/bundle/default/resources.json { - "state_version": 3, "features": { "deployment_history": {} + }, + "state": {} +} + +=== Inspect the remote backup's original pre-migration resource IDs, state, and dependencies +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.first.json + +>>> cat .databricks/resources.backup.first.json +{ + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "state": { + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[TWO_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [ONE_ID] + }, + "task_key": "run_one" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.one", + "label": "${resources.jobs.one.id}" + } + ] } + } } +=== The backup equals the pre-migration resources.json byte for byte +>>> cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json + +=== Read back the completed migration and deploy versions with their recorded resource state +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","last_version_id":"2","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index e64ca6febe6..202c648f0d2 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -1,38 +1,36 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" -trace $CLI bundle deploy -dms_plan -trace print_requests.py --dms //api/2.0/bundle --oneline -trace jq '{state_version, features}' .databricks/bundle/default/resources.json +title "Config disables recording even when the environment enables it" +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy 2>&1 | contains.py '!Current Deployment Version:' '!Enabling deployment history' +replace_ids.py +trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json +trace jq '{state_version, serial, features}' .databricks/bundle/default/resources.json -title "Enabling recording is refused because the existing deployment is not recorded" -musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle --oneline +title "Config enables recording despite the disabled environment: register version 2, then deploy version 3" +export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false +update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' +update_file.py databricks.yml 'name: one' 'name: updated' +trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=2' '?version=3' +trace find.py 'resources[.]json[.]wal' --expect 0 +trace jq '{features, state}' .databricks/bundle/default/resources.json -title "Wiping the local cache still refuses after pulling the non-DMS state from the workspace" -rm -rf .databricks -musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +title "Inspect the remote backup's original pre-migration resource IDs, state, and dependencies" +trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json +trace cat .databricks/resources.backup.first.json -title "A leftover local WAL with the wrong lineage is still refused" -mkdir -p .databricks/bundle/default -printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal -musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +title "The backup equals the pre-migration resources.json byte for byte" +trace cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json -title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone" -# Clear the stale WAL first: it was refused, not deleted, and destroy runs without recording, so it -# would otherwise try to replay it and fail on the lineage mismatch. -rm -f .databricks/bundle/default/resources.json.wal -# Keep the mismatched setting here to verify that destroy ignores configuration and follows state. -trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve -trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle --oneline -trace find.py 'resources[.]json[.]wal' --expect 0 -trace jq '{state_version, features}' .databricks/bundle/default/resources.json -DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +title "Read back the completed migration and deploy versions with their recorded resource state" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') add_repl "$deployment_id" DEPLOYMENT_ID +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/3" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -c ' + sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, + state: (.state | fromjson | {name: .state.name, + tasks: [.state.tasks[]? | {task_key, job_id: .run_job_task.job_id}], + depends_on: (.depends_on // [])})})' -# The recorded requests are asserted above via print_requests.py; drop whatever the run added after -# that (telemetry, and the state probes of the non-recording variant) so nothing is left to compare. rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/existing-state/test.toml b/acceptance/bundle/dms/existing-state/test.toml index 7499498f419..4da5c0d15a3 100644 --- a/acceptance/bundle/dms/existing-state/test.toml +++ b/acceptance/bundle/dms/existing-state/test.toml @@ -1,3 +1,17 @@ -# This test deploys without recording first, then turns it on per-command, so it opts out of -# the parent's DMS=true and drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY inline instead. +# This test checks config precedence with conflicting environment settings, +# so it drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY instead of the parent's matrix. EnvMatrix.DMS = [""] + +# acceptance/bundle/test.toml removes Current Deployment Version for shared DMS/non-DMS goldens. +# Inherited Repls append and cannot be disabled here. +# Mask at Order -1 before the parent's default 0, then restore at +1. +# This lets these tests assert both migration and deployment links. +[[Repls]] +Old = '(?m)^Current Deployment Version: ' +New = '__CURRENT_DEPLOYMENT_VERSION__: ' +Order = -1 + +[[Repls]] +Old = '(?m)^__CURRENT_DEPLOYMENT_VERSION__: ' +New = 'Current Deployment Version: ' +Order = 1 diff --git a/acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl b/acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl new file mode 100644 index 00000000000..6c29e76684e --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: dms-repeated-migration-$UNIQUE_NAME + +experimental: + deployment_history: false + +sync: + exclude: + - LOG.* + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/repeated-migration/out.test.toml b/acceptance/bundle/dms/repeated-migration/out.test.toml new file mode 100644 index 00000000000..84b22ba6735 --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/repeated-migration/output.txt b/acceptance/bundle/dms/repeated-migration/output.txt new file mode 100644 index 00000000000..037db3bdd27 --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/output.txt @@ -0,0 +1,82 @@ + +=== Setup (LOG.setup) +Current: job=updated; DMS version=3; old backup=one. + +=== 1. Before +Expect: old backup=one; latest snapshot=updated. + +>>> gron.py .databricks/resources.backup.first.json +json.state.resources.jobs.one.state.name = "one"; + +>>> gron.py .databricks/resources.before-repeat.json +json.state.resources.jobs.one.state.name = "updated"; +PASS: names checked + +=== 2. Repeat deploy +Expect: register=4; deploy=5; name=repeated; created=0. + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/resources.json --file .databricks/resources.before-repeat.json --format AUTO --overwrite + +>>> update_file.py databricks.yml name: updated name: repeated + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/files... +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 +Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=5 +Updated jobs.one +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +PASS: repeat deployed + +=== 3. DMS records +Expect: versions complete; original ID preserved; name=repeated. + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +version 4: previous=3 type=DEPLOY status=COMPLETED completion=SUCCESS +version 5: previous=4 type=DEPLOY status=COMPLETED completion=SUCCESS + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/4 -o json +operation 4: key=jobs.one id=[ONE_ID] action=INITIAL_REGISTER status=SUCCEEDED + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/5 -o json +operation 5: key=jobs.one id=[ONE_ID] action=UPDATE status=SUCCEEDED + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +resource: key=jobs.one id=[ONE_ID] + last_action=UPDATE version=5 name=repeated + tasks=[] dependencies=[] + +=== 4. Backup copy +Expect: backup=updated; bytes match latest and differ from first. + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.repeat.json + +>>> cat .databricks/resources.backup.repeat.json +{ + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "resources": { + "resources.jobs.one": { + "id": "[ONE_ID]", + "name": "updated" + } + } +} + +>>> cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json + +>>> musterr cmp -s .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json +PASS: backup checked + +=== A remote history marker prevents repeat registration despite a higher unmarked local serial +>>> cp .databricks/resources.before-repeat.json .databricks/bundle/default/resources.json + +>>> update_file.py .databricks/bundle/default/resources.json "serial": 3 "serial": 99 + +>>> [CLI] bundle deploy -qq + +>>> [CLI] bundle summary -o json +{"latest_version_id":5} diff --git a/acceptance/bundle/dms/repeated-migration/script b/acceptance/bundle/dms/repeated-migration/script new file mode 100644 index 00000000000..c61afcf5ef4 --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/script @@ -0,0 +1,59 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-repeated-migration-${UNIQUE_NAME}/default/state" +title "Setup (LOG.setup)\nCurrent: job=updated; DMS version=3; old backup=one.\n" +{ + trace $CLI bundle deploy + trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py + trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' + trace update_file.py databricks.yml 'name: one' 'name: updated' + trace $CLI bundle deploy + deployment_id=$(trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_path/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') + add_repl "$deployment_id" DEPLOYMENT_ID + trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json + trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py > .databricks/resources.before-repeat.json +} &> LOG.setup + +title "1. Before\nExpect: old backup=one; latest snapshot=updated.\n" +trace gron.py .databricks/resources.backup.first.json | grep '\.state\.name = ' | contains.py '"one"' +trace gron.py .databricks/resources.before-repeat.json | grep '\.state\.name = ' | contains.py '"updated"' +echo "PASS: names checked" + +title "2. Repeat deploy\nExpect: register=4; deploy=5; name=repeated; created=0.\n" +trace MSYS_NO_PATHCONV=1 $CLI workspace import "$state_path/resources.json" --file .databricks/resources.before-repeat.json --format AUTO --overwrite +trace update_file.py databricks.yml 'name: updated' 'name: repeated' +trace $CLI bundle deploy 2>&1 | contains.py 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=4' '?version=5' 'Resources: 0 created' '!Created ' +echo "PASS: repeat deployed" + +title "3. DMS records\nExpect: versions complete; original ID preserved; name=repeated.\n" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | + jq 'map(select((.version_id | tonumber) >= 4))' | + jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' | + jq -r '.[] | "version \(.version_id): previous=\(.previous_version_id) type=\(.version_type | ltrimstr("VERSION_TYPE_")) status=\(.status | ltrimstr("VERSION_STATUS_")) completion=\(.completion_reason | ltrimstr("VERSION_COMPLETE_"))"' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/4" -o json | + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' | + jq -r '.[] | "operation 4: key=\(.resource_key) id=\(.resource_id) action=\(.action_type | ltrimstr("OPERATION_ACTION_TYPE_")) status=\(.status | ltrimstr("OPERATION_STATUS_"))"' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/5" -o json | + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' | + jq -r '.[] | "operation 5: key=\(.resource_key) id=\(.resource_id) action=\(.action_type | ltrimstr("OPERATION_ACTION_TYPE_")) status=\(.status | ltrimstr("OPERATION_STATUS_"))"' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, + state: (.state | fromjson | {name: .state.name, + tasks: [.state.tasks[]? | {task_key, job_id: .run_job_task.job_id}], + depends_on: (.depends_on // [])})})' | + jq -r '.[] | "resource: key=\(.resource_key) id=\(.resource_id)\n last_action=\(.last_action_type | ltrimstr("OPERATION_ACTION_TYPE_")) version=\(.last_version_id) name=\(.state.name)\n tasks=\(.state.tasks | tojson) dependencies=\(.state.depends_on | tojson)"' + +title "4. Backup copy\nExpect: backup=updated; bytes match latest and differ from first.\n" +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.repeat.json +trace cat .databricks/resources.backup.repeat.json | contains.py '"name": "updated"' '!"name": "repeated"' | jq '{state_version, cli_version, lineage, serial, resources: (.state | map_values({id: .__id__, name: .state.name}))}' +trace cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json +trace musterr cmp -s .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json +echo "PASS: backup checked" + +title "A remote history marker prevents repeat registration despite a higher unmarked local serial" +trace cp .databricks/resources.before-repeat.json .databricks/bundle/default/resources.json +trace update_file.py .databricks/bundle/default/resources.json '"serial": 3' '"serial": 99' +trace $CLI bundle deploy -qq +trace $CLI bundle summary -o json | jq -c '.bundle.deployment | {latest_version_id}' + +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/repeated-migration/test.toml b/acceptance/bundle/dms/repeated-migration/test.toml new file mode 100644 index 00000000000..8182a51630b --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/test.toml @@ -0,0 +1,13 @@ +# acceptance/bundle/test.toml removes Current Deployment Version for shared DMS/non-DMS goldens. +# Inherited Repls append and cannot be disabled here. +# Mask at Order -1 before the parent's default 0, then restore at +1. +# This lets these tests assert both migration and deployment links. +[[Repls]] +Old = '(?m)^Current Deployment Version: ' +New = '__CURRENT_DEPLOYMENT_VERSION__: ' +Order = -1 + +[[Repls]] +Old = '(?m)^__CURRENT_DEPLOYMENT_VERSION__: ' +New = 'Current Deployment Version: ' +Order = 1 diff --git a/acceptance/bundle/migrate/auto/dms-migration/output.txt b/acceptance/bundle/migrate/auto/dms-migration/output.txt index ba6f36cc8bf..d57bcbe4a55 100644 --- a/acceptance/bundle/migrate/auto/dms-migration/output.txt +++ b/acceptance/bundle/migrate/auto/dms-migration/output.txt @@ -2,19 +2,24 @@ === Replay a prior terraform deployment (captured from a real terraform run) >>> replay_tfstate.py terraform-requests.json terraform.tfstate -=== Direct + deployment-history requested against a terraform state is rejected before commit ->>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true errcode [CLI] bundle deploy +=== Terraform migrates to direct deployment with history enabled and preserves the job ID +>>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Error: deployment history setting (true) does not match the existing state (false) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Migrated 1 resource to direct deployment engine. +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged +>>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) +.databricks/bundle/default/resources.json +.databricks/bundle/default/terraform/terraform.tfstate.backup -Exit code: 1 +>>> find.py resources[.]json[.]wal --expect 0 ->>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) -.databricks/bundle/default/terraform/terraform.tfstate +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle summary -o json +{"job_id":"[TEST_JOB_ID]","version":4} ->>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state -ID Type Language Path -[OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json --format AUTO +{"features":{"deployment_history":{}},"state":{}} diff --git a/acceptance/bundle/migrate/auto/dms-migration/script b/acceptance/bundle/migrate/auto/dms-migration/script index 7ef1ca09f94..e2254672221 100644 --- a/acceptance/bundle/migrate/auto/dms-migration/script +++ b/acceptance/bundle/migrate/auto/dms-migration/script @@ -2,13 +2,13 @@ export DATABRICKS_BUNDLE_ENGINE= title "Replay a prior terraform deployment (captured from a real terraform run)" trace replay_tfstate.py terraform-requests.json terraform.tfstate +original_job_id=$(read_id.py test_job) rm -f "$OUT_REQUESTS" -title "Direct + deployment-history requested against a terraform state is rejected before commit" -# Migration produces a plain direct state (no deployment_history feature), so requesting -# deployment history while migrating is a mismatch. It is rejected after the in-memory migration -# but before CommitMigration writes or pushes anything, so the bundle stays on terraform with no -# leaked direct state - migrate first (plain deploy), then enable deployment history. -trace DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true errcode $CLI bundle deploy +title "Terraform migrates to direct deployment with history enabled and preserves the job ID" +trace DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' -trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state" +trace find.py 'resources[.]json[.]wal' --expect 0 +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle summary -o json | + jq -ce --arg id "$original_job_id" '{job_id: .resources.jobs.test_job.id, version: .bundle.deployment.latest_version_id} | select(.job_id == $id)' +trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state/resources.json" --format AUTO | jq -c '{features, state}' diff --git a/bundle/bundle.go b/bundle/bundle.go index 68654bead7c..a22099cd236 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -194,6 +194,9 @@ type Bundle struct { // leave the terraform state untouched. MigratingToDirect bool + // MigratingToDMS registers the existing direct state before deploying with history enabled. + MigratingToDMS bool + // AllowTerraformEngineConfig reports a terraform bundle.engine as a warning instead // of an error and skips the DATABRICKS_BUNDLE_ENGINE check. Set by read-only commands // (bundle summary) so users can still inspect a bundle that pins the removed engine. diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 6fd5511b79b..498e1b15d47 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -54,9 +54,9 @@ type Experimental struct { // service (DMS), which records deployment history and tracks what changed // across deployments. // - // Only for bundles with no deployed resources yet: DMS becomes the source of - // truth for their state. - // An explicit value takes priority over the environment setting. + // Existing resources are registered when deployment history is enabled, and + // DMS becomes the source of truth for their state. An explicit value takes + // priority over the environment setting. DeploymentHistory *bool `json:"deployment_history,omitempty"` } diff --git a/bundle/deploy/dmsmigration/migrate.go b/bundle/deploy/dmsmigration/migrate.go new file mode 100644 index 00000000000..d6f574bcbdc --- /dev/null +++ b/bundle/deploy/dmsmigration/migrate.go @@ -0,0 +1,77 @@ +package dmsmigration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strconv" + "strings" + + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Migrate records every resource from the open resources.json state in an initial registration version. +// The caller must hold the deployment lock and have created the DMS deployment. +func Migrate(ctx context.Context, db *dstate.DeploymentState) error { + keys := slices.Sorted(maps.Keys(db.Data.State)) + states := make([]json.RawMessage, len(keys)) + operations := make([]bundledeployments.StagedOperation, len(keys)) + for i, key := range keys { + entry := db.Data.State[key] + state, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + return fmt.Errorf("serializing migration state for %s: %w", key, err) + } + states[i] = state + operations[i] = bundledeployments.StagedOperation{ + ResourceKey: strings.TrimPrefix(key, dms.StatePrefix), + ActionType: bundledeployments.OperationActionTypeOperationActionTypeInitialRegister, + } + } + + versionID := max(db.Data.Serial, db.VersionID) + 1 + previousVersionID := "" + if db.VersionID > 0 { + previousVersionID = strconv.Itoa(db.VersionID) + } + w := cmdctx.WorkspaceClient(ctx) + _, err := w.BundleDeployments.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ + Parent: dms.DeploymentName(db.DeploymentID), + VersionId: strconv.Itoa(versionID), + Version: bundledeployments.Version{ + CliVersion: build.GetInfo().Version, + VersionType: dms.VersionTypeDeploy, + PreviousVersionId: previousVersionID, + Operations: operations, + }, + }) + if err != nil { + return fmt.Errorf("creating migration version: %w", err) + } + + buffer := dms.StartOperationBuffer(ctx, db.DeploymentID, versionID) + for i, key := range keys { + buffer.RecordOperation(ctx, key, false, db.Data.State[key].ID, states[i]) + } + recordErr := buffer.Drain() + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if recordErr != nil { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + _, completeErr := w.BundleDeployments.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: dms.VersionName(db.DeploymentID, versionID), + CompletionReason: reason, + }) + if err := errors.Join(recordErr, completeErr); err != nil { + return fmt.Errorf("recording migration version: %w", err) + } + + return db.UseDeploymentHistory(versionID) +} diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 57e10b8d4b4..f9a90a1cd77 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -16,6 +16,21 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } +// UseDeploymentHistory persists the DMS marker after all existing resources have been recorded. +func (db *DeploymentState) UseDeploymentHistory(versionID int) error { + db.AssertOpenedForRead() + db.mu.Lock() + defer db.mu.Unlock() + + db.recordsHistory = true + db.VersionID = versionID + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}) + } + db.Data.Features[FeatureDeploymentHistory] = struct{}{} + return db.unlockedSave() +} + // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS // resources.json does not track state. The service is queried for state for resources and then they // are filled in-place in DeploymentState.Data.State: diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 13c5121f569..e791b26b393 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deploy" + "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/metadata" @@ -294,6 +295,27 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } } + if b.MigratingToDMS { + if err := statemgmt.BackupRemoteResourcesState(ctx, b); err != nil { + logdiag.LogError(ctx, err) + return + } + createOrUpdateDeployment(ctx, b, dmsDeployment) + if logdiag.HasError(ctx) { + return + } + cmdio.LogString(ctx, "Enabling deployment history...") + if err := dmsmigration.Migrate(ctx, &b.DeploymentBundle.StateDB); err != nil { + logdiag.LogError(ctx, err) + return + } + logDeploymentVersion(ctx, b, "Deployment history enabled") + statemgmt.PushResourcesState(ctx, b) + if logdiag.HasError(ctx) { + return + } + } + // Upgrade from read (opened by process.go, or by Migrate) to write mode. After approval // and the migration commit above, so a declined deploy never opens a WAL it must discard, // and the migration's tf+1 push lands before this advances the WAL header to tf+2. @@ -321,12 +343,14 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { - firstDeploy := b.DeploymentBundle.StateDB.DeploymentID == "" - createOrUpdateDeployment(ctx, b, dmsDeployment) + injectDeploymentIdInPlan := b.DeploymentBundle.StateDB.DeploymentID == "" || b.MigratingToDMS + if !b.MigratingToDMS { + createOrUpdateDeployment(ctx, b, dmsDeployment) + } if logdiag.HasError(ctx) { return } - if firstDeploy { + if injectDeploymentIdInPlan { if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { logdiag.LogError(ctx, err) return @@ -345,7 +369,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logdiag.LogError(ctx, err) return } - logDeploymentVersion(ctx, b) + logDeploymentVersion(ctx, b, "Current Deployment Version") } } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index abd89f03f2f..095e4a149b0 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -165,7 +165,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version // logDeploymentVersion logs the deployment version URL. Workspace ID is omitted // so the page stays clickable in a terminal and redirects correctly without it. -func logDeploymentVersion(ctx context.Context, b *bundle.Bundle) { +func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, message string) { deploymentID := b.DeploymentBundle.StateDB.DeploymentID version := b.DeploymentBundle.StateDB.VersionID if version == 0 { @@ -177,11 +177,11 @@ func logDeploymentVersion(ctx context.Context, b *bundle.Bundle) { // Only the link is lost, so report the version without it rather than failing // a deploy over it. log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) - cmdio.LogString(ctx, fmt.Sprintf("Current Deployment Version: %s version %d", deploymentID, version)) + cmdio.LogString(ctx, fmt.Sprintf("%s: %s version %d", message, deploymentID, version)) return } - cmdio.LogString(ctx, "Current Deployment Version: "+workspaceurls.DeploymentURL(*baseURL, deploymentID, version)) + cmdio.LogString(ctx, message+": "+workspaceurls.DeploymentURL(*baseURL, deploymentID, version)) } // deploymentMetadata describes the bundle this deploy came from and where it diff --git a/bundle/statemgmt/state_push.go b/bundle/statemgmt/state_push.go index 03036ba712d..9df720f6157 100644 --- a/bundle/statemgmt/state_push.go +++ b/bundle/statemgmt/state_push.go @@ -3,6 +3,7 @@ package statemgmt import ( "context" "errors" + "fmt" "io/fs" "os" @@ -43,6 +44,27 @@ func PushResourcesState(ctx context.Context, b *bundle.Bundle) { } } +// BackupRemoteResourcesState preserves the remote state before deployment history replaces it. +func BackupRemoteResourcesState(ctx context.Context, b *bundle.Bundle) error { + f, err := deploy.StateFiler(ctx, b) + if err != nil { + return fmt.Errorf("backing up resource state: creating state filer: %w", err) + } + + remotePath, _ := b.StateFilenameDirect(ctx) + reader, err := f.Read(ctx, remotePath) + if err != nil { + return fmt.Errorf("backing up resource state: reading %s: %w", remotePath, err) + } + defer reader.Close() + + backupPath := remotePath + ".backup" + if err := f.Write(ctx, backupPath, reader, filer.OverwriteIfExists); err != nil { + return fmt.Errorf("backing up resource state: writing %s: %w", backupPath, err) + } + return nil +} + func BackupRemoteTerraformState(ctx context.Context, b *bundle.Bundle) { f, err := deploy.StateFiler(ctx, b) if err != nil { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 6ccb3367db7..8a3b64b2854 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -24,6 +24,7 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/atomicfile" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" @@ -236,6 +237,23 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } + if opts.Deploy && stateDesc.IsLocal && stateDesc.Engine.IsDirect() && configuresDeploymentHistory(ctx, b) { + // Migration follows the remote history marker rather than a conflicting local copy. + for _, remoteState := range stateDesc.AllStates { + if remoteState.IsLocal || !remoteState.Engine.IsDirect() || (remoteState.IsDMS() && stateDesc.IsDMS()) { + continue + } + _, localPath := b.StateFilenameDirect(ctx) + if err := atomicfile.Write(localPath, remoteState.Content, 0o600, atomicfile.MkDir(0o700)); err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + remoteState.AllStates = stateDesc.AllStates + stateDesc = remoteState + break + } + } + b.MigratingToDirect = !stateDesc.Engine.IsDirect() // Tag the user agent with the engine this run actually uses. On the auto-migration @@ -363,14 +381,21 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } - if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting { + b.MigratingToDMS = opts.Deploy && stateDesc.Engine.IsDirect() && !stateDesc.IsDMS() && configuresDeploymentHistory(ctx, b) + // TODO: Support applying plans computed against direct state after migration to DMS. + // This restriction blocks automatic migration with --plan once deployment history reaches GA. + if b.MigratingToDMS && opts.ReadPlanPath != "" { + return b, stateDesc, errors.New("migrating to deployment history does not support --plan; run 'bundle deploy' without --plan") + } + if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting && !b.MigratingToDMS { if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, opts.Deploy || opts.PreDeployChecks); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } } - if needDirectState && stateDesc.IsDMS() { + // Repeated migrations may already have a DMS deployment even when the state lacks its history marker. + if needDirectState && (stateDesc.IsDMS() || b.MigratingToDMS) { var err error dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { @@ -379,15 +404,18 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } // Stamp the deployment and the version this run records onto every job and pipeline so - // the plan carries them. version_id is always known (last recorded + 1); deployment_id - // does not exist until a first deploy creates it, so it is left off here and the deploy - // phase stamps the created id. + // the plan carries them. The deployment version follows the last recorded version or + // the initial registration version; deployment_id does not exist until a first deploy + // creates it, so it is left off here and the deploy phase stamps the created id. lastVersionID, err := parseLastVersionID(dmsDeployment) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } nextVersion := lastVersionID + 1 + if b.MigratingToDMS { + nextVersion = max(b.DeploymentBundle.StateDB.Data.Serial, lastVersionID) + 2 + } muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} if dmsDeploymentID != "" { bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { @@ -404,7 +432,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { + if b.MigratingToDMS { + b.DeploymentBundle.StateDB.DeploymentID = dmsDeploymentID + b.DeploymentBundle.StateDB.VersionID = lastVersionID + } else if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go index 86aef24d83c..bc52c392cf7 100644 --- a/libs/testserver/bundledeployments.go +++ b/libs/testserver/bundledeployments.go @@ -2,6 +2,7 @@ package testserver import ( "bytes" + "cmp" "encoding/json" "fmt" "path" @@ -576,6 +577,27 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re return Response{Body: body} } +// ListVersions returns recorded versions in descending numeric order. +func (s *FakeWorkspace) ListVersions(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + versions := make([]bundledeployments.Version, 0, len(d.Versions)) + for _, version := range d.Versions { + versions = append(versions, *version) + } + slices.SortFunc(versions, func(a, b bundledeployments.Version) int { + aID, _ := strconv.ParseInt(a.VersionId, 10, 64) + bID, _ := strconv.ParseInt(b.VersionId, 10, 64) + return cmp.Compare(bID, aID) + }) + return Response{Body: bundledeployments.ListVersionsResponse{Versions: versions}} +} + func (s *FakeWorkspace) ListResources(deploymentID string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index f2d9cff5109..c8d4ef74655 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -468,6 +468,9 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.ListVersions(req.Vars["deployment_id"]) + }) server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) }) From 160411191ceaf8b034e3fbcfb80c70c3782aee52 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 13:10:32 +0000 Subject: [PATCH 02/25] Explain remote-state authority during DMS migration --- cmd/bundle/utils/process.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 8a3b64b2854..068e2fe69a7 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -237,8 +237,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } + // For migration, invalidate the selected local state and use remote state as + // the source of truth. Relying only on the remote view keeps migration logic + // simple and consistent. if opts.Deploy && stateDesc.IsLocal && stateDesc.Engine.IsDirect() && configuresDeploymentHistory(ctx, b) { - // Migration follows the remote history marker rather than a conflicting local copy. for _, remoteState := range stateDesc.AllStates { if remoteState.IsLocal || !remoteState.Engine.IsDirect() || (remoteState.IsDMS() && stateDesc.IsDMS()) { continue From 74f87dbf86cc45799ebd969501c8203ddbd71251 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 13:16:17 +0000 Subject: [PATCH 03/25] Verify DMS migration invalidates newer local state --- acceptance/bundle/dms/existing-state/output.txt | 9 ++++++++- acceptance/bundle/dms/existing-state/script | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index adf7cb6a09f..4e67bb8006f 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -16,6 +16,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "features": null } +=== Remote serial 1 is authoritative despite a newer local serial 99 +>>> update_file.py .databricks/bundle/default/resources.json "serial": 1 "serial": 99 + +>>> jq -c {serial} .databricks/bundle/default/resources.json +{"serial":99} + === Config enables recording despite the disabled environment: register version 2, then deploy version 3 >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... @@ -28,8 +34,9 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> find.py resources[.]json[.]wal --expect 0 ->>> jq {features, state} .databricks/bundle/default/resources.json +>>> jq {serial, features, state} .databricks/bundle/default/resources.json { + "serial": 3, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 202c648f0d2..b44548842f3 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -6,13 +6,17 @@ replace_ids.py trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json trace jq '{state_version, serial, features}' .databricks/bundle/default/resources.json +title "Remote serial 1 is authoritative despite a newer local serial 99" +trace update_file.py .databricks/bundle/default/resources.json '"serial": 1' '"serial": 99' +trace jq -c '{serial}' .databricks/bundle/default/resources.json + title "Config enables recording despite the disabled environment: register version 2, then deploy version 3" export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' update_file.py databricks.yml 'name: one' 'name: updated' trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=2' '?version=3' trace find.py 'resources[.]json[.]wal' --expect 0 -trace jq '{features, state}' .databricks/bundle/default/resources.json +trace jq '{serial, features, state}' .databricks/bundle/default/resources.json title "Inspect the remote backup's original pre-migration resource IDs, state, and dependencies" trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json From e9ca163df0a8468f81353ab15f6f7a1f15e90713 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 13:26:12 +0000 Subject: [PATCH 04/25] Assert the exact diff between migration backups --- .../bundle/dms/repeated-migration/output.txt | 23 ++++++++++++++++++- .../bundle/dms/repeated-migration/script | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/dms/repeated-migration/output.txt b/acceptance/bundle/dms/repeated-migration/output.txt index 037db3bdd27..9b744c6cf18 100644 --- a/acceptance/bundle/dms/repeated-migration/output.txt +++ b/acceptance/bundle/dms/repeated-migration/output.txt @@ -68,7 +68,28 @@ Expect: backup=updated; bytes match latest and differ from first. >>> cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json ->>> musterr cmp -s .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json +>>> diff.py .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json +--- .databricks/resources.backup.first.json ++++ .databricks/resources.backup.repeat.json +@@ -3,5 +3,5 @@ + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", +- "serial": 1, ++ "serial": 3, + "state": { + "resources.jobs.one": { +@@ -15,5 +15,5 @@ + "format": "MULTI_TASK", + "max_concurrent_runs": 1, +- "name": "one", ++ "name": "updated", + "queue": { + "enabled": true +@@ -22,3 +22,3 @@ + } + } +-} ++} PASS: backup checked === A remote history marker prevents repeat registration despite a higher unmarked local serial diff --git a/acceptance/bundle/dms/repeated-migration/script b/acceptance/bundle/dms/repeated-migration/script index c61afcf5ef4..6d3d0118d8b 100644 --- a/acceptance/bundle/dms/repeated-migration/script +++ b/acceptance/bundle/dms/repeated-migration/script @@ -47,7 +47,7 @@ title "4. Backup copy\nExpect: backup=updated; bytes match latest and differ fro trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.repeat.json trace cat .databricks/resources.backup.repeat.json | contains.py '"name": "updated"' '!"name": "repeated"' | jq '{state_version, cli_version, lineage, serial, resources: (.state | map_values({id: .__id__, name: .state.name}))}' trace cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json -trace musterr cmp -s .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json +trace diff.py .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json echo "PASS: backup checked" title "A remote history marker prevents repeat registration despite a higher unmarked local serial" From b5a31b82231358b54da4d2aec8f87b5a6d256318 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 13:40:08 +0000 Subject: [PATCH 05/25] Use default state selection for DMS migration --- .../bundle/dms/existing-state/output.txt | 24 +++++++++---------- acceptance/bundle/dms/existing-state/script | 12 +++++----- .../bundle/dms/repeated-migration/output.txt | 12 ++-------- .../bundle/dms/repeated-migration/script | 7 +----- cmd/bundle/utils/process.go | 20 ---------------- 5 files changed, 21 insertions(+), 54 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 4e67bb8006f..184e735bb4b 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -16,18 +16,18 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "features": null } -=== Remote serial 1 is authoritative despite a newer local serial 99 ->>> update_file.py .databricks/bundle/default/resources.json "serial": 1 "serial": 99 +=== Normal state selection prefers local serial 2 over remote serial 1 +>>> update_file.py .databricks/bundle/default/resources.json "serial": 1 "serial": 2 >>> jq -c {serial} .databricks/bundle/default/resources.json -{"serial":99} +{"serial":2} -=== Config enables recording despite the disabled environment: register version 2, then deploy version 3 +=== Config enables recording despite the disabled environment: register version 3, then deploy version 4 >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Enabling deployment history... -Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 -Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 +Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 Updated jobs.one Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -36,7 +36,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> jq {serial, features, state} .databricks/bundle/default/resources.json { - "serial": 3, + "serial": 4, "features": { "deployment_history": {} }, @@ -101,18 +101,18 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } } } -=== The backup equals the pre-migration resources.json byte for byte +=== The backup equals the remote pre-migration snapshot byte for byte >>> cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json === Read back the completed migration and deploy versions with their recorded resource state >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json -[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] +[{"version_id":"3","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"4","previous_version_id":"3","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json [{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/4 -o json [{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json -[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","last_version_id":"2","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"4","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","last_version_id":"3","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index b44548842f3..e82d5b7ff90 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -6,15 +6,15 @@ replace_ids.py trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json trace jq '{state_version, serial, features}' .databricks/bundle/default/resources.json -title "Remote serial 1 is authoritative despite a newer local serial 99" -trace update_file.py .databricks/bundle/default/resources.json '"serial": 1' '"serial": 99' +title "Normal state selection prefers local serial 2 over remote serial 1" +trace update_file.py .databricks/bundle/default/resources.json '"serial": 1' '"serial": 2' trace jq -c '{serial}' .databricks/bundle/default/resources.json -title "Config enables recording despite the disabled environment: register version 2, then deploy version 3" +title "Config enables recording despite the disabled environment: register version 3, then deploy version 4" export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' update_file.py databricks.yml 'name: one' 'name: updated' -trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=2' '?version=3' +trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=3' '?version=4' trace find.py 'resources[.]json[.]wal' --expect 0 trace jq '{serial, features, state}' .databricks/bundle/default/resources.json @@ -22,15 +22,15 @@ title "Inspect the remote backup's original pre-migration resource IDs, state, a trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json trace cat .databricks/resources.backup.first.json -title "The backup equals the pre-migration resources.json byte for byte" +title "The backup equals the remote pre-migration snapshot byte for byte" trace cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json title "Read back the completed migration and deploy versions with their recorded resource state" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/3" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/4" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -c ' sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, state: (.state | fromjson | {name: .state.name, diff --git a/acceptance/bundle/dms/repeated-migration/output.txt b/acceptance/bundle/dms/repeated-migration/output.txt index 9b744c6cf18..195e3121cc0 100644 --- a/acceptance/bundle/dms/repeated-migration/output.txt +++ b/acceptance/bundle/dms/repeated-migration/output.txt @@ -17,6 +17,8 @@ Expect: register=4; deploy=5; name=repeated; created=0. >>> MSYS_NO_PATHCONV=1 [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/resources.json --file .databricks/resources.before-repeat.json --format AUTO --overwrite +>>> cp .databricks/resources.before-repeat.json .databricks/bundle/default/resources.json + >>> update_file.py databricks.yml name: updated name: repeated >>> [CLI] bundle deploy @@ -91,13 +93,3 @@ Expect: backup=updated; bytes match latest and differ from first. -} +} PASS: backup checked - -=== A remote history marker prevents repeat registration despite a higher unmarked local serial ->>> cp .databricks/resources.before-repeat.json .databricks/bundle/default/resources.json - ->>> update_file.py .databricks/bundle/default/resources.json "serial": 3 "serial": 99 - ->>> [CLI] bundle deploy -qq - ->>> [CLI] bundle summary -o json -{"latest_version_id":5} diff --git a/acceptance/bundle/dms/repeated-migration/script b/acceptance/bundle/dms/repeated-migration/script index 6d3d0118d8b..ec3287dc91e 100644 --- a/acceptance/bundle/dms/repeated-migration/script +++ b/acceptance/bundle/dms/repeated-migration/script @@ -21,6 +21,7 @@ echo "PASS: names checked" title "2. Repeat deploy\nExpect: register=4; deploy=5; name=repeated; created=0.\n" trace MSYS_NO_PATHCONV=1 $CLI workspace import "$state_path/resources.json" --file .databricks/resources.before-repeat.json --format AUTO --overwrite +trace cp .databricks/resources.before-repeat.json .databricks/bundle/default/resources.json trace update_file.py databricks.yml 'name: updated' 'name: repeated' trace $CLI bundle deploy 2>&1 | contains.py 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=4' '?version=5' 'Resources: 0 created' '!Created ' echo "PASS: repeat deployed" @@ -50,10 +51,4 @@ trace cmp .databricks/resources.before-repeat.json .databricks/resources.backup. trace diff.py .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json echo "PASS: backup checked" -title "A remote history marker prevents repeat registration despite a higher unmarked local serial" -trace cp .databricks/resources.before-repeat.json .databricks/bundle/default/resources.json -trace update_file.py .databricks/bundle/default/resources.json '"serial": 3' '"serial": 99' -trace $CLI bundle deploy -qq -trace $CLI bundle summary -o json | jq -c '.bundle.deployment | {latest_version_id}' - rm -f "$OUT_REQUESTS" diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 068e2fe69a7..043bae532d9 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -24,7 +24,6 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" - "github.com/databricks/cli/libs/atomicfile" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" @@ -237,25 +236,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } - // For migration, invalidate the selected local state and use remote state as - // the source of truth. Relying only on the remote view keeps migration logic - // simple and consistent. - if opts.Deploy && stateDesc.IsLocal && stateDesc.Engine.IsDirect() && configuresDeploymentHistory(ctx, b) { - for _, remoteState := range stateDesc.AllStates { - if remoteState.IsLocal || !remoteState.Engine.IsDirect() || (remoteState.IsDMS() && stateDesc.IsDMS()) { - continue - } - _, localPath := b.StateFilenameDirect(ctx) - if err := atomicfile.Write(localPath, remoteState.Content, 0o600, atomicfile.MkDir(0o700)); err != nil { - logdiag.LogError(ctx, err) - return b, stateDesc, root.ErrAlreadyPrinted - } - remoteState.AllStates = stateDesc.AllStates - stateDesc = remoteState - break - } - } - b.MigratingToDirect = !stateDesc.Engine.IsDirect() // Tag the user agent with the engine this run actually uses. On the auto-migration From 3d4929aa7538a9e8378fa1e2432a69ddc7d466a8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 14:29:41 +0000 Subject: [PATCH 06/25] Clarify migration and deployment version offsets --- cmd/bundle/utils/process.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 043bae532d9..8cb211d3474 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -396,6 +396,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } nextVersion := lastVersionID + 1 if b.MigratingToDMS { + // The migration itself uses +1; the deployment uses the following version (+2). nextVersion = max(b.DeploymentBundle.StateDB.Data.Serial, lastVersionID) + 2 } muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} From 4871cbf1a447a436e13339c018766723c5bf6804 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 15:37:40 +0000 Subject: [PATCH 07/25] Prepare deployment history migration before planning --- .../bundle/dms/existing-state/output.txt | 8 +++ acceptance/bundle/dms/existing-state/script | 10 ++- .../databricks.yml.tmpl | 18 +++++ .../dms/migration-not-committed/out.test.toml | 2 + .../dms/migration-not-committed/output.txt | 47 +++++++++++++ .../bundle/dms/migration-not-committed/script | 32 +++++++++ .../dms/migration-not-committed/test.toml | 2 + bundle/bundle.go | 5 +- bundle/deploy/dmsmigration/migrate.go | 38 ++++++++--- bundle/deploy/dmsmigration/migrate_test.go | 68 +++++++++++++++++++ bundle/direct/dstate/dms.go | 8 ++- bundle/direct/dstate/dms_test.go | 27 ++++++++ bundle/phases/deploy.go | 25 ++++--- bundle/statemgmt/direct_migration.go | 11 ++- cmd/bundle/utils/process.go | 34 ++++------ 15 files changed, 286 insertions(+), 49 deletions(-) create mode 100644 acceptance/bundle/dms/migration-not-committed/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/migration-not-committed/out.test.toml create mode 100644 acceptance/bundle/dms/migration-not-committed/output.txt create mode 100644 acceptance/bundle/dms/migration-not-committed/script create mode 100644 acceptance/bundle/dms/migration-not-committed/test.toml create mode 100644 bundle/deploy/dmsmigration/migrate_test.go diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 184e735bb4b..26acb56ad78 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -22,6 +22,14 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq -c {serial} .databricks/bundle/default/resources.json {"serial":2} +=== Preview prepares serial 3 and history in memory without committing migration +>>> [CLI] bundle plan -o json +{"serial":3,"features":{"deployment_history":{}}} + +>>> cmp .databricks/resources.before-preview.json .databricks/bundle/default/resources.json + +>>> print_requests.py --dms //api/2.0/bundle //workspace/import //workspace-files/import-file + === Config enables recording despite the disabled environment: register version 3, then deploy version 4 >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index e82d5b7ff90..78dd6bf0dd2 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -10,10 +10,18 @@ title "Normal state selection prefers local serial 2 over remote serial 1" trace update_file.py .databricks/bundle/default/resources.json '"serial": 1' '"serial": 2' trace jq -c '{serial}' .databricks/bundle/default/resources.json -title "Config enables recording despite the disabled environment: register version 3, then deploy version 4" export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' update_file.py databricks.yml 'name: one' 'name: updated' + +title "Preview prepares serial 3 and history in memory without committing migration" +cp .databricks/bundle/default/resources.json .databricks/resources.before-preview.json +rm -f "$OUT_REQUESTS" +trace $CLI bundle plan -o json | jq -c '{serial, features}' +trace cmp .databricks/resources.before-preview.json .databricks/bundle/default/resources.json +trace print_requests.py --dms //api/2.0/bundle //workspace/import //workspace-files/import-file + +title "Config enables recording despite the disabled environment: register version 3, then deploy version 4" trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=3' '?version=4' trace find.py 'resources[.]json[.]wal' --expect 0 trace jq '{serial, features, state}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/migration-not-committed/databricks.yml.tmpl b/acceptance/bundle/dms/migration-not-committed/databricks.yml.tmpl new file mode 100644 index 00000000000..11d945e06b0 --- /dev/null +++ b/acceptance/bundle/dms/migration-not-committed/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: dms-migration-not-committed-$UNIQUE_NAME + +experimental: + deployment_history: false + +sync: + exclude: + - LOG.* + +resources: + jobs: + job: + name: migration-not-committed + schemas: + schema: + name: dms_migration_not_committed_$UNIQUE_NAME + catalog_name: main diff --git a/acceptance/bundle/dms/migration-not-committed/out.test.toml b/acceptance/bundle/dms/migration-not-committed/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/migration-not-committed/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/migration-not-committed/output.txt b/acceptance/bundle/dms/migration-not-committed/output.txt new file mode 100644 index 00000000000..775a2cbed0f --- /dev/null +++ b/acceptance/bundle/dms/migration-not-committed/output.txt @@ -0,0 +1,47 @@ + +=== Setup (LOG.setup): legacy resources exist; migration would recreate the schema +=== 1. Planning fails after preparation; no migration is committed +>>> fault.py GET /api/2.2/jobs/get* 403 0 1 + +>>> musterr [CLI] bundle deploy -qq +Error: cannot plan resources.jobs.job: reading id="[JOB_ID]": Fault injected by test. (403 INJECTED) + +Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[JOB_ID] +HTTP Status: 403 Forbidden +API error_code: INJECTED +API message: Fault injected by test. + +Error: planning failed + + +>>> cmp .databricks/resources.before-migration.json .databricks/bundle/default/resources.json + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-not-committed-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.after-failure.json + +>>> cmp .databricks/resources.before-migration.json .databricks/resources.after-failure.json + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/dms-migration-not-committed-[UNIQUE_NAME]/default/state -o json +[] + +>>> print_requests.py --dms //api/2.0/bundle + +=== 2. Destructive deployment is declined; migration stays uncommitted +>>> musterr [CLI] bundle deploy -qq + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.schema +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + + +>>> cmp .databricks/resources.before-migration.json .databricks/bundle/default/resources.json + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-not-committed-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.after-decline.json + +>>> cmp .databricks/resources.before-migration.json .databricks/resources.after-decline.json + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/dms-migration-not-committed-[UNIQUE_NAME]/default/state -o json +[] + +>>> print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/migration-not-committed/script b/acceptance/bundle/dms/migration-not-committed/script new file mode 100644 index 00000000000..fd564728c72 --- /dev/null +++ b/acceptance/bundle/dms/migration-not-committed/script @@ -0,0 +1,32 @@ +envsubst < databricks.yml.tmpl > databricks.yml +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-not-committed-${UNIQUE_NAME}/default/state" +other_catalog="dms_other_${UNIQUE_NAME}" + +title "Setup (LOG.setup): legacy resources exist; migration would recreate the schema" +{ + $CLI bundle deploy -qq + DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py + cp .databricks/bundle/default/resources.json .databricks/resources.before-migration.json + $CLI catalogs create "$other_catalog" + update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' + update_file.py databricks.yml 'catalog_name: main' "catalog_name: ${other_catalog}" +} &> LOG.setup +trap 'update_file.py databricks.yml "deployment_history: true" "deployment_history: false" >> LOG.cleanup 2>&1; $CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1; $CLI catalogs delete "$other_catalog" --force >> LOG.cleanup 2>&1; rm -f "$OUT_REQUESTS"' EXIT +rm -f "$OUT_REQUESTS" + +title "1. Planning fails after preparation; no migration is committed" +trace fault.py 'GET /api/2.2/jobs/get*' 403 0 1 +trace musterr $CLI bundle deploy -qq +trace cmp .databricks/resources.before-migration.json .databricks/bundle/default/resources.json +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.after-failure.json +trace cmp .databricks/resources.before-migration.json .databricks/resources.after-failure.json +trace MSYS_NO_PATHCONV=1 $CLI workspace list "$state_path" -o json | jq -c '[.[].path | select(endswith("/resources.json.backup") or endswith("/resources.deployment.json"))]' +trace print_requests.py --dms //api/2.0/bundle + +title "2. Destructive deployment is declined; migration stays uncommitted" +trace musterr $CLI bundle deploy -qq +trace cmp .databricks/resources.before-migration.json .databricks/bundle/default/resources.json +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.after-decline.json +trace cmp .databricks/resources.before-migration.json .databricks/resources.after-decline.json +trace MSYS_NO_PATHCONV=1 $CLI workspace list "$state_path" -o json | jq -c '[.[].path | select(endswith("/resources.json.backup") or endswith("/resources.deployment.json"))]' +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/migration-not-committed/test.toml b/acceptance/bundle/dms/migration-not-committed/test.toml new file mode 100644 index 00000000000..7123b8a9421 --- /dev/null +++ b/acceptance/bundle/dms/migration-not-committed/test.toml @@ -0,0 +1,2 @@ +# Fault injection exercises a real planner failure against the local testserver. +Cloud = false diff --git a/bundle/bundle.go b/bundle/bundle.go index a22099cd236..2239ab88e5e 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/metadata" @@ -194,8 +195,8 @@ type Bundle struct { // leave the terraform state untouched. MigratingToDirect bool - // MigratingToDMS registers the existing direct state before deploying with history enabled. - MigratingToDMS bool + // MigratingToDMS holds deployment-history state prepared before planning and committed after approval. + MigratingToDMS *dmsmigration.Migration // AllowTerraformEngineConfig reports a terraform bundle.engine as a warning instead // of an error and skips the DATABRICKS_BUNDLE_ENGINE check. Set by read-only commands diff --git a/bundle/deploy/dmsmigration/migrate.go b/bundle/deploy/dmsmigration/migrate.go index d6f574bcbdc..f00d46aaec3 100644 --- a/bundle/deploy/dmsmigration/migrate.go +++ b/bundle/deploy/dmsmigration/migrate.go @@ -17,14 +17,34 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// Migrate records every resource from the open resources.json state in an initial registration version. +// Migration retains the original state until deployment history is committed. +type Migration struct { + Snapshot dstate.Database + previousVersionID int + versionID int +} + +// Prepare switches planning to DMS while leaving the original state unchanged on disk. +func Prepare(db *dstate.DeploymentState, deploymentID string, lastVersionID int) *Migration { + // Migration uses max(state serial, last version)+1; deployment follows at the next version. + m := &Migration{ + Snapshot: db.Data, + previousVersionID: lastVersionID, + versionID: max(db.Data.Serial, lastVersionID) + 1, + } + db.DeploymentID = deploymentID + db.PrepareDeploymentHistory(m.versionID) + return m +} + +// Commit records the original state in its reserved initial registration version. // The caller must hold the deployment lock and have created the DMS deployment. -func Migrate(ctx context.Context, db *dstate.DeploymentState) error { - keys := slices.Sorted(maps.Keys(db.Data.State)) +func (m *Migration) Commit(ctx context.Context, db *dstate.DeploymentState) error { + keys := slices.Sorted(maps.Keys(m.Snapshot.State)) states := make([]json.RawMessage, len(keys)) operations := make([]bundledeployments.StagedOperation, len(keys)) for i, key := range keys { - entry := db.Data.State[key] + entry := m.Snapshot.State[key] state, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) if err != nil { return fmt.Errorf("serializing migration state for %s: %w", key, err) @@ -36,10 +56,10 @@ func Migrate(ctx context.Context, db *dstate.DeploymentState) error { } } - versionID := max(db.Data.Serial, db.VersionID) + 1 + versionID := m.versionID previousVersionID := "" - if db.VersionID > 0 { - previousVersionID = strconv.Itoa(db.VersionID) + if m.previousVersionID > 0 { + previousVersionID = strconv.Itoa(m.previousVersionID) } w := cmdctx.WorkspaceClient(ctx) _, err := w.BundleDeployments.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ @@ -58,7 +78,7 @@ func Migrate(ctx context.Context, db *dstate.DeploymentState) error { buffer := dms.StartOperationBuffer(ctx, db.DeploymentID, versionID) for i, key := range keys { - buffer.RecordOperation(ctx, key, false, db.Data.State[key].ID, states[i]) + buffer.RecordOperation(ctx, key, false, m.Snapshot.State[key].ID, states[i]) } recordErr := buffer.Drain() reason := bundledeployments.VersionCompleteVersionCompleteSuccess @@ -73,5 +93,5 @@ func Migrate(ctx context.Context, db *dstate.DeploymentState) error { return fmt.Errorf("recording migration version: %w", err) } - return db.UseDeploymentHistory(versionID) + return db.Persist() } diff --git a/bundle/deploy/dmsmigration/migrate_test.go b/bundle/deploy/dmsmigration/migrate_test.go new file mode 100644 index 00000000000..044b2070dff --- /dev/null +++ b/bundle/deploy/dmsmigration/migrate_test.go @@ -0,0 +1,68 @@ +package dmsmigration_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle/deploy/dmsmigration" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareMigration(t *testing.T) { + tests := []struct { + name string + serial int + lastVersion int + registration int + }{ + { + name: "first registration", + serial: 2, + registration: 3, + }, + { + name: "service ahead", + serial: 2, + lastVersion: 5, + registration: 6, + }, + { + name: "file ahead", + serial: 8, + lastVersion: 3, + registration: 9, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "resources.json") + original := dstate.NewDatabase("lineage", tt.serial) + original.Features = map[string]struct{}{} + original.State["resources.jobs.foo"] = dstate.ResourceEntry{ID: "original-id", State: json.RawMessage(`{"name":"original"}`)} + var db dstate.DeploymentState + db.OpenWithData(statePath, original) + require.NoError(t, db.Persist()) + before, err := os.ReadFile(statePath) + require.NoError(t, err) + + migration := dmsmigration.Prepare(&db, "123", tt.lastVersion) + + assert.True(t, db.IsDeploymentMetadataService()) + assert.Equal(t, tt.registration, db.GetSerial()) + assert.Equal(t, "123", db.DeploymentID) + assert.Contains(t, db.StateFeatures(), dstate.FeatureDeploymentHistory) + assert.NotContains(t, migration.Snapshot.Features, dstate.FeatureDeploymentHistory) + assert.Equal(t, tt.serial, migration.Snapshot.Serial) + assert.Equal(t, original.State, migration.Snapshot.State) + assert.Equal(t, "original-id", db.GetResourceID("resources.jobs.foo")) + after, err := os.ReadFile(statePath) + require.NoError(t, err) + assert.Equal(t, before, after) + assert.NoFileExists(t, statePath+".wal") + }) + } +} diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index f9a90a1cd77..97738f6be1e 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,6 +3,7 @@ package dstate import ( "encoding/json" "fmt" + "maps" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/dms" @@ -16,19 +17,20 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// UseDeploymentHistory persists the DMS marker after all existing resources have been recorded. -func (db *DeploymentState) UseDeploymentHistory(versionID int) error { +// PrepareDeploymentHistory enables in-memory DMS planning without persisting its marker. +// versionID is reserved for registration; the deployment plan follows that version. +func (db *DeploymentState) PrepareDeploymentHistory(versionID int) { db.AssertOpenedForRead() db.mu.Lock() defer db.mu.Unlock() db.recordsHistory = true db.VersionID = versionID + db.Data.Features = maps.Clone(db.Data.Features) if db.Data.Features == nil { db.Data.Features = make(map[string]struct{}) } db.Data.Features[FeatureDeploymentHistory] = struct{}{} - return db.unlockedSave() } // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index ab4e1bc017f..6fe44d24ff9 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -2,6 +2,8 @@ package dstate import ( "encoding/json" + "os" + "path/filepath" "testing" "github.com/databricks/cli/bundle/deployplan" @@ -10,6 +12,31 @@ import ( "github.com/stretchr/testify/require" ) +func TestPrepareDeploymentHistory(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "resources.json") + original := NewDatabase("lineage", 2) + original.Features = map[string]struct{}{} + original.State["resources.jobs.foo"] = ResourceEntry{ID: "123", State: json.RawMessage(`{"name":"original"}`)} + var db DeploymentState + db.OpenWithData(statePath, original) + require.NoError(t, db.Persist()) + before, err := os.ReadFile(statePath) + require.NoError(t, err) + + db.PrepareDeploymentHistory(3) + + assert.True(t, db.IsDeploymentMetadataService()) + assert.Equal(t, 3, db.GetSerial()) + assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistory) + assert.NotContains(t, original.Features, FeatureDeploymentHistory) + assert.Equal(t, original.State, db.Data.State) + assert.Equal(t, "123", db.GetResourceID("resources.jobs.foo")) + after, err := os.ReadFile(statePath) + require.NoError(t, err) + assert.Equal(t, before, after) + assert.NoFileExists(t, statePath+".wal") +} + func TestApplyDMSState(t *testing.T) { // The service stores state as an opaque string, so the envelope the write path sent // arrives verbatim. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index e791b26b393..b306838c54b 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -10,7 +10,6 @@ import ( "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deploy" - "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/metadata" @@ -295,17 +294,27 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } } - if b.MigratingToDMS { + if b.MigratingToDMS != nil { if err := statemgmt.BackupRemoteResourcesState(ctx, b); err != nil { logdiag.LogError(ctx, err) return } + } + + // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. + // A first deploy's id did not exist at plan time - the version and any existing id were stamped + // then - so stamp the one just created into the plan the apply reads. + injectDeploymentIdInPlan := b.DeploymentBundle.StateDB.DeploymentID == "" + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { return } + } + + if b.MigratingToDMS != nil { cmdio.LogString(ctx, "Enabling deployment history...") - if err := dmsmigration.Migrate(ctx, &b.DeploymentBundle.StateDB); err != nil { + if err := b.MigratingToDMS.Commit(ctx, &b.DeploymentBundle.StateDB); err != nil { logdiag.LogError(ctx, err) return } @@ -339,17 +348,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. - // A first deploy's id did not exist at plan time - the version and any existing id were stamped - // then - so stamp the one just created into the plan the apply reads. if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { - injectDeploymentIdInPlan := b.DeploymentBundle.StateDB.DeploymentID == "" || b.MigratingToDMS - if !b.MigratingToDMS { - createOrUpdateDeployment(ctx, b, dmsDeployment) - } - if logdiag.HasError(ctx) { - return - } if injectDeploymentIdInPlan { if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index f3d632f138f..2afe357a332 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -138,8 +138,15 @@ func CleanupTerraformStateAfterMigration(ctx context.Context, b *bundle.Bundle) // this: it commits the migration as part of its teardown (files.Delete owns the remote terraform // state, and a destroy that migrates only to tear down is not adoption worth recording). func CommitMigration(ctx context.Context, b *bundle.Bundle) { - count := len(b.DeploymentBundle.StateDB.ExportState(ctx)) - if err := b.DeploymentBundle.StateDB.Persist(); err != nil { + db := &b.DeploymentBundle.StateDB + if b.MigratingToDMS != nil { + // Commit the converted direct snapshot before DMS registration can replace it. + var directState dstate.DeploymentState + directState.OpenWithData(db.Path, b.MigratingToDMS.Snapshot) + db = &directState + } + count := len(db.ExportState(ctx)) + if err := db.Persist(); err != nil { logdiag.LogError(ctx, fmt.Errorf("persisting migrated direct state: %w", err)) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 8cb211d3474..575e8cae7f3 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" @@ -363,13 +364,13 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } - b.MigratingToDMS = opts.Deploy && stateDesc.Engine.IsDirect() && !stateDesc.IsDMS() && configuresDeploymentHistory(ctx, b) + migrateToDMS := (opts.Deploy || opts.PreDeployChecks) && stateDesc.Engine.IsDirect() && !stateDesc.IsDMS() && configuresDeploymentHistory(ctx, b) // TODO: Support applying plans computed against direct state after migration to DMS. // This restriction blocks automatic migration with --plan once deployment history reaches GA. - if b.MigratingToDMS && opts.ReadPlanPath != "" { + if migrateToDMS && opts.ReadPlanPath != "" { return b, stateDesc, errors.New("migrating to deployment history does not support --plan; run 'bundle deploy' without --plan") } - if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting && !b.MigratingToDMS { + if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting && !migrateToDMS { if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, opts.Deploy || opts.PreDeployChecks); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted @@ -377,7 +378,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } // Repeated migrations may already have a DMS deployment even when the state lacks its history marker. - if needDirectState && (stateDesc.IsDMS() || b.MigratingToDMS) { + if needDirectState && (stateDesc.IsDMS() || migrateToDMS) { var err error dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { @@ -394,11 +395,17 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } - nextVersion := lastVersionID + 1 - if b.MigratingToDMS { - // The migration itself uses +1; the deployment uses the following version (+2). - nextVersion = max(b.DeploymentBundle.StateDB.Data.Serial, lastVersionID) + 2 + // StateDB.Open builds the DMS client from the workspace client on the context. + if !cmdctx.HasWorkspaceClient(ctx) { + ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) + } + if migrateToDMS { + b.MigratingToDMS = dmsmigration.Prepare(&b.DeploymentBundle.StateDB, dmsDeploymentID, lastVersionID) + } else if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted } + nextVersion := b.DeploymentBundle.StateDB.GetSerial() + 1 muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} if dmsDeploymentID != "" { bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { @@ -411,17 +418,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } - // StateDB.Open builds the DMS client from the workspace client on the context. - if !cmdctx.HasWorkspaceClient(ctx) { - ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) - } - if b.MigratingToDMS { - b.DeploymentBundle.StateDB.DeploymentID = dmsDeploymentID - b.DeploymentBundle.StateDB.VersionID = lastVersionID - } else if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { - logdiag.LogError(ctx, err) - return b, stateDesc, root.ErrAlreadyPrinted - } } if needDirectState { From c3457102e3492d1428f972007ca8d48f6fa88ca8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 16:03:55 +0000 Subject: [PATCH 08/25] Commit DMS migration separately before deployment --- .../bundle/dms/existing-state/output.txt | 17 +--- acceptance/bundle/dms/existing-state/script | 7 -- .../databricks.yml.tmpl | 6 +- .../out.test.toml | 0 .../dms/migration-before-deploy/output.txt | 71 ++++++++++++++ .../bundle/dms/migration-before-deploy/script | 45 +++++++++ .../test.toml | 0 .../dms/migration-not-committed/output.txt | 47 --------- .../bundle/dms/migration-not-committed/script | 32 ------- .../bundle/dms/repeated-migration/output.txt | 2 +- .../migrate/auto/dms-migration/output.txt | 7 +- bundle/bundle.go | 4 - bundle/deploy/dmsmigration/migrate.go | 39 ++------ bundle/deploy/dmsmigration/migrate_test.go | 57 +++++++---- bundle/direct/dstate/dms.go | 8 +- bundle/direct/dstate/dms_test.go | 12 ++- bundle/phases/deploy.go | 39 ++------ bundle/phases/dms_migration.go | 96 +++++++++++++++++++ bundle/phases/dms_migration_test.go | 54 +++++++++++ bundle/statemgmt/direct_migration.go | 11 +-- cmd/bundle/utils/process.go | 94 ++++++------------ cmd/bundle/utils/process_test.go | 3 +- libs/dms/metadata.go | 48 ++++++++++ 23 files changed, 426 insertions(+), 273 deletions(-) rename acceptance/bundle/dms/{migration-not-committed => migration-before-deploy}/databricks.yml.tmpl (53%) rename acceptance/bundle/dms/{migration-not-committed => migration-before-deploy}/out.test.toml (100%) create mode 100644 acceptance/bundle/dms/migration-before-deploy/output.txt create mode 100644 acceptance/bundle/dms/migration-before-deploy/script rename acceptance/bundle/dms/{migration-not-committed => migration-before-deploy}/test.toml (100%) delete mode 100644 acceptance/bundle/dms/migration-not-committed/output.txt delete mode 100644 acceptance/bundle/dms/migration-not-committed/script create mode 100644 bundle/phases/dms_migration.go create mode 100644 bundle/phases/dms_migration_test.go diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 26acb56ad78..76a8aa728df 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -22,23 +22,16 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq -c {serial} .databricks/bundle/default/resources.json {"serial":2} -=== Preview prepares serial 3 and history in memory without committing migration ->>> [CLI] bundle plan -o json -{"serial":3,"features":{"deployment_history":{}}} - ->>> cmp .databricks/resources.before-preview.json .databricks/bundle/default/resources.json - ->>> print_requests.py --dms //api/2.0/bundle //workspace/import //workspace-files/import-file - === Config enables recording despite the disabled environment: register version 3, then deploy version 4 >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Enabling deployment history... Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 Updated jobs.one +Updated jobs.two Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 1 unchanged +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged >>> find.py resources[.]json[.]wal --expect 0 @@ -120,7 +113,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged [{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/4 -o json -[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json -[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"4","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","last_version_id":"3","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"4","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"4","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 78dd6bf0dd2..0e001957e97 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -14,13 +14,6 @@ export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' update_file.py databricks.yml 'name: one' 'name: updated' -title "Preview prepares serial 3 and history in memory without committing migration" -cp .databricks/bundle/default/resources.json .databricks/resources.before-preview.json -rm -f "$OUT_REQUESTS" -trace $CLI bundle plan -o json | jq -c '{serial, features}' -trace cmp .databricks/resources.before-preview.json .databricks/bundle/default/resources.json -trace print_requests.py --dms //api/2.0/bundle //workspace/import //workspace-files/import-file - title "Config enables recording despite the disabled environment: register version 3, then deploy version 4" trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=3' '?version=4' trace find.py 'resources[.]json[.]wal' --expect 0 diff --git a/acceptance/bundle/dms/migration-not-committed/databricks.yml.tmpl b/acceptance/bundle/dms/migration-before-deploy/databricks.yml.tmpl similarity index 53% rename from acceptance/bundle/dms/migration-not-committed/databricks.yml.tmpl rename to acceptance/bundle/dms/migration-before-deploy/databricks.yml.tmpl index 11d945e06b0..a0af2a05d62 100644 --- a/acceptance/bundle/dms/migration-not-committed/databricks.yml.tmpl +++ b/acceptance/bundle/dms/migration-before-deploy/databricks.yml.tmpl @@ -1,5 +1,5 @@ bundle: - name: dms-migration-not-committed-$UNIQUE_NAME + name: dms-migration-before-deploy-$UNIQUE_NAME experimental: deployment_history: false @@ -11,8 +11,8 @@ sync: resources: jobs: job: - name: migration-not-committed + name: migration-before-deploy schemas: schema: - name: dms_migration_not_committed_$UNIQUE_NAME + name: dms_migration_before_deploy_$UNIQUE_NAME catalog_name: main diff --git a/acceptance/bundle/dms/migration-not-committed/out.test.toml b/acceptance/bundle/dms/migration-before-deploy/out.test.toml similarity index 100% rename from acceptance/bundle/dms/migration-not-committed/out.test.toml rename to acceptance/bundle/dms/migration-before-deploy/out.test.toml diff --git a/acceptance/bundle/dms/migration-before-deploy/output.txt b/acceptance/bundle/dms/migration-before-deploy/output.txt new file mode 100644 index 00000000000..4d2618f2f4f --- /dev/null +++ b/acceptance/bundle/dms/migration-before-deploy/output.txt @@ -0,0 +1,71 @@ + +=== Setup (LOG.setup): legacy resources exist; deployment would recreate the schema +=== 1. Migration commits and DMS is reread before ordinary planning fails +>>> fault.py GET /api/2.2/jobs/get* 403 0 1 + +>>> musterr [CLI] bundle deploy -qq +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 +Error: cannot plan resources.jobs.job: reading id="[JOB_ID]": Fault injected by test. (403 INJECTED) + +Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[JOB_ID] +HTTP Status: 403 Forbidden +API error_code: INJECTED +API message: Fault injected by test. + +Error: planning failed + + +>>> print_requests.py --dms --get //api/2.0/bundle //jobs/get //workspace/import //workspace-files/import-file --oneline +{"method":"POST","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete"} +{"method":"POST","path":"/Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} +{"method":"GET","path":"/api/2.2/jobs/get"} + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.marker.json + +>>> jq -c {serial, features, state} .databricks/resources.marker.json +{"serial":2,"features":{"deployment_history":{}},"state":{}} + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.json + +>>> cmp .databricks/resources.before-migration.json .databricks/resources.backup.json + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json +[{"resource_key":"jobs.job","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"schemas.schema","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] + +=== 2. Destructive deployment is declined; committed migration is not repeated +>>> musterr [CLI] bundle deploy -qq + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.schema +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + + +>>> cmp .databricks/resources.marker.json .databricks/bundle/default/resources.json + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.after-decline.json + +>>> cmp .databricks/resources.marker.json .databricks/resources.after-decline.json + +>>> print_requests.py --dms //api/2.0/bundle + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +=== 3. Approval creates ordinary deployment version 3, not another registration +>>> [CLI] bundle deploy --auto-approve -qq + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.schema + +>>> [CLI] bundle summary -o json +{"latest_version_id":3} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] diff --git a/acceptance/bundle/dms/migration-before-deploy/script b/acceptance/bundle/dms/migration-before-deploy/script new file mode 100644 index 00000000000..fa0250749de --- /dev/null +++ b/acceptance/bundle/dms/migration-before-deploy/script @@ -0,0 +1,45 @@ +envsubst < databricks.yml.tmpl > databricks.yml +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-before-deploy-${UNIQUE_NAME}/default/state" +other_catalog="dms_other_${UNIQUE_NAME}" + +title "Setup (LOG.setup): legacy resources exist; deployment would recreate the schema" +{ + $CLI bundle deploy -qq + DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py + cp .databricks/bundle/default/resources.json .databricks/resources.before-migration.json + $CLI catalogs create "$other_catalog" + update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' + update_file.py databricks.yml 'catalog_name: main' "catalog_name: ${other_catalog}" +} &> LOG.setup +trap '$CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1; $CLI catalogs delete "$other_catalog" --force >> LOG.cleanup 2>&1; rm -f "$OUT_REQUESTS"' EXIT +rm -f "$OUT_REQUESTS" + +title "1. Migration commits and DMS is reread before ordinary planning fails" +trace fault.py 'GET /api/2.2/jobs/get*' 403 0 1 +trace musterr $CLI bundle deploy -qq +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_path/resources.deployment.json" -o json | jq -r '.object_id') +add_repl "$deployment_id" DEPLOYMENT_ID +trace print_requests.py --dms --get //api/2.0/bundle //jobs/get //workspace/import //workspace-files/import-file --oneline | + MSYS_NO_PATHCONV=1 jq -c --arg marker "$state_path/resources.json" ' + (.body.multipart_form.path // .body.path // (.path | ltrimstr("/api/2.0/workspace-files/import-file"))) as $destination | + select((.path | endswith("/complete")) or (.path | endswith("/resources")) or .path == "/api/2.2/jobs/get" or $destination == $marker) | + {method, path: (if $destination == $marker then $destination else .path end)}' +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.marker.json +trace jq -c '{serial, features, state}' .databricks/resources.marker.json +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.json +trace cmp .databricks/resources.before-migration.json .databricks/resources.backup.json +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'map({version_id, status, completion_reason})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, action_type, status})' + +title "2. Destructive deployment is declined; committed migration is not repeated" +trace musterr $CLI bundle deploy -qq +trace cmp .databricks/resources.marker.json .databricks/bundle/default/resources.json +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.after-decline.json +trace cmp .databricks/resources.marker.json .databricks/resources.after-decline.json +trace print_requests.py --dms //api/2.0/bundle +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'map({version_id, status, completion_reason})' + +title "3. Approval creates ordinary deployment version 3, not another registration" +trace $CLI bundle deploy --auto-approve -qq +trace $CLI bundle summary -o json | jq -c '.bundle.deployment | {latest_version_id}' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, status, completion_reason})' diff --git a/acceptance/bundle/dms/migration-not-committed/test.toml b/acceptance/bundle/dms/migration-before-deploy/test.toml similarity index 100% rename from acceptance/bundle/dms/migration-not-committed/test.toml rename to acceptance/bundle/dms/migration-before-deploy/test.toml diff --git a/acceptance/bundle/dms/migration-not-committed/output.txt b/acceptance/bundle/dms/migration-not-committed/output.txt deleted file mode 100644 index 775a2cbed0f..00000000000 --- a/acceptance/bundle/dms/migration-not-committed/output.txt +++ /dev/null @@ -1,47 +0,0 @@ - -=== Setup (LOG.setup): legacy resources exist; migration would recreate the schema -=== 1. Planning fails after preparation; no migration is committed ->>> fault.py GET /api/2.2/jobs/get* 403 0 1 - ->>> musterr [CLI] bundle deploy -qq -Error: cannot plan resources.jobs.job: reading id="[JOB_ID]": Fault injected by test. (403 INJECTED) - -Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[JOB_ID] -HTTP Status: 403 Forbidden -API error_code: INJECTED -API message: Fault injected by test. - -Error: planning failed - - ->>> cmp .databricks/resources.before-migration.json .databricks/bundle/default/resources.json - ->>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-not-committed-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.after-failure.json - ->>> cmp .databricks/resources.before-migration.json .databricks/resources.after-failure.json - ->>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/dms-migration-not-committed-[UNIQUE_NAME]/default/state -o json -[] - ->>> print_requests.py --dms //api/2.0/bundle - -=== 2. Destructive deployment is declined; migration stays uncommitted ->>> musterr [CLI] bundle deploy -qq - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.schema -Error: the deployment requires destructive actions, but the current console does not support prompting. -Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. -To proceed, use --auto-approve after reviewing the plan above. - - ->>> cmp .databricks/resources.before-migration.json .databricks/bundle/default/resources.json - ->>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-not-committed-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.after-decline.json - ->>> cmp .databricks/resources.before-migration.json .databricks/resources.after-decline.json - ->>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/dms-migration-not-committed-[UNIQUE_NAME]/default/state -o json -[] - ->>> print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/migration-not-committed/script b/acceptance/bundle/dms/migration-not-committed/script deleted file mode 100644 index fd564728c72..00000000000 --- a/acceptance/bundle/dms/migration-not-committed/script +++ /dev/null @@ -1,32 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml -state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-not-committed-${UNIQUE_NAME}/default/state" -other_catalog="dms_other_${UNIQUE_NAME}" - -title "Setup (LOG.setup): legacy resources exist; migration would recreate the schema" -{ - $CLI bundle deploy -qq - DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py - cp .databricks/bundle/default/resources.json .databricks/resources.before-migration.json - $CLI catalogs create "$other_catalog" - update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' - update_file.py databricks.yml 'catalog_name: main' "catalog_name: ${other_catalog}" -} &> LOG.setup -trap 'update_file.py databricks.yml "deployment_history: true" "deployment_history: false" >> LOG.cleanup 2>&1; $CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1; $CLI catalogs delete "$other_catalog" --force >> LOG.cleanup 2>&1; rm -f "$OUT_REQUESTS"' EXIT -rm -f "$OUT_REQUESTS" - -title "1. Planning fails after preparation; no migration is committed" -trace fault.py 'GET /api/2.2/jobs/get*' 403 0 1 -trace musterr $CLI bundle deploy -qq -trace cmp .databricks/resources.before-migration.json .databricks/bundle/default/resources.json -trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.after-failure.json -trace cmp .databricks/resources.before-migration.json .databricks/resources.after-failure.json -trace MSYS_NO_PATHCONV=1 $CLI workspace list "$state_path" -o json | jq -c '[.[].path | select(endswith("/resources.json.backup") or endswith("/resources.deployment.json"))]' -trace print_requests.py --dms //api/2.0/bundle - -title "2. Destructive deployment is declined; migration stays uncommitted" -trace musterr $CLI bundle deploy -qq -trace cmp .databricks/resources.before-migration.json .databricks/bundle/default/resources.json -trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.after-decline.json -trace cmp .databricks/resources.before-migration.json .databricks/resources.after-decline.json -trace MSYS_NO_PATHCONV=1 $CLI workspace list "$state_path" -o json | jq -c '[.[].path | select(endswith("/resources.json.backup") or endswith("/resources.deployment.json"))]' -trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/repeated-migration/output.txt b/acceptance/bundle/dms/repeated-migration/output.txt index 195e3121cc0..c568d1fef40 100644 --- a/acceptance/bundle/dms/repeated-migration/output.txt +++ b/acceptance/bundle/dms/repeated-migration/output.txt @@ -22,9 +22,9 @@ Expect: register=4; deploy=5; name=repeated; created=0. >>> update_file.py databricks.yml name: updated name: repeated >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/files... Enabling deployment history... Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/files... Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=5 Updated jobs.one Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/migrate/auto/dms-migration/output.txt b/acceptance/bundle/migrate/auto/dms-migration/output.txt index d57bcbe4a55..63b0d3da59d 100644 --- a/acceptance/bundle/migrate/auto/dms-migration/output.txt +++ b/acceptance/bundle/migrate/auto/dms-migration/output.txt @@ -5,12 +5,13 @@ === Terraform migrates to direct deployment with history enabled and preserves the job ID >>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Migrated 1 resource to direct deployment engine. Enabling deployment history... Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Updated jobs.test_job Files: 1 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) .databricks/bundle/default/resources.json @@ -19,7 +20,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> find.py resources[.]json[.]wal --expect 0 >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle summary -o json -{"job_id":"[TEST_JOB_ID]","version":4} +{"job_id":"[TEST_JOB_ID]","version":5} >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json --format AUTO {"features":{"deployment_history":{}},"state":{}} diff --git a/bundle/bundle.go b/bundle/bundle.go index 2239ab88e5e..68654bead7c 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -16,7 +16,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/metadata" @@ -195,9 +194,6 @@ type Bundle struct { // leave the terraform state untouched. MigratingToDirect bool - // MigratingToDMS holds deployment-history state prepared before planning and committed after approval. - MigratingToDMS *dmsmigration.Migration - // AllowTerraformEngineConfig reports a terraform bundle.engine as a warning instead // of an error and skips the DATABRICKS_BUNDLE_ENGINE check. Set by read-only commands // (bundle summary) so users can still inspect a bundle that pins the removed engine. diff --git a/bundle/deploy/dmsmigration/migrate.go b/bundle/deploy/dmsmigration/migrate.go index f00d46aaec3..b706748f307 100644 --- a/bundle/deploy/dmsmigration/migrate.go +++ b/bundle/deploy/dmsmigration/migrate.go @@ -17,34 +17,14 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// Migration retains the original state until deployment history is committed. -type Migration struct { - Snapshot dstate.Database - previousVersionID int - versionID int -} - -// Prepare switches planning to DMS while leaving the original state unchanged on disk. -func Prepare(db *dstate.DeploymentState, deploymentID string, lastVersionID int) *Migration { - // Migration uses max(state serial, last version)+1; deployment follows at the next version. - m := &Migration{ - Snapshot: db.Data, - previousVersionID: lastVersionID, - versionID: max(db.Data.Serial, lastVersionID) + 1, - } - db.DeploymentID = deploymentID - db.PrepareDeploymentHistory(m.versionID) - return m -} - -// Commit records the original state in its reserved initial registration version. +// Migrate records every resource from the open direct state in an initial registration version. // The caller must hold the deployment lock and have created the DMS deployment. -func (m *Migration) Commit(ctx context.Context, db *dstate.DeploymentState) error { - keys := slices.Sorted(maps.Keys(m.Snapshot.State)) +func Migrate(ctx context.Context, db *dstate.DeploymentState) error { + keys := slices.Sorted(maps.Keys(db.Data.State)) states := make([]json.RawMessage, len(keys)) operations := make([]bundledeployments.StagedOperation, len(keys)) for i, key := range keys { - entry := m.Snapshot.State[key] + entry := db.Data.State[key] state, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) if err != nil { return fmt.Errorf("serializing migration state for %s: %w", key, err) @@ -56,10 +36,11 @@ func (m *Migration) Commit(ctx context.Context, db *dstate.DeploymentState) erro } } - versionID := m.versionID + // Migration uses max(state serial, last version)+1; deployment follows at the next version. + versionID := max(db.Data.Serial, db.VersionID) + 1 previousVersionID := "" - if m.previousVersionID > 0 { - previousVersionID = strconv.Itoa(m.previousVersionID) + if db.VersionID > 0 { + previousVersionID = strconv.Itoa(db.VersionID) } w := cmdctx.WorkspaceClient(ctx) _, err := w.BundleDeployments.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ @@ -78,7 +59,7 @@ func (m *Migration) Commit(ctx context.Context, db *dstate.DeploymentState) erro buffer := dms.StartOperationBuffer(ctx, db.DeploymentID, versionID) for i, key := range keys { - buffer.RecordOperation(ctx, key, false, m.Snapshot.State[key].ID, states[i]) + buffer.RecordOperation(ctx, key, false, db.Data.State[key].ID, states[i]) } recordErr := buffer.Drain() reason := bundledeployments.VersionCompleteVersionCompleteSuccess @@ -93,5 +74,5 @@ func (m *Migration) Commit(ctx context.Context, db *dstate.DeploymentState) erro return fmt.Errorf("recording migration version: %w", err) } - return db.Persist() + return db.UseDeploymentHistory(versionID) } diff --git a/bundle/deploy/dmsmigration/migrate_test.go b/bundle/deploy/dmsmigration/migrate_test.go index 044b2070dff..5e00f71da2b 100644 --- a/bundle/deploy/dmsmigration/migrate_test.go +++ b/bundle/deploy/dmsmigration/migrate_test.go @@ -4,15 +4,22 @@ import ( "encoding/json" "os" "path/filepath" + "strconv" "testing" "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -func TestPrepareMigration(t *testing.T) { +func TestMigrate(t *testing.T) { tests := []struct { name string serial int @@ -40,29 +47,43 @@ func TestPrepareMigration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { statePath := filepath.Join(t.TempDir(), "resources.json") - original := dstate.NewDatabase("lineage", tt.serial) - original.Features = map[string]struct{}{} - original.State["resources.jobs.foo"] = dstate.ResourceEntry{ID: "original-id", State: json.RawMessage(`{"name":"original"}`)} var db dstate.DeploymentState - db.OpenWithData(statePath, original) - require.NoError(t, db.Persist()) - before, err := os.ReadFile(statePath) - require.NoError(t, err) + db.OpenWithData(statePath, dstate.NewDatabase("lineage", tt.serial)) + db.DeploymentID = "123" + db.VersionID = tt.lastVersion + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockBundleDeploymentsAPI() + previousVersion := "" + if tt.lastVersion > 0 { + previousVersion = strconv.Itoa(tt.lastVersion) + } + api.EXPECT().CreateVersion(mock.Anything, bundledeployments.CreateVersionRequest{ + Parent: dms.DeploymentName("123"), + VersionId: strconv.Itoa(tt.registration), + Version: bundledeployments.Version{ + CliVersion: build.GetInfo().Version, + VersionType: dms.VersionTypeDeploy, + PreviousVersionId: previousVersion, + Operations: []bundledeployments.StagedOperation{}, + }, + }).Return(&bundledeployments.Version{}, nil).Once() + api.EXPECT().CompleteVersion(mock.Anything, bundledeployments.CompleteVersionRequest{ + Name: dms.VersionName("123", tt.registration), + CompletionReason: bundledeployments.VersionCompleteVersionCompleteSuccess, + }).Return(&bundledeployments.Version{}, nil).Once() + ctx := cmdctx.SetWorkspaceClient(t.Context(), m.WorkspaceClient) - migration := dmsmigration.Prepare(&db, "123", tt.lastVersion) + require.NoError(t, dmsmigration.Migrate(ctx, &db)) assert.True(t, db.IsDeploymentMetadataService()) assert.Equal(t, tt.registration, db.GetSerial()) - assert.Equal(t, "123", db.DeploymentID) - assert.Contains(t, db.StateFeatures(), dstate.FeatureDeploymentHistory) - assert.NotContains(t, migration.Snapshot.Features, dstate.FeatureDeploymentHistory) - assert.Equal(t, tt.serial, migration.Snapshot.Serial) - assert.Equal(t, original.State, migration.Snapshot.State) - assert.Equal(t, "original-id", db.GetResourceID("resources.jobs.foo")) - after, err := os.ReadFile(statePath) + content, err := os.ReadFile(statePath) require.NoError(t, err) - assert.Equal(t, before, after) - assert.NoFileExists(t, statePath+".wal") + var persisted dstate.Database + require.NoError(t, json.Unmarshal(content, &persisted)) + assert.Contains(t, persisted.Features, dstate.FeatureDeploymentHistory) + assert.Equal(t, tt.registration, persisted.Serial) + assert.Empty(t, persisted.State) }) } } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 97738f6be1e..f9a90a1cd77 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,7 +3,6 @@ package dstate import ( "encoding/json" "fmt" - "maps" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/dms" @@ -17,20 +16,19 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// PrepareDeploymentHistory enables in-memory DMS planning without persisting its marker. -// versionID is reserved for registration; the deployment plan follows that version. -func (db *DeploymentState) PrepareDeploymentHistory(versionID int) { +// UseDeploymentHistory persists the DMS marker after all existing resources have been recorded. +func (db *DeploymentState) UseDeploymentHistory(versionID int) error { db.AssertOpenedForRead() db.mu.Lock() defer db.mu.Unlock() db.recordsHistory = true db.VersionID = versionID - db.Data.Features = maps.Clone(db.Data.Features) if db.Data.Features == nil { db.Data.Features = make(map[string]struct{}) } db.Data.Features[FeatureDeploymentHistory] = struct{}{} + return db.unlockedSave() } // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 6fe44d24ff9..4729a4f1c46 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestPrepareDeploymentHistory(t *testing.T) { +func TestUseDeploymentHistory(t *testing.T) { statePath := filepath.Join(t.TempDir(), "resources.json") original := NewDatabase("lineage", 2) original.Features = map[string]struct{}{} @@ -23,17 +23,21 @@ func TestPrepareDeploymentHistory(t *testing.T) { before, err := os.ReadFile(statePath) require.NoError(t, err) - db.PrepareDeploymentHistory(3) + require.NoError(t, db.UseDeploymentHistory(3)) assert.True(t, db.IsDeploymentMetadataService()) assert.Equal(t, 3, db.GetSerial()) assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistory) - assert.NotContains(t, original.Features, FeatureDeploymentHistory) assert.Equal(t, original.State, db.Data.State) assert.Equal(t, "123", db.GetResourceID("resources.jobs.foo")) after, err := os.ReadFile(statePath) require.NoError(t, err) - assert.Equal(t, before, after) + assert.NotEqual(t, before, after) + var persisted Database + require.NoError(t, json.Unmarshal(after, &persisted)) + assert.Equal(t, 3, persisted.Serial) + assert.Contains(t, persisted.Features, FeatureDeploymentHistory) + assert.Empty(t, persisted.State) assert.NoFileExists(t, statePath+".wal") } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index b306838c54b..9de3596aca7 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -294,37 +294,6 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } } - if b.MigratingToDMS != nil { - if err := statemgmt.BackupRemoteResourcesState(ctx, b); err != nil { - logdiag.LogError(ctx, err) - return - } - } - - // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. - // A first deploy's id did not exist at plan time - the version and any existing id were stamped - // then - so stamp the one just created into the plan the apply reads. - injectDeploymentIdInPlan := b.DeploymentBundle.StateDB.DeploymentID == "" - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { - createOrUpdateDeployment(ctx, b, dmsDeployment) - if logdiag.HasError(ctx) { - return - } - } - - if b.MigratingToDMS != nil { - cmdio.LogString(ctx, "Enabling deployment history...") - if err := b.MigratingToDMS.Commit(ctx, &b.DeploymentBundle.StateDB); err != nil { - logdiag.LogError(ctx, err) - return - } - logDeploymentVersion(ctx, b, "Deployment history enabled") - statemgmt.PushResourcesState(ctx, b) - if logdiag.HasError(ctx) { - return - } - } - // Upgrade from read (opened by process.go, or by Migrate) to write mode. After approval // and the migration commit above, so a declined deploy never opens a WAL it must discard, // and the migration's tf+1 push lands before this advances the WAL header to tf+2. @@ -348,7 +317,15 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } + // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. + // A first deploy's id did not exist at plan time - the version and any existing id were stamped + // then - so stamp the one just created into the plan the apply reads. if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + injectDeploymentIdInPlan := b.DeploymentBundle.StateDB.DeploymentID == "" + createOrUpdateDeployment(ctx, b, dmsDeployment) + if logdiag.HasError(ctx) { + return + } if injectDeploymentIdInPlan { if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go new file mode 100644 index 00000000000..3869dfbdc49 --- /dev/null +++ b/bundle/phases/dms_migration.go @@ -0,0 +1,96 @@ +package phases + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/dmsmigration" + "github.com/databricks/cli/bundle/deploy/lock" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/logdiag" +) + +// MigrateToDMS commits deployment history before ordinary deployment planning begins. +func MigrateToDMS(ctx context.Context, b *bundle.Bundle) { + bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalDeploy)) + if logdiag.HasError(ctx) { + return + } + defer bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) + + db := &b.DeploymentBundle.StateDB + defer func() { + if _, err := db.Finalize(ctx); err != nil { + logdiag.LogError(ctx, err) + } + }() + + // Another deploy may have completed since the command's initial state read. + state := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) + if logdiag.HasError(ctx) { + return + } + if state.IsDMS() { + b.MigratingToDirect = false + return + } + if b.MigratingToDirect { + if !state.Engine.IsDirect() { + statemgmt.CommitMigration(ctx, b) + if logdiag.HasError(ctx) { + return + } + state = statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) + if logdiag.HasError(ctx) { + return + } + } + b.MigratingToDirect = false + } + if _, err := db.Finalize(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if !state.Engine.IsDirect() { + logdiag.LogError(ctx, fmt.Errorf("deployment state changed to %s while enabling deployment history; retry the deployment", state.Engine)) + return + } + _, localPath := b.StateFilenameDirect(ctx) + if err := db.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { + logdiag.LogError(ctx, err) + return + } + deploymentID, deployment, err := dms.ReadDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + if err != nil { + logdiag.LogError(ctx, err) + return + } + lastVersion, err := dms.ParseLastVersionID(deployment) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := statemgmt.BackupRemoteResourcesState(ctx, b); err != nil { + logdiag.LogError(ctx, err) + return + } + db.DeploymentID = deploymentID + db.VersionID = lastVersion + createOrUpdateDeployment(ctx, b, deployment) + if logdiag.HasError(ctx) { + return + } + ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) + cmdio.LogString(ctx, "Enabling deployment history...") + if err := dmsmigration.Migrate(ctx, db); err != nil { + logdiag.LogError(ctx, err) + return + } + logDeploymentVersion(ctx, b, "Deployment history enabled") + statemgmt.PushResourcesState(ctx, b) +} diff --git a/bundle/phases/dms_migration_test.go b/bundle/phases/dms_migration_test.go new file mode 100644 index 00000000000..d6f11e8667a --- /dev/null +++ b/bundle/phases/dms_migration_test.go @@ -0,0 +1,54 @@ +package phases_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrateToDMSAlreadyRecorded(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/workspace/export", func(req testserver.Request) any { + return testserver.Response{StatusCode: 404, Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST"}} + }) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + MigratingToDirect: true, + Config: config.Root{ + Bundle: config.Bundle{ + Target: "default", + Deployment: config.Deployment{Lock: config.Lock{Enabled: new(false)}}, + }, + Workspace: config.Workspace{StatePath: "/Workspace/state"}, + }, + } + b.SetWorkpaceClient(client) + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + _, statePath := b.StateFilenameDirect(ctx) + content := []byte(`{"state_version":3,"serial":2,"lineage":"lineage","features":{"deployment_history":{}},"state":{}}`) + require.NoError(t, os.MkdirAll(filepath.Dir(statePath), 0o700)) + require.NoError(t, os.WriteFile(statePath, content, 0o600)) + b.DeploymentBundle.StateDB.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) + + phases.MigrateToDMS(ctx, b) + + require.False(t, logdiag.HasError(ctx), "%v", logdiag.FlushCollected(ctx)) + assert.False(t, b.MigratingToDirect) + assert.False(t, b.DeploymentBundle.StateDB.IsOpen()) + after, err := os.ReadFile(statePath) + require.NoError(t, err) + assert.Equal(t, content, after) +} diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 2afe357a332..f3d632f138f 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -138,15 +138,8 @@ func CleanupTerraformStateAfterMigration(ctx context.Context, b *bundle.Bundle) // this: it commits the migration as part of its teardown (files.Delete owns the remote terraform // state, and a destroy that migrates only to tear down is not adoption worth recording). func CommitMigration(ctx context.Context, b *bundle.Bundle) { - db := &b.DeploymentBundle.StateDB - if b.MigratingToDMS != nil { - // Commit the converted direct snapshot before DMS registration can replace it. - var directState dstate.DeploymentState - directState.OpenWithData(db.Path, b.MigratingToDMS.Snapshot) - db = &directState - } - count := len(db.ExportState(ctx)) - if err := db.Persist(); err != nil { + count := len(b.DeploymentBundle.StateDB.ExportState(ctx)) + if err := b.DeploymentBundle.StateDB.Persist(); err != nil { logdiag.LogError(ctx, fmt.Errorf("persisting migrated direct state: %w", err)) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 575e8cae7f3..034fec06968 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -4,9 +4,7 @@ import ( "context" "errors" "fmt" - "path" "path/filepath" - "strconv" "time" "github.com/databricks/cli/bundle" @@ -14,7 +12,6 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" - "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" @@ -35,8 +32,6 @@ import ( "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/databricks/databricks-sdk-go/useragent" "github.com/spf13/cobra" @@ -350,37 +345,47 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle cmd.SetContext(ctx) } + migrateToDMS := opts.Deploy && stateDesc.Engine.IsDirect() && !stateDesc.IsDMS() && configuresDeploymentHistory(ctx, b) + // TODO: Support applying plans computed against direct state after migration to DMS. + // This restriction blocks automatic migration with --plan once deployment history reaches GA. + if migrateToDMS && opts.ReadPlanPath != "" { + return b, stateDesc, errors.New("migrating to deployment history does not support --plan; run 'bundle deploy' without --plan") + } + if migrateToDMS { + phases.MigrateToDMS(ctx, b) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + stateDesc = statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + resolveDeploymentHistory(ctx, b, stateDesc) + } + // Open direct engine state once for all subsequent operations (ExportState, CalculatePlan, Apply, etc.) // A migrated-from-Terraform state is already open (seeded in memory above), so skip the disk open. needDirectState := stateDesc.Engine.IsDirect() && needsState var localPath string - if needDirectState && !b.DeploymentBundle.StateDB.IsOpen() { + if needDirectState { _, localPath = b.StateFilenameDirect(ctx) - if !stateDesc.IsDMS() { + if !b.DeploymentBundle.StateDB.IsOpen() && !stateDesc.IsDMS() { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } } } - - migrateToDMS := (opts.Deploy || opts.PreDeployChecks) && stateDesc.Engine.IsDirect() && !stateDesc.IsDMS() && configuresDeploymentHistory(ctx, b) - // TODO: Support applying plans computed against direct state after migration to DMS. - // This restriction blocks automatic migration with --plan once deployment history reaches GA. - if migrateToDMS && opts.ReadPlanPath != "" { - return b, stateDesc, errors.New("migrating to deployment history does not support --plan; run 'bundle deploy' without --plan") - } - if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting && !migrateToDMS { + if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting { if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, opts.Deploy || opts.PreDeployChecks); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } } - // Repeated migrations may already have a DMS deployment even when the state lacks its history marker. - if needDirectState && (stateDesc.IsDMS() || migrateToDMS) { + if needDirectState && stateDesc.IsDMS() { var err error - dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + dmsDeploymentID, dmsDeployment, err = dms.ReadDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted @@ -390,7 +395,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // the plan carries them. The deployment version follows the last recorded version or // the initial registration version; deployment_id does not exist until a first deploy // creates it, so it is left off here and the deploy phase stamps the created id. - lastVersionID, err := parseLastVersionID(dmsDeployment) + lastVersionID, err := dms.ParseLastVersionID(dmsDeployment) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted @@ -399,9 +404,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } - if migrateToDMS { - b.MigratingToDMS = dmsmigration.Prepare(&b.DeploymentBundle.StateDB, dmsDeploymentID, lastVersionID) - } else if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } @@ -594,49 +597,6 @@ func ValidateEngineSetting(ctx context.Context, b *bundle.Bundle) error { return nil } -// Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json -// -// TODO: a deployment is only usable when both the node and the service's record exist, and a -// half-created one - node present, record missing - blocks the bundle here even though -// CreateDeployment already recovers from it. Move this behind a dms.ReadDeployment(ctx, statePath) -// that returns an empty id and version unless both halves are there, leaving the caller to call -// CreateDeployment to create or finalize it. -// -// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a -// workspace lookup to turn the node into an id, then a get by that id. -func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { - nodePath := path.Join(statePath, dms.DeploymentNodeName) - - obj, err := w.Workspace.GetStatusByPath(ctx, nodePath) - if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { - return "", nil, nil - } - if err != nil { - return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) - } - deploymentID := strconv.FormatInt(obj.ObjectId, 10) - deployment, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ - Name: dms.DeploymentName(deploymentID), - }) - if err != nil { - return "", nil, err - } - return deploymentID, deployment, nil -} - -// parseLastVersionID parses the deployment's last recorded version, which the service reports -// as a string. It returns 0 when the deployment does not exist yet or has no recorded version. -func parseLastVersionID(dmsDeployment *bundledeployments.Deployment) (int, error) { - if dmsDeployment == nil || dmsDeployment.LastVersionId == "" { - return 0, nil - } - v, err := strconv.Atoi(dmsDeployment.LastVersionId) - if err != nil { - return 0, fmt.Errorf("failed to parse last_version_id %q: %w", dmsDeployment.LastVersionId, err) - } - return v, nil -} - // OpenDirectStateForRead opens the direct-engine state database read-only. When the bundle // records deployment history the local state file is only a tombstone, so the resources are // read from the deployment metadata service instead. @@ -649,11 +609,11 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st return enforceDeploymentHistorySetting(ctx, b, stateDesc, false) } - dmsDeploymentID, dmsDeployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + dmsDeploymentID, dmsDeployment, err := dms.ReadDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { return err } - lastVersionID, err := parseLastVersionID(dmsDeployment) + lastVersionID, err := dms.ParseLastVersionID(dmsDeployment) if err != nil { return err } diff --git a/cmd/bundle/utils/process_test.go b/cmd/bundle/utils/process_test.go index b216f1102d2..e52fc4ab6f4 100644 --- a/cmd/bundle/utils/process_test.go +++ b/cmd/bundle/utils/process_test.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -81,7 +82,7 @@ func TestParseLastVersionID(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseLastVersionID(tt.deployment) + got, err := dms.ParseLastVersionID(tt.deployment) if tt.wantErr { require.Error(t, err) return diff --git a/libs/dms/metadata.go b/libs/dms/metadata.go index f9e23a67db0..c91ac4d5be5 100644 --- a/libs/dms/metadata.go +++ b/libs/dms/metadata.go @@ -1,11 +1,16 @@ package dms import ( + "context" + "errors" "fmt" + "path" "strconv" "strings" "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -85,6 +90,49 @@ func NextVersion(lastVersionID string) (int, error) { // Keep it equal to the service's DEPLOYMENT_NODE_NAME. const DeploymentNodeName = "resources.deployment.json" +// ReadDeployment looks up the deployment node and its service record. +// +// TODO: a deployment is only usable when both the node and the service's record exist, and a +// half-created one - node present, record missing - blocks the bundle here even though +// CreateDeployment already recovers from it. Recover here with an empty id and version unless +// both halves are there, leaving the caller to call +// CreateDeployment to create or finalize it. +// +// TODO: ask the service for a lookup by state path, so this is one round trip rather than two - a +// workspace lookup to turn the node into an id, then a get by that id. +func ReadDeployment(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { + nodePath := path.Join(statePath, DeploymentNodeName) + + obj, err := w.Workspace.GetStatusByPath(ctx, nodePath) + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil, nil + } + if err != nil { + return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + deploymentID := strconv.FormatInt(obj.ObjectId, 10) + deployment, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: DeploymentName(deploymentID), + }) + if err != nil { + return "", nil, err + } + return deploymentID, deployment, nil +} + +// ParseLastVersionID parses the deployment's last recorded version, which the service reports +// as a string. It returns 0 when the deployment does not exist yet or has no recorded version. +func ParseLastVersionID(dmsDeployment *bundledeployments.Deployment) (int, error) { + if dmsDeployment == nil || dmsDeployment.LastVersionId == "" { + return 0, nil + } + v, err := strconv.Atoi(dmsDeployment.LastVersionId) + if err != nil { + return 0, fmt.Errorf("failed to parse last_version_id %q: %w", dmsDeployment.LastVersionId, err) + } + return v, nil +} + // DeploymentName, VersionName and OperationName build the resource names the // service uses, so callers pass only ids. func DeploymentName(deploymentID string) string { From 6ee4adb9813dd197a24ee2df0a2dd706969faef0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 16:56:26 +0000 Subject: [PATCH 09/25] Close migration state explicitly before normal DMS reads --- bundle/direct/dstate/state.go | 13 ++++++++++++ bundle/direct/dstate/state_test.go | 31 +++++++++++++++++++++++++++++ bundle/phases/dms_migration.go | 25 ++++++++++++----------- bundle/phases/dms_migration_test.go | 5 ++++- cmd/bundle/utils/process.go | 11 +++------- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ac8f8226c4b..009f2e2dd84 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -885,6 +885,19 @@ func (db *DeploymentState) Persist() error { return db.unlockedSave() } +// Close discards read-only in-memory state so the next Open rereads its backend. +// It is safe to call on a closed state. Writable state must be finalized instead. +func (db *DeploymentState) Close() { + db.mu.Lock() + defer db.mu.Unlock() + + if db.Path == "" { + return + } + db.AssertOpenedForRead() + db.reset() +} + // IsOpen reports whether the state has been opened (for read or write). It lets // callers probe the state without risking the panic in AssertOpenedForReadOrWrite, // e.g. code paths shared with the terraform engine where the state DB is never opened. diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 72ec263f7b7..963f51b5d28 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -49,6 +49,37 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { assert.ErrorIs(t, err, os.ErrNotExist) } +func TestCloseReadState(t *testing.T) { + path := filepath.Join(t.TempDir(), "resources.json") + var db DeploymentState + db.OpenWithData(path, NewDatabase("lineage", 1)) + require.NoError(t, db.Persist()) + before, err := os.ReadFile(path) + require.NoError(t, err) + db.Data.Serial = 99 + + db.Close() + db.Close() + + assert.False(t, db.IsOpen()) + after, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, before, after) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) + assert.Equal(t, 1, db.GetSerial()) + db.Close() +} + +func TestCloseRefusesWritableState(t *testing.T) { + var db DeploymentState + db.OpenWithData(filepath.Join(t.TempDir(), "resources.json"), NewDatabase("lineage", 1)) + require.NoError(t, db.UpgradeToWrite()) + defer mustFinalize(t, &db) + + assert.PanicsWithValue(t, "internal error: DeploymentState must be opened in read mode", db.Close) + assert.True(t, db.IsOpen()) +} + func TestDataForFileSerial(t *testing.T) { tests := []struct { name string diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go index 3869dfbdc49..f63165f7e58 100644 --- a/bundle/phases/dms_migration.go +++ b/bundle/phases/dms_migration.go @@ -3,6 +3,7 @@ package phases import ( "context" "fmt" + "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/dmsmigration" @@ -15,8 +16,8 @@ import ( "github.com/databricks/cli/libs/logdiag" ) -// MigrateToDMS commits deployment history before ordinary deployment planning begins. -func MigrateToDMS(ctx context.Context, b *bundle.Bundle) { +// MigrateToDMS commits deployment history and returns its state descriptor before ordinary deployment planning begins. +func MigrateToDMS(ctx context.Context, b *bundle.Bundle) (state *statemgmt.StateDesc) { bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalDeploy)) if logdiag.HasError(ctx) { return @@ -24,14 +25,10 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle) { defer bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) db := &b.DeploymentBundle.StateDB - defer func() { - if _, err := db.Finalize(ctx); err != nil { - logdiag.LogError(ctx, err) - } - }() + defer db.Close() // Another deploy may have completed since the command's initial state read. - state := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) + state = statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) if logdiag.HasError(ctx) { return } @@ -52,10 +49,7 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle) { } b.MigratingToDirect = false } - if _, err := db.Finalize(ctx); err != nil { - logdiag.LogError(ctx, err) - return - } + db.Close() if !state.Engine.IsDirect() { logdiag.LogError(ctx, fmt.Errorf("deployment state changed to %s while enabling deployment history; retry the deployment", state.Engine)) return @@ -93,4 +87,11 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle) { } logDeploymentVersion(ctx, b, "Deployment history enabled") statemgmt.PushResourcesState(ctx, b) + state.Serial = db.GetSerial() + state.Lineage = db.Data.Lineage + state.Features = db.StateFeatures() + state.SourcePath = filepath.ToSlash(db.Path) + state.IsLocal = true + state.Content = nil + return } diff --git a/bundle/phases/dms_migration_test.go b/bundle/phases/dms_migration_test.go index d6f11e8667a..4e2219d3720 100644 --- a/bundle/phases/dms_migration_test.go +++ b/bundle/phases/dms_migration_test.go @@ -43,9 +43,12 @@ func TestMigrateToDMSAlreadyRecorded(t *testing.T) { require.NoError(t, os.WriteFile(statePath, content, 0o600)) b.DeploymentBundle.StateDB.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) - phases.MigrateToDMS(ctx, b) + state := phases.MigrateToDMS(ctx, b) require.False(t, logdiag.HasError(ctx), "%v", logdiag.FlushCollected(ctx)) + require.NotNil(t, state) + assert.True(t, state.IsDMS()) + assert.Equal(t, 2, state.Serial) assert.False(t, b.MigratingToDirect) assert.False(t, b.DeploymentBundle.StateDB.IsOpen()) after, err := os.ReadFile(statePath) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 034fec06968..930534b8a32 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -352,24 +352,19 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, errors.New("migrating to deployment history does not support --plan; run 'bundle deploy' without --plan") } if migrateToDMS { - phases.MigrateToDMS(ctx, b) + stateDesc = phases.MigrateToDMS(ctx, b) if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } - stateDesc = statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted - } - resolveDeploymentHistory(ctx, b, stateDesc) } // Open direct engine state once for all subsequent operations (ExportState, CalculatePlan, Apply, etc.) // A migrated-from-Terraform state is already open (seeded in memory above), so skip the disk open. needDirectState := stateDesc.Engine.IsDirect() && needsState var localPath string - if needDirectState { + if needDirectState && !b.DeploymentBundle.StateDB.IsOpen() { _, localPath = b.StateFilenameDirect(ctx) - if !b.DeploymentBundle.StateDB.IsOpen() && !stateDesc.IsDMS() { + if !stateDesc.IsDMS() { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted From 207e2f02a6d8e9a40a83c0120034d3060dc8f70a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 17:00:38 +0000 Subject: [PATCH 10/25] Clarify deployment annotation cases next to mutators --- cmd/bundle/utils/process.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 930534b8a32..fe469894d49 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -386,10 +386,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } - // Stamp the deployment and the version this run records onto every job and pipeline so - // the plan carries them. The deployment version follows the last recorded version or - // the initial registration version; deployment_id does not exist until a first deploy - // creates it, so it is left off here and the deploy phase stamps the created id. lastVersionID, err := dms.ParseLastVersionID(dmsDeployment) if err != nil { logdiag.LogError(ctx, err) @@ -404,6 +400,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } nextVersion := b.DeploymentBundle.StateDB.GetSerial() + 1 + // Stamp jobs and pipelines for the plan: + // 1. Existing: deployment ID is known; version = last recorded + 1. + // 2. New: version = 1; only here is the ID stamped later, after deployment creation. + // 3. Migrated: deployment ID is known; version = migration version + 1. muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} if dmsDeploymentID != "" { bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { From f35c8d96d47bde657ee7a6d7787741eae448bdf1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 17:09:19 +0000 Subject: [PATCH 11/25] Reuse loaded state for migration and skip locking recorded state --- bundle/phases/dms_migration.go | 44 ++++++++--------------------- bundle/phases/dms_migration_test.go | 32 +++++++-------------- cmd/bundle/utils/process.go | 17 ++++++----- 3 files changed, 29 insertions(+), 64 deletions(-) diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go index f63165f7e58..11fc9050ec3 100644 --- a/bundle/phases/dms_migration.go +++ b/bundle/phases/dms_migration.go @@ -2,13 +2,11 @@ package phases import ( "context" - "fmt" "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/deploy/lock" - "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" @@ -16,8 +14,13 @@ import ( "github.com/databricks/cli/libs/logdiag" ) -// MigrateToDMS commits deployment history and returns its state descriptor before ordinary deployment planning begins. -func MigrateToDMS(ctx context.Context, b *bundle.Bundle) (state *statemgmt.StateDesc) { +// MigrateToDMS commits the already-open direct state to DMS and updates its descriptor before deployment planning. +func MigrateToDMS(ctx context.Context, b *bundle.Bundle, state *statemgmt.StateDesc) { + if state.IsDMS() { + b.MigratingToDirect = false + return + } + bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalDeploy)) if logdiag.HasError(ctx) { return @@ -25,40 +28,16 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle) (state *statemgmt.State defer bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) db := &b.DeploymentBundle.StateDB + db.AssertOpenedForRead() defer db.Close() - // Another deploy may have completed since the command's initial state read. - state = statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) - if logdiag.HasError(ctx) { - return - } - if state.IsDMS() { - b.MigratingToDirect = false - return - } if b.MigratingToDirect { - if !state.Engine.IsDirect() { - statemgmt.CommitMigration(ctx, b) - if logdiag.HasError(ctx) { - return - } - state = statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) - if logdiag.HasError(ctx) { - return - } + statemgmt.CommitMigration(ctx, b) + if logdiag.HasError(ctx) { + return } b.MigratingToDirect = false } - db.Close() - if !state.Engine.IsDirect() { - logdiag.LogError(ctx, fmt.Errorf("deployment state changed to %s while enabling deployment history; retry the deployment", state.Engine)) - return - } - _, localPath := b.StateFilenameDirect(ctx) - if err := db.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { - logdiag.LogError(ctx, err) - return - } deploymentID, deployment, err := dms.ReadDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { logdiag.LogError(ctx, err) @@ -93,5 +72,4 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle) (state *statemgmt.State state.SourcePath = filepath.ToSlash(db.Path) state.IsLocal = true state.Content = nil - return } diff --git a/bundle/phases/dms_migration_test.go b/bundle/phases/dms_migration_test.go index 4e2219d3720..5b86aaf1f56 100644 --- a/bundle/phases/dms_migration_test.go +++ b/bundle/phases/dms_migration_test.go @@ -1,7 +1,6 @@ package phases_test import ( - "os" "path/filepath" "testing" @@ -9,49 +8,38 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/testserver" - "github.com/databricks/databricks-sdk-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestMigrateToDMSAlreadyRecorded(t *testing.T) { - server := testserver.New(t) - server.Handle("GET", "/api/2.0/workspace/export", func(req testserver.Request) any { - return testserver.Response{StatusCode: 404, Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST"}} - }) - client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) - require.NoError(t, err) b := &bundle.Bundle{ BundleRootPath: t.TempDir(), MigratingToDirect: true, Config: config.Root{ Bundle: config.Bundle{ Target: "default", - Deployment: config.Deployment{Lock: config.Lock{Enabled: new(false)}}, + Deployment: config.Deployment{Lock: config.Lock{Enabled: new(true)}}, }, Workspace: config.Workspace{StatePath: "/Workspace/state"}, }, } - b.SetWorkpaceClient(client) ctx := logdiag.InitContext(t.Context()) logdiag.SetCollect(ctx, true) - _, statePath := b.StateFilenameDirect(ctx) - content := []byte(`{"state_version":3,"serial":2,"lineage":"lineage","features":{"deployment_history":{}},"state":{}}`) - require.NoError(t, os.MkdirAll(filepath.Dir(statePath), 0o700)) - require.NoError(t, os.WriteFile(statePath, content, 0o600)) - b.DeploymentBundle.StateDB.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) + b.DeploymentBundle.StateDB.OpenWithData(filepath.Join(b.BundleRootPath, "resources.json"), dstate.NewDatabase("lineage", 2)) + state := &statemgmt.StateDesc{ + Serial: 2, + Features: map[string]struct{}{dstate.FeatureDeploymentHistory: {}}, + } - state := phases.MigrateToDMS(ctx, b) + phases.MigrateToDMS(ctx, b, state) require.False(t, logdiag.HasError(ctx), "%v", logdiag.FlushCollected(ctx)) - require.NotNil(t, state) assert.True(t, state.IsDMS()) assert.Equal(t, 2, state.Serial) assert.False(t, b.MigratingToDirect) - assert.False(t, b.DeploymentBundle.StateDB.IsOpen()) - after, err := os.ReadFile(statePath) - require.NoError(t, err) - assert.Equal(t, content, after) + assert.Nil(t, b.Locker) + assert.True(t, b.DeploymentBundle.StateDB.IsOpen()) } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index fe469894d49..84ebf2020fd 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -351,19 +351,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if migrateToDMS && opts.ReadPlanPath != "" { return b, stateDesc, errors.New("migrating to deployment history does not support --plan; run 'bundle deploy' without --plan") } - if migrateToDMS { - stateDesc = phases.MigrateToDMS(ctx, b) - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted - } - } - // Open direct engine state once for all subsequent operations (ExportState, CalculatePlan, Apply, etc.) // A migrated-from-Terraform state is already open (seeded in memory above), so skip the disk open. needDirectState := stateDesc.Engine.IsDirect() && needsState - var localPath string if needDirectState && !b.DeploymentBundle.StateDB.IsOpen() { - _, localPath = b.StateFilenameDirect(ctx) + _, localPath := b.StateFilenameDirect(ctx) if !stateDesc.IsDMS() { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { logdiag.LogError(ctx, err) @@ -371,6 +363,12 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } } + if migrateToDMS { + phases.MigrateToDMS(ctx, b, stateDesc) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting { if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, opts.Deploy || opts.PreDeployChecks); err != nil { logdiag.LogError(ctx, err) @@ -395,6 +393,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } + _, localPath := b.StateFilenameDirect(ctx) if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted From 3ddeba6f1b5e6b9c12d36f89bccbaec231794a4f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 19 Oct 2026 17:19:37 +0000 Subject: [PATCH 12/25] Require direct state before enabling deployment history --- .../migrate/auto/dms-migration/output.txt | 23 ++++++++++++++++--- .../bundle/migrate/auto/dms-migration/script | 11 ++++++++- bundle/phases/dms_migration.go | 15 ++++++------ bundle/phases/dms_migration_test.go | 15 ++++++++++++ cmd/bundle/utils/process.go | 5 ++++ 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/migrate/auto/dms-migration/output.txt b/acceptance/bundle/migrate/auto/dms-migration/output.txt index 63b0d3da59d..58cb5d081d3 100644 --- a/acceptance/bundle/migrate/auto/dms-migration/output.txt +++ b/acceptance/bundle/migrate/auto/dms-migration/output.txt @@ -2,15 +2,32 @@ === Replay a prior terraform deployment (captured from a real terraform run) >>> replay_tfstate.py terraform-requests.json terraform.tfstate -=== Terraform migrates to direct deployment with history enabled and preserves the job ID ->>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +=== Deployment history rejects Terraform state before conversion, for both plan and deploy +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true musterr [CLI] bundle plan +Error: deployment history is not supported with Terraform state; first migrate to direct deployment with deployment history disabled + +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true musterr [CLI] bundle deploy +Error: deployment history is not supported with Terraform state; first migrate to direct deployment with deployment history disabled + +>>> find.py resources[.]json|tfstate[.]backup --expect 0 + +>>> print_requests.py --dms //api/2.0/bundle //jobs + +=== First migrate Terraform state to direct with deployment history disabled +>>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Migrated 1 resource to direct deployment engine. +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +=== Then enable deployment history on the direct state, preserving the job ID +>>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Enabling deployment history... Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Updated jobs.test_job -Files: 1 uploaded, 0 deleted +Files: 0 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) diff --git a/acceptance/bundle/migrate/auto/dms-migration/script b/acceptance/bundle/migrate/auto/dms-migration/script index e2254672221..a5d1a0c53f9 100644 --- a/acceptance/bundle/migrate/auto/dms-migration/script +++ b/acceptance/bundle/migrate/auto/dms-migration/script @@ -5,7 +5,16 @@ trace replay_tfstate.py terraform-requests.json terraform.tfstate original_job_id=$(read_id.py test_job) rm -f "$OUT_REQUESTS" -title "Terraform migrates to direct deployment with history enabled and preserves the job ID" +title "Deployment history rejects Terraform state before conversion, for both plan and deploy" +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true musterr $CLI bundle plan +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true musterr $CLI bundle deploy +trace find.py 'resources[.]json|tfstate[.]backup' --expect 0 +trace print_requests.py --dms //api/2.0/bundle //jobs + +title "First migrate Terraform state to direct with deployment history disabled" +trace DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle deploy + +title "Then enable deployment history on the direct state, preserving the job ID" trace DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace find.py 'resources[.]json[.]wal' --expect 0 diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go index 11fc9050ec3..05606fe879c 100644 --- a/bundle/phases/dms_migration.go +++ b/bundle/phases/dms_migration.go @@ -2,6 +2,7 @@ package phases import ( "context" + "errors" "path/filepath" "github.com/databricks/cli/bundle" @@ -14,8 +15,15 @@ import ( "github.com/databricks/cli/libs/logdiag" ) +// ErrDMSRequiresDirectState rejects enabling history before Terraform state is migrated to direct. +var ErrDMSRequiresDirectState = errors.New("deployment history is not supported with Terraform state; first migrate to direct deployment with deployment history disabled") + // MigrateToDMS commits the already-open direct state to DMS and updates its descriptor before deployment planning. func MigrateToDMS(ctx context.Context, b *bundle.Bundle, state *statemgmt.StateDesc) { + if !state.Engine.IsDirect() { + logdiag.LogError(ctx, ErrDMSRequiresDirectState) + return + } if state.IsDMS() { b.MigratingToDirect = false return @@ -31,13 +39,6 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle, state *statemgmt.StateD db.AssertOpenedForRead() defer db.Close() - if b.MigratingToDirect { - statemgmt.CommitMigration(ctx, b) - if logdiag.HasError(ctx) { - return - } - b.MigratingToDirect = false - } deploymentID, deployment, err := dms.ReadDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/phases/dms_migration_test.go b/bundle/phases/dms_migration_test.go index 5b86aaf1f56..ac0dcef1687 100644 --- a/bundle/phases/dms_migration_test.go +++ b/bundle/phases/dms_migration_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/statemgmt" @@ -14,6 +15,20 @@ import ( "github.com/stretchr/testify/require" ) +func TestMigrateToDMSRejectsTerraformBeforeLock(t *testing.T) { + b := &bundle.Bundle{} + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + + phases.MigrateToDMS(ctx, b, &statemgmt.StateDesc{Engine: engine.EngineTerraform}) + + require.True(t, logdiag.HasError(ctx)) + diags := logdiag.FlushCollected(ctx) + require.Len(t, diags, 1) + assert.Equal(t, phases.ErrDMSRequiresDirectState.Error(), diags[0].Summary) + assert.Nil(t, b.Locker) +} + func TestMigrateToDMSAlreadyRecorded(t *testing.T) { b := &bundle.Bundle{ BundleRootPath: t.TempDir(), diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 84ebf2020fd..24e68f9827b 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -250,6 +250,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // Record the engine the resolved state uses now, so deploy telemetry reports // it even when the deploy fails or is cancelled before deployCore runs. b.Metrics.StateEngine = stateDesc.Engine.ThisOrDefault() + + // TODO: Support the full Terraform -> direct -> DMS migration path at GA. + if (opts.Deploy || opts.PreDeployChecks) && !stateDesc.Engine.IsDirect() && configuresDeploymentHistory(ctx, b) { + return b, stateDesc, phases.ErrDMSRequiresDirectState + } } // --plan applies a precomputed plan, so it skips Build and PreDeployChecks; a plain From b540bdfdc86dcfdf296aa4a23c37e201e2856528 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:02:29 +0000 Subject: [PATCH 13/25] Clarify skipped deployment ID annotation for new deployments --- cmd/bundle/utils/process.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 24e68f9827b..3f05ca81eb6 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -406,7 +406,8 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle nextVersion := b.DeploymentBundle.StateDB.GetSerial() + 1 // Stamp jobs and pipelines for the plan: // 1. Existing: deployment ID is known; version = last recorded + 1. - // 2. New: version = 1; only here is the ID stamped later, after deployment creation. + // 2. New: stamp version 1, but skip AnnotateDeployment below because no ID exists yet. + // The deploy phase creates the deployment and stamps its ID. // 3. Migrated: deployment ID is known; version = migration version + 1. muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} if dmsDeploymentID != "" { From 6c5f9a8bcdac65e30c70c51c3b1680b47ee29edc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:06:57 +0000 Subject: [PATCH 14/25] Inline DMS registration in the migration phase --- bundle/deploy/dmsmigration/migrate.go | 78 ------------------- bundle/deploy/dmsmigration/migrate_test.go | 89 ---------------------- bundle/phases/dms_migration.go | 72 ++++++++++++++++- bundle/phases/dms_migration_test.go | 87 +++++++++++++++++++-- 4 files changed, 152 insertions(+), 174 deletions(-) delete mode 100644 bundle/deploy/dmsmigration/migrate.go delete mode 100644 bundle/deploy/dmsmigration/migrate_test.go diff --git a/bundle/deploy/dmsmigration/migrate.go b/bundle/deploy/dmsmigration/migrate.go deleted file mode 100644 index b706748f307..00000000000 --- a/bundle/deploy/dmsmigration/migrate.go +++ /dev/null @@ -1,78 +0,0 @@ -package dmsmigration - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" - "strconv" - "strings" - - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/internal/build" - "github.com/databricks/cli/libs/cmdctx" - "github.com/databricks/cli/libs/dms" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" -) - -// Migrate records every resource from the open direct state in an initial registration version. -// The caller must hold the deployment lock and have created the DMS deployment. -func Migrate(ctx context.Context, db *dstate.DeploymentState) error { - keys := slices.Sorted(maps.Keys(db.Data.State)) - states := make([]json.RawMessage, len(keys)) - operations := make([]bundledeployments.StagedOperation, len(keys)) - for i, key := range keys { - entry := db.Data.State[key] - state, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) - if err != nil { - return fmt.Errorf("serializing migration state for %s: %w", key, err) - } - states[i] = state - operations[i] = bundledeployments.StagedOperation{ - ResourceKey: strings.TrimPrefix(key, dms.StatePrefix), - ActionType: bundledeployments.OperationActionTypeOperationActionTypeInitialRegister, - } - } - - // Migration uses max(state serial, last version)+1; deployment follows at the next version. - versionID := max(db.Data.Serial, db.VersionID) + 1 - previousVersionID := "" - if db.VersionID > 0 { - previousVersionID = strconv.Itoa(db.VersionID) - } - w := cmdctx.WorkspaceClient(ctx) - _, err := w.BundleDeployments.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ - Parent: dms.DeploymentName(db.DeploymentID), - VersionId: strconv.Itoa(versionID), - Version: bundledeployments.Version{ - CliVersion: build.GetInfo().Version, - VersionType: dms.VersionTypeDeploy, - PreviousVersionId: previousVersionID, - Operations: operations, - }, - }) - if err != nil { - return fmt.Errorf("creating migration version: %w", err) - } - - buffer := dms.StartOperationBuffer(ctx, db.DeploymentID, versionID) - for i, key := range keys { - buffer.RecordOperation(ctx, key, false, db.Data.State[key].ID, states[i]) - } - recordErr := buffer.Drain() - reason := bundledeployments.VersionCompleteVersionCompleteSuccess - if recordErr != nil { - reason = bundledeployments.VersionCompleteVersionCompleteFailure - } - _, completeErr := w.BundleDeployments.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ - Name: dms.VersionName(db.DeploymentID, versionID), - CompletionReason: reason, - }) - if err := errors.Join(recordErr, completeErr); err != nil { - return fmt.Errorf("recording migration version: %w", err) - } - - return db.UseDeploymentHistory(versionID) -} diff --git a/bundle/deploy/dmsmigration/migrate_test.go b/bundle/deploy/dmsmigration/migrate_test.go deleted file mode 100644 index 5e00f71da2b..00000000000 --- a/bundle/deploy/dmsmigration/migrate_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package dmsmigration_test - -import ( - "encoding/json" - "os" - "path/filepath" - "strconv" - "testing" - - "github.com/databricks/cli/bundle/deploy/dmsmigration" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/internal/build" - "github.com/databricks/cli/libs/cmdctx" - "github.com/databricks/cli/libs/dms" - "github.com/databricks/databricks-sdk-go/experimental/mocks" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestMigrate(t *testing.T) { - tests := []struct { - name string - serial int - lastVersion int - registration int - }{ - { - name: "first registration", - serial: 2, - registration: 3, - }, - { - name: "service ahead", - serial: 2, - lastVersion: 5, - registration: 6, - }, - { - name: "file ahead", - serial: 8, - lastVersion: 3, - registration: 9, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - statePath := filepath.Join(t.TempDir(), "resources.json") - var db dstate.DeploymentState - db.OpenWithData(statePath, dstate.NewDatabase("lineage", tt.serial)) - db.DeploymentID = "123" - db.VersionID = tt.lastVersion - m := mocks.NewMockWorkspaceClient(t) - api := m.GetMockBundleDeploymentsAPI() - previousVersion := "" - if tt.lastVersion > 0 { - previousVersion = strconv.Itoa(tt.lastVersion) - } - api.EXPECT().CreateVersion(mock.Anything, bundledeployments.CreateVersionRequest{ - Parent: dms.DeploymentName("123"), - VersionId: strconv.Itoa(tt.registration), - Version: bundledeployments.Version{ - CliVersion: build.GetInfo().Version, - VersionType: dms.VersionTypeDeploy, - PreviousVersionId: previousVersion, - Operations: []bundledeployments.StagedOperation{}, - }, - }).Return(&bundledeployments.Version{}, nil).Once() - api.EXPECT().CompleteVersion(mock.Anything, bundledeployments.CompleteVersionRequest{ - Name: dms.VersionName("123", tt.registration), - CompletionReason: bundledeployments.VersionCompleteVersionCompleteSuccess, - }).Return(&bundledeployments.Version{}, nil).Once() - ctx := cmdctx.SetWorkspaceClient(t.Context(), m.WorkspaceClient) - - require.NoError(t, dmsmigration.Migrate(ctx, &db)) - - assert.True(t, db.IsDeploymentMetadataService()) - assert.Equal(t, tt.registration, db.GetSerial()) - content, err := os.ReadFile(statePath) - require.NoError(t, err) - var persisted dstate.Database - require.NoError(t, json.Unmarshal(content, &persisted)) - assert.Contains(t, persisted.Features, dstate.FeatureDeploymentHistory) - assert.Equal(t, tt.registration, persisted.Serial) - assert.Empty(t, persisted.State) - }) - } -} diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go index 05606fe879c..45256152909 100644 --- a/bundle/phases/dms_migration.go +++ b/bundle/phases/dms_migration.go @@ -2,17 +2,25 @@ package phases import ( "context" + "encoding/json" "errors" + "fmt" + "maps" "path/filepath" + "slices" + "strconv" + "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/deploy/lock" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) // ErrDMSRequiresDirectState rejects enabling history before Terraform state is migrated to direct. @@ -61,7 +69,7 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle, state *statemgmt.StateD } ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) cmdio.LogString(ctx, "Enabling deployment history...") - if err := dmsmigration.Migrate(ctx, db); err != nil { + if err := registerMigrationState(ctx, db); err != nil { logdiag.LogError(ctx, err) return } @@ -74,3 +82,63 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle, state *statemgmt.StateD state.IsLocal = true state.Content = nil } + +// registerMigrationState records every resource from the open direct state in an initial registration version. +// The caller must hold the deployment lock and have created the DMS deployment. +func registerMigrationState(ctx context.Context, db *dstate.DeploymentState) error { + keys := slices.Sorted(maps.Keys(db.Data.State)) + states := make([]json.RawMessage, len(keys)) + operations := make([]bundledeployments.StagedOperation, len(keys)) + for i, key := range keys { + entry := db.Data.State[key] + state, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + return fmt.Errorf("serializing migration state for %s: %w", key, err) + } + states[i] = state + operations[i] = bundledeployments.StagedOperation{ + ResourceKey: strings.TrimPrefix(key, dms.StatePrefix), + ActionType: bundledeployments.OperationActionTypeOperationActionTypeInitialRegister, + } + } + + // Migration uses max(state serial, last version)+1; deployment follows at the next version. + versionID := max(db.Data.Serial, db.VersionID) + 1 + previousVersionID := "" + if db.VersionID > 0 { + previousVersionID = strconv.Itoa(db.VersionID) + } + w := cmdctx.WorkspaceClient(ctx) + _, err := w.BundleDeployments.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ + Parent: dms.DeploymentName(db.DeploymentID), + VersionId: strconv.Itoa(versionID), + Version: bundledeployments.Version{ + CliVersion: build.GetInfo().Version, + VersionType: dms.VersionTypeDeploy, + PreviousVersionId: previousVersionID, + Operations: operations, + }, + }) + if err != nil { + return fmt.Errorf("creating migration version: %w", err) + } + + buffer := dms.StartOperationBuffer(ctx, db.DeploymentID, versionID) + for i, key := range keys { + buffer.RecordOperation(ctx, key, false, db.Data.State[key].ID, states[i]) + } + recordErr := buffer.Drain() + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if recordErr != nil { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + _, completeErr := w.BundleDeployments.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: dms.VersionName(db.DeploymentID, versionID), + CompletionReason: reason, + }) + if err := errors.Join(recordErr, completeErr); err != nil { + return fmt.Errorf("recording migration version: %w", err) + } + + return db.UseDeploymentHistory(versionID) +} diff --git a/bundle/phases/dms_migration_test.go b/bundle/phases/dms_migration_test.go index ac0dcef1687..2837e2b3040 100644 --- a/bundle/phases/dms_migration_test.go +++ b/bundle/phases/dms_migration_test.go @@ -1,17 +1,25 @@ -package phases_test +package phases import ( + "encoding/json" + "os" "path/filepath" + "strconv" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -20,12 +28,12 @@ func TestMigrateToDMSRejectsTerraformBeforeLock(t *testing.T) { ctx := logdiag.InitContext(t.Context()) logdiag.SetCollect(ctx, true) - phases.MigrateToDMS(ctx, b, &statemgmt.StateDesc{Engine: engine.EngineTerraform}) + MigrateToDMS(ctx, b, &statemgmt.StateDesc{Engine: engine.EngineTerraform}) require.True(t, logdiag.HasError(ctx)) diags := logdiag.FlushCollected(ctx) require.Len(t, diags, 1) - assert.Equal(t, phases.ErrDMSRequiresDirectState.Error(), diags[0].Summary) + assert.Equal(t, ErrDMSRequiresDirectState.Error(), diags[0].Summary) assert.Nil(t, b.Locker) } @@ -49,7 +57,7 @@ func TestMigrateToDMSAlreadyRecorded(t *testing.T) { Features: map[string]struct{}{dstate.FeatureDeploymentHistory: {}}, } - phases.MigrateToDMS(ctx, b, state) + MigrateToDMS(ctx, b, state) require.False(t, logdiag.HasError(ctx), "%v", logdiag.FlushCollected(ctx)) assert.True(t, state.IsDMS()) @@ -58,3 +66,72 @@ func TestMigrateToDMSAlreadyRecorded(t *testing.T) { assert.Nil(t, b.Locker) assert.True(t, b.DeploymentBundle.StateDB.IsOpen()) } + +func TestRegisterMigrationState(t *testing.T) { + tests := []struct { + name string + serial int + lastVersion int + registration int + }{ + { + name: "first registration", + serial: 2, + registration: 3, + }, + { + name: "service ahead", + serial: 2, + lastVersion: 5, + registration: 6, + }, + { + name: "file ahead", + serial: 8, + lastVersion: 3, + registration: 9, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "resources.json") + var db dstate.DeploymentState + db.OpenWithData(statePath, dstate.NewDatabase("lineage", tt.serial)) + db.DeploymentID = "123" + db.VersionID = tt.lastVersion + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockBundleDeploymentsAPI() + previousVersion := "" + if tt.lastVersion > 0 { + previousVersion = strconv.Itoa(tt.lastVersion) + } + api.EXPECT().CreateVersion(mock.Anything, bundledeployments.CreateVersionRequest{ + Parent: dms.DeploymentName("123"), + VersionId: strconv.Itoa(tt.registration), + Version: bundledeployments.Version{ + CliVersion: build.GetInfo().Version, + VersionType: dms.VersionTypeDeploy, + PreviousVersionId: previousVersion, + Operations: []bundledeployments.StagedOperation{}, + }, + }).Return(&bundledeployments.Version{}, nil).Once() + api.EXPECT().CompleteVersion(mock.Anything, bundledeployments.CompleteVersionRequest{ + Name: dms.VersionName("123", tt.registration), + CompletionReason: bundledeployments.VersionCompleteVersionCompleteSuccess, + }).Return(&bundledeployments.Version{}, nil).Once() + ctx := cmdctx.SetWorkspaceClient(t.Context(), m.WorkspaceClient) + + require.NoError(t, registerMigrationState(ctx, &db)) + + assert.True(t, db.IsDeploymentMetadataService()) + assert.Equal(t, tt.registration, db.GetSerial()) + content, err := os.ReadFile(statePath) + require.NoError(t, err) + var persisted dstate.Database + require.NoError(t, json.Unmarshal(content, &persisted)) + assert.Contains(t, persisted.Features, dstate.FeatureDeploymentHistory) + assert.Equal(t, tt.registration, persisted.Serial) + assert.Empty(t, persisted.State) + }) + } +} From 51a97d686a34e7f1bc37a238ccae8b15383032e6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:10:39 +0000 Subject: [PATCH 15/25] Clarify version annotation runs for new deployments --- cmd/bundle/utils/process.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 3f05ca81eb6..d95d775a7aa 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -406,8 +406,8 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle nextVersion := b.DeploymentBundle.StateDB.GetSerial() + 1 // Stamp jobs and pipelines for the plan: // 1. Existing: deployment ID is known; version = last recorded + 1. - // 2. New: stamp version 1, but skip AnnotateDeployment below because no ID exists yet. - // The deploy phase creates the deployment and stamps its ID. + // 2. New: AnnotateDeploymentVersion stamps version 1 now; the deploy phase + // stamps the deployment ID after creating the deployment. // 3. Migrated: deployment ID is known; version = migration version + 1. muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} if dmsDeploymentID != "" { From ee3746f2d61348b939eb0b9a062dc7cf39c6a971 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:14:45 +0000 Subject: [PATCH 16/25] Rename deployment history marker method --- bundle/direct/dstate/dms.go | 4 ++-- bundle/direct/dstate/dms_test.go | 4 ++-- bundle/phases/dms_migration.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index f9a90a1cd77..00f5ac29253 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -16,8 +16,8 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// UseDeploymentHistory persists the DMS marker after all existing resources have been recorded. -func (db *DeploymentState) UseDeploymentHistory(versionID int) error { +// MarkDeploymentHistory persists the DMS marker after all existing resources have been recorded. +func (db *DeploymentState) MarkDeploymentHistory(versionID int) error { db.AssertOpenedForRead() db.mu.Lock() defer db.mu.Unlock() diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 4729a4f1c46..4a967a8a512 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestUseDeploymentHistory(t *testing.T) { +func TestMarkDeploymentHistory(t *testing.T) { statePath := filepath.Join(t.TempDir(), "resources.json") original := NewDatabase("lineage", 2) original.Features = map[string]struct{}{} @@ -23,7 +23,7 @@ func TestUseDeploymentHistory(t *testing.T) { before, err := os.ReadFile(statePath) require.NoError(t, err) - require.NoError(t, db.UseDeploymentHistory(3)) + require.NoError(t, db.MarkDeploymentHistory(3)) assert.True(t, db.IsDeploymentMetadataService()) assert.Equal(t, 3, db.GetSerial()) diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go index 45256152909..2d7da1e7782 100644 --- a/bundle/phases/dms_migration.go +++ b/bundle/phases/dms_migration.go @@ -140,5 +140,5 @@ func registerMigrationState(ctx context.Context, db *dstate.DeploymentState) err return fmt.Errorf("recording migration version: %w", err) } - return db.UseDeploymentHistory(versionID) + return db.MarkDeploymentHistory(versionID) } From bd53e852cf996859b70c9a9dcf6f5d94d1260dcc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:18:04 +0000 Subject: [PATCH 17/25] Remove obsolete Terraform migration flag reset from DMS --- bundle/phases/dms_migration.go | 1 - bundle/phases/dms_migration_test.go | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go index 2d7da1e7782..9a3181596c6 100644 --- a/bundle/phases/dms_migration.go +++ b/bundle/phases/dms_migration.go @@ -33,7 +33,6 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle, state *statemgmt.StateD return } if state.IsDMS() { - b.MigratingToDirect = false return } diff --git a/bundle/phases/dms_migration_test.go b/bundle/phases/dms_migration_test.go index 2837e2b3040..39cd2e769c5 100644 --- a/bundle/phases/dms_migration_test.go +++ b/bundle/phases/dms_migration_test.go @@ -39,8 +39,7 @@ func TestMigrateToDMSRejectsTerraformBeforeLock(t *testing.T) { func TestMigrateToDMSAlreadyRecorded(t *testing.T) { b := &bundle.Bundle{ - BundleRootPath: t.TempDir(), - MigratingToDirect: true, + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -62,7 +61,6 @@ func TestMigrateToDMSAlreadyRecorded(t *testing.T) { require.False(t, logdiag.HasError(ctx), "%v", logdiag.FlushCollected(ctx)) assert.True(t, state.IsDMS()) assert.Equal(t, 2, state.Serial) - assert.False(t, b.MigratingToDirect) assert.Nil(t, b.Locker) assert.True(t, b.DeploymentBundle.StateDB.IsOpen()) } From aebedd55b2d2a01160f301bd9c5198c4247c63c8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:27:16 +0000 Subject: [PATCH 18/25] Remove marker persistence unit test covered by acceptance --- bundle/direct/dstate/dms_test.go | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 4a967a8a512..ab4e1bc017f 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -2,8 +2,6 @@ package dstate import ( "encoding/json" - "os" - "path/filepath" "testing" "github.com/databricks/cli/bundle/deployplan" @@ -12,35 +10,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestMarkDeploymentHistory(t *testing.T) { - statePath := filepath.Join(t.TempDir(), "resources.json") - original := NewDatabase("lineage", 2) - original.Features = map[string]struct{}{} - original.State["resources.jobs.foo"] = ResourceEntry{ID: "123", State: json.RawMessage(`{"name":"original"}`)} - var db DeploymentState - db.OpenWithData(statePath, original) - require.NoError(t, db.Persist()) - before, err := os.ReadFile(statePath) - require.NoError(t, err) - - require.NoError(t, db.MarkDeploymentHistory(3)) - - assert.True(t, db.IsDeploymentMetadataService()) - assert.Equal(t, 3, db.GetSerial()) - assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistory) - assert.Equal(t, original.State, db.Data.State) - assert.Equal(t, "123", db.GetResourceID("resources.jobs.foo")) - after, err := os.ReadFile(statePath) - require.NoError(t, err) - assert.NotEqual(t, before, after) - var persisted Database - require.NoError(t, json.Unmarshal(after, &persisted)) - assert.Equal(t, 3, persisted.Serial) - assert.Contains(t, persisted.Features, FeatureDeploymentHistory) - assert.Empty(t, persisted.State) - assert.NoFileExists(t, statePath+".wal") -} - func TestApplyDMSState(t *testing.T) { // The service stores state as an opaque string, so the envelope the write path sent // arrives verbatim. From f6a63c2db1d3a286c0e19226ecbd2b6c616e1148 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:31:39 +0000 Subject: [PATCH 19/25] Only refresh migration descriptor fields used downstream --- bundle/phases/dms_migration.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go index 9a3181596c6..b405d6fb4f9 100644 --- a/bundle/phases/dms_migration.go +++ b/bundle/phases/dms_migration.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "maps" - "path/filepath" "slices" "strconv" "strings" @@ -74,12 +73,10 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle, state *statemgmt.StateD } logDeploymentVersion(ctx, b, "Deployment history enabled") statemgmt.PushResourcesState(ctx, b) + // Keep the descriptor at the committed version. Downstream checks Features via + // IsDMS() to validate the history setting and reopen resource state from DMS. state.Serial = db.GetSerial() - state.Lineage = db.Data.Lineage state.Features = db.StateFeatures() - state.SourcePath = filepath.ToSlash(db.Path) - state.IsLocal = true - state.Content = nil } // registerMigrationState records every resource from the open direct state in an initial registration version. From accee1874d8192da001273e5264bbf36a2042255 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:36:10 +0000 Subject: [PATCH 20/25] Explain close-state tests with numbered expectations --- bundle/direct/dstate/state_test.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 963f51b5d28..11f825fbfe7 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -50,34 +50,41 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { } func TestCloseReadState(t *testing.T) { + // 1. Disk has serial 1; change only the in-memory copy to serial 99. path := filepath.Join(t.TempDir(), "resources.json") var db DeploymentState db.OpenWithData(path, NewDatabase("lineage", 1)) require.NoError(t, db.Persist()) - before, err := os.ReadFile(path) + persistedBeforeClose, err := os.ReadFile(path) require.NoError(t, err) db.Data.Serial = 99 + // 2. Closing twice is safe and leaves the database closed. db.Close() db.Close() + assert.False(t, db.IsOpen(), "Close must leave the database closed") - assert.False(t, db.IsOpen()) - after, err := os.ReadFile(path) + // 3. Close does not write the in-memory changes to disk. + persistedAfterClose, err := os.ReadFile(path) require.NoError(t, err) - assert.Equal(t, before, after) + assert.Equal(t, persistedBeforeClose, persistedAfterClose, "Close must not change resources.json") + + // 4. Reopening reads serial 1 from disk, not the discarded serial 99. require.NoError(t, db.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), OpenDmsArgs{})) - assert.Equal(t, 1, db.GetSerial()) + assert.Equal(t, 1, db.GetSerial(), "Open must reload the persisted serial") db.Close() } func TestCloseRefusesWritableState(t *testing.T) { + // 1. Enter write mode. Writable state must use Finalize, not Close. var db DeploymentState db.OpenWithData(filepath.Join(t.TempDir(), "resources.json"), NewDatabase("lineage", 1)) require.NoError(t, db.UpgradeToWrite()) defer mustFinalize(t, &db) + // 2. Reject Close without closing the database, so Finalize can still finish. assert.PanicsWithValue(t, "internal error: DeploymentState must be opened in read mode", db.Close) - assert.True(t, db.IsOpen()) + assert.True(t, db.IsOpen(), "A rejected Close must leave writable state open for Finalize") } func TestDataForFileSerial(t *testing.T) { From e6f058f4eace1f111c36a7c37830a2e229a0b815 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:39:06 +0000 Subject: [PATCH 21/25] Simplify migration unit tests around uncovered cases --- bundle/phases/dms_migration_test.go | 109 +++++++++------------------- 1 file changed, 35 insertions(+), 74 deletions(-) diff --git a/bundle/phases/dms_migration_test.go b/bundle/phases/dms_migration_test.go index 39cd2e769c5..59956a3797c 100644 --- a/bundle/phases/dms_migration_test.go +++ b/bundle/phases/dms_migration_test.go @@ -1,20 +1,16 @@ package phases import ( - "encoding/json" - "os" + "context" "path/filepath" "strconv" "testing" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt" - "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/cmdctx" - "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -24,112 +20,77 @@ import ( ) func TestMigrateToDMSRejectsTerraformBeforeLock(t *testing.T) { + // 1. Try migration with Terraform state and locking enabled by default. b := &bundle.Bundle{} ctx := logdiag.InitContext(t.Context()) logdiag.SetCollect(ctx, true) MigrateToDMS(ctx, b, &statemgmt.StateDesc{Engine: engine.EngineTerraform}) + // 2. Reject the state before creating a deployment lock. require.True(t, logdiag.HasError(ctx)) - diags := logdiag.FlushCollected(ctx) - require.Len(t, diags, 1) - assert.Equal(t, ErrDMSRequiresDirectState.Error(), diags[0].Summary) - assert.Nil(t, b.Locker) + assert.Nil(t, b.Locker, "Terraform state must be rejected before locking") } -func TestMigrateToDMSAlreadyRecorded(t *testing.T) { - b := &bundle.Bundle{ - BundleRootPath: t.TempDir(), - Config: config.Root{ - Bundle: config.Bundle{ - Target: "default", - Deployment: config.Deployment{Lock: config.Lock{Enabled: new(true)}}, - }, - Workspace: config.Workspace{StatePath: "/Workspace/state"}, - }, - } +func TestMigrateToDMSSkipsAlreadyRecordedBeforeLock(t *testing.T) { + // 1. State already uses DMS; no open database or workspace client is needed. + b := &bundle.Bundle{} ctx := logdiag.InitContext(t.Context()) logdiag.SetCollect(ctx, true) - b.DeploymentBundle.StateDB.OpenWithData(filepath.Join(b.BundleRootPath, "resources.json"), dstate.NewDatabase("lineage", 2)) state := &statemgmt.StateDesc{ - Serial: 2, Features: map[string]struct{}{dstate.FeatureDeploymentHistory: {}}, } MigrateToDMS(ctx, b, state) + // 2. Return successfully without acquiring a migration lock. require.False(t, logdiag.HasError(ctx), "%v", logdiag.FlushCollected(ctx)) - assert.True(t, state.IsDMS()) - assert.Equal(t, 2, state.Serial) - assert.Nil(t, b.Locker) - assert.True(t, b.DeploymentBundle.StateDB.IsOpen()) + assert.Nil(t, b.Locker, "Already-recorded state must not acquire a migration lock") } -func TestRegisterMigrationState(t *testing.T) { +func TestRegisterMigrationStateVersionNumbers(t *testing.T) { + // Acceptance tests cover registration and persistence. These cases cover unequal counters. tests := []struct { - name string - serial int - lastVersion int - registration int + name string + stateSerial int + serviceVersion int + wantVersion int }{ { - name: "first registration", - serial: 2, - registration: 3, + name: "service version is newer", + stateSerial: 2, + serviceVersion: 5, + wantVersion: 6, }, { - name: "service ahead", - serial: 2, - lastVersion: 5, - registration: 6, - }, - { - name: "file ahead", - serial: 8, - lastVersion: 3, - registration: 9, + name: "state serial is newer", + stateSerial: 8, + serviceVersion: 3, + wantVersion: 9, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // 1. Open state with deliberately different file and service counters. statePath := filepath.Join(t.TempDir(), "resources.json") var db dstate.DeploymentState - db.OpenWithData(statePath, dstate.NewDatabase("lineage", tt.serial)) + db.OpenWithData(statePath, dstate.NewDatabase("lineage", tt.stateSerial)) db.DeploymentID = "123" - db.VersionID = tt.lastVersion + db.VersionID = tt.serviceVersion + + // 2. Registration uses max(file, service)+1, but its predecessor is the service version. m := mocks.NewMockWorkspaceClient(t) api := m.GetMockBundleDeploymentsAPI() - previousVersion := "" - if tt.lastVersion > 0 { - previousVersion = strconv.Itoa(tt.lastVersion) - } - api.EXPECT().CreateVersion(mock.Anything, bundledeployments.CreateVersionRequest{ - Parent: dms.DeploymentName("123"), - VersionId: strconv.Itoa(tt.registration), - Version: bundledeployments.Version{ - CliVersion: build.GetInfo().Version, - VersionType: dms.VersionTypeDeploy, - PreviousVersionId: previousVersion, - Operations: []bundledeployments.StagedOperation{}, - }, - }).Return(&bundledeployments.Version{}, nil).Once() - api.EXPECT().CompleteVersion(mock.Anything, bundledeployments.CompleteVersionRequest{ - Name: dms.VersionName("123", tt.registration), - CompletionReason: bundledeployments.VersionCompleteVersionCompleteSuccess, - }).Return(&bundledeployments.Version{}, nil).Once() + api.EXPECT().CreateVersion(mock.Anything, mock.Anything). + Run(func(_ context.Context, request bundledeployments.CreateVersionRequest) { + assert.Equal(t, strconv.Itoa(tt.wantVersion), request.VersionId, "Migration must follow the larger counter") + assert.Equal(t, strconv.Itoa(tt.serviceVersion), request.Version.PreviousVersionId, "Predecessor must be the actual DMS version") + }).Return(&bundledeployments.Version{}, nil).Once() + api.EXPECT().CompleteVersion(mock.Anything, mock.Anything).Return(&bundledeployments.Version{}, nil).Once() ctx := cmdctx.SetWorkspaceClient(t.Context(), m.WorkspaceClient) + // 3. Run registration; the mock checks the version numbers above. require.NoError(t, registerMigrationState(ctx, &db)) - - assert.True(t, db.IsDeploymentMetadataService()) - assert.Equal(t, tt.registration, db.GetSerial()) - content, err := os.ReadFile(statePath) - require.NoError(t, err) - var persisted dstate.Database - require.NoError(t, json.Unmarshal(content, &persisted)) - assert.Contains(t, persisted.Features, dstate.FeatureDeploymentHistory) - assert.Equal(t, tt.registration, persisted.Serial) - assert.Empty(t, persisted.State) }) } } From f1fa38b974c839e676224fadaadcd294990aa04d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:43:00 +0000 Subject: [PATCH 22/25] Explain migration state update in plain language --- bundle/phases/dms_migration.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/phases/dms_migration.go b/bundle/phases/dms_migration.go index b405d6fb4f9..15bee44d7d5 100644 --- a/bundle/phases/dms_migration.go +++ b/bundle/phases/dms_migration.go @@ -73,8 +73,8 @@ func MigrateToDMS(ctx context.Context, b *bundle.Bundle, state *statemgmt.StateD } logDeploymentVersion(ctx, b, "Deployment history enabled") statemgmt.PushResourcesState(ctx, b) - // Keep the descriptor at the committed version. Downstream checks Features via - // IsDMS() to validate the history setting and reopen resource state from DMS. + // Remember the migration version and that state now lives in DMS. + // The deployment code uses this flag to accept the history setting and read resources from DMS. state.Serial = db.GetSerial() state.Features = db.StateFeatures() } From d97a6988a1cfe42db565da5507d9697b7209849f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Oct 2026 11:54:00 +0000 Subject: [PATCH 23/25] Simplify migration acceptance scenarios and visible explanations --- .../bundle/dms/existing-state/output.txt | 47 +++++++------- acceptance/bundle/dms/existing-state/script | 29 +++++---- .../dms/migration-before-deploy/output.txt | 17 +++-- .../bundle/dms/migration-before-deploy/script | 13 ++-- .../bundle/dms/repeated-migration/output.txt | 63 +++++++++++-------- .../bundle/dms/repeated-migration/script | 34 +++++----- 6 files changed, 111 insertions(+), 92 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 76a8aa728df..e6a1802e69c 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -1,5 +1,6 @@ -=== Config disables recording even when the environment enables it +=== 1. Create two jobs without history (direct state serial 1) +Config disables history even though the environment enables it. >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Created jobs.one @@ -7,44 +8,40 @@ Created jobs.two Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +=== Save the original resources.json for the backup check >>> cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json ->>> jq {state_version, serial, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "serial": 1, - "features": null -} - -=== Normal state selection prefers local serial 2 over remote serial 1 ->>> update_file.py .databricks/bundle/default/resources.json "serial": 1 "serial": 2 +=== 2. Enable history and rename job one +Expect: version 2 registers old state; version 3 applies the new name; no new jobs. +>>> update_file.py databricks.yml deployment_history: false deployment_history: true ->>> jq -c {serial} .databricks/bundle/default/resources.json -{"serial":2} +>>> update_file.py databricks.yml name: one name: updated -=== Config enables recording despite the disabled environment: register version 3, then deploy version 4 >>> [CLI] bundle deploy Enabling deployment history... -Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... -Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 +Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 Updated jobs.one Updated jobs.two Files: 1 uploaded, 0 deleted Resources: 0 created, 2 changed, 0 deleted, 0 unchanged +=== 3. Check local state +Expect: serial=3; history enabled; no resource state or WAL. >>> find.py resources[.]json[.]wal --expect 0 >>> jq {serial, features, state} .databricks/bundle/default/resources.json { - "serial": 4, + "serial": 3, "features": { "deployment_history": {} }, "state": {} } -=== Inspect the remote backup's original pre-migration resource IDs, state, and dependencies +=== 4. Check backup +Expect: original IDs, names and dependencies; exact pre-migration bytes. >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.first.json >>> cat .databricks/resources.backup.first.json @@ -102,18 +99,22 @@ Resources: 0 created, 2 changed, 0 deleted, 0 unchanged } } } -=== The backup equals the remote pre-migration snapshot byte for byte >>> cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json -=== Read back the completed migration and deploy versions with their recorded resource state +=== 5. Check DMS records +Expect: completed versions 2 and 3; original job IDs and dependency preserved. +=== Version history: migration 2 completes before deployment 3 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json -[{"version_id":"3","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"4","previous_version_id":"3","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] +[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json +=== Migration 2: register the original job IDs (INITIAL_REGISTER) +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json [{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/4 -o json +=== Deployment 3: record updates separately +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json [{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] +=== Current resources: new name, original IDs, and job two still depends on job one >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json -[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"4","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"4","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 0e001957e97..79b645cb252 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -1,37 +1,36 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Config disables recording even when the environment enables it" +title "1. Create two jobs without history (direct state serial 1)\nConfig disables history even though the environment enables it." trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy 2>&1 | contains.py '!Current Deployment Version:' '!Enabling deployment history' replace_ids.py +title "Save the original resources.json for the backup check" trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json -trace jq '{state_version, serial, features}' .databricks/bundle/default/resources.json - -title "Normal state selection prefers local serial 2 over remote serial 1" -trace update_file.py .databricks/bundle/default/resources.json '"serial": 1' '"serial": 2' -trace jq -c '{serial}' .databricks/bundle/default/resources.json +title "2. Enable history and rename job one\nExpect: version 2 registers old state; version 3 applies the new name; no new jobs." export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false -update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' -update_file.py databricks.yml 'name: one' 'name: updated' +trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' +trace update_file.py databricks.yml 'name: one' 'name: updated' +trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=2' '?version=3' 'Resources: 0 created' -title "Config enables recording despite the disabled environment: register version 3, then deploy version 4" -trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=3' '?version=4' +title "3. Check local state\nExpect: serial=3; history enabled; no resource state or WAL." trace find.py 'resources[.]json[.]wal' --expect 0 trace jq '{serial, features, state}' .databricks/bundle/default/resources.json -title "Inspect the remote backup's original pre-migration resource IDs, state, and dependencies" +title "4. Check backup\nExpect: original IDs, names and dependencies; exact pre-migration bytes." trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json trace cat .databricks/resources.backup.first.json - -title "The backup equals the remote pre-migration snapshot byte for byte" trace cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json -title "Read back the completed migration and deploy versions with their recorded resource state" +title "5. Check DMS records\nExpect: completed versions 2 and 3; original job IDs and dependency preserved." deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') add_repl "$deployment_id" DEPLOYMENT_ID +title "Version history: migration 2 completes before deployment 3" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' +title "Migration 2: register the original job IDs (INITIAL_REGISTER)" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Deployment 3: record updates separately" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/3" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/4" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Current resources: new name, original IDs, and job two still depends on job one" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -c ' sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, state: (.state | fromjson | {name: .state.name, diff --git a/acceptance/bundle/dms/migration-before-deploy/output.txt b/acceptance/bundle/dms/migration-before-deploy/output.txt index 4d2618f2f4f..2e8d58a6f37 100644 --- a/acceptance/bundle/dms/migration-before-deploy/output.txt +++ b/acceptance/bundle/dms/migration-before-deploy/output.txt @@ -1,6 +1,8 @@ -=== Setup (LOG.setup): legacy resources exist; deployment would recreate the schema -=== 1. Migration commits and DMS is reread before ordinary planning fails +=== Setup (LOG.setup): create a job and schema without history +Next deploy enables history and recreates the schema in another catalog, requiring approval. +=== 1. Fail deployment planning +Expect: migration version 2 stays complete; no deployment version is created. >>> fault.py GET /api/2.2/jobs/get* 403 0 1 >>> musterr [CLI] bundle deploy -qq @@ -16,12 +18,14 @@ API message: Fault injected by test. Error: planning failed +=== Request order: complete migration -> upload marker -> read DMS -> plan jobs >>> print_requests.py --dms --get //api/2.0/bundle //jobs/get //workspace/import //workspace-files/import-file --oneline {"method":"POST","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete"} {"method":"POST","path":"/Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json"} {"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} {"method":"GET","path":"/api/2.2/jobs/get"} +=== Remote state: version 2 marker; original resources preserved in the backup >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.marker.json >>> jq -c {serial, features, state} .databricks/resources.marker.json @@ -31,13 +35,15 @@ Error: planning failed >>> cmp .databricks/resources.before-migration.json .databricks/resources.backup.json +=== Migration result: only completed version 2, with INITIAL_REGISTER operations >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json [{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json [{"resource_key":"jobs.job","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"schemas.schema","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] -=== 2. Destructive deployment is declined; committed migration is not repeated +=== 2. Retry without approval +Expect: deployment declined; migration stays at version 2 and is not repeated. >>> musterr [CLI] bundle deploy -qq This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: @@ -47,18 +53,21 @@ Deleting data assets such as schemas, pipelines, or volumes may cause permanent To proceed, use --auto-approve after reviewing the plan above. +=== Local and remote markers must stay byte-for-byte unchanged >>> cmp .databricks/resources.marker.json .databricks/bundle/default/resources.json >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-before-deploy-[UNIQUE_NAME]/default/state/resources.json --format AUTO --file .databricks/resources.after-decline.json >>> cmp .databricks/resources.marker.json .databricks/resources.after-decline.json +=== No DMS writes and no extra version on the declined retry >>> print_requests.py --dms //api/2.0/bundle >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json [{"version_id":"2","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] -=== 3. Approval creates ordinary deployment version 3, not another registration +=== 3. Approve deployment +Expect: ordinary deployment version 3; version 2 remains the only migration. >>> [CLI] bundle deploy --auto-approve -qq This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: diff --git a/acceptance/bundle/dms/migration-before-deploy/script b/acceptance/bundle/dms/migration-before-deploy/script index fa0250749de..18b106a7c48 100644 --- a/acceptance/bundle/dms/migration-before-deploy/script +++ b/acceptance/bundle/dms/migration-before-deploy/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-before-deploy-${UNIQUE_NAME}/default/state" other_catalog="dms_other_${UNIQUE_NAME}" -title "Setup (LOG.setup): legacy resources exist; deployment would recreate the schema" +title "Setup (LOG.setup): create a job and schema without history\nNext deploy enables history and recreates the schema in another catalog, requiring approval." { $CLI bundle deploy -qq DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py @@ -14,32 +14,37 @@ title "Setup (LOG.setup): legacy resources exist; deployment would recreate the trap '$CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1; $CLI catalogs delete "$other_catalog" --force >> LOG.cleanup 2>&1; rm -f "$OUT_REQUESTS"' EXIT rm -f "$OUT_REQUESTS" -title "1. Migration commits and DMS is reread before ordinary planning fails" +title "1. Fail deployment planning\nExpect: migration version 2 stays complete; no deployment version is created." trace fault.py 'GET /api/2.2/jobs/get*' 403 0 1 trace musterr $CLI bundle deploy -qq deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_path/resources.deployment.json" -o json | jq -r '.object_id') add_repl "$deployment_id" DEPLOYMENT_ID +title "Request order: complete migration -> upload marker -> read DMS -> plan jobs" trace print_requests.py --dms --get //api/2.0/bundle //jobs/get //workspace/import //workspace-files/import-file --oneline | MSYS_NO_PATHCONV=1 jq -c --arg marker "$state_path/resources.json" ' (.body.multipart_form.path // .body.path // (.path | ltrimstr("/api/2.0/workspace-files/import-file"))) as $destination | select((.path | endswith("/complete")) or (.path | endswith("/resources")) or .path == "/api/2.2/jobs/get" or $destination == $marker) | {method, path: (if $destination == $marker then $destination else .path end)}' +title "Remote state: version 2 marker; original resources preserved in the backup" trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.marker.json trace jq -c '{serial, features, state}' .databricks/resources.marker.json trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.json trace cmp .databricks/resources.before-migration.json .databricks/resources.backup.json +title "Migration result: only completed version 2, with INITIAL_REGISTER operations" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'map({version_id, status, completion_reason})' trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, action_type, status})' -title "2. Destructive deployment is declined; committed migration is not repeated" +title "2. Retry without approval\nExpect: deployment declined; migration stays at version 2 and is not repeated." trace musterr $CLI bundle deploy -qq +title "Local and remote markers must stay byte-for-byte unchanged" trace cmp .databricks/resources.marker.json .databricks/bundle/default/resources.json trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format AUTO --file .databricks/resources.after-decline.json trace cmp .databricks/resources.marker.json .databricks/resources.after-decline.json +title "No DMS writes and no extra version on the declined retry" trace print_requests.py --dms //api/2.0/bundle trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'map({version_id, status, completion_reason})' -title "3. Approval creates ordinary deployment version 3, not another registration" +title "3. Approve deployment\nExpect: ordinary deployment version 3; version 2 remains the only migration." trace $CLI bundle deploy --auto-approve -qq trace $CLI bundle summary -o json | jq -c '.bundle.deployment | {latest_version_id}' trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, status, completion_reason})' diff --git a/acceptance/bundle/dms/repeated-migration/output.txt b/acceptance/bundle/dms/repeated-migration/output.txt index c568d1fef40..420a829d71b 100644 --- a/acceptance/bundle/dms/repeated-migration/output.txt +++ b/acceptance/bundle/dms/repeated-migration/output.txt @@ -2,18 +2,17 @@ === Setup (LOG.setup) Current: job=updated; DMS version=3; old backup=one. -=== 1. Before -Expect: old backup=one; latest snapshot=updated. +=== 1. Compare the old backup with the latest saved state +Expect: old backup name=one; latest state name=updated. >>> gron.py .databricks/resources.backup.first.json json.state.resources.jobs.one.state.name = "one"; >>> gron.py .databricks/resources.before-repeat.json json.state.resources.jobs.one.state.name = "updated"; -PASS: names checked -=== 2. Repeat deploy -Expect: register=4; deploy=5; name=repeated; created=0. +=== 2. Restore saved state without its history flag and deploy again +Expect: migration=4; deployment=5; name=repeated; no new jobs. >>> MSYS_NO_PATHCONV=1 [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/resources.json --file .databricks/resources.before-repeat.json --format AUTO --overwrite @@ -29,47 +28,60 @@ Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version Updated jobs.one Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged -PASS: repeat deployed -=== 3. DMS records -Expect: versions complete; original ID preserved; name=repeated. +=== 3. Check DMS records +Expect: completed migration 4 and deployment 5; original ID preserved; name=repeated. >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json -version 4: previous=3 type=DEPLOY status=COMPLETED completion=SUCCESS -version 5: previous=4 type=DEPLOY status=COMPLETED completion=SUCCESS +[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"4","previous_version_id":"3","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"5","previous_version_id":"4","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] +=== Migration 4: register the existing job ID again (INITIAL_REGISTER) >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/4 -o json -operation 4: key=jobs.one id=[ONE_ID] action=INITIAL_REGISTER status=SUCCEEDED +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] +=== Deployment 5: update the job name without creating a replacement >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/5 -o json -operation 5: key=jobs.one id=[ONE_ID] action=UPDATE status=SUCCEEDED +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] +=== Current resource: original job ID, name=repeated, last version=5 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json -resource: key=jobs.one id=[ONE_ID] - last_action=UPDATE version=5 name=repeated - tasks=[] dependencies=[] +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_version_id":"5","name":"repeated"}] -=== 4. Backup copy -Expect: backup=updated; bytes match latest and differ from first. +=== 4. Check backup was overwritten +Expect: name=updated (before this deploy), not one or repeated. >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.repeat.json >>> cat .databricks/resources.backup.repeat.json { - "state_version": 3, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 3, - "resources": { - "resources.jobs.one": { - "id": "[ONE_ID]", - "name": "updated" + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "state": { + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "updated", + "queue": { + "enabled": true } + } } + } } +=== Backup matches the restored snapshot byte for byte >>> cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json +=== Compare backups: serial 1 -> 3 and job name one -> updated >>> diff.py .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json --- .databricks/resources.backup.first.json +++ .databricks/resources.backup.repeat.json @@ -92,4 +104,3 @@ Expect: backup=updated; bytes match latest and differ from first. } -} +} -PASS: backup checked diff --git a/acceptance/bundle/dms/repeated-migration/script b/acceptance/bundle/dms/repeated-migration/script index ec3287dc91e..99d363df7f3 100644 --- a/acceptance/bundle/dms/repeated-migration/script +++ b/acceptance/bundle/dms/repeated-migration/script @@ -14,41 +14,35 @@ title "Setup (LOG.setup)\nCurrent: job=updated; DMS version=3; old backup=one.\n trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py > .databricks/resources.before-repeat.json } &> LOG.setup -title "1. Before\nExpect: old backup=one; latest snapshot=updated.\n" +title "1. Compare the old backup with the latest saved state\nExpect: old backup name=one; latest state name=updated.\n" trace gron.py .databricks/resources.backup.first.json | grep '\.state\.name = ' | contains.py '"one"' trace gron.py .databricks/resources.before-repeat.json | grep '\.state\.name = ' | contains.py '"updated"' -echo "PASS: names checked" -title "2. Repeat deploy\nExpect: register=4; deploy=5; name=repeated; created=0.\n" +title "2. Restore saved state without its history flag and deploy again\nExpect: migration=4; deployment=5; name=repeated; no new jobs.\n" trace MSYS_NO_PATHCONV=1 $CLI workspace import "$state_path/resources.json" --file .databricks/resources.before-repeat.json --format AUTO --overwrite trace cp .databricks/resources.before-repeat.json .databricks/bundle/default/resources.json trace update_file.py databricks.yml 'name: updated' 'name: repeated' trace $CLI bundle deploy 2>&1 | contains.py 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=4' '?version=5' 'Resources: 0 created' '!Created ' -echo "PASS: repeat deployed" -title "3. DMS records\nExpect: versions complete; original ID preserved; name=repeated.\n" +title "3. Check DMS records\nExpect: completed migration 4 and deployment 5; original ID preserved; name=repeated.\n" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | - jq 'map(select((.version_id | tonumber) >= 4))' | - jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' | - jq -r '.[] | "version \(.version_id): previous=\(.previous_version_id) type=\(.version_type | ltrimstr("VERSION_TYPE_")) status=\(.status | ltrimstr("VERSION_STATUS_")) completion=\(.completion_reason | ltrimstr("VERSION_COMPLETE_"))"' + jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' +title "Migration 4: register the existing job ID again (INITIAL_REGISTER)" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/4" -o json | - jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' | - jq -r '.[] | "operation 4: key=\(.resource_key) id=\(.resource_id) action=\(.action_type | ltrimstr("OPERATION_ACTION_TYPE_")) status=\(.status | ltrimstr("OPERATION_STATUS_"))"' + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Deployment 5: update the job name without creating a replacement" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/5" -o json | - jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' | - jq -r '.[] | "operation 5: key=\(.resource_key) id=\(.resource_id) action=\(.action_type | ltrimstr("OPERATION_ACTION_TYPE_")) status=\(.status | ltrimstr("OPERATION_STATUS_"))"' + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Current resource: original job ID, name=repeated, last version=5" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | - jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, - state: (.state | fromjson | {name: .state.name, - tasks: [.state.tasks[]? | {task_key, job_id: .run_job_task.job_id}], - depends_on: (.depends_on // [])})})' | - jq -r '.[] | "resource: key=\(.resource_key) id=\(.resource_id)\n last_action=\(.last_action_type | ltrimstr("OPERATION_ACTION_TYPE_")) version=\(.last_version_id) name=\(.state.name)\n tasks=\(.state.tasks | tojson) dependencies=\(.state.depends_on | tojson)"' + jq -c 'map({resource_key, resource_id, last_version_id, name: (.state | fromjson | .state.name)})' -title "4. Backup copy\nExpect: backup=updated; bytes match latest and differ from first.\n" +title "4. Check backup was overwritten\nExpect: name=updated (before this deploy), not one or repeated.\n" trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format AUTO --file .databricks/resources.backup.repeat.json -trace cat .databricks/resources.backup.repeat.json | contains.py '"name": "updated"' '!"name": "repeated"' | jq '{state_version, cli_version, lineage, serial, resources: (.state | map_values({id: .__id__, name: .state.name}))}' +trace cat .databricks/resources.backup.repeat.json | contains.py '"name": "updated"' '!"name": "one"' '!"name": "repeated"' +title "Backup matches the restored snapshot byte for byte" trace cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json +title "Compare backups: serial 1 -> 3 and job name one -> updated" trace diff.py .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json -echo "PASS: backup checked" rm -f "$OUT_REQUESTS" From 3d85450aa0383d03656d6d2c9460d345f46b081d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 21 Oct 2026 11:04:01 +0000 Subject: [PATCH 24/25] Separate migration acceptance from existing-state lifecycle checks --- .../dms/existing-state/databricks.yml.tmpl | 9 -- .../bundle/dms/existing-state/output.txt | 153 ++++++------------ acceptance/bundle/dms/existing-state/script | 57 +++---- .../bundle/dms/existing-state/test.toml | 18 +-- .../bundle/dms/migration/databricks.yml.tmpl | 16 ++ acceptance/bundle/dms/migration/out.test.toml | 2 + acceptance/bundle/dms/migration/output.txt | 120 ++++++++++++++ acceptance/bundle/dms/migration/script | 40 +++++ acceptance/bundle/dms/migration/test.toml | 17 ++ 9 files changed, 275 insertions(+), 157 deletions(-) create mode 100644 acceptance/bundle/dms/migration/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/migration/out.test.toml create mode 100644 acceptance/bundle/dms/migration/output.txt create mode 100644 acceptance/bundle/dms/migration/script create mode 100644 acceptance/bundle/dms/migration/test.toml diff --git a/acceptance/bundle/dms/existing-state/databricks.yml.tmpl b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl index ff21609cb89..29addd6433e 100644 --- a/acceptance/bundle/dms/existing-state/databricks.yml.tmpl +++ b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl @@ -1,16 +1,7 @@ bundle: name: dms-existing-state-$UNIQUE_NAME -experimental: - deployment_history: false - resources: jobs: one: name: one - two: - name: two - tasks: - - task_key: run_one - run_job_task: - job_id: ${resources.jobs.one.id} diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index e6a1802e69c..aace51a1b44 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -1,120 +1,73 @@ -=== 1. Create two jobs without history (direct state serial 1) -Config disables history even though the environment enables it. ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +=== 1. Deploy without history: existing state tracks one job +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Created jobs.one -Created jobs.two Files: 2 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan -o json +{ + "plan_version": [PLAN_VERSION], + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1 +} -=== Save the original resources.json for the backup check ->>> cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== 2. Enable history and rename job one -Expect: version 2 registers old state; version 3 applies the new name; no new jobs. ->>> update_file.py databricks.yml deployment_history: false deployment_history: true +>>> print_requests.py --dms //api/2.0/bundle --oneline ->>> update_file.py databricks.yml name: one name: updated +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": null +} ->>> [CLI] bundle deploy +=== 2. Enable history on the existing deployment +Expect: migration succeeds without recreating the job. +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Enabling deployment history... Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... -Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 Updated jobs.one -Updated jobs.two -Files: 1 uploaded, 0 deleted -Resources: 0 created, 2 changed, 0 deleted, 0 unchanged +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged -=== 3. Check local state -Expect: serial=3; history enabled; no resource state or WAL. ->>> find.py resources[.]json[.]wal --expect 0 +=== 3. Remove local state and deploy again +Expect: remote DMS state is loaded; no migration or resource creation. +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged ->>> jq {serial, features, state} .databricks/bundle/default/resources.json -{ - "serial": 3, - "features": { - "deployment_history": {} - }, - "state": {} -} +=== 4. A leftover local WAL is refused: DMS deployments do not use WALs +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: unexpected WAL file found at [TEST_TMP_DIR]/.databricks/bundle/default/resources.json.wal: this deployment records deployment history, which does not write one -=== 4. Check backup -Expect: original IDs, names and dependencies; exact pre-migration bytes. ->>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.first.json ->>> cat .databricks/resources.backup.first.json -{ - "state_version": 3, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "state": { - "resources.jobs.one": { - "__id__": "[ONE_ID]", - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.two": { - "__id__": "[TWO_ID]", - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "two", - "queue": { - "enabled": true - }, - "tasks": [ - { - "run_job_task": { - "job_id": [ONE_ID] - }, - "task_key": "run_one" - } - ] - }, - "depends_on": [ - { - "node": "resources.jobs.one", - "label": "${resources.jobs.one.id}" - } - ] - } - } -} ->>> cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json +=== 5. Destroy follows recorded state; redeploy starts a new history +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default -=== 5. Check DMS records -Expect: completed versions 2 and 3; original job IDs and dependency preserved. -=== Version history: migration 2 completes before deployment 3 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json -[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] +Destroy: 1 deleted -=== Migration 2: register the original job IDs (INITIAL_REGISTER) ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json -[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Created jobs.one +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -=== Deployment 3: record updates separately ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json -[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] +>>> find.py resources[.]json[.]wal --expect 0 -=== Current resources: new name, original IDs, and job two still depends on job one ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json -[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + } +} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 79b645cb252..80f964ad89b 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -1,40 +1,33 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "1. Create two jobs without history (direct state serial 1)\nConfig disables history even though the environment enables it." -trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy 2>&1 | contains.py '!Current Deployment Version:' '!Enabling deployment history' -replace_ids.py -title "Save the original resources.json for the backup check" -trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json +title "1. Deploy without history: existing state tracks one job" +trace $CLI bundle deploy +dms_plan +trace print_requests.py --dms //api/2.0/bundle --oneline +trace jq '{state_version, features}' .databricks/bundle/default/resources.json -title "2. Enable history and rename job one\nExpect: version 2 registers old state; version 3 applies the new name; no new jobs." -export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false -trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' -trace update_file.py databricks.yml 'name: one' 'name: updated' -trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=2' '?version=3' 'Resources: 0 created' +title "2. Enable history on the existing deployment\nExpect: migration succeeds without recreating the job." +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID -title "3. Check local state\nExpect: serial=3; history enabled; no resource state or WAL." -trace find.py 'resources[.]json[.]wal' --expect 0 -trace jq '{serial, features, state}' .databricks/bundle/default/resources.json +title "3. Remove local state and deploy again\nExpect: remote DMS state is loaded; no migration or resource creation." +rm -rf .databricks +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy -title "4. Check backup\nExpect: original IDs, names and dependencies; exact pre-migration bytes." -trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json -trace cat .databricks/resources.backup.first.json -trace cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json +title "4. A leftover local WAL is refused: DMS deployments do not use WALs" +mkdir -p .databricks/bundle/default +printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal +musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy -title "5. Check DMS records\nExpect: completed versions 2 and 3; original job IDs and dependency preserved." -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') -add_repl "$deployment_id" DEPLOYMENT_ID -title "Version history: migration 2 completes before deployment 3" -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' -title "Migration 2: register the original job IDs (INITIAL_REGISTER)" -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' -title "Deployment 3: record updates separately" -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/3" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' -title "Current resources: new name, original IDs, and job two still depends on job one" -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -c ' - sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, - state: (.state | fromjson | {name: .state.name, - tasks: [.state.tasks[]? | {task_key, job_id: .run_job_task.job_id}], - depends_on: (.depends_on // [])})})' +title "5. Destroy follows recorded state; redeploy starts a new history" +# The rejected WAL remains on disk. Remove it before destroying the recorded deployment. +rm -f .databricks/bundle/default/resources.json.wal +# Turn history off to check that destroy still follows the recorded backend. +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle destroy --auto-approve +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace find.py 'resources[.]json[.]wal' --expect 0 +trace jq '{state_version, features}' .databricks/bundle/default/resources.json +# Registration payloads, versions, and backups are covered by the migration test. rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/existing-state/test.toml b/acceptance/bundle/dms/existing-state/test.toml index 4da5c0d15a3..7499498f419 100644 --- a/acceptance/bundle/dms/existing-state/test.toml +++ b/acceptance/bundle/dms/existing-state/test.toml @@ -1,17 +1,3 @@ -# This test checks config precedence with conflicting environment settings, -# so it drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY instead of the parent's matrix. +# This test deploys without recording first, then turns it on per-command, so it opts out of +# the parent's DMS=true and drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY inline instead. EnvMatrix.DMS = [""] - -# acceptance/bundle/test.toml removes Current Deployment Version for shared DMS/non-DMS goldens. -# Inherited Repls append and cannot be disabled here. -# Mask at Order -1 before the parent's default 0, then restore at +1. -# This lets these tests assert both migration and deployment links. -[[Repls]] -Old = '(?m)^Current Deployment Version: ' -New = '__CURRENT_DEPLOYMENT_VERSION__: ' -Order = -1 - -[[Repls]] -Old = '(?m)^__CURRENT_DEPLOYMENT_VERSION__: ' -New = 'Current Deployment Version: ' -Order = 1 diff --git a/acceptance/bundle/dms/migration/databricks.yml.tmpl b/acceptance/bundle/dms/migration/databricks.yml.tmpl new file mode 100644 index 00000000000..bb879a94254 --- /dev/null +++ b/acceptance/bundle/dms/migration/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: dms-migration-$UNIQUE_NAME + +experimental: + deployment_history: false + +resources: + jobs: + one: + name: one + two: + name: two + tasks: + - task_key: run_one + run_job_task: + job_id: ${resources.jobs.one.id} diff --git a/acceptance/bundle/dms/migration/out.test.toml b/acceptance/bundle/dms/migration/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/dms/migration/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/migration/output.txt b/acceptance/bundle/dms/migration/output.txt new file mode 100644 index 00000000000..808dea82789 --- /dev/null +++ b/acceptance/bundle/dms/migration/output.txt @@ -0,0 +1,120 @@ + +=== 1. Create two jobs without history (direct state serial 1) +Config disables history even though the environment enables it. +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/files... +Created jobs.one +Created jobs.two +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Save the original resources.json for the backup check +>>> cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json + +=== 2. Enable history and rename job one +Expect: version 2 registers old state; version 3 applies the new name; no new jobs. +>>> update_file.py databricks.yml deployment_history: false deployment_history: true + +>>> update_file.py databricks.yml name: one name: updated + +>>> [CLI] bundle deploy +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/files... +Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 +Updated jobs.one +Updated jobs.two +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + +=== 3. Check local state +Expect: serial=3; history enabled; no resource state or WAL. +>>> find.py resources[.]json[.]wal --expect 0 + +>>> jq {serial, features, state} .databricks/bundle/default/resources.json +{ + "serial": 3, + "features": { + "deployment_history": {} + }, + "state": {} +} + +=== 4. Check backup +Expect: original IDs, names and dependencies; exact pre-migration bytes. +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/state/resources.json.backup --format AUTO --file .databricks/resources.backup.first.json + +>>> cat .databricks/resources.backup.first.json +{ + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "state": { + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[TWO_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-migration-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [ONE_ID] + }, + "task_key": "run_one" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.one", + "label": "${resources.jobs.one.id}" + } + ] + } + } +} +>>> cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json + +=== 5. Check DMS records +Expect: completed versions 2 and 3; original job IDs and dependency preserved. +=== Version history: migration 2 completes before deployment 3 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +=== Migration 2: register the original job IDs (INITIAL_REGISTER) +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] + +=== Deployment 3: record updates separately +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] + +=== Current resources: new name, original IDs, and job two still depends on job one +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] diff --git a/acceptance/bundle/dms/migration/script b/acceptance/bundle/dms/migration/script new file mode 100644 index 00000000000..9e1b6010261 --- /dev/null +++ b/acceptance/bundle/dms/migration/script @@ -0,0 +1,40 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "1. Create two jobs without history (direct state serial 1)\nConfig disables history even though the environment enables it." +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy 2>&1 | contains.py '!Current Deployment Version:' '!Enabling deployment history' +replace_ids.py +title "Save the original resources.json for the backup check" +trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json + +title "2. Enable history and rename job one\nExpect: version 2 registers old state; version 3 applies the new name; no new jobs." +export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false +trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' +trace update_file.py databricks.yml 'name: one' 'name: updated' +trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=2' '?version=3' 'Resources: 0 created' + +title "3. Check local state\nExpect: serial=3; history enabled; no resource state or WAL." +trace find.py 'resources[.]json[.]wal' --expect 0 +trace jq '{serial, features, state}' .databricks/bundle/default/resources.json + +title "4. Check backup\nExpect: original IDs, names and dependencies; exact pre-migration bytes." +trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-${UNIQUE_NAME}/default/state/resources.json.backup" --format AUTO --file .databricks/resources.backup.first.json +trace cat .databricks/resources.backup.first.json +trace cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json + +title "5. Check DMS records\nExpect: completed versions 2 and 3; original job IDs and dependency preserved." +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-migration-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID +title "Version history: migration 2 completes before deployment 3" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' +title "Migration 2: register the original job IDs (INITIAL_REGISTER)" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Deployment 3: record updates separately" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/3" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +title "Current resources: new name, original IDs, and job two still depends on job one" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -c ' + sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, + state: (.state | fromjson | {name: .state.name, + tasks: [.state.tasks[]? | {task_key, job_id: .run_job_task.job_id}], + depends_on: (.depends_on // [])})})' + +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/migration/test.toml b/acceptance/bundle/dms/migration/test.toml new file mode 100644 index 00000000000..4da5c0d15a3 --- /dev/null +++ b/acceptance/bundle/dms/migration/test.toml @@ -0,0 +1,17 @@ +# This test checks config precedence with conflicting environment settings, +# so it drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY instead of the parent's matrix. +EnvMatrix.DMS = [""] + +# acceptance/bundle/test.toml removes Current Deployment Version for shared DMS/non-DMS goldens. +# Inherited Repls append and cannot be disabled here. +# Mask at Order -1 before the parent's default 0, then restore at +1. +# This lets these tests assert both migration and deployment links. +[[Repls]] +Old = '(?m)^Current Deployment Version: ' +New = '__CURRENT_DEPLOYMENT_VERSION__: ' +Order = -1 + +[[Repls]] +Old = '(?m)^__CURRENT_DEPLOYMENT_VERSION__: ' +New = 'Current Deployment Version: ' +Order = 1 From 836970baebbad42d50c6636a8db183c24d2f4941 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 21 Oct 2026 12:51:07 +0000 Subject: [PATCH 25/25] Restore fresh DMS deployment assertion after destroy --- acceptance/bundle/dms/existing-state/output.txt | 7 +++++++ acceptance/bundle/dms/existing-state/script | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index aace51a1b44..a239b1b14c1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -56,12 +56,19 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted +=== Fresh history: version 1 creates the job without migration >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Created jobs.one Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID_2]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID_2]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID_2]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID_2]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + >>> find.py resources[.]json[.]wal --expect 0 >>> jq {state_version, features} .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 80f964ad89b..2a9a48abc90 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -25,9 +25,15 @@ title "5. Destroy follows recorded state; redeploy starts a new history" rm -f .databricks/bundle/default/resources.json.wal # Turn history off to check that destroy still follows the recorded backend. trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle destroy --auto-approve +title "Fresh history: version 1 creates the job without migration" +rm -f "$OUT_REQUESTS" trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline trace find.py 'resources[.]json[.]wal' --expect 0 trace jq '{state_version, features}' .databricks/bundle/default/resources.json +DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID -# Registration payloads, versions, and backups are covered by the migration test. +# Drop telemetry and state probes recorded after the fresh-deployment assertion above. rm -f "$OUT_REQUESTS"