diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index d40adbca178..6ad674342c3 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -184,8 +184,8 @@ def print_recorded_state(filename, target): # printed state matches a non-recording run without every caller piping through nostamp. data["state"] = scrub(get_recorded_state(target)) - # The service owns the version and the file persists no serial, so take it from the deployment. - # Rebuilt in header order, since the file has no serial key to overwrite in place. + # The file's serial is a copy of the DMS version to ensure outdated resource state is not read. + # Rebuild in header order with the authoritative serial from the deployment. last_version_id = get_last_version_id(target) serial = int(last_version_id) if last_version_id else 0 diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index e1f52e355ff..3ef52219df5 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -97,6 +97,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index d25afa59255..c1b9beb1dec 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -89,6 +89,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 4bf5a7efa43..0a6cbabcbce 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -161,6 +161,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/existing-state/databricks.yml.tmpl b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl index 29addd6433e..ff21609cb89 100644 --- a/acceptance/bundle/dms/existing-state/databricks.yml.tmpl +++ b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl @@ -1,7 +1,16 @@ bundle: name: dms-existing-state-$UNIQUE_NAME +experimental: + deployment_history: false + resources: jobs: one: name: one + two: + name: two + tasks: + - task_key: run_one + run_job_task: + job_id: ${resources.jobs.one.id} diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 4057525509b..0001ca7a7b5 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -1,78 +1,111 @@ -=== Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS ->>> [CLI] bundle deploy +=== Config disables recording even when the environment enables it +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Created jobs.one +Created jobs.two Files: 2 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> [CLI] bundle plan -o json -{ - "plan_version": [PLAN_VERSION], - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1 -} +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +>>> cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json ->>> print_requests.py --dms //api/2.0/bundle --oneline - ->>> jq {state_version, features} .databricks/bundle/default/resources.json +>>> jq {state_version, serial, features} .databricks/bundle/default/resources.json { "state_version": 3, + "serial": 1, "features": null } -=== Enabling recording is refused because the existing deployment is not recorded ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: deployment history setting (true) does not match the existing state (false) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over - - ->>> print_requests.py --dms //api/2.0/bundle --oneline - -=== Wiping the local cache still refuses after pulling the non-DMS state from the workspace ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: deployment history setting (true) does not match the existing state (false) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over - - -=== A leftover local WAL with the wrong lineage is still refused ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: reading state from [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: WAL recovery failed: WAL lineage ("stale") does not match state lineage ("[UUID]") - - -=== Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.one - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default - -Destroy: 1 deleted - ->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +=== Config enables recording despite the disabled environment: register version 2, then deploy version 3 +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... -Created jobs.one -Files: 2 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=2 +Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=3 +Updated jobs.one +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> find.py resources[.]json[.]wal --expect 0 ->>> jq {state_version, features} .databricks/bundle/default/resources.json +>>> jq {features, state} .databricks/bundle/default/resources.json { - "state_version": 3, "features": { "deployment_history": {} + }, + "state": {} +} + +=== Inspect the remote backup's original pre-migration resource IDs, state, and dependencies +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json.backup --format RAW --file .databricks/resources.backup.first.json + +>>> cat .databricks/resources.backup.first.json +{ + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "state": { + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[TWO_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [ONE_ID] + }, + "task_key": "run_one" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.one", + "label": "${resources.jobs.one.id}" + } + ] } + } } +=== The backup equals the pre-migration resources.json byte for byte +>>> cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json + +=== Read back the completed migration and deploy versions with their recorded resource state +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +[{"version_id":"2","previous_version_id":null,"version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"},{"version_id":"3","previous_version_id":"2","version_type":"VERSION_TYPE_DEPLOY","status":"VERSION_STATUS_COMPLETED","completion_reason":"VERSION_COMPLETE_SUCCESS"}] + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","status":"OPERATION_STATUS_SUCCEEDED"}] + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/3 -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","action_type":"OPERATION_ACTION_TYPE_UPDATE","status":"OPERATION_STATUS_SUCCEEDED"}] + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +[{"resource_key":"jobs.one","resource_id":"[ONE_ID]","last_action_type":"OPERATION_ACTION_TYPE_UPDATE","last_version_id":"3","state":{"name":"updated","tasks":[],"depends_on":[]}},{"resource_key":"jobs.two","resource_id":"[TWO_ID]","last_action_type":"OPERATION_ACTION_TYPE_INITIAL_REGISTER","last_version_id":"2","state":{"name":"two","tasks":[{"task_key":"run_one","job_id":[ONE_ID]}],"depends_on":[{"node":"resources.jobs.one","label":"${resources.jobs.one.id}"}]}}] diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index e64ca6febe6..dec0741aafc 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -1,38 +1,36 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" -trace $CLI bundle deploy -dms_plan -trace print_requests.py --dms //api/2.0/bundle --oneline -trace jq '{state_version, features}' .databricks/bundle/default/resources.json +title "Config disables recording even when the environment enables it" +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy 2>&1 | contains.py '!Current Deployment Version:' '!Enabling deployment history' +replace_ids.py +trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json +trace jq '{state_version, serial, features}' .databricks/bundle/default/resources.json -title "Enabling recording is refused because the existing deployment is not recorded" -musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle --oneline +title "Config enables recording despite the disabled environment: register version 2, then deploy version 3" +export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false +update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' +update_file.py databricks.yml 'name: one' 'name: updated' +trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=2' '?version=3' +trace find.py 'resources[.]json[.]wal' --expect 0 +trace jq '{features, state}' .databricks/bundle/default/resources.json -title "Wiping the local cache still refuses after pulling the non-DMS state from the workspace" -rm -rf .databricks -musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +title "Inspect the remote backup's original pre-migration resource IDs, state, and dependencies" +trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.json.backup" --format RAW --file .databricks/resources.backup.first.json +trace cat .databricks/resources.backup.first.json -title "A leftover local WAL with the wrong lineage is still refused" -mkdir -p .databricks/bundle/default -printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal -musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy +title "The backup equals the pre-migration resources.json byte for byte" +trace cmp .databricks/resources.before-dms.json .databricks/resources.backup.first.json -title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone" -# Clear the stale WAL first: it was refused, not deleted, and destroy runs without recording, so it -# would otherwise try to replay it and fail on the lineage mismatch. -rm -f .databricks/bundle/default/resources.json.wal -# Keep the mismatched setting here to verify that destroy ignores configuration and follows state. -trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve -trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle --oneline -trace find.py 'resources[.]json[.]wal' --expect 0 -trace jq '{state_version, features}' .databricks/bundle/default/resources.json -DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +title "Read back the completed migration and deploy versions with their recorded resource state" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') add_repl "$deployment_id" DEPLOYMENT_ID +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/3" -o json | jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -c ' + sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, + state: (.state | fromjson | {name: .state.name, + tasks: [.state.tasks[]? | {task_key, job_id: .run_job_task.job_id}], + depends_on: (.depends_on // [])})})' -# The recorded requests are asserted above via print_requests.py; drop whatever the run added after -# that (telemetry, and the state probes of the non-recording variant) so nothing is left to compare. rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/existing-state/test.toml b/acceptance/bundle/dms/existing-state/test.toml index 7499498f419..4da5c0d15a3 100644 --- a/acceptance/bundle/dms/existing-state/test.toml +++ b/acceptance/bundle/dms/existing-state/test.toml @@ -1,3 +1,17 @@ -# This test deploys without recording first, then turns it on per-command, so it opts out of -# the parent's DMS=true and drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY inline instead. +# This test checks config precedence with conflicting environment settings, +# so it drives DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY instead of the parent's matrix. EnvMatrix.DMS = [""] + +# acceptance/bundle/test.toml removes Current Deployment Version for shared DMS/non-DMS goldens. +# Inherited Repls append and cannot be disabled here. +# Mask at Order -1 before the parent's default 0, then restore at +1. +# This lets these tests assert both migration and deployment links. +[[Repls]] +Old = '(?m)^Current Deployment Version: ' +New = '__CURRENT_DEPLOYMENT_VERSION__: ' +Order = -1 + +[[Repls]] +Old = '(?m)^__CURRENT_DEPLOYMENT_VERSION__: ' +New = 'Current Deployment Version: ' +Order = 1 diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index fd8d9a72e61..8203cdf34f2 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -90,6 +90,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, @@ -181,6 +182,7 @@ API message: Fault injected by test. "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index ddbd5707b6c..860460c3375 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -90,6 +90,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, @@ -176,6 +177,7 @@ Files: 2 uploaded, 0 deleted "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index c73181e7247..47ef6cd6eac 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -43,6 +43,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, @@ -97,6 +98,7 @@ Files: 1 uploaded, 0 deleted "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index f090abba0fe..10c8a496c2b 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -68,6 +68,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, @@ -133,6 +134,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index c0b7a35688e..c2fe06a747e 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -48,6 +48,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, @@ -197,6 +198,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 4d417d6adb7..dc976dadead 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -37,6 +37,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, @@ -79,6 +80,7 @@ Files: 2 uploaded, 0 deleted "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 7719a98aed7..4decd10218b 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -98,6 +98,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index a35bb951a32..9f61f4a9274 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -90,6 +90,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl b/acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl new file mode 100644 index 00000000000..6c29e76684e --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: dms-repeated-migration-$UNIQUE_NAME + +experimental: + deployment_history: false + +sync: + exclude: + - LOG.* + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/repeated-migration/out.test.toml b/acceptance/bundle/dms/repeated-migration/out.test.toml new file mode 100644 index 00000000000..84b22ba6735 --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/repeated-migration/output.txt b/acceptance/bundle/dms/repeated-migration/output.txt new file mode 100644 index 00000000000..03ce38d7602 --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/output.txt @@ -0,0 +1,72 @@ + +=== Setup (LOG.setup) +Current: job=updated; DMS version=3; old backup=one. + +=== 1. Before +Expect: old backup=one; latest snapshot=updated. + +>>> gron.py .databricks/resources.backup.first.json +json.state.resources.jobs.one.state.name = "one"; + +>>> gron.py .databricks/resources.before-repeat.json +json.state.resources.jobs.one.state.name = "updated"; +PASS: names checked + +=== 2. Repeat deploy +Expect: register=4; deploy=5; name=repeated; created=0. + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/resources.json --file .databricks/resources.before-repeat.json --format AUTO --overwrite + +>>> update_file.py databricks.yml name: updated name: repeated + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/files... +Enabling deployment history... +Deployment history enabled: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=4 +Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=5 +Updated jobs.one +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +PASS: repeat deployed + +=== 3. DMS records +Expect: versions complete; original ID preserved; name=repeated. + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-versions deployments/[DEPLOYMENT_ID] -o json +version 4: previous=3 type=DEPLOY status=COMPLETED completion=SUCCESS +version 5: previous=4 type=DEPLOY status=COMPLETED completion=SUCCESS + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/4 -o json +operation 4: key=jobs.one id=[ONE_ID] action=INITIAL_REGISTER status=SUCCEEDED + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/5 -o json +operation 5: key=jobs.one id=[ONE_ID] action=UPDATE status=SUCCEEDED + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +resource: key=jobs.one id=[ONE_ID] + last_action=UPDATE version=5 name=repeated + tasks=[] dependencies=[] + +=== 4. Backup copy +Expect: backup=updated; bytes match latest and differ from first. + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-repeated-migration-[UNIQUE_NAME]/default/state/resources.json.backup --format RAW --file .databricks/resources.backup.repeat.json + +>>> cat .databricks/resources.backup.repeat.json +{ + "state_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "resources": { + "resources.jobs.one": { + "id": "[ONE_ID]", + "name": "updated" + } + } +} + +>>> cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json + +>>> musterr cmp -s .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json +PASS: backup checked diff --git a/acceptance/bundle/dms/repeated-migration/script b/acceptance/bundle/dms/repeated-migration/script new file mode 100644 index 00000000000..54ace9fe35e --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/script @@ -0,0 +1,53 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-repeated-migration-${UNIQUE_NAME}/default/state" +title "Setup (LOG.setup)\nCurrent: job=updated; DMS version=3; old backup=one.\n" +{ + trace $CLI bundle deploy + trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false replace_ids.py + trace update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' + trace update_file.py databricks.yml 'name: one' 'name: updated' + trace $CLI bundle deploy + deployment_id=$(trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "$state_path/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') + add_repl "$deployment_id" DEPLOYMENT_ID + trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format RAW --file .databricks/resources.backup.first.json + trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py > .databricks/resources.before-repeat.json +} &> LOG.setup + +title "1. Before\nExpect: old backup=one; latest snapshot=updated.\n" +trace gron.py .databricks/resources.backup.first.json | grep '\.state\.name = ' | contains.py '"one"' +trace gron.py .databricks/resources.before-repeat.json | grep '\.state\.name = ' | contains.py '"updated"' +echo "PASS: names checked" + +title "2. Repeat deploy\nExpect: register=4; deploy=5; name=repeated; created=0.\n" +trace MSYS_NO_PATHCONV=1 $CLI workspace import "$state_path/resources.json" --file .databricks/resources.before-repeat.json --format AUTO --overwrite +trace update_file.py databricks.yml 'name: updated' 'name: repeated' +trace $CLI bundle deploy 2>&1 | contains.py 'Deployment history enabled: ' 'Current Deployment Version: ' '?version=4' '?version=5' 'Resources: 0 created' '!Created ' +echo "PASS: repeat deployed" + +title "3. DMS records\nExpect: versions complete; original ID preserved; name=repeated.\n" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-versions "deployments/${deployment_id}" -o json | + jq 'map(select((.version_id | tonumber) >= 4))' | + jq -c 'sort_by(.version_id | tonumber) | map({version_id, previous_version_id, version_type, status, completion_reason})' | + jq -r '.[] | "version \(.version_id): previous=\(.previous_version_id) type=\(.version_type | ltrimstr("VERSION_TYPE_")) status=\(.status | ltrimstr("VERSION_STATUS_")) completion=\(.completion_reason | ltrimstr("VERSION_COMPLETE_"))"' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/4" -o json | + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' | + jq -r '.[] | "operation 4: key=\(.resource_key) id=\(.resource_id) action=\(.action_type | ltrimstr("OPERATION_ACTION_TYPE_")) status=\(.status | ltrimstr("OPERATION_STATUS_"))"' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/5" -o json | + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, action_type, status})' | + jq -r '.[] | "operation 5: key=\(.resource_key) id=\(.resource_id) action=\(.action_type | ltrimstr("OPERATION_ACTION_TYPE_")) status=\(.status | ltrimstr("OPERATION_STATUS_"))"' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | + jq -c 'sort_by(.resource_key) | map({resource_key, resource_id, last_action_type, last_version_id, + state: (.state | fromjson | {name: .state.name, + tasks: [.state.tasks[]? | {task_key, job_id: .run_job_task.job_id}], + depends_on: (.depends_on // [])})})' | + jq -r '.[] | "resource: key=\(.resource_key) id=\(.resource_id)\n last_action=\(.last_action_type | ltrimstr("OPERATION_ACTION_TYPE_")) version=\(.last_version_id) name=\(.state.name)\n tasks=\(.state.tasks | tojson) dependencies=\(.state.depends_on | tojson)"' + +title "4. Backup copy\nExpect: backup=updated; bytes match latest and differ from first.\n" +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json.backup" --format RAW --file .databricks/resources.backup.repeat.json +trace cat .databricks/resources.backup.repeat.json | contains.py '"name": "updated"' '!"name": "repeated"' | jq '{state_version, cli_version, lineage, serial, resources: (.state | map_values({id: .__id__, name: .state.name}))}' +trace cmp .databricks/resources.before-repeat.json .databricks/resources.backup.repeat.json +trace musterr cmp -s .databricks/resources.backup.first.json .databricks/resources.backup.repeat.json +echo "PASS: backup checked" + +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/repeated-migration/test.toml b/acceptance/bundle/dms/repeated-migration/test.toml new file mode 100644 index 00000000000..8182a51630b --- /dev/null +++ b/acceptance/bundle/dms/repeated-migration/test.toml @@ -0,0 +1,13 @@ +# acceptance/bundle/test.toml removes Current Deployment Version for shared DMS/non-DMS goldens. +# Inherited Repls append and cannot be disabled here. +# Mask at Order -1 before the parent's default 0, then restore at +1. +# This lets these tests assert both migration and deployment links. +[[Repls]] +Old = '(?m)^Current Deployment Version: ' +New = '__CURRENT_DEPLOYMENT_VERSION__: ' +Order = -1 + +[[Repls]] +Old = '(?m)^__CURRENT_DEPLOYMENT_VERSION__: ' +New = 'Current Deployment Version: ' +Order = 1 diff --git a/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl b/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl new file mode 100644 index 00000000000..a9b1eada1f9 --- /dev/null +++ b/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: dms-stale-local-state-$UNIQUE_NAME + +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/stale-local-state/out.test.toml b/acceptance/bundle/dms/stale-local-state/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/dms/stale-local-state/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/stale-local-state/output.txt b/acceptance/bundle/dms/stale-local-state/output.txt new file mode 100644 index 00000000000..32b359dbf87 --- /dev/null +++ b/acceptance/bundle/dms/stale-local-state/output.txt @@ -0,0 +1,31 @@ + +=== Setup (LOG.setup): reconstruct full version 1 without a history marker, then deploy version 2 +>>> jq -c {serial, features} .databricks/resources.v1.json +{"serial":1,"features":null} + +>>> jq -c {serial, features} .databricks/resources.v2.json +{"serial":2,"features":{"deployment_history":{}}} + +=== 1. Config false and env false: newer remote state replaces the local copy and reads DMS +>>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json + +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json +Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state. +{"job_id":"[FOO_ID]"} + +>>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json + +>>> print_requests.py --dms //resources --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} + +=== 2. Config removed and env false: restore the same snapshot, then read remote DMS state +>>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json + +>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json +Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state. +{"job_id":"[FOO_ID]"} + +>>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json + +>>> print_requests.py --dms //resources --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} diff --git a/acceptance/bundle/dms/stale-local-state/script b/acceptance/bundle/dms/stale-local-state/script new file mode 100644 index 00000000000..ee16e686128 --- /dev/null +++ b/acceptance/bundle/dms/stale-local-state/script @@ -0,0 +1,33 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Setup (LOG.setup): reconstruct full version 1 without a history marker, then deploy version 2" +{ + $CLI bundle deploy + DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py > .databricks/resources.v1.json + DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py + update_file.py databricks.yml 'name: foo' 'name: updated' + $CLI bundle deploy + cp .databricks/bundle/default/resources.json .databricks/resources.v2.json + deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-local-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') + add_repl "$deployment_id" DEPLOYMENT_ID +} &> LOG.setup +trace jq -c '{serial, features}' .databricks/resources.v1.json +trace jq -c '{serial, features}' .databricks/resources.v2.json +rm -f "$OUT_REQUESTS" + +title "1. Config false and env false: newer remote state replaces the local copy and reads DMS" +update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' +trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}' +trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json +trace print_requests.py --dms //resources --get --oneline + +title "2. Config removed and env false: restore the same snapshot, then read remote DMS state" +update_file.py databricks.yml $'experimental:\n deployment_history: false\n\n' '' +trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}' +trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json +trace print_requests.py --dms //resources --get --oneline + +DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve &> LOG.cleanup +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/stale-local-state/test.toml b/acceptance/bundle/dms/stale-local-state/test.toml new file mode 100644 index 00000000000..fa9adf06e5f --- /dev/null +++ b/acceptance/bundle/dms/stale-local-state/test.toml @@ -0,0 +1 @@ +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 71c39e8e24b..e9fbc70f832 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -33,6 +33,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, @@ -73,7 +74,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> jq {serial} .databricks/bundle/default/resources.json { - "serial": null + "serial": 2 } === Replaying the older plan is rejected: same deployment, outdated version diff --git a/acceptance/bundle/dms/state-resolution/databricks.yml.tmpl b/acceptance/bundle/dms/state-resolution/databricks.yml.tmpl new file mode 100644 index 00000000000..3927e7b160e --- /dev/null +++ b/acceptance/bundle/dms/state-resolution/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: dms-state-resolution-$UNIQUE_NAME + +experimental: + deployment_history: true + +sync: + exclude: + - LOG.* + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/state-resolution/out.test.toml b/acceptance/bundle/dms/state-resolution/out.test.toml new file mode 100644 index 00000000000..84b22ba6735 --- /dev/null +++ b/acceptance/bundle/dms/state-resolution/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/state-resolution/output.txt b/acceptance/bundle/dms/state-resolution/output.txt new file mode 100644 index 00000000000..b2307d2f738 --- /dev/null +++ b/acceptance/bundle/dms/state-resolution/output.txt @@ -0,0 +1,50 @@ + +=== Setup (LOG.setup) +Current: DMS enabled; remote reference snapshot saved. + +=== 1. Stale cache +Expect: remote beats serial 999/wrong lineage; cache matches remote. + +>>> jq {serial, lineage} .databricks/bundle/default/resources.json +{ + "serial": 999, + "lineage": "stale-[UUID]" +} + +>>> [CLI] bundle debug states --force-pull +resources.json: remote direct state serial=1 lineage="[UUID]" + +>>> cmp .databricks/resources.remote.json .databricks/bundle/default/resources.json +PASS: stale cache replaced + +=== 2. Invalid cache +Input: YAML in JSON cache paths. +Expect: use remote state; overwrite broken JSON; remove Terraform cache; no resource changes. + +>>> cp databricks.yml .databricks/bundle/default/resources.json + +>>> mkdir -p .databricks/bundle/default/terraform + +>>> cp databricks.yml .databricks/bundle/default/terraform/terraform.tfstate + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> test ! -e .databricks/bundle/default/terraform/terraform.tfstate + +>>> cmp .databricks/resources.remote.json .databricks/bundle/default/resources.json +PASS: invalid caches handled + +=== 3. Missing remote +Expect: bad JSON fails; local cache bytes remain unchanged. + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace delete /Workspace/Users/[USERNAME]/.bundle/dms-state-resolution-[UNIQUE_NAME]/default/state/resources.json + +>>> cp databricks.yml .databricks/bundle/default/resources.json + +>>> musterr [CLI] bundle plan +Error: invalid character 'b' looking for beginning of value + + +>>> cmp databricks.yml .databricks/bundle/default/resources.json +PASS: failure preserved cache diff --git a/acceptance/bundle/dms/state-resolution/script b/acceptance/bundle/dms/state-resolution/script new file mode 100644 index 00000000000..284e212dd88 --- /dev/null +++ b/acceptance/bundle/dms/state-resolution/script @@ -0,0 +1,40 @@ +envsubst < databricks.yml.tmpl > databricks.yml +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-state-resolution-${UNIQUE_NAME}/default/state" + +title "Setup (LOG.setup)\nCurrent: DMS enabled; remote reference snapshot saved.\n" +{ + trace $CLI bundle deploy + trace cp .databricks/bundle/default/resources.json .databricks/resources.remote.json +} &> LOG.setup + +title "1. Stale cache\nExpect: remote beats serial 999/wrong lineage; cache matches remote.\n" +{ + trace update_file.py .databricks/bundle/default/resources.json '"serial": 1,' '"serial": 999,' + trace update_file.py .databricks/bundle/default/resources.json '"lineage": "' '"lineage": "stale-' +} &> LOG.stale-cache +trace jq '{serial, lineage}' .databricks/bundle/default/resources.json +trace $CLI bundle debug states --force-pull 2>&1 | contains.py 'remote direct state' '!local ' '!stale-' '!serial=999' +trace cmp .databricks/resources.remote.json .databricks/bundle/default/resources.json +echo "PASS: stale cache replaced" + +title "2. Invalid cache\nInput: YAML in JSON cache paths.\nExpect: use remote state; overwrite broken JSON; remove Terraform cache; no resource changes.\n" +trace cp databricks.yml .databricks/bundle/default/resources.json +trace mkdir -p .databricks/bundle/default/terraform +trace cp databricks.yml .databricks/bundle/default/terraform/terraform.tfstate +trace $CLI bundle plan | contains.py 'Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged' +trace test ! -e .databricks/bundle/default/terraform/terraform.tfstate +trace cmp .databricks/resources.remote.json .databricks/bundle/default/resources.json +echo "PASS: invalid caches handled" + +title "3. Missing remote\nExpect: bad JSON fails; local cache bytes remain unchanged.\n" +trace MSYS_NO_PATHCONV=1 $CLI workspace delete "$state_path/resources.json" +trace cp databricks.yml .databricks/bundle/default/resources.json +trace musterr $CLI bundle plan 2>&1 | contains.py 'Error' 'invalid character' +trace cmp databricks.yml .databricks/bundle/default/resources.json +echo "PASS: failure preserved cache" + +{ + trace MSYS_NO_PATHCONV=1 $CLI workspace import "$state_path/resources.json" --file .databricks/resources.remote.json --format AUTO --overwrite + trace $CLI bundle destroy --auto-approve +} &> LOG.cleanup +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index f7c42c00f00..f2ecc3f8cbc 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -90,6 +90,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { "deployment_history": {} }, @@ -192,6 +193,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 2, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/state/bad_json_local/out.test.toml b/acceptance/bundle/state/bad_json_local/out.test.toml index a927a5fbc06..19b5b15d11f 100644 --- a/acceptance/bundle/state/bad_json_local/out.test.toml +++ b/acceptance/bundle/state/bad_json_local/out.test.toml @@ -1,2 +1,2 @@ Cloud = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/bad_json_local/test.toml b/acceptance/bundle/state/bad_json_local/test.toml new file mode 100644 index 00000000000..2d998a533d8 --- /dev/null +++ b/acceptance/bundle/state/bad_json_local/test.toml @@ -0,0 +1,2 @@ +# Exercises local file-backed state; DMS remote-state behavior is covered in bundle/dms/state-resolution. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/engine_mismatch/out.test.toml b/acceptance/bundle/state/engine_mismatch/out.test.toml index a927a5fbc06..19b5b15d11f 100644 --- a/acceptance/bundle/state/engine_mismatch/out.test.toml +++ b/acceptance/bundle/state/engine_mismatch/out.test.toml @@ -1,2 +1,2 @@ Cloud = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/engine_mismatch/test.toml b/acceptance/bundle/state/engine_mismatch/test.toml index 601384fdf96..99279d4342f 100644 --- a/acceptance/bundle/state/engine_mismatch/test.toml +++ b/acceptance/bundle/state/engine_mismatch/test.toml @@ -1 +1,4 @@ +# Exercises local file-backed state; DMS remote-state behavior is covered in bundle/dms/state-resolution. +EnvMatrix.DMS = [""] + Ignore = [".databricks"] diff --git a/acceptance/bundle/state/force_pull_commands/out.test.toml b/acceptance/bundle/state/force_pull_commands/out.test.toml index a927a5fbc06..19b5b15d11f 100644 --- a/acceptance/bundle/state/force_pull_commands/out.test.toml +++ b/acceptance/bundle/state/force_pull_commands/out.test.toml @@ -1,2 +1,2 @@ Cloud = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/force_pull_commands/test.toml b/acceptance/bundle/state/force_pull_commands/test.toml index 7b9e47c1f4f..8fb588b210f 100644 --- a/acceptance/bundle/state/force_pull_commands/test.toml +++ b/acceptance/bundle/state/force_pull_commands/test.toml @@ -1,3 +1,6 @@ +# Exercises local file-backed state; DMS remote-state behavior is covered in bundle/dms/state-resolution. +EnvMatrix.DMS = [""] + Cloud = false RecordRequests = true diff --git a/acceptance/bundle/state/lineage_different/out.test.toml b/acceptance/bundle/state/lineage_different/out.test.toml index a927a5fbc06..19b5b15d11f 100644 --- a/acceptance/bundle/state/lineage_different/out.test.toml +++ b/acceptance/bundle/state/lineage_different/out.test.toml @@ -1,2 +1,2 @@ Cloud = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/lineage_different/test.toml b/acceptance/bundle/state/lineage_different/test.toml index 0f5f0266936..f721346f39c 100644 --- a/acceptance/bundle/state/lineage_different/test.toml +++ b/acceptance/bundle/state/lineage_different/test.toml @@ -1,3 +1,6 @@ +# Exercises local file-backed state; DMS remote-state behavior is covered in bundle/dms/state-resolution. +EnvMatrix.DMS = [""] + RecordRequests = false IncludeRequestHeaders = ["User-Agent"] Ignore = [".databricks"] diff --git a/acceptance/bundle/state/same_serial/out.test.toml b/acceptance/bundle/state/same_serial/out.test.toml index a927a5fbc06..19b5b15d11f 100644 --- a/acceptance/bundle/state/same_serial/out.test.toml +++ b/acceptance/bundle/state/same_serial/out.test.toml @@ -1,2 +1,2 @@ Cloud = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/same_serial/test.toml b/acceptance/bundle/state/same_serial/test.toml index 601384fdf96..99279d4342f 100644 --- a/acceptance/bundle/state/same_serial/test.toml +++ b/acceptance/bundle/state/same_serial/test.toml @@ -1 +1,4 @@ +# Exercises local file-backed state; DMS remote-state behavior is covered in bundle/dms/state-resolution. +EnvMatrix.DMS = [""] + Ignore = [".databricks"] diff --git a/bundle/bundle.go b/bundle/bundle.go index 68654bead7c..a22099cd236 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -194,6 +194,9 @@ type Bundle struct { // leave the terraform state untouched. MigratingToDirect bool + // MigratingToDMS registers the existing direct state before deploying with history enabled. + MigratingToDMS bool + // AllowTerraformEngineConfig reports a terraform bundle.engine as a warning instead // of an error and skips the DATABRICKS_BUNDLE_ENGINE check. Set by read-only commands // (bundle summary) so users can still inspect a bundle that pins the removed engine. diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 03cd2ba30d4..498e1b15d47 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -54,9 +54,10 @@ type Experimental struct { // service (DMS), which records deployment history and tracks what changed // across deployments. // - // Only for bundles with no deployed resources yet: DMS becomes the source of - // truth for their state. - DeploymentHistory bool `json:"deployment_history,omitempty"` + // Existing resources are registered when deployment history is enabled, and + // DMS becomes the source of truth for their state. An explicit value takes + // priority over the environment setting. + DeploymentHistory *bool `json:"deployment_history,omitempty"` } type Python struct { diff --git a/bundle/deploy/dmsmigration/migrate.go b/bundle/deploy/dmsmigration/migrate.go new file mode 100644 index 00000000000..d6f574bcbdc --- /dev/null +++ b/bundle/deploy/dmsmigration/migrate.go @@ -0,0 +1,77 @@ +package dmsmigration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strconv" + "strings" + + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Migrate records every resource from the open resources.json state in an initial registration version. +// The caller must hold the deployment lock and have created the DMS deployment. +func Migrate(ctx context.Context, db *dstate.DeploymentState) error { + keys := slices.Sorted(maps.Keys(db.Data.State)) + states := make([]json.RawMessage, len(keys)) + operations := make([]bundledeployments.StagedOperation, len(keys)) + for i, key := range keys { + entry := db.Data.State[key] + state, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + return fmt.Errorf("serializing migration state for %s: %w", key, err) + } + states[i] = state + operations[i] = bundledeployments.StagedOperation{ + ResourceKey: strings.TrimPrefix(key, dms.StatePrefix), + ActionType: bundledeployments.OperationActionTypeOperationActionTypeInitialRegister, + } + } + + versionID := max(db.Data.Serial, db.VersionID) + 1 + previousVersionID := "" + if db.VersionID > 0 { + previousVersionID = strconv.Itoa(db.VersionID) + } + w := cmdctx.WorkspaceClient(ctx) + _, err := w.BundleDeployments.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ + Parent: dms.DeploymentName(db.DeploymentID), + VersionId: strconv.Itoa(versionID), + Version: bundledeployments.Version{ + CliVersion: build.GetInfo().Version, + VersionType: dms.VersionTypeDeploy, + PreviousVersionId: previousVersionID, + Operations: operations, + }, + }) + if err != nil { + return fmt.Errorf("creating migration version: %w", err) + } + + buffer := dms.StartOperationBuffer(ctx, db.DeploymentID, versionID) + for i, key := range keys { + buffer.RecordOperation(ctx, key, false, db.Data.State[key].ID, states[i]) + } + recordErr := buffer.Drain() + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if recordErr != nil { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + _, completeErr := w.BundleDeployments.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: dms.VersionName(db.DeploymentID, versionID), + CompletionReason: reason, + }) + if err := errors.Join(recordErr, completeErr); err != nil { + return fmt.Errorf("recording migration version: %w", err) + } + + return db.UseDeploymentHistory(versionID) +} diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 57e10b8d4b4..f9a90a1cd77 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -16,6 +16,21 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } +// UseDeploymentHistory persists the DMS marker after all existing resources have been recorded. +func (db *DeploymentState) UseDeploymentHistory(versionID int) error { + db.AssertOpenedForRead() + db.mu.Lock() + defer db.mu.Unlock() + + db.recordsHistory = true + db.VersionID = versionID + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}) + } + db.Data.Features[FeatureDeploymentHistory] = struct{}{} + return db.unlockedSave() +} + // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS // resources.json does not track state. The service is queried for state for resources and then they // are filled in-place in DeploymentState.Data.State: diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index a0be3d80629..1f9688bf453 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -111,8 +111,9 @@ type DeploymentState struct { // the deployment, which Open cannot know about. DeploymentID string - // VersionID is the DMS counterpart of Data.Serial - same meaning, but owned by the service - // rather than the file, so it lives outside Data and is never written to resources.json. + // VersionID is the service-owned counter for recorded deployments. A copy is stored in + // resources.json as Serial to ensure outdated resource state is not read. + // Open always takes the authoritative value from the service. VersionID int } @@ -139,8 +140,8 @@ type Header struct { Lineage string `json:"lineage"` - // Serial counts state writes. Omitted for recorded deployments, where the service's version is - // the source of truth and Open sets this from it. + // Serial counts state writes. For recorded deployments it is a copy of the current DMS version + // to ensure outdated resource state is not read; the service remains the source of truth. Serial int `json:"serial,omitempty"` // Features maps each feature flag this state depends on to a (currently empty) @@ -401,9 +402,9 @@ func (db *DeploymentState) StateFeatures() map[string]struct{} { return maps.Clone(db.Data.Features) } -// GetSerial reports the counter a saved plan is validated against: the recorded version for a -// deployment that records history, since its state file persists no serial of its own, and the -// state serial otherwise. The two mean the same thing and advance together. +// GetSerial reports the counter a saved plan is validated against: the service's version for a +// deployment that records history, and the state serial otherwise. When the deployment uses the +// deployment metadata service, the serial from resources.json is not used for plan validation. func (db *DeploymentState) GetSerial() int { db.mu.Lock() defer db.mu.Unlock() @@ -465,9 +466,8 @@ type ( // Open reads the deployment state from disk, recovering the WAL when withRecovery is set. // When withDeploymentHistory is set it builds a DMS client from wsClient and reads resources from // the service instead, with dmsDeploymentID the id the service holds (empty before the first -// recorded deploy); lineage and serial still come from the file, since that is what the write path -// increments. Open only reads through the client - InitializeOperationBuffer installs the write path once a -// version exists. +// recorded deploy). Lineage comes from the file. Open only reads through the client - +// InitializeOperationBuffer installs the write path once a version exists. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeployment OpenDmsArgs) error { db.mu.Lock() defer db.mu.Unlock() @@ -603,8 +603,8 @@ Run "databricks bundle destroy" first, then deploy again with deployment history } } - // The service owns the version number, so it is read from the deployment rather than from - // the state file, which persists none. No deployment yet means no versions, hence zero. + // The service owns the version number; the state file stores a copy. + // No deployment yet means no versions, hence zero. db.VersionID = dmsDeployment.LastVersionID } @@ -992,9 +992,9 @@ func (db *DeploymentState) dataForFile() Database { header.Features = make(map[string]struct{}) } header.Features[FeatureDeploymentHistory] = struct{}{} - // Under DMS the serial is managed by the deployment's last_version_id, so it is not - // persisted here; Open sets it from the service. - header.Serial = 0 + // Store a copy of the current version to ensure outdated resource state is not read. + // Open still takes the authoritative version from the service. + header.Serial = db.VersionID return Database{ Header: header, diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index c8d78d150d6..3760d472199 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -7,6 +7,10 @@ import ( "testing" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,6 +49,94 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { assert.ErrorIs(t, err, os.ErrNotExist) } +func TestDataForFileSerial(t *testing.T) { + tests := []struct { + name string + recording bool + versionID int + wantSerial int + }{ + {name: "direct state keeps its serial", versionID: 7, wantSerial: 3}, + {name: "recorded state caches the current version", recording: true, versionID: 7, wantSerial: 7}, + {name: "no recorded version omits the serial", recording: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db := DeploymentState{ + Data: NewDatabase("test-lineage", 3), + recordsHistory: tt.recording, + VersionID: tt.versionID, + } + db.Data.State["resources.jobs.foo"] = ResourceEntry{ID: "123", State: json.RawMessage(`{"name":"foo"}`)} + + snapshot := db.dataForFile() + assert.Equal(t, tt.wantSerial, snapshot.Serial) + assert.Equal(t, 3, db.Data.Serial) + assert.Len(t, db.Data.State, 1) + if tt.recording { + assert.Empty(t, snapshot.State) + assert.Contains(t, snapshot.Features, FeatureDeploymentHistory) + } else { + assert.Equal(t, db.Data.State, snapshot.State) + } + data, err := json.Marshal(snapshot) + require.NoError(t, err) + if tt.wantSerial == 0 { + assert.NotContains(t, string(data), `"serial"`) + } + }) + } +} + +func TestOpenUsesDMSVersionForSerial(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/bundle/deployments/123/resources", func(req testserver.Request) any { + return bundledeployments.ListResourcesResponse{} + }) + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + + tests := []struct { + name string + omitSerial bool + cachedSerial int + deploymentID string + lastVersionID int + }{ + {name: "legacy missing serial", omitSerial: true, deploymentID: "123", lastVersionID: 7}, + {name: "legacy zero serial", deploymentID: "123", lastVersionID: 7}, + {name: "cached serial behind service", cachedSerial: 2, deploymentID: "123", lastVersionID: 7}, + {name: "cached serial ahead of service", cachedSerial: 99, deploymentID: "123", lastVersionID: 7}, + {name: "no deployment has version zero", cachedSerial: 99}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "resources.json") + seed := map[string]any{ + "state_version": currentStateVersion, + "lineage": "test-lineage", + "features": map[string]struct{}{FeatureDeploymentHistory: {}}, + "state": map[string]ResourceEntry{}, + } + if !tt.omitSerial { + seed["serial"] = tt.cachedSerial + } + data, err := json.Marshal(seed) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, data, 0o600)) + + ctx := cmdctx.SetWorkspaceClient(t.Context(), client) + var db DeploymentState + require.NoError(t, db.Open(ctx, path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(true), OpenDmsArgs{ + DeploymentID: tt.deploymentID, LastVersionID: tt.lastVersionID, + })) + assert.Equal(t, tt.lastVersionID, db.GetSerial()) + mustFinalize(t, &db) + }) + } +} + func TestExportStateFromDataJobRunJobID(t *testing.T) { data := Database{ State: map[string]ResourceEntry{ diff --git a/bundle/env/dms.go b/bundle/env/dms.go index 2474b468da4..a92d9f09a07 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -8,11 +8,12 @@ import "context" const DeploymentHistoryVariable = "DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY" // RecordsDeploymentHistory reports whether recording is on, from config or env var. Single predicate -// for all recording code paths, keeping them in sync. The env var only counts as "true": anything +// for all recording code paths, keeping them in sync. Explicit config takes priority over the env var. +// The env var only counts as "true": anything // else - a typo like "TRUE" or "yes" - leaves recording off rather than silently enabling it. -func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { - if configured { - return true +func RecordsDeploymentHistory(ctx context.Context, configured *bool) bool { + if configured != nil { + return *configured } value, _ := get(ctx, []string{DeploymentHistoryVariable}) return value == "true" diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go index 469b7707e85..5bfad6df39a 100644 --- a/bundle/env/dms_test.go +++ b/bundle/env/dms_test.go @@ -22,20 +22,34 @@ func TestDeploymentHistoryEnv(t *testing.T) { {"yes", false}, } { ctx := env.Set(t.Context(), DeploymentHistoryVariable, tc.value) - assert.Equal(t, tc.want, RecordsDeploymentHistory(ctx, false), "value %q", tc.value) + assert.Equal(t, tc.want, RecordsDeploymentHistory(ctx, nil), "value %q", tc.value) } } func TestDeploymentHistoryEnvUnset(t *testing.T) { - assert.False(t, RecordsDeploymentHistory(t.Context(), false)) + assert.False(t, RecordsDeploymentHistory(t.Context(), nil)) } func TestRecordsDeploymentHistory(t *testing.T) { - // The bundle setting alone is enough, and so is the environment; the env var - // exists so the acceptance suite can record without touching every databricks.yml. - assert.True(t, RecordsDeploymentHistory(t.Context(), true)) - assert.False(t, RecordsDeploymentHistory(t.Context(), false)) - - ctx := env.Set(t.Context(), DeploymentHistoryVariable, "true") - assert.True(t, RecordsDeploymentHistory(ctx, false)) + for _, tc := range []struct { + name string + configured *bool + value string + want bool + }{ + {"config true env true", new(true), "true", true}, + {"config true env false", new(true), "false", true}, + {"config true env unset", new(true), "", true}, + {"config false env true", new(false), "true", false}, + {"config false env false", new(false), "false", false}, + {"config false env unset", new(false), "", false}, + {"config unset env true", nil, "true", true}, + {"config unset env false", nil, "false", false}, + {"config unset env unset", nil, "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := env.Set(t.Context(), DeploymentHistoryVariable, tc.value) + assert.Equal(t, tc.want, RecordsDeploymentHistory(ctx, tc.configured)) + }) + } } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 13c5121f569..e791b26b393 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deploy" + "github.com/databricks/cli/bundle/deploy/dmsmigration" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/metadata" @@ -294,6 +295,27 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } } + if b.MigratingToDMS { + if err := statemgmt.BackupRemoteResourcesState(ctx, b); err != nil { + logdiag.LogError(ctx, err) + return + } + createOrUpdateDeployment(ctx, b, dmsDeployment) + if logdiag.HasError(ctx) { + return + } + cmdio.LogString(ctx, "Enabling deployment history...") + if err := dmsmigration.Migrate(ctx, &b.DeploymentBundle.StateDB); err != nil { + logdiag.LogError(ctx, err) + return + } + logDeploymentVersion(ctx, b, "Deployment history enabled") + statemgmt.PushResourcesState(ctx, b) + if logdiag.HasError(ctx) { + return + } + } + // Upgrade from read (opened by process.go, or by Migrate) to write mode. After approval // and the migration commit above, so a declined deploy never opens a WAL it must discard, // and the migration's tf+1 push lands before this advances the WAL header to tf+2. @@ -321,12 +343,14 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { - firstDeploy := b.DeploymentBundle.StateDB.DeploymentID == "" - createOrUpdateDeployment(ctx, b, dmsDeployment) + injectDeploymentIdInPlan := b.DeploymentBundle.StateDB.DeploymentID == "" || b.MigratingToDMS + if !b.MigratingToDMS { + createOrUpdateDeployment(ctx, b, dmsDeployment) + } if logdiag.HasError(ctx) { return } - if firstDeploy { + if injectDeploymentIdInPlan { if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { logdiag.LogError(ctx, err) return @@ -345,7 +369,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logdiag.LogError(ctx, err) return } - logDeploymentVersion(ctx, b) + logDeploymentVersion(ctx, b, "Current Deployment Version") } } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index abd89f03f2f..095e4a149b0 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -165,7 +165,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version // logDeploymentVersion logs the deployment version URL. Workspace ID is omitted // so the page stays clickable in a terminal and redirects correctly without it. -func logDeploymentVersion(ctx context.Context, b *bundle.Bundle) { +func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, message string) { deploymentID := b.DeploymentBundle.StateDB.DeploymentID version := b.DeploymentBundle.StateDB.VersionID if version == 0 { @@ -177,11 +177,11 @@ func logDeploymentVersion(ctx context.Context, b *bundle.Bundle) { // Only the link is lost, so report the version without it rather than failing // a deploy over it. log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) - cmdio.LogString(ctx, fmt.Sprintf("Current Deployment Version: %s version %d", deploymentID, version)) + cmdio.LogString(ctx, fmt.Sprintf("%s: %s version %d", message, deploymentID, version)) return } - cmdio.LogString(ctx, "Current Deployment Version: "+workspaceurls.DeploymentURL(*baseURL, deploymentID, version)) + cmdio.LogString(ctx, message+": "+workspaceurls.DeploymentURL(*baseURL, deploymentID, version)) } // deploymentMetadata describes the bundle this deploy came from and where it diff --git a/bundle/statemgmt/state_pull.go b/bundle/statemgmt/state_pull.go index 4ce9494c894..1fed9fe4980 100644 --- a/bundle/statemgmt/state_pull.go +++ b/bundle/statemgmt/state_pull.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deploy" + bundleenv "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/atomicfile" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" @@ -34,6 +35,7 @@ type StateDesc struct { // additional fields describing state: SourcePath string Content []byte + readErr error Engine engine.EngineType `json:"-"` IsLocal bool `json:"-"` @@ -73,19 +75,17 @@ func (s *StateDesc) IsDMS() bool { return ok } -func localRead(ctx context.Context, fullPath string, engine engine.EngineType) *StateDesc { +func localRead(fullPath string, engine engine.EngineType) *StateDesc { content, err := os.ReadFile(fullPath) - if err != nil { - if !errors.Is(err, fs.ErrNotExist) { - logdiag.LogError(ctx, fmt.Errorf("reading %s: %w", filepath.ToSlash(fullPath), err)) - } + if errors.Is(err, fs.ErrNotExist) { return nil } state := &StateDesc{} - err = json.Unmarshal(content, state) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("parsing %s: %w", filepath.ToSlash(fullPath), err)) + state.readErr = fmt.Errorf("reading %s: %w", filepath.ToSlash(fullPath), err) + } else if err := json.Unmarshal(content, state); err != nil { + state.readErr = fmt.Errorf("parsing %s: %w", filepath.ToSlash(fullPath), err) } state.SourcePath = filepath.ToSlash(fullPath) @@ -134,7 +134,8 @@ func filerRead(ctx context.Context, f filer.Filer, path string, engine engine.En return state } -// PullResourcesState determines correct state to use by reading all 4 states (terraform/direct, local/remote). +// PullResourcesState selects the newest Terraform or direct state. +// Deployment history selects only remote state and writes a local copy when present. func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) *StateDesc { var err error @@ -142,7 +143,37 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always _, localPathDirect := b.StateFilenameDirect(ctx) _, localPathTerraform := b.StateFilenameTerraform(ctx) - states := readStates(ctx, b, alwaysPull) + var configured *bool + if b.Config.Experimental != nil { + configured = b.Config.Experimental.DeploymentHistory + } + historyEnabled := bundleenv.RecordsDeploymentHistory(ctx, configured) + states := readStates(ctx, b, AlwaysPull(bool(alwaysPull) || historyEnabled)) + if logdiag.HasError(ctx) { + return nil + } + // DMS owns recorded resource state; a remote marker's serial is only a checkpoint. + remoteRecorded := slices.ContainsFunc(states, func(state *StateDesc) bool { + return !state.IsLocal && state.Engine.IsDirect() && state.IsDMS() + }) + if historyEnabled || remoteRecorded { + states = slices.DeleteFunc(states, func(state *StateDesc) bool { + if !state.IsLocal { + return false + } + if !state.Engine.IsDirect() { + if err := os.Remove(state.SourcePath); err != nil && !errors.Is(err, fs.ErrNotExist) { + logdiag.LogError(ctx, fmt.Errorf("removing cached resource state %s: %w", state.SourcePath, err)) + } + } + return true + }) + } + for _, state := range states { + if state.readErr != nil { + logdiag.LogError(ctx, state.readErr) + } + } if logdiag.HasError(ctx) { return nil @@ -207,8 +238,8 @@ func readStates(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) [] return nil } - directLocalState := localRead(ctx, localPathDirect, engine.EngineDirect) - terraformLocalState := localRead(ctx, localPathTerraform, engine.EngineTerraform) + directLocalState := localRead(localPathDirect, engine.EngineDirect) + terraformLocalState := localRead(localPathTerraform, engine.EngineTerraform) if (directLocalState == nil && terraformLocalState == nil) || alwaysPull { f, err := deploy.StateFiler(ctx, b) diff --git a/bundle/statemgmt/state_pull_test.go b/bundle/statemgmt/state_pull_test.go new file mode 100644 index 00000000000..43fce551793 --- /dev/null +++ b/bundle/statemgmt/state_pull_test.go @@ -0,0 +1,82 @@ +package statemgmt_test + +import ( + "os" + "path" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPullResourcesStateRemoteRecordedCheckpoint(t *testing.T) { + const recordedMarker = `{"state_version":3,"cli_version":"0.1.0","lineage":"recorded-lineage","serial":2,"features":{"deployment_history":{}},"state":{}}` + const directSnapshot = `{"state_version":3,"lineage":"direct-lineage","serial":99,"state":{"resources.jobs.foo":{"__id__":"456","state":{"name":"foo"}}}}` + const statePath = "/Workspace/state" + tests := []struct { + name string + configured *bool + }{ + {name: "removed"}, + {name: "disabled", configured: new(false)}, + {name: "enabled", configured: new(true)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := testserver.New(t) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Bundle: config.Bundle{Target: "test"}, + Workspace: config.Workspace{StatePath: statePath}, + Experimental: &config.Experimental{DeploymentHistory: tt.configured}, + }, + } + b.SetWorkpaceClient(client) + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + _, localPath := b.StateFilenameDirect(ctx) + writeLocalState(t, localPath, directSnapshot) + _, terraformPath := b.StateFilenameTerraform(ctx) + writeLocalState(t, terraformPath, "invalid JSON") + server.Handle("GET", "/api/2.0/workspace/get-status", func(req testserver.Request) any { + requestedPath := req.URL.Query().Get("path") + if requestedPath == path.Join(statePath, "resources.json") { + return workspace.ObjectInfo{Path: requestedPath, ObjectType: workspace.ObjectTypeFile} + } + return testserver.Response{StatusCode: 404, Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST"}} + }) + server.Handle("GET", "/api/2.0/workspace/export", func(req testserver.Request) any { + return []byte(recordedMarker) + }) + + state := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true)) + require.False(t, logdiag.HasError(ctx), "%v", logdiag.FlushCollected(ctx)) + require.NotNil(t, state) + assert.True(t, state.IsDMS()) + assert.False(t, state.IsLocal) + assert.Equal(t, 2, state.Serial) + assert.Equal(t, "recorded-lineage", state.Lineage) + content, err := os.ReadFile(localPath) + require.NoError(t, err) + assert.JSONEq(t, recordedMarker, string(content)) + }) + } +} + +// writeLocalState prepares a local snapshot without running a deployment. +func writeLocalState(t *testing.T, filename, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0o700)) + require.NoError(t, os.WriteFile(filename, []byte(content), 0o600)) +} diff --git a/bundle/statemgmt/state_push.go b/bundle/statemgmt/state_push.go index 03036ba712d..9df720f6157 100644 --- a/bundle/statemgmt/state_push.go +++ b/bundle/statemgmt/state_push.go @@ -3,6 +3,7 @@ package statemgmt import ( "context" "errors" + "fmt" "io/fs" "os" @@ -43,6 +44,27 @@ func PushResourcesState(ctx context.Context, b *bundle.Bundle) { } } +// BackupRemoteResourcesState preserves the remote state before deployment history replaces it. +func BackupRemoteResourcesState(ctx context.Context, b *bundle.Bundle) error { + f, err := deploy.StateFiler(ctx, b) + if err != nil { + return fmt.Errorf("backing up resource state: creating state filer: %w", err) + } + + remotePath, _ := b.StateFilenameDirect(ctx) + reader, err := f.Read(ctx, remotePath) + if err != nil { + return fmt.Errorf("backing up resource state: reading %s: %w", remotePath, err) + } + defer reader.Close() + + backupPath := remotePath + ".backup" + if err := f.Write(ctx, backupPath, reader, filer.OverwriteIfExists); err != nil { + return fmt.Errorf("backing up resource state: writing %s: %w", backupPath, err) + } + return nil +} + func BackupRemoteTerraformState(ctx context.Context, b *bundle.Bundle) { f, err := deploy.StateFiler(ctx, b) if err != nil { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index cbc8cf93212..04989740c69 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -363,14 +363,21 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } - if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting { + b.MigratingToDMS = opts.Deploy && stateDesc.Engine.IsDirect() && !stateDesc.IsDMS() && configuresDeploymentHistory(ctx, b) + // TODO: Support applying plans computed against direct state after migration to DMS. + // This restriction blocks automatic migration with --plan once deployment history reaches GA. + if b.MigratingToDMS && opts.ReadPlanPath != "" { + return b, stateDesc, errors.New("migrating to deployment history does not support --plan; run 'bundle deploy' without --plan") + } + if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting && !b.MigratingToDMS { if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, opts.Deploy || opts.PreDeployChecks); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } } - if needDirectState && stateDesc.IsDMS() { + // Repeated migrations may already have a DMS deployment even when the state lacks its history marker. + if needDirectState && (stateDesc.IsDMS() || b.MigratingToDMS) { var err error dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { @@ -379,15 +386,18 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } // Stamp the deployment and the version this run records onto every job and pipeline so - // the plan carries them. version_id is always known (last recorded + 1); deployment_id - // does not exist until a first deploy creates it, so it is left off here and the deploy - // phase stamps the created id. + // the plan carries them. The deployment version follows the last recorded version or + // the initial registration version; deployment_id does not exist until a first deploy + // creates it, so it is left off here and the deploy phase stamps the created id. lastVersionID, err := parseLastVersionID(dmsDeployment) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } nextVersion := lastVersionID + 1 + if b.MigratingToDMS { + nextVersion = max(b.DeploymentBundle.StateDB.Data.Serial, lastVersionID) + 2 + } muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} if dmsDeploymentID != "" { bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { @@ -404,7 +414,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { + if b.MigratingToDMS { + b.DeploymentBundle.StateDB.DeploymentID = dmsDeploymentID + b.DeploymentBundle.StateDB.VersionID = lastVersionID + } else if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } @@ -692,8 +705,12 @@ Run "databricks bundle destroy" first, then deploy again with deployment history return nil } +// configuresDeploymentHistory reports whether the configuration or environment enables history. func configuresDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { - configured := b.Config.Experimental != nil && b.Config.Experimental.DeploymentHistory + var configured *bool + if b.Config.Experimental != nil { + configured = b.Config.Experimental.DeploymentHistory + } return bundleenv.RecordsDeploymentHistory(ctx, configured) } diff --git a/cmd/bundle/utils/process_test.go b/cmd/bundle/utils/process_test.go index 080229889b6..b216f1102d2 100644 --- a/cmd/bundle/utils/process_test.go +++ b/cmd/bundle/utils/process_test.go @@ -14,14 +14,14 @@ import ( func TestResolveDeploymentHistory(t *testing.T) { b := &bundle.Bundle{Config: config.Root{ - Experimental: &config.Experimental{DeploymentHistory: true}, + Experimental: &config.Experimental{DeploymentHistory: new(true)}, }} newState := &statemgmt.StateDesc{} assert.True(t, resolveDeploymentHistory(t.Context(), b, newState)) assert.True(t, newState.IsDMS()) assert.False(t, resolveDeploymentHistory(t.Context(), b, &statemgmt.StateDesc{SourcePath: "resources.json"})) - b.Config.Experimental.DeploymentHistory = false + b.Config.Experimental.DeploymentHistory = new(false) assert.True(t, resolveDeploymentHistory(t.Context(), b, &statemgmt.StateDesc{ SourcePath: "resources.json", Features: map[string]struct{}{dstate.FeatureDeploymentHistory: {}}, diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go index b4019b75d0a..464c0c4183d 100644 --- a/libs/testserver/bundledeployments.go +++ b/libs/testserver/bundledeployments.go @@ -2,6 +2,7 @@ package testserver import ( "bytes" + "cmp" "encoding/json" "fmt" "path" @@ -576,6 +577,27 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re return Response{Body: body} } +// ListVersions returns recorded versions in descending numeric order. +func (s *FakeWorkspace) ListVersions(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + versions := make([]bundledeployments.Version, 0, len(d.Versions)) + for _, version := range d.Versions { + versions = append(versions, *version) + } + slices.SortFunc(versions, func(a, b bundledeployments.Version) int { + aID, _ := strconv.ParseInt(a.VersionId, 10, 64) + bID, _ := strconv.ParseInt(b.VersionId, 10, 64) + return cmp.Compare(bID, aID) + }) + return Response{Body: bundledeployments.ListVersionsResponse{Versions: versions}} +} + func (s *FakeWorkspace) ListResources(deploymentID string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a2d0be14760..3623b9ae857 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -467,6 +467,9 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.ListVersions(req.Vars["deployment_id"]) + }) server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) })