From eaab2acdebe22932f7cd85a350c743df92e4952f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 12:50:48 +0200 Subject: [PATCH 1/5] bundle: clean up the migration WAL when deployment migrate fails Co-authored-by: Isaac --- .../retry-after-failure/databricks.yml | 14 ++ .../command/retry-after-failure/out.test.toml | 2 + .../command/retry-after-failure/output.txt | 19 ++ .../command/retry-after-failure/script | 13 ++ .../terraform-requests.json | 36 +++ .../retry-after-failure/terraform.tfstate | 210 ++++++++++++++++++ cmd/bundle/deployment/migrate.go | 2 + 7 files changed, 296 insertions(+) create mode 100644 acceptance/bundle/migrate/command/retry-after-failure/databricks.yml create mode 100644 acceptance/bundle/migrate/command/retry-after-failure/out.test.toml create mode 100644 acceptance/bundle/migrate/command/retry-after-failure/output.txt create mode 100644 acceptance/bundle/migrate/command/retry-after-failure/script create mode 100644 acceptance/bundle/migrate/command/retry-after-failure/terraform-requests.json create mode 100644 acceptance/bundle/migrate/command/retry-after-failure/terraform.tfstate diff --git a/acceptance/bundle/migrate/command/retry-after-failure/databricks.yml b/acceptance/bundle/migrate/command/retry-after-failure/databricks.yml new file mode 100644 index 00000000000..483a78843e5 --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: migrate-removed-test + +resources: + jobs: + job_a: {name: "Job A"} + job_b: + name: "Job B" + tasks: + - task_key: main + existing_cluster_id: "1234-567890-abcde123" + python_wheel_task: + package_name: "${resources.jobs.ghost.id}" + entry_point: main diff --git a/acceptance/bundle/migrate/command/retry-after-failure/out.test.toml b/acceptance/bundle/migrate/command/retry-after-failure/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/command/retry-after-failure/output.txt b/acceptance/bundle/migrate/command/retry-after-failure/output.txt new file mode 100644 index 00000000000..3587c73e569 --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/output.txt @@ -0,0 +1,19 @@ + +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +>>> [CLI] bundle deployment migrate +Error: resources.jobs.job_b: cannot resolve field "tasks[0].python_wheel_task.package_name" (template "${resources.jobs.ghost.id}"): jobs.job_b field tasks[0].python_wheel_task.package_name: method A: index 0 out of range (len 0); method B: cannot look up "resources.jobs.ghost.id": databricks_job.ghost not found in TF state + +>>> ls .databricks/bundle/default +sync-snapshots +terraform + +>>> [CLI] bundle deployment migrate +Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json + +Validate the migration by running "databricks bundle plan", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy". + +To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate + diff --git a/acceptance/bundle/migrate/command/retry-after-failure/script b/acceptance/bundle/migrate/command/retry-after-failure/script new file mode 100644 index 00000000000..3123066b0ac --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/script @@ -0,0 +1,13 @@ +export DATABRICKS_BUNDLE_ENGINE= + +trace replay_tfstate.py terraform-requests.json terraform.tfstate + +# The reference cannot be converted, so the first migration fails after the WAL is created +musterr trace $CLI bundle deployment migrate + +# Nothing of the failed attempt may be left behind +trace ls .databricks/bundle/default + +update_file.py databricks.yml '${resources.jobs.ghost.id}' 'test_pkg' + +trace $CLI bundle deployment migrate diff --git a/acceptance/bundle/migrate/command/retry-after-failure/terraform-requests.json b/acceptance/bundle/migrate/command/retry-after-failure/terraform-requests.json new file mode 100644 index 00000000000..20d41dc71cf --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/terraform-requests.json @@ -0,0 +1,36 @@ +[ + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Job A", + "queue": { + "enabled": true + } + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Job B", + "queue": { + "enabled": true + } + } + } +] diff --git a/acceptance/bundle/migrate/command/retry-after-failure/terraform.tfstate b/acceptance/bundle/migrate/command/retry-after-failure/terraform.tfstate new file mode 100644 index 00000000000..6a8ad85c9a1 --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/terraform.tfstate @@ -0,0 +1,210 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 1, + "lineage": "37f6c843-91a5-3d5c-457c-2c43d9630444", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_job", + "name": "job_a", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8790614038615498000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "Job A", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8790614038615498000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + }, + { + "mode": "managed", + "type": "databricks_job", + "name": "job_b", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8790614038615799000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "Job B", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8790614038615799000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + } + ], + "check_results": null +} diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index f329ec7962c..ebf25cd8cc0 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -133,6 +133,8 @@ New bundles use the direct engine automatically, no migration is needed.`) defer func() { if tempStatePathAutoRemove { _ = os.Remove(tempStatePath) + // UpgradeToWrite creates the WAL with O_EXCL; a leftover one fails every retry. + _ = os.Remove(tempStatePath + ".wal") } }() From d9c506da4f1335b282033d63745671c2bdf24f4c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 12:51:30 +0200 Subject: [PATCH 2/5] Add changelog fragment Co-authored-by: Isaac --- .nextchanges/bundles/migrate-wal-cleanup.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/migrate-wal-cleanup.md diff --git a/.nextchanges/bundles/migrate-wal-cleanup.md b/.nextchanges/bundles/migrate-wal-cleanup.md new file mode 100644 index 00000000000..9db7e129f5b --- /dev/null +++ b/.nextchanges/bundles/migrate-wal-cleanup.md @@ -0,0 +1 @@ +* Fix `bundle deployment migrate` failing on retry with a leftover temporary WAL file after a failed migration. ([#6973](https://github.com/databricks/cli/pull/6973)) From 9ffb69c149aa91ac39c9775339f72df25c867e9f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 13:19:17 +0200 Subject: [PATCH 3/5] Close the state DB before removing the temp WAL (Windows) Co-authored-by: Isaac --- cmd/bundle/deployment/migrate.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index ebf25cd8cc0..abdc36c207b 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -132,6 +132,8 @@ New bundles use the direct engine automatically, no migration is needed.`) defer func() { if tempStatePathAutoRemove { + // Finalize closes the open WAL handle, which Windows requires before removal. + _, _ = stateDB.Finalize(ctx) _ = os.Remove(tempStatePath) // UpgradeToWrite creates the WAL with O_EXCL; a leftover one fails every retry. _ = os.Remove(tempStatePath + ".wal") From ef999881d34d89a4e01137fe8187b2867281ea20 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Oct 2026 14:35:10 +0200 Subject: [PATCH 4/5] Use DeploymentState.Discard for the temp state cleanup Co-authored-by: Isaac --- cmd/bundle/deployment/migrate.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index abdc36c207b..e8ad0b8bb5a 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -132,11 +132,8 @@ New bundles use the direct engine automatically, no migration is needed.`) defer func() { if tempStatePathAutoRemove { - // Finalize closes the open WAL handle, which Windows requires before removal. - _, _ = stateDB.Finalize(ctx) - _ = os.Remove(tempStatePath) // UpgradeToWrite creates the WAL with O_EXCL; a leftover one fails every retry. - _ = os.Remove(tempStatePath + ".wal") + stateDB.Discard() } }() From 67fcc358307e6585cb6e57e3a63554dd3c0a942b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 12 Oct 2026 13:00:11 +0200 Subject: [PATCH 5/5] Also remove the temp state Finalize may have written Co-authored-by: Isaac --- cmd/bundle/deployment/migrate.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index e8ad0b8bb5a..bc15e534e35 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -134,6 +134,8 @@ New bundles use the direct engine automatically, no migration is needed.`) if tempStatePathAutoRemove { // UpgradeToWrite creates the WAL with O_EXCL; a leftover one fails every retry. stateDB.Discard() + // Discard is a no-op once Finalize has run, but Finalize may have written the temp state. + dstate.RemoveFiles(tempStatePath) } }()