bundle: clean up the migration WAL when deployment migrate fails

.nextchanges/bundles/migrate-wal-cleanup.md

@@ -0,0 +1 @@
+* Fix `bundle deployment migrate` failing on retry with a leftover temporary WAL file after a failed migration. ([#6973](https://github.com/databricks/cli/pull/6973))

acceptance/bundle/migrate/command/retry-after-failure/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+  name: migrate-removed-test
+
+resources:
+  jobs:
+    job_a: {name: "Job A"}
+    job_b:
+      name: "Job B"
+      tasks:
+        - task_key: main
+          existing_cluster_id: "1234-567890-abcde123"
+          python_wheel_task:
+            package_name: "${resources.jobs.ghost.id}"
+            entry_point: main

acceptance/bundle/migrate/command/retry-after-failure/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/migrate/command/retry-after-failure/output.txt

@@ -0,0 +1,19 @@
+
+>>> replay_tfstate.py terraform-requests.json terraform.tfstate
+
+>>> [CLI] bundle deployment migrate
+Error: resources.jobs.job_b: cannot resolve field "tasks[0].python_wheel_task.package_name" (template "${resources.jobs.ghost.id}"): jobs.job_b field tasks[0].python_wheel_task.package_name: method A: index 0 out of range (len 0); method B: cannot look up "resources.jobs.ghost.id": databricks_job.ghost not found in TF state
+
+>>> ls .databricks/bundle/default
+sync-snapshots
+terraform
+
+>>> [CLI] bundle deployment migrate
+Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
+
+Validate the migration by running "databricks bundle plan", there should be no actions planned.
+
+The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy".
+
+To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate
+

acceptance/bundle/migrate/command/retry-after-failure/script

@@ -0,0 +1,13 @@
+export DATABRICKS_BUNDLE_ENGINE=
+
+trace replay_tfstate.py terraform-requests.json terraform.tfstate
+
+# The reference cannot be converted, so the first migration fails after the WAL is created
+musterr trace $CLI bundle deployment migrate
+
+# Nothing of the failed attempt may be left behind
+trace ls .databricks/bundle/default
+
+update_file.py databricks.yml '${resources.jobs.ghost.id}' 'test_pkg'
+
+trace $CLI bundle deployment migrate

acceptance/bundle/migrate/command/retry-after-failure/terraform-requests.json

@@ -0,0 +1,36 @@
+[
+  {
+    "method": "POST",
+    "path": "/api/2.2/jobs/create",
+    "body": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "Job A",
+      "queue": {
+        "enabled": true
+      }
+    }
+  },
+  {
+    "method": "POST",
+    "path": "/api/2.2/jobs/create",
+    "body": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "Job B",
+      "queue": {
+        "enabled": true
+      }
+    }
+  }
+]

acceptance/bundle/migrate/command/retry-after-failure/terraform.tfstate

@@ -0,0 +1,210 @@
+{
+  "version": 4,
+  "terraform_version": "1.5.5",
+  "serial": 1,
+  "lineage": "37f6c843-91a5-3d5c-457c-2c43d9630444",
+  "outputs": {},
+  "resources": [
+    {
+      "mode": "managed",
+      "type": "databricks_job",
+      "name": "job_a",
+      "provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
+      "instances": [
+        {
+          "schema_version": 2,
+          "attributes": {
+            "always_running": false,
+            "budget_policy_id": null,
+            "continuous": [],
+            "control_run_state": false,
+            "dbt_task": [],
+            "deployment": [
+              {
+                "deployment_id": "",
+                "kind": "BUNDLE",
+                "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json",
+                "version_id": ""
+              }
+            ],
+            "description": null,
+            "edit_mode": "UI_LOCKED",
+            "email_notifications": [
+              {
+                "no_alert_for_skipped_runs": false,
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ],
+            "environment": [],
+            "existing_cluster_id": null,
+            "format": "MULTI_TASK",
+            "git_source": [],
+            "health": [],
+            "id": "8790614038615498000",
+            "job_cluster": [],
+            "library": [],
+            "max_concurrent_runs": 1,
+            "max_retries": 0,
+            "min_retry_interval_millis": 0,
+            "name": "Job A",
+            "new_cluster": [],
+            "notebook_task": [],
+            "notification_settings": [],
+            "parameter": [],
+            "parent_path": null,
+            "performance_target": null,
+            "pipeline_task": [],
+            "provider_config": [
+              {
+                "workspace_id": "900800700600"
+              }
+            ],
+            "python_wheel_task": [],
+            "queue": [
+              {
+                "enabled": true
+              }
+            ],
+            "retry_on_timeout": false,
+            "run_as": [
+              {
+                "group_name": "",
+                "service_principal_name": "",
+                "user_name": "tester@databricks.com"
+              }
+            ],
+            "run_job_task": [],
+            "schedule": [],
+            "spark_jar_task": [],
+            "spark_python_task": [],
+            "spark_submit_task": [],
+            "tags": null,
+            "task": [],
+            "timeout_seconds": 0,
+            "timeouts": null,
+            "trigger": [],
+            "triggers": [],
+            "url": "[DATABRICKS_URL]/#job/8790614038615498000",
+            "usage_policy_id": null,
+            "webhook_notifications": [
+              {
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ]
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ=="
+        }
+      ]
+    },
+    {
+      "mode": "managed",
+      "type": "databricks_job",
+      "name": "job_b",
+      "provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
+      "instances": [
+        {
+          "schema_version": 2,
+          "attributes": {
+            "always_running": false,
+            "budget_policy_id": null,
+            "continuous": [],
+            "control_run_state": false,
+            "dbt_task": [],
+            "deployment": [
+              {
+                "deployment_id": "",
+                "kind": "BUNDLE",
+                "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json",
+                "version_id": ""
+              }
+            ],
+            "description": null,
+            "edit_mode": "UI_LOCKED",
+            "email_notifications": [
+              {
+                "no_alert_for_skipped_runs": false,
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ],
+            "environment": [],
+            "existing_cluster_id": null,
+            "format": "MULTI_TASK",
+            "git_source": [],
+            "health": [],
+            "id": "8790614038615799000",
+            "job_cluster": [],
+            "library": [],
+            "max_concurrent_runs": 1,
+            "max_retries": 0,
+            "min_retry_interval_millis": 0,
+            "name": "Job B",
+            "new_cluster": [],
+            "notebook_task": [],
+            "notification_settings": [],
+            "parameter": [],
+            "parent_path": null,
+            "performance_target": null,
+            "pipeline_task": [],
+            "provider_config": [
+              {
+                "workspace_id": "900800700600"
+              }
+            ],
+            "python_wheel_task": [],
+            "queue": [
+              {
+                "enabled": true
+              }
+            ],
+            "retry_on_timeout": false,
+            "run_as": [
+              {
+                "group_name": "",
+                "service_principal_name": "",
+                "user_name": "tester@databricks.com"
+              }
+            ],
+            "run_job_task": [],
+            "schedule": [],
+            "spark_jar_task": [],
+            "spark_python_task": [],
+            "spark_submit_task": [],
+            "tags": null,
+            "task": [],
+            "timeout_seconds": 0,
+            "timeouts": null,
+            "trigger": [],
+            "triggers": [],
+            "url": "[DATABRICKS_URL]/#job/8790614038615799000",
+            "usage_policy_id": null,
+            "webhook_notifications": [
+              {
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ]
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ=="
+        }
+      ]
+    }
+  ],
+  "check_results": null
+}

cmd/bundle/deployment/migrate.go

@@ -132,7 +132,10 @@ New bundles use the direct engine automatically, no migration is needed.`)
 
 		defer func() {
 			if tempStatePathAutoRemove {
-				_ = os.Remove(tempStatePath)
+				// UpgradeToWrite creates the WAL with O_EXCL; a leftover one fails every retry.
+				stateDB.Discard()
+				// Discard is a no-op once Finalize has run, but Finalize may have written the temp state.
+				dstate.RemoveFiles(tempStatePath)
 			}
 		}()