diff --git a/.nextchanges/bundles/migrate-wal-cleanup.md b/.nextchanges/bundles/migrate-wal-cleanup.md new file mode 100644 index 00000000000..9db7e129f5b --- /dev/null +++ b/.nextchanges/bundles/migrate-wal-cleanup.md @@ -0,0 +1 @@ +* Fix `bundle deployment migrate` failing on retry with a leftover temporary WAL file after a failed migration. ([#6973](https://github.com/databricks/cli/pull/6973)) diff --git a/acceptance/bundle/migrate/command/retry-after-failure/databricks.yml b/acceptance/bundle/migrate/command/retry-after-failure/databricks.yml new file mode 100644 index 00000000000..483a78843e5 --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: migrate-removed-test + +resources: + jobs: + job_a: {name: "Job A"} + job_b: + name: "Job B" + tasks: + - task_key: main + existing_cluster_id: "1234-567890-abcde123" + python_wheel_task: + package_name: "${resources.jobs.ghost.id}" + entry_point: main diff --git a/acceptance/bundle/migrate/command/retry-after-failure/out.test.toml b/acceptance/bundle/migrate/command/retry-after-failure/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/command/retry-after-failure/output.txt b/acceptance/bundle/migrate/command/retry-after-failure/output.txt new file mode 100644 index 00000000000..3587c73e569 --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/output.txt @@ -0,0 +1,19 @@ + +>>> replay_tfstate.py terraform-requests.json terraform.tfstate + +>>> [CLI] bundle deployment migrate +Error: resources.jobs.job_b: cannot resolve field "tasks[0].python_wheel_task.package_name" (template "${resources.jobs.ghost.id}"): jobs.job_b field tasks[0].python_wheel_task.package_name: method A: index 0 out of range (len 0); method B: cannot look up "resources.jobs.ghost.id": databricks_job.ghost not found in TF state + +>>> ls .databricks/bundle/default +sync-snapshots +terraform + +>>> [CLI] bundle deployment migrate +Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json + +Validate the migration by running "databricks bundle plan", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy". + +To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate + diff --git a/acceptance/bundle/migrate/command/retry-after-failure/script b/acceptance/bundle/migrate/command/retry-after-failure/script new file mode 100644 index 00000000000..3123066b0ac --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/script @@ -0,0 +1,13 @@ +export DATABRICKS_BUNDLE_ENGINE= + +trace replay_tfstate.py terraform-requests.json terraform.tfstate + +# The reference cannot be converted, so the first migration fails after the WAL is created +musterr trace $CLI bundle deployment migrate + +# Nothing of the failed attempt may be left behind +trace ls .databricks/bundle/default + +update_file.py databricks.yml '${resources.jobs.ghost.id}' 'test_pkg' + +trace $CLI bundle deployment migrate diff --git a/acceptance/bundle/migrate/command/retry-after-failure/terraform-requests.json b/acceptance/bundle/migrate/command/retry-after-failure/terraform-requests.json new file mode 100644 index 00000000000..20d41dc71cf --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/terraform-requests.json @@ -0,0 +1,36 @@ +[ + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Job A", + "queue": { + "enabled": true + } + } + }, + { + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Job B", + "queue": { + "enabled": true + } + } + } +] diff --git a/acceptance/bundle/migrate/command/retry-after-failure/terraform.tfstate b/acceptance/bundle/migrate/command/retry-after-failure/terraform.tfstate new file mode 100644 index 00000000000..6a8ad85c9a1 --- /dev/null +++ b/acceptance/bundle/migrate/command/retry-after-failure/terraform.tfstate @@ -0,0 +1,210 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 1, + "lineage": "37f6c843-91a5-3d5c-457c-2c43d9630444", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_job", + "name": "job_a", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8790614038615498000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "Job A", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8790614038615498000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + }, + { + "mode": "managed", + "type": "databricks_job", + "name": "job_b", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 2, + "attributes": { + "always_running": false, + "budget_policy_id": null, + "continuous": [], + "control_run_state": false, + "dbt_task": [], + "deployment": [ + { + "deployment_id": "", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/migrate-removed-test/default/state/metadata.json", + "version_id": "" + } + ], + "description": null, + "edit_mode": "UI_LOCKED", + "email_notifications": [ + { + "no_alert_for_skipped_runs": false, + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ], + "environment": [], + "existing_cluster_id": null, + "format": "MULTI_TASK", + "git_source": [], + "health": [], + "id": "8790614038615799000", + "job_cluster": [], + "library": [], + "max_concurrent_runs": 1, + "max_retries": 0, + "min_retry_interval_millis": 0, + "name": "Job B", + "new_cluster": [], + "notebook_task": [], + "notification_settings": [], + "parameter": [], + "parent_path": null, + "performance_target": null, + "pipeline_task": [], + "provider_config": [ + { + "workspace_id": "900800700600" + } + ], + "python_wheel_task": [], + "queue": [ + { + "enabled": true + } + ], + "retry_on_timeout": false, + "run_as": [ + { + "group_name": "", + "service_principal_name": "", + "user_name": "tester@databricks.com" + } + ], + "run_job_task": [], + "schedule": [], + "spark_jar_task": [], + "spark_python_task": [], + "spark_submit_task": [], + "tags": null, + "task": [], + "timeout_seconds": 0, + "timeouts": null, + "trigger": [], + "triggers": [], + "url": "[DATABRICKS_URL]/#job/8790614038615799000", + "usage_policy_id": null, + "webhook_notifications": [ + { + "on_duration_warning_threshold_exceeded": [], + "on_failure": [], + "on_start": [], + "on_streaming_backlog_exceeded": [], + "on_success": [] + } + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ==" + } + ] + } + ], + "check_results": null +} diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index f329ec7962c..bc15e534e35 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -132,7 +132,10 @@ New bundles use the direct engine automatically, no migration is needed.`) defer func() { if tempStatePathAutoRemove { - _ = os.Remove(tempStatePath) + // UpgradeToWrite creates the WAL with O_EXCL; a leftover one fails every retry. + stateDB.Discard() + // Discard is a no-op once Finalize has run, but Finalize may have written the temp state. + dstate.RemoveFiles(tempStatePath) } }()