Allow changing deployment history on existing file state

acceptance/bundle/dms/existing-state/output.txt

@@ -1,78 +1,26 @@
 
-=== Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS
->>> [CLI] bundle deploy
+=== Enable history on existing remote file state without replacing the job
+>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
-Created jobs.one
-Files: 2 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
 
->>> [CLI] bundle plan -o json
+>>> jq {features: (.features // {}), job_id: .state["resources.jobs.one"].__id__} .databricks/bundle/default/resources.json
 {
-  "plan_version": [PLAN_VERSION],
-  "cli_version": "[CLI_VERSION]",
-  "lineage": "[UUID]",
-  "serial": 1
-}
-
->>> [CLI] bundle plan
-Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
-
->>> print_requests.py --dms //api/2.0/bundle --oneline
-
->>> jq {state_version, features} .databricks/bundle/default/resources.json
-{
-  "state_version": 3,
-  "features": null
+  "features": {
+    "deployment_history_shadow": {}
+  },
+  "job_id": "[ONE_ID]"
 }
 
-=== Enabling recording is refused because the existing deployment is not recorded
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
-Error: deployment history setting (true) does not match the existing state (false)
-
-Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over
-
-
->>> print_requests.py --dms //api/2.0/bundle --oneline
-
-=== Wiping the local cache still refuses after pulling the non-DMS state from the workspace
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
-Error: deployment history setting (true) does not match the existing state (false)
-
-Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over
-
-
-=== A leftover local WAL with the wrong lineage is still refused
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
-Error: reading state from [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: WAL recovery failed: WAL lineage ("stale") does not match state lineage ("[UUID]")
-
-
-=== Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve
-The following resources will be deleted:
-  delete resources.jobs.one
-
-All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default
-
-Destroy: 1 deleted
-
->>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
+=== Disable history without replacing the job
+>>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
-Created jobs.one
-Files: 2 uploaded, 0 deleted
-Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
-
->>> print_requests.py --dms //api/2.0/bundle --oneline
-{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.one"}], "version_type": "VERSION_TYPE_DEPLOY"}}
-{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[ONE_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
 
->>> find.py resources[.]json[.]wal --expect 0
-
->>> jq {state_version, features} .databricks/bundle/default/resources.json
+>>> jq {features: (.features // {}), job_id: .state["resources.jobs.one"].__id__} .databricks/bundle/default/resources.json
 {
-  "state_version": 3,
-  "features": {
-    "deployment_history_shadow": {}
-  }
+  "features": {},
+  "job_id": "[ONE_ID]"
 }

acceptance/bundle/dms/existing-state/script

@@ -1,38 +1,16 @@
 envsubst < databricks.yml.tmpl > databricks.yml
 
-title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS"
-trace $CLI bundle deploy
-dms_plan
-trace print_requests.py --dms //api/2.0/bundle --oneline
-trace jq '{state_version, features}' .databricks/bundle/default/resources.json
-
-title "Enabling recording is refused because the existing deployment is not recorded"
-musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
-trace print_requests.py --dms //api/2.0/bundle --oneline
-
-title "Wiping the local cache still refuses after pulling the non-DMS state from the workspace"
+# Create ordinary file state, then force the next command to pull it from the workspace.
+$CLI bundle deploy &> LOG.setup
+read_id.py one > /dev/null
 rm -rf .databricks
-musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
 
-title "A leftover local WAL with the wrong lineage is still refused"
-mkdir -p .databricks/bundle/default
-printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history_shadow":{}}}\n' > .databricks/bundle/default/resources.json.wal
-musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
-
-title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone"
-# Clear the stale WAL first: it was refused, not deleted, and destroy runs without recording, so it
-# would otherwise try to replay it and fail on the lineage mismatch.
-rm -f .databricks/bundle/default/resources.json.wal
-# Keep the mismatched setting here to verify that destroy ignores configuration and follows state.
-trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve
+title "Enable history on existing remote file state without replacing the job"
 trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
-trace print_requests.py --dms //api/2.0/bundle --oneline
-trace find.py 'resources[.]json[.]wal' --expect 0
-trace jq '{state_version, features}' .databricks/bundle/default/resources.json
-DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py
-deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
-add_repl "$deployment_id" DEPLOYMENT_ID
+trace jq '{features: (.features // {}), job_id: .state["resources.jobs.one"].__id__}' .databricks/bundle/default/resources.json
+
+title "Disable history without replacing the job"
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle deploy
+trace jq '{features: (.features // {}), job_id: .state["resources.jobs.one"].__id__}' .databricks/bundle/default/resources.json
 
-# The recorded requests are asserted above via print_requests.py; drop whatever the run added after
-# that (telemetry, and the state probes of the non-recording variant) so nothing is left to compare.
 rm -f "$OUT_REQUESTS"

acceptance/bundle/migrate/auto/dms-migration/output.txt

@@ -2,19 +2,27 @@
 === Replay a prior terraform deployment (captured from a real terraform run)
 >>> replay_tfstate.py terraform-requests.json terraform.tfstate
 
-=== Direct + deployment-history requested against a terraform state is rejected before commit
->>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true errcode [CLI] bundle deploy
+=== Terraform state migrates to file-backed direct state with shadow history
+>>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy
 Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
-Error: deployment history setting (true) does not match the existing state (false)
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Migrated 1 resource to direct deployment engine.
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
 
-Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over
-
-
-Exit code: 1
+>>> jq {features, job_id: .state["resources.jobs.test_job"].__id__} .databricks/bundle/default/resources.json
+{
+  "features": {
+    "deployment_history_shadow": {}
+  },
+  "job_id": "[TEST_JOB_ID]"
+}
 
 >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal)
-.databricks/bundle/default/terraform/terraform.tfstate
+.databricks/bundle/default/resources.json
+.databricks/bundle/default/terraform/terraform.tfstate.backup
 
->>> MSYS_NO_PATHCONV=1 [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state
-ID                   Type  Language  Path
-[OBJECT_ID]  FILE            /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate
+>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup -o json
+{
+  "object_type": "FILE"
+}

acceptance/bundle/migrate/auto/dms-migration/script

@@ -2,13 +2,11 @@ export DATABRICKS_BUNDLE_ENGINE=
 
 title "Replay a prior terraform deployment (captured from a real terraform run)"
 trace replay_tfstate.py terraform-requests.json terraform.tfstate
+read_id.py test_job > /dev/null
 rm -f "$OUT_REQUESTS"
 
-title "Direct + deployment-history requested against a terraform state is rejected before commit"
-# Migration produces a plain direct state (no deployment_history feature), so requesting
-# deployment history while migrating is a mismatch. It is rejected after the in-memory migration
-# but before CommitMigration writes or pushes anything, so the bundle stays on terraform with no
-# leaked direct state - migrate first (plain deploy), then enable deployment history.
-trace DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true errcode $CLI bundle deploy
+title "Terraform state migrates to file-backed direct state with shadow history"
+trace DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy
+trace jq '{features, job_id: .state["resources.jobs.test_job"].__id__}' .databricks/bundle/default/resources.json
 trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)'
-trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state"
+trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state/terraform.tfstate.backup" -o json | jq '{object_type}'

bundle/statemgmt/state_pull.go

@@ -77,12 +77,6 @@ func (s *StateDesc) IsDMS() bool {
 	return ok
 }
 
-// RecordsHistory includes both DMS-authoritative state and deployment history shadow.
-func (s *StateDesc) RecordsHistory() bool {
-	_, bestEffort := s.Features["deployment_history_shadow"]
-	return bestEffort || s.IsDMS()
-}
-
 func localRead(ctx context.Context, fullPath string, engine engine.EngineType) *StateDesc {
 	content, err := os.ReadFile(fullPath)
 	if err != nil {

cmd/bundle/utils/process.go

@@ -359,13 +359,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle
 			}
 		}
 
-		if stateDesc.Engine.IsDirect() && (opts.Deploy || opts.PreDeployChecks) {
-			if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, true); err != nil {
-				logdiag.LogError(ctx, err)
-				return b, stateDesc, root.ErrAlreadyPrinted
-			}
-		}
-
 		stateMode := dstate.WritingFile
 		switch {
 		case stateDesc.IsDMS():
@@ -668,9 +661,6 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st
 		if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil {
 			return err
 		}
-		if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, false); err != nil {
-			return err
-		}
 		if configuresDeploymentHistory(ctx, b) {
 			if _, err := prepareDeploymentHistoryShadow(ctx, b); err != nil {
 				log.Warnf(ctx, "Could not read deployment history: %s", err)
@@ -694,32 +684,6 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st
 	if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WritingDeploymentHistoryOnly, dstate.WithRecovery(false), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil {
 		return err
 	}
-	return enforceDeploymentHistorySetting(ctx, b, stateDesc, false)
-}
-
-func enforceDeploymentHistorySetting(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc, requireMatch bool) error {
-	if stateDesc.SourcePath == "" {
-		return nil
-	}
-	configured := configuresDeploymentHistory(ctx, b)
-	recorded := stateDesc.RecordsHistory()
-	if b.DeploymentBundle.StateDB.IsOpen() {
-		recorded = b.DeploymentBundle.StateDB.RecordsHistory()
-	}
-	if configured == recorded || (!configured && !stateDesc.IsDMS()) {
-		return nil
-	}
-	if requireMatch {
-		return fmt.Errorf(`deployment history setting (%t) does not match the existing state (%t)
-
-Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over`, configured, recorded)
-	}
-	if configured {
-		return errors.New(`enabling experimental.deployment_history for an existing deployment is not supported
-
-Run "databricks bundle destroy" first, then deploy again with deployment history enabled`)
-	}
-	log.Warnf(ctx, "Deployment history setting (%t) does not match the existing state (%t). Using the existing state.", configured, recorded)
 	return nil
 }