bundle: keep recording deployment history when the setting is removed

.nextchanges/bundles/7088.md

@@ -0,0 +1 @@
+* Keep recording deployment history for a deployment when `experimental.deployment_history` is removed from the configuration; only an explicit `false` is rejected. ([#7088](https://github.com/databricks/cli/pull/7088))

acceptance/bundle/dms/sticky/databricks.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: dms-sticky-$UNIQUE_NAME
+experimental:
+  deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: sticky

acceptance/bundle/dms/sticky/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = [""]

acceptance/bundle/dms/sticky/output.txt

@@ -0,0 +1,38 @@
+
+=== Deploy with recording on
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-sticky-[UNIQUE_NAME]/default/files...
+Created jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Drop the setting: the deployment keeps recording, so the next deploy creates version 2
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-sticky-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --dms //api/2.0/bundle/deployments --oneline --del-body state,resource_id
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+
+=== Wiping the local cache does not change that: the recording state is pulled from the workspace
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== An explicit false still conflicts with the state
+>>> musterr [CLI] bundle deploy
+Error: deployment history setting (false) does not match the existing state (true)
+
+Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over
+
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-sticky-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/dms/sticky/script

@@ -0,0 +1,32 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+title "Deploy with recording on"
+trace $CLI bundle deploy
+rm -f "$OUT_REQUESTS"
+
+title "Drop the setting: the deployment keeps recording, so the next deploy creates version 2"
+update_file.py databricks.yml 'experimental:
+  deployment_history: true
+' ''
+update_file.py databricks.yml 'name: sticky' 'name: sticky-renamed'
+trace $CLI bundle deploy
+trace print_requests.py --dms //api/2.0/bundle/deployments --oneline --del-body state,resource_id
+
+title "Wiping the local cache does not change that: the recording state is pulled from the workspace"
+rm -rf .databricks
+trace $CLI bundle plan
+rm -f "$OUT_REQUESTS"
+
+title "An explicit false still conflicts with the state"
+update_file.py databricks.yml 'bundle:' 'experimental:
+  deployment_history: false
+
+bundle:'
+trace musterr $CLI bundle deploy
+update_file.py databricks.yml 'experimental:
+  deployment_history: false
+
+' ''
+
+trace $CLI bundle destroy --auto-approve
+rm -f "$OUT_REQUESTS"

acceptance/bundle/dms/sticky/test.toml

@@ -0,0 +1,2 @@
+# The setting is driven from databricks.yml alone, so the env var must not turn it back on.
+EnvMatrix.DMS = [""]

bundle/statemgmt/state_pull.go

@@ -252,9 +252,9 @@ func readStates(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) ([
 		})
 
 		// Whether the state records deployment history is only known once it is read, but the
-		// lookup is needed right after; going by the config overlaps it with the reads. A state
-		// that disagrees with the config is either rejected or looked up again by the caller.
-		if ConfiguresDeploymentHistory(ctx, b) {
+		// lookup is needed right after; going by the config and the local state overlaps it with
+		// the reads. Without either, a recording state is looked up again by the caller.
+		if ConfiguresDeploymentHistory(ctx, b) || (directLocalState != nil && directLocalState.IsDMS()) {
 			wg.Go(func() {
 				id, err := dms.LookupDeploymentID(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
 				deployment = &DeploymentLookup{ID: id, Err: err}

cmd/bundle/utils/process.go

@@ -677,6 +677,11 @@ func enforceDeploymentHistorySetting(ctx context.Context, b *bundle.Bundle, stat
 	if configured == recorded {
 		return nil
 	}
+	// Recording sticks to a deployment that has it: leaving the setting out keeps it on, and only
+	// an explicit false conflicts with the state.
+	if recorded && (b.Config.Experimental == nil || b.Config.Experimental.DeploymentHistory == nil) {
+		return nil
+	}
 	if requireMatch {
 		return fmt.Errorf(`deployment history setting (%t) does not match the existing state (%t)