Keep state for a bound resource whose update fails under DMS

acceptance/bundle/dms/record-op-fault/output.txt

@@ -1,11 +1,106 @@
 
+=== Declining the first bind creates no deployment or version
+>>> print_requests.py --dms --method POST //api/2.0/bundle
+
+=== A failed resource update records the existing job state and completes its version
+>>> fault.py POST /api/2.2/jobs/reset 403 0 1 PERMISSION_DENIED
+
+>>> musterr [CLI] bundle deployment bind foo [JOB_ID] --auto-approve
+Error: cannot bind_and_update resources.jobs.foo: updating id=[JOB_ID]: Fault injected by test. (403 PERMISSION_DENIED)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/reset
+HTTP Status: 403 Forbidden
+API error_code: PERMISSION_DENIED
+API message: Fault injected by test.
+
+
+>>> [CLI] jobs get [JOB_ID]
+{
+  "name": "target-[UNIQUE_NAME]"
+}
+
+>>> read_state.py jobs foo id name
+jobs foo id='[JOB_ID]' name='target-[UNIQUE_NAME]'
+
+>>> print_requests.py --dms --method POST //versions
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "operations": [
+      {
+        "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE",
+        "resource_key": "jobs.foo"
+      }
+    ],
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_FAILURE"
+  }
+}
+
+=== A failed bind still persists the remote deployment-history marker
+>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-record-op-fault-[UNIQUE_NAME]/default/state/resources.json
+
+>>> jq -e .features | has("deployment_history") LOG.remote-state
+true
+
+=== Deploy finishes applying the configuration to the bound job
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-op-fault-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 5 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --dms --method POST //versions
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "operations": [
+      {
+        "action_type": "OPERATION_ACTION_TYPE_UPDATE",
+        "resource_key": "jobs.foo"
+      }
+    ],
+    "previous_version_id": "1",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+>>> read_state.py jobs foo id name
+jobs foo id='[JOB_ID]' name='foo'
+
+=== Bind a second job with the first job omitted from configuration
+>>> update_file.py databricks.yml foo bar
+
 === The service rejects the operation upload, so recording the bind fails
 >>> fault.py PATCH /api/2.0/bundle/* 500 0 1
 
->>> musterr [CLI] bundle deployment bind foo [JOB_ID] --auto-approve
-Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INJECTED)
+>>> musterr [CLI] bundle deployment bind bar [BAR_ID] --auto-approve
+Error: recording operation for resources.jobs.bar: Fault injected by test. (500 INJECTED)
 
-Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus
+Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.bar?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus
 HTTP Status: 500 Internal Server Error
 API error_code: INJECTED
 API message: Fault injected by test.
@@ -17,23 +112,68 @@ API message: Fault injected by test.
   "method": "POST",
   "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
   "q": {
-    "version_id": "1"
+    "version_id": "3"
   },
   "body": {
     "cli_version": "[CLI_VERSION]",
     "operations": [
       {
         "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE",
-        "resource_key": "jobs.foo"
+        "resource_key": "jobs.bar"
       }
     ],
+    "previous_version_id": "2",
     "version_type": "VERSION_TYPE_DEPLOY"
   }
 }
 {
   "method": "POST",
-  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete",
   "body": {
     "completion_reason": "VERSION_COMPLETE_FAILURE"
   }
 }
+
+=== Retrying the bind succeeds without a dangling version
+>>> [CLI] bundle deployment bind bar [BAR_ID] --auto-approve
+Successfully bound job with an id '[BAR_ID]'
+Bind changes have been applied to your workspace
+
+>>> print_requests.py --dms --method POST //versions
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
+  "q": {
+    "version_id": "4"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "operations": [
+      {
+        "action_type": "OPERATION_ACTION_TYPE_BIND",
+        "resource_key": "jobs.bar"
+      }
+    ],
+    "previous_version_id": "3",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/4/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+=== Both resources remain managed; binding bar did not delete foo
+>>> read_state.py jobs foo id name
+jobs foo id='[JOB_ID]' name='foo'
+
+>>> read_state.py jobs bar id name
+jobs bar id='[BAR_ID]' name='bar'
+
+>>> [CLI] bundle plan
+delete jobs.foo
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged

acceptance/bundle/dms/record-op-fault/script

@@ -3,9 +3,45 @@ envsubst < databricks.yml.tmpl > databricks.yml
 job_id=$($CLI jobs create --json '{"name": "target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
 add_repl "$job_id" JOB_ID
 
+title "Declining the first bind creates no deployment or version"
+trace musterr $CLI bundle deployment bind foo "$job_id" &> LOG.confirm
+contains.py 'requires user confirmation' < LOG.confirm > LOG.confirm.checked
+trace print_requests.py --dms --method POST //api/2.0/bundle
+
+title "A failed resource update records the existing job state and completes its version"
+trace fault.py "POST /api/2.2/jobs/reset" 403 0 1 PERMISSION_DENIED
+trace musterr $CLI bundle deployment bind foo "$job_id" --auto-approve
+trace $CLI jobs get "$job_id" | jq '{name: .settings.name}'
+trace read_state.py jobs foo id name
+trace print_requests.py --dms --method POST //versions
+
+title "A failed bind still persists the remote deployment-history marker"
+state_path=$($CLI bundle validate -o json | jq -r '.workspace.state_path')
+trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" > LOG.remote-state
+trace jq -e '.features | has("deployment_history")' LOG.remote-state
+
+title "Deploy finishes applying the configuration to the bound job"
+trace $CLI bundle deploy --auto-approve
+trace print_requests.py --dms --method POST //versions
+trace read_state.py jobs foo id name
+
+title "Bind a second job with the first job omitted from configuration"
+trace update_file.py databricks.yml foo bar
+bar_id=$($CLI jobs create --json '{"name": "second-target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
+add_repl "$bar_id" BAR_ID
+
 title "The service rejects the operation upload, so recording the bind fails"
 trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1
-trace musterr $CLI bundle deployment bind foo "$job_id" --auto-approve
+trace musterr $CLI bundle deployment bind bar "$bar_id" --auto-approve
 
 title "The version is still completed - with failure - rather than left holding its lease"
 trace print_requests.py --dms --method POST //versions
+
+title "Retrying the bind succeeds without a dangling version"
+trace $CLI bundle deployment bind bar "$bar_id" --auto-approve
+trace print_requests.py --dms --method POST //versions
+
+title "Both resources remain managed; binding bar did not delete foo"
+trace read_state.py jobs foo id name
+trace read_state.py jobs bar id name
+trace $CLI bundle plan

bundle/direct/apply.go

@@ -45,7 +45,18 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState,
 		log.Infof(ctx, "Bound %s id=%#v", d.ResourceKey, planEntry.ID)
 		return d.saveState(ctx, db, planEntry.ID, newState, d.DependsOn)
 	case deployplan.BindAndUpdate:
-		return d.Update(ctx, db, planEntry.ID, newState, planEntry)
+		err := d.Update(ctx, db, planEntry.ID, newState, planEntry)
+		// A failed wait may already have recorded the updated state.
+		if err == nil || db.GetResourceID(d.ResourceKey) != "" {
+			return err
+		}
+		// DMS requires state for an existing resource even when its update fails. Keep the
+		// remote baseline; a later deploy can finish applying the configuration.
+		remoteState, stateErr := d.Adapter.RemapState(planEntry.RemoteState)
+		if stateErr != nil {
+			return errors.Join(err, stateErr)
+		}
+		return errors.Join(err, d.saveState(ctx, db, planEntry.ID, remoteState, d.DependsOn))
 	default:
 		// The remaining actions act on a resource already in state.
 	}