diff --git a/acceptance/bundle/dms/record-op-fault/output.txt b/acceptance/bundle/dms/record-op-fault/output.txt index 4648e7f6a0d..f52cd01e5fa 100644 --- a/acceptance/bundle/dms/record-op-fault/output.txt +++ b/acceptance/bundle/dms/record-op-fault/output.txt @@ -1,11 +1,106 @@ +=== Declining the first bind creates no deployment or version +>>> print_requests.py --dms --method POST //api/2.0/bundle + +=== A failed resource update records the existing job state and completes its version +>>> fault.py POST /api/2.2/jobs/reset 403 0 1 PERMISSION_DENIED + +>>> musterr [CLI] bundle deployment bind foo [JOB_ID] --auto-approve +Error: cannot bind_and_update resources.jobs.foo: updating id=[JOB_ID]: Fault injected by test. (403 PERMISSION_DENIED) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/reset +HTTP Status: 403 Forbidden +API error_code: PERMISSION_DENIED +API message: Fault injected by test. + + +>>> [CLI] jobs get [JOB_ID] +{ + "name": "target-[UNIQUE_NAME]" +} + +>>> read_state.py jobs foo id name +jobs foo id='[JOB_ID]' name='target-[UNIQUE_NAME]' + +>>> print_requests.py --dms --method POST //versions +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "operations": [ + { + "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE", + "resource_key": "jobs.foo" + } + ], + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== A failed bind still persists the remote deployment-history marker +>>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-record-op-fault-[UNIQUE_NAME]/default/state/resources.json + +>>> jq -e .features | has("deployment_history") LOG.remote-state +true + +=== Deploy finishes applying the configuration to the bound job +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-op-fault-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms --method POST //versions +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "operations": [ + { + "action_type": "OPERATION_ACTION_TYPE_UPDATE", + "resource_key": "jobs.foo" + } + ], + "previous_version_id": "1", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +>>> read_state.py jobs foo id name +jobs foo id='[JOB_ID]' name='foo' + +=== Bind a second job with the first job omitted from configuration +>>> update_file.py databricks.yml foo bar + === The service rejects the operation upload, so recording the bind fails >>> fault.py PATCH /api/2.0/bundle/* 500 0 1 ->>> musterr [CLI] bundle deployment bind foo [JOB_ID] --auto-approve -Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INJECTED) +>>> musterr [CLI] bundle deployment bind bar [BAR_ID] --auto-approve +Error: recording operation for resources.jobs.bar: Fault injected by test. (500 INJECTED) -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.bar?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus HTTP Status: 500 Internal Server Error API error_code: INJECTED API message: Fault injected by test. @@ -17,23 +112,68 @@ API message: Fault injected by test. "method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": { - "version_id": "1" + "version_id": "3" }, "body": { "cli_version": "[CLI_VERSION]", "operations": [ { "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE", - "resource_key": "jobs.foo" + "resource_key": "jobs.bar" } ], + "previous_version_id": "2", "version_type": "VERSION_TYPE_DEPLOY" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": { "completion_reason": "VERSION_COMPLETE_FAILURE" } } + +=== Retrying the bind succeeds without a dangling version +>>> [CLI] bundle deployment bind bar [BAR_ID] --auto-approve +Successfully bound job with an id '[BAR_ID]' +Bind changes have been applied to your workspace + +>>> print_requests.py --dms --method POST //versions +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", + "q": { + "version_id": "4" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "operations": [ + { + "action_type": "OPERATION_ACTION_TYPE_BIND", + "resource_key": "jobs.bar" + } + ], + "previous_version_id": "3", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/4/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Both resources remain managed; binding bar did not delete foo +>>> read_state.py jobs foo id name +jobs foo id='[JOB_ID]' name='foo' + +>>> read_state.py jobs bar id name +jobs bar id='[BAR_ID]' name='bar' + +>>> [CLI] bundle plan +delete jobs.foo + +Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged diff --git a/acceptance/bundle/dms/record-op-fault/script b/acceptance/bundle/dms/record-op-fault/script index fdc8afc00bf..68e3c89d56c 100644 --- a/acceptance/bundle/dms/record-op-fault/script +++ b/acceptance/bundle/dms/record-op-fault/script @@ -3,9 +3,45 @@ envsubst < databricks.yml.tmpl > databricks.yml job_id=$($CLI jobs create --json '{"name": "target-'$UNIQUE_NAME'"}' | jq -r '.job_id') add_repl "$job_id" JOB_ID +title "Declining the first bind creates no deployment or version" +trace musterr $CLI bundle deployment bind foo "$job_id" &> LOG.confirm +contains.py 'requires user confirmation' < LOG.confirm > LOG.confirm.checked +trace print_requests.py --dms --method POST //api/2.0/bundle + +title "A failed resource update records the existing job state and completes its version" +trace fault.py "POST /api/2.2/jobs/reset" 403 0 1 PERMISSION_DENIED +trace musterr $CLI bundle deployment bind foo "$job_id" --auto-approve +trace $CLI jobs get "$job_id" | jq '{name: .settings.name}' +trace read_state.py jobs foo id name +trace print_requests.py --dms --method POST //versions + +title "A failed bind still persists the remote deployment-history marker" +state_path=$($CLI bundle validate -o json | jq -r '.workspace.state_path') +trace MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" > LOG.remote-state +trace jq -e '.features | has("deployment_history")' LOG.remote-state + +title "Deploy finishes applying the configuration to the bound job" +trace $CLI bundle deploy --auto-approve +trace print_requests.py --dms --method POST //versions +trace read_state.py jobs foo id name + +title "Bind a second job with the first job omitted from configuration" +trace update_file.py databricks.yml foo bar +bar_id=$($CLI jobs create --json '{"name": "second-target-'$UNIQUE_NAME'"}' | jq -r '.job_id') +add_repl "$bar_id" BAR_ID + title "The service rejects the operation upload, so recording the bind fails" trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 -trace musterr $CLI bundle deployment bind foo "$job_id" --auto-approve +trace musterr $CLI bundle deployment bind bar "$bar_id" --auto-approve title "The version is still completed - with failure - rather than left holding its lease" trace print_requests.py --dms --method POST //versions + +title "Retrying the bind succeeds without a dangling version" +trace $CLI bundle deployment bind bar "$bar_id" --auto-approve +trace print_requests.py --dms --method POST //versions + +title "Both resources remain managed; binding bar did not delete foo" +trace read_state.py jobs foo id name +trace read_state.py jobs bar id name +trace $CLI bundle plan diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 486b044a124..7898f4559bb 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -45,7 +45,18 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, log.Infof(ctx, "Bound %s id=%#v", d.ResourceKey, planEntry.ID) return d.saveState(ctx, db, planEntry.ID, newState, d.DependsOn) case deployplan.BindAndUpdate: - return d.Update(ctx, db, planEntry.ID, newState, planEntry) + err := d.Update(ctx, db, planEntry.ID, newState, planEntry) + // A failed wait may already have recorded the updated state. + if err == nil || db.GetResourceID(d.ResourceKey) != "" { + return err + } + // DMS requires state for an existing resource even when its update fails. Keep the + // remote baseline; a later deploy can finish applying the configuration. + remoteState, stateErr := d.Adapter.RemapState(planEntry.RemoteState) + if stateErr != nil { + return errors.Join(err, stateErr) + } + return errors.Join(err, d.saveState(ctx, db, planEntry.ID, remoteState, d.DependsOn)) default: // The remaining actions act on a resource already in state. }