From c32b54dbcf9eb11ec1e448e5841e38f09ad5adda Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 14 Oct 2026 14:57:02 +0000 Subject: [PATCH] Support bundle deployment bind for DMS-recorded deployments With experimental.deployment_history the service is the source of truth, so bind no longer stages a change for the next deploy. It plans the bound resource (and its permissions/grants) as Bind or BindAndUpdate, asks for approval if the workspace would change, and applies it immediately in its own version, recorded with the OPERATION_ACTION_TYPE_BIND / BIND_AND_UPDATE action types. Other resources are left untouched; a dependency that would change rejects the bind. Unbind under DMS stays unsupported: the service has no unbind action type yet. Co-authored-by: Isaac --- acceptance/bundle/deployment/bind/test.toml | 3 +- .../bundle/dms/bind-unbind-unsupported/script | 14 -- .../bundle/dms/bind/databricks.yml.tmpl | 23 +++ acceptance/bundle/dms/bind/out.test.toml | 2 + acceptance/bundle/dms/bind/output.txt | 167 ++++++++++++++++++ acceptance/bundle/dms/bind/script | 56 ++++++ .../dms/record-op-fault/databricks.yml.tmpl | 9 + .../out.test.toml | 0 .../bundle/dms/record-op-fault/output.txt | 39 ++++ acceptance/bundle/dms/record-op-fault/script | 11 ++ .../bundle/dms/record-op-fault/test.toml | 3 + .../databricks.yml.tmpl | 2 +- .../dms/unbind-unsupported/out.test.toml | 2 + .../output.txt | 9 +- .../bundle/dms/unbind-unsupported/script | 6 + .../test.toml | 2 +- bundle/phases/bind.go | 53 ++++-- bundle/phases/bind_dms.go | 118 +++++++++++++ bundle/phases/bind_dms_test.go | 35 ++++ bundle/phases/bind_test.go | 47 +++++ bundle/phases/deploy.go | 7 - bundle/phases/dms.go | 12 +- bundle/phases/dms_test.go | 6 + cmd/bundle/deployment/bind.go | 9 +- cmd/bundle/deployment/bind_resource.go | 30 +++- 25 files changed, 607 insertions(+), 58 deletions(-) delete mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/script create mode 100644 acceptance/bundle/dms/bind/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/bind/out.test.toml create mode 100644 acceptance/bundle/dms/bind/output.txt create mode 100644 acceptance/bundle/dms/bind/script create mode 100644 acceptance/bundle/dms/record-op-fault/databricks.yml.tmpl rename acceptance/bundle/dms/{bind-unbind-unsupported => record-op-fault}/out.test.toml (100%) create mode 100644 acceptance/bundle/dms/record-op-fault/output.txt create mode 100644 acceptance/bundle/dms/record-op-fault/script create mode 100644 acceptance/bundle/dms/record-op-fault/test.toml rename acceptance/bundle/dms/{bind-unbind-unsupported => unbind-unsupported}/databricks.yml.tmpl (71%) create mode 100644 acceptance/bundle/dms/unbind-unsupported/out.test.toml rename acceptance/bundle/dms/{bind-unbind-unsupported => unbind-unsupported}/output.txt (51%) create mode 100644 acceptance/bundle/dms/unbind-unsupported/script rename acceptance/bundle/dms/{bind-unbind-unsupported => unbind-unsupported}/test.toml (58%) create mode 100644 bundle/phases/bind_dms.go create mode 100644 bundle/phases/bind_dms_test.go create mode 100644 bundle/phases/bind_test.go diff --git a/acceptance/bundle/deployment/bind/test.toml b/acceptance/bundle/deployment/bind/test.toml index b693e4ded26..634ca8b4cae 100644 --- a/acceptance/bundle/deployment/bind/test.toml +++ b/acceptance/bundle/deployment/bind/test.toml @@ -1,2 +1,3 @@ -# Bind operations are not yet supported by the Deployment Metadata Service (DMS) +# These tests cover the file-based bind, which stages its change for the next deploy. With the +# Deployment Metadata Service (DMS) a bind is applied immediately, so it is covered by bundle/dms/bind. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/script b/acceptance/bundle/dms/bind-unbind-unsupported/script deleted file mode 100644 index 03d9cf34a31..00000000000 --- a/acceptance/bundle/dms/bind-unbind-unsupported/script +++ /dev/null @@ -1,14 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -# The gate that refuses bind while recording deployment history is on the config, so it fires -# before any deploy. Create a real workspace job so bind gets past fetching the resource and -# reaches the gate, instead of failing because the resource does not exist. -title "bind is refused while recording deployment history" -job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id') -add_repl "$job_id" JOB_ID -trace musterr $CLI bundle deployment bind foo "$job_id" - -# unbind refuses when it opens an existing recording state, so it needs a deployment first. -title "Deploy so the bundle records state, then unbind is refused too" -trace $CLI bundle deploy -trace musterr $CLI bundle deployment unbind foo diff --git a/acceptance/bundle/dms/bind/databricks.yml.tmpl b/acceptance/bundle/dms/bind/databricks.yml.tmpl new file mode 100644 index 00000000000..c7c7061aa22 --- /dev/null +++ b/acceptance/bundle/dms/bind/databricks.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: dms-bind-$UNIQUE_NAME +experimental: + deployment_history: true + +targets: + default: + default: true + first_job: {} + +resources: + schemas: + sc: + name: dms_bind_$UNIQUE_NAME + catalog_name: main + jobs: + foo: + name: foo + permissions: + - group_name: users + level: CAN_MANAGE + bar: + name: bar diff --git a/acceptance/bundle/dms/bind/out.test.toml b/acceptance/bundle/dms/bind/out.test.toml new file mode 100644 index 00000000000..84b22ba6735 --- /dev/null +++ b/acceptance/bundle/dms/bind/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/bind/output.txt b/acceptance/bundle/dms/bind/output.txt new file mode 100644 index 00000000000..3d7adf73e5c --- /dev/null +++ b/acceptance/bundle/dms/bind/output.txt @@ -0,0 +1,167 @@ + +>>> [CLI] schemas create dms_bind_[UNIQUE_NAME] main +{ + "full_name": "main.dms_bind_[UNIQUE_NAME]" +} + +=== A missing resource cannot be bound +>>> musterr [CLI] bundle deployment bind sc main.missing_[UNIQUE_NAME] --auto-approve +Error: schema with an id 'main.missing_[UNIQUE_NAME]' is not found + +=== Bind sc, the first bind: it creates the deployment. The config already matches, so this is a plain bind that records the resource without touching it +>>> [CLI] bundle deployment bind sc main.dms_bind_[UNIQUE_NAME] +Successfully bound schema with an id 'main.dms_bind_[UNIQUE_NAME]' +Bind changes have been applied to your workspace + +=== Version 1 records only sc, with the plain bind action type; the jobs are left untouched +>>> print_requests.py --dms --method POST //versions --nostamp +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "operations": [ + { + "action_type": "OPERATION_ACTION_TYPE_BIND", + "resource_key": "schemas.sc" + } + ], + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== An update requires confirmation and leaves the workspace unchanged without it +>>> [CLI] jobs get [FOO_ID] +{ + "name": "bind-target-[UNIQUE_NAME]" +} + +=== Bind foo into the existing deployment: its config differs, so this is a bind-and-update that applies the config now +>>> [CLI] bundle deployment bind foo [FOO_ID] --auto-approve +Successfully bound job with an id '[FOO_ID]' +Bind changes have been applied to your workspace + +=== Version 2 records only foo and its permissions (bar is untouched) and chains to version 1 +>>> print_requests.py --dms --method POST //versions --nostamp +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "operations": [ + { + "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE", + "resource_key": "jobs.foo" + }, + { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_key": "jobs.foo.permissions" + } + ], + "previous_version_id": "1", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The bound job has already been updated +>>> [CLI] jobs get [FOO_ID] +{ + "name": "foo" +} + +=== An already-bound resource cannot be rebound +>>> musterr [CLI] bundle deployment bind foo [FOO_ID] --auto-approve +Error: resources.jobs.foo is already bound to ID "[FOO_ID]"; rebinding is not supported with deployment history + + +=== sc and foo are now managed; bar is still an unmanaged create +>>> [CLI] bundle plan +create jobs.bar + +Plan: 1 to add, 0 to change, 0 to delete, 3 unchanged + +=== A job can be the first bind of a new deployment +>>> [CLI] bundle deployment bind foo [FIRST_ID] -t first_job --auto-approve +Successfully bound job with an id '[FIRST_ID]' +Bind changes have been applied to your workspace + +>>> print_requests.py --dms --method POST //versions --nostamp +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "operations": [ + { + "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE", + "resource_key": "jobs.foo" + }, + { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_key": "jobs.foo.permissions" + } + ], + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The job points to the deployment that recorded its adoption, at version 1 +>>> [CLI] jobs get [FIRST_ID] +{ + "kind": "BUNDLE", + "version_id": "1" +} + +>>> [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +true + +=== Test cleanup: the bound resources are now bundle-managed, so destroy removes them +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + delete resources.schemas.sc + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.sc + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-bind-[UNIQUE_NAME]/default + +Destroy: 2 deleted + +>>> [CLI] bundle destroy -t first_job --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-bind-[UNIQUE_NAME]/first_job + +Destroy: 1 deleted diff --git a/acceptance/bundle/dms/bind/script b/acceptance/bundle/dms/bind/script new file mode 100644 index 00000000000..d278df82d88 --- /dev/null +++ b/acceptance/bundle/dms/bind/script @@ -0,0 +1,56 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# sc's config matches this existing schema exactly, so binding it changes nothing (a plain bind). +trace $CLI schemas create "dms_bind_$UNIQUE_NAME" main | jq '{full_name}' +# foo's config ("foo") differs from this existing job, so binding it applies an update. +foo_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id') +add_repl "$foo_id" FOO_ID + +cleanup() { + title "Test cleanup: the bound resources are now bundle-managed, so destroy removes them" + trace $CLI bundle destroy --auto-approve + if [[ -n "${first_id:-}" ]]; then + trace $CLI bundle destroy -t first_job --auto-approve + fi +} +trap cleanup EXIT + +title "A missing resource cannot be bound" +trace musterr $CLI bundle deployment bind sc "main.missing_$UNIQUE_NAME" --auto-approve + +title "Bind sc, the first bind: it creates the deployment. The config already matches, so this is a plain bind that records the resource without touching it" +trace $CLI bundle deployment bind sc "main.dms_bind_$UNIQUE_NAME" + +title "Version 1 records only sc, with the plain bind action type; the jobs are left untouched" +trace print_requests.py --dms --method POST //versions --nostamp + +title "An update requires confirmation and leaves the workspace unchanged without it" +trace musterr $CLI bundle deployment bind foo "$foo_id" &> LOG.confirm +contains.py 'requires user confirmation' 'Plan: create resources.jobs.foo.permissions' < LOG.confirm > LOG.confirm.checked +trace $CLI jobs get "$foo_id" | jq '{name: .settings.name}' + +title "Bind foo into the existing deployment: its config differs, so this is a bind-and-update that applies the config now" +trace $CLI bundle deployment bind foo "$foo_id" --auto-approve + +title "Version 2 records only foo and its permissions (bar is untouched) and chains to version 1" +trace print_requests.py --dms --method POST //versions --nostamp + +title "The bound job has already been updated" +trace $CLI jobs get "$foo_id" | jq '{name: .settings.name}' + +title "An already-bound resource cannot be rebound" +trace musterr $CLI bundle deployment bind foo "$foo_id" --auto-approve + +title "sc and foo are now managed; bar is still an unmanaged create" +trace $CLI bundle plan + +title "A job can be the first bind of a new deployment" +first_id=$($CLI jobs create --json '{"name": "first-bind-'$UNIQUE_NAME'"}' | jq -r '.job_id') +add_repl "$first_id" FIRST_ID +trace $CLI bundle deployment bind foo "$first_id" -t first_job --auto-approve +trace print_requests.py --dms --method POST //versions --nostamp + +title "The job points to the deployment that recorded its adoption, at version 1" +trace $CLI jobs get "$first_id" | jq '.settings.deployment | {kind, version_id}' +deployment_id=$($CLI jobs get "$first_id" | jq -r '.settings.deployment.deployment_id') +trace $CLI bundle-deployments list-resources "deployments/$deployment_id" -o json | jq -e --arg id "$first_id" 'any(.[]; .resource_key == "jobs.foo" and .resource_id == $id)' diff --git a/acceptance/bundle/dms/record-op-fault/databricks.yml.tmpl b/acceptance/bundle/dms/record-op-fault/databricks.yml.tmpl new file mode 100644 index 00000000000..235760efa24 --- /dev/null +++ b/acceptance/bundle/dms/record-op-fault/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-record-op-fault-$UNIQUE_NAME +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml b/acceptance/bundle/dms/record-op-fault/out.test.toml similarity index 100% rename from acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml rename to acceptance/bundle/dms/record-op-fault/out.test.toml diff --git a/acceptance/bundle/dms/record-op-fault/output.txt b/acceptance/bundle/dms/record-op-fault/output.txt new file mode 100644 index 00000000000..4648e7f6a0d --- /dev/null +++ b/acceptance/bundle/dms/record-op-fault/output.txt @@ -0,0 +1,39 @@ + +=== The service rejects the operation upload, so recording the bind fails +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1 + +>>> musterr [CLI] bundle deployment bind foo [JOB_ID] --auto-approve +Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INJECTED) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus +HTTP Status: 500 Internal Server Error +API error_code: INJECTED +API message: Fault injected by test. + + +=== The version is still completed - with failure - rather than left holding its lease +>>> print_requests.py --dms --method POST //versions +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "operations": [ + { + "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE", + "resource_key": "jobs.foo" + } + ], + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} diff --git a/acceptance/bundle/dms/record-op-fault/script b/acceptance/bundle/dms/record-op-fault/script new file mode 100644 index 00000000000..fdc8afc00bf --- /dev/null +++ b/acceptance/bundle/dms/record-op-fault/script @@ -0,0 +1,11 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +job_id=$($CLI jobs create --json '{"name": "target-'$UNIQUE_NAME'"}' | jq -r '.job_id') +add_repl "$job_id" JOB_ID + +title "The service rejects the operation upload, so recording the bind fails" +trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 +trace musterr $CLI bundle deployment bind foo "$job_id" --auto-approve + +title "The version is still completed - with failure - rather than left holding its lease" +trace print_requests.py --dms --method POST //versions diff --git a/acceptance/bundle/dms/record-op-fault/test.toml b/acceptance/bundle/dms/record-op-fault/test.toml new file mode 100644 index 00000000000..24f1970a8f2 --- /dev/null +++ b/acceptance/bundle/dms/record-op-fault/test.toml @@ -0,0 +1,3 @@ +# Local only: the case needs an injected failure (DMS must reject the operation upload once), and +# fault.py registers its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl b/acceptance/bundle/dms/unbind-unsupported/databricks.yml.tmpl similarity index 71% rename from acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl rename to acceptance/bundle/dms/unbind-unsupported/databricks.yml.tmpl index e60f4605610..6ba28905b76 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl +++ b/acceptance/bundle/dms/unbind-unsupported/databricks.yml.tmpl @@ -1,5 +1,5 @@ bundle: - name: dms-bind-unbind-$UNIQUE_NAME + name: dms-unbind-$UNIQUE_NAME experimental: deployment_history: true diff --git a/acceptance/bundle/dms/unbind-unsupported/out.test.toml b/acceptance/bundle/dms/unbind-unsupported/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/unbind-unsupported/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt b/acceptance/bundle/dms/unbind-unsupported/output.txt similarity index 51% rename from acceptance/bundle/dms/bind-unbind-unsupported/output.txt rename to acceptance/bundle/dms/unbind-unsupported/output.txt index d2d0a366778..a1391c4aa08 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt +++ b/acceptance/bundle/dms/unbind-unsupported/output.txt @@ -1,12 +1,7 @@ -=== bind is refused while recording deployment history ->>> musterr [CLI] bundle deployment bind foo [JOB_ID] -Error: bind is not supported for a bundle target that records deployment history - - -=== Deploy so the bundle records state, then unbind is refused too +=== Deploy so the bundle records state, then unbind is refused >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-unbind-[UNIQUE_NAME]/default/files... Created jobs.foo Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/dms/unbind-unsupported/script b/acceptance/bundle/dms/unbind-unsupported/script new file mode 100644 index 00000000000..d72aafe94b5 --- /dev/null +++ b/acceptance/bundle/dms/unbind-unsupported/script @@ -0,0 +1,6 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# unbind refuses when it opens an existing recording state, so it needs a deployment first. +title "Deploy so the bundle records state, then unbind is refused" +trace $CLI bundle deploy +trace musterr $CLI bundle deployment unbind foo diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/test.toml b/acceptance/bundle/dms/unbind-unsupported/test.toml similarity index 58% rename from acceptance/bundle/dms/bind-unbind-unsupported/test.toml rename to acceptance/bundle/dms/unbind-unsupported/test.toml index 140ee938238..267d07f61de 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/test.toml +++ b/acceptance/bundle/dms/unbind-unsupported/test.toml @@ -1,4 +1,4 @@ RecordRequests = false -# bind/unbind refusal is a client-side gate on the recording config; it needs no real +# The unbind refusal is a client-side gate on the recording config; it needs no real # workspace, so this stays local-only rather than inheriting the suite's cloud run. Cloud = false diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 7b7d056f879..3107dc527c0 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -47,13 +47,13 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *BindOptions, stateDesc *s return } + resourceKey := fmt.Sprintf("resources.%s.%s", opts.Group, opts.ResourceKey) if stateDesc.IsDMS() { - logdiag.LogError(ctx, errors.New("bind is not supported for a bundle target that records deployment history")) + bindWithHistory(ctx, b, resourceKey, opts.ResourceId, opts.AutoApprove) return } // Direct engine: import into temp state, run plan, check for changes. - resourceKey := fmt.Sprintf("resources.%s.%s", opts.Group, opts.ResourceKey) _, statePath := b.StateFilenameDirect(ctx) result, err := b.DeploymentBundle.Bind(ctx, b.WorkspaceClient(ctx), &b.Config, statePath, resourceKey, opts.ResourceId) @@ -62,7 +62,7 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *BindOptions, stateDesc *s return } - if !confirmBindPlan(ctx, resourceKey, result.Plan, opts.AutoApprove) { + if !confirmBindPlan(ctx, resourceKey, result.Plan, opts.AutoApprove, false) { result.Cancel() return } @@ -86,24 +86,37 @@ func jsonDump(ctx context.Context, v any, field string) string { return string(b) } -// confirmBindPlan shows the planned changes to the bound resource and, unless autoApprove, asks -// the user to approve them. It reports whether the bind should proceed. -func confirmBindPlan(ctx context.Context, resourceKey string, plan *deployplan.Plan, autoApprove bool) bool { - entry := plan.Plan[resourceKey] - if autoApprove || entry == nil || entry.Action == deployplan.Skip || entry.Action == deployplan.Undefined { +// confirmBindPlan shows the planned changes and, unless autoApprove, asks the user to approve them. +// It reports whether the bind should proceed. An immediate bind (DMS) also applies the bound +// resource's permissions and grants now, so their changes need approval too. +func confirmBindPlan(ctx context.Context, resourceKey string, plan *deployplan.Plan, autoApprove, immediate bool) bool { + if autoApprove { return true } - - cmdio.LogString(ctx, fmt.Sprintf("Plan: %s %s", entry.Action, resourceKey)) - if len(entry.Changes) > 0 { - cmdio.LogString(ctx, "\nChanges detected:") - for _, field := range slices.Sorted(maps.Keys(entry.Changes)) { - change := entry.Changes[field] - if change.Action != deployplan.Skip { - cmdio.LogString(ctx, fmt.Sprintf(" ~ %s: %v -> %v", field, jsonDump(ctx, change.Remote, field), jsonDump(ctx, change.New, field))) + changesWorkspace := false + for _, action := range plan.GetActions() { + if !immediate && action.ResourceKey != resourceKey { + continue + } + if action.ActionType == deployplan.Skip || action.ActionType == deployplan.Bind || action.ActionType == deployplan.Undefined { + continue + } + changesWorkspace = true + entry := plan.Plan[action.ResourceKey] + cmdio.LogString(ctx, fmt.Sprintf("Plan: %s %s", entry.Action, action.ResourceKey)) + if len(entry.Changes) > 0 { + cmdio.LogString(ctx, "\nChanges detected:") + for _, field := range slices.Sorted(maps.Keys(entry.Changes)) { + change := entry.Changes[field] + if change.Action != deployplan.Skip { + cmdio.LogString(ctx, fmt.Sprintf(" ~ %s: %v -> %v", field, jsonDump(ctx, change.Remote, field), jsonDump(ctx, change.New, field))) + } } + cmdio.LogString(ctx, "") } - cmdio.LogString(ctx, "") + } + if !changesWorkspace { + return true } if !cmdio.IsPromptSupported(ctx) { @@ -111,7 +124,11 @@ func confirmBindPlan(ctx context.Context, resourceKey string, plan *deployplan.P return false } - ans, err := cmdio.AskYesOrNo(ctx, "Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'.") + prompt := "Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'." + if immediate { + prompt = "Confirm bind? The changes will be applied to the workspace now." + } + ans, err := cmdio.AskYesOrNo(ctx, prompt) if err != nil { logdiag.LogError(ctx, err) return false diff --git a/bundle/phases/bind_dms.go b/bundle/phases/bind_dms.go new file mode 100644 index 00000000000..897522df2cd --- /dev/null +++ b/bundle/phases/bind_dms.go @@ -0,0 +1,118 @@ +package phases + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/metadata" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// bindWithHistory applies adoption immediately and records it in DMS, the deployment's +// authoritative state. The caller opens StateDB before entering this phase. +func bindWithHistory(ctx context.Context, b *bundle.Bundle, resourceKey, resourceID string, autoApprove bool) { + wsc := b.WorkspaceClient(ctx) + // The operation buffer records each write through the workspace client on the context. + if !cmdctx.HasWorkspaceClient(ctx) { + ctx = cmdctx.SetWorkspaceClient(ctx, wsc) + } + + db := &b.DeploymentBundle.StateDB + versionStarted := false + defer func() { + if _, err := db.Finalize(ctx); err != nil { + logdiag.LogError(ctx, err) + } + if versionStarted { + // Push the state marker even after a partial failure, as deploy does. + statemgmt.PushResourcesState(ctx, b) + } + if _, err := db.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } + }() + + if existingID := db.GetResourceID(resourceKey); existingID != "" { + logdiag.LogError(ctx, fmt.Errorf("%s is already bound to ID %q; rebinding is not supported with deployment history", resourceKey, existingID)) + return + } + + // Stamp before planning to avoid reporting deployment metadata as drift. + muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(db.VersionID + 1)} + if db.DeploymentID != "" { + muts = append(muts, metadata.AnnotateDeployment(db.DeploymentID)) + } + bundle.ApplySeqContext(ctx, b, muts...) + if logdiag.HasError(ctx) { + return + } + + b.DeploymentBundle.BindKey = resourceKey + b.DeploymentBundle.BindID = resourceID + plan, err := b.DeploymentBundle.CalculatePlan(ctx, wsc, &b.Config) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := selectBindPlan(plan, resourceKey); err != nil { + logdiag.LogError(ctx, err) + return + } + + if !confirmBindPlan(ctx, resourceKey, plan, autoApprove, true) { + return + } + + var deployment *bundledeployments.Deployment + if db.DeploymentID != "" { + deployment, err = wsc.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: dms.DeploymentName(db.DeploymentID), + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + } + + if err := db.UpgradeToWrite(); err != nil { + logdiag.LogError(ctx, err) + return + } + + createOrUpdateDeployment(ctx, b, deployment) + if logdiag.HasError(ctx) { + return + } + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + logdiag.LogError(ctx, err) + return + } + versionStarted = true + + b.DeploymentBundle.Apply(ctx, wsc, plan, false) +} + +// selectBindPlan includes the bound resource and its children, keeping unchanged dependencies +// for reference resolution. A bind must not deploy changes to other resources. +func selectBindPlan(plan *deployplan.Plan, resourceKey string) error { + plan.FilterToSelected([]string{strings.TrimPrefix(resourceKey, "resources.")}) + for _, action := range plan.GetActions() { + if action.ResourceKey == resourceKey || strings.HasPrefix(action.ResourceKey, resourceKey+".") || action.ActionType == deployplan.Skip { + continue + } + return fmt.Errorf("cannot bind %s: dependency %s requires %s; deploy or bind the dependency first", resourceKey, action.ResourceKey, action.ActionType) + } + return nil +} diff --git a/bundle/phases/bind_dms_test.go b/bundle/phases/bind_dms_test.go new file mode 100644 index 00000000000..0dc0c880b6a --- /dev/null +++ b/bundle/phases/bind_dms_test.go @@ -0,0 +1,35 @@ +package phases + +import ( + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSelectBindPlan(t *testing.T) { + for _, action := range []deployplan.ActionType{deployplan.Skip, deployplan.Create, deployplan.Update, deployplan.Recreate} { + t.Run(string(action), func(t *testing.T) { + plan := deployplan.NewPlanDirect() + plan.Plan = map[string]*deployplan.PlanEntry{ + "resources.jobs.foo": { + Action: deployplan.Bind, + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.dependency"}}, + }, + "resources.jobs.foo.permissions": {Action: deployplan.Create}, + "resources.jobs.dependency": {Action: action}, + "resources.jobs.unrelated": {Action: deployplan.Update}, + } + err := selectBindPlan(plan, "resources.jobs.foo") + if action != deployplan.Skip { + require.ErrorContains(t, err, "dependency resources.jobs.dependency requires "+string(action)) + return + } + require.NoError(t, err) + assert.Contains(t, plan.Plan, "resources.jobs.foo.permissions") + assert.Contains(t, plan.Plan, "resources.jobs.dependency") + assert.NotContains(t, plan.Plan, "resources.jobs.unrelated") + }) + } +} diff --git a/bundle/phases/bind_test.go b/bundle/phases/bind_test.go new file mode 100644 index 00000000000..fa9d67d17cd --- /dev/null +++ b/bundle/phases/bind_test.go @@ -0,0 +1,47 @@ +package phases + +import ( + "bytes" + "io" + "strings" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/logdiag" + "github.com/stretchr/testify/assert" +) + +func TestConfirmBindPlan(t *testing.T) { + for _, tc := range []struct { + name string + action deployplan.ActionType + child deployplan.ActionType + immediate bool + autoApprove bool + want bool + }{ + {"unchanged", deployplan.Bind, deployplan.Skip, true, false, true}, + {"update requires approval", deployplan.BindAndUpdate, deployplan.Skip, true, false, false}, + {"permissions require approval", deployplan.Bind, deployplan.Create, true, false, false}, + {"approved permissions", deployplan.Bind, deployplan.Create, true, true, true}, + {"deferred permissions", deployplan.Skip, deployplan.Create, false, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + stderr := &bytes.Buffer{} + ctx := logdiag.InitContext(t.Context()) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, strings.NewReader(""), io.Discard, stderr, "", "")) + plan := deployplan.NewPlanDirect() + plan.Plan = map[string]*deployplan.PlanEntry{ + "resources.jobs.foo": {Action: tc.action}, + "resources.jobs.foo.permissions": {Action: tc.child}, + } + assert.Equal(t, tc.want, confirmBindPlan(ctx, "resources.jobs.foo", plan, tc.autoApprove, tc.immediate)) + assert.Equal(t, !tc.want, logdiag.HasError(ctx)) + if !tc.want { + assert.Contains(t, stderr.String(), "requires user confirmation") + } + }) + } +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 13c5121f569..8194afa535b 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -321,17 +321,10 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { - firstDeploy := b.DeploymentBundle.StateDB.DeploymentID == "" createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { return } - if firstDeploy { - if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { - logdiag.LogError(ctx, err) - return - } - } // Only create a version when the plan has at least one operation to record, so it moves // with serial. Done after the prompt, so a declined deploy never claims a number. diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index abd89f03f2f..8ce28cc3967 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -51,6 +51,10 @@ func actionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType switch a { case deployplan.Create: return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil + case deployplan.Bind: + return bundledeployments.OperationActionTypeOperationActionTypeBind, nil + case deployplan.BindAndUpdate: + return bundledeployments.OperationActionTypeOperationActionTypeBindAndUpdate, nil case deployplan.Update: return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil case deployplan.UpdateWithID: @@ -79,7 +83,8 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu return } deploymentID := db.StateDB.DeploymentID - if deploymentID == "" { + firstDeploy := deploymentID == "" + if firstDeploy { dep := metadata.Deployment() dep.InitialParentPath = b.Config.Workspace.StatePath created, err := w.BundleDeployments.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{Deployment: dep}) @@ -110,6 +115,11 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { b.Config.Bundle.Deployment.DeploymentID = deploymentID }) + if firstDeploy { + if err := db.StampDeploymentIdForFirstVersion(deploymentID); err != nil { + logdiag.LogError(ctx, err) + } + } } // startVersion claims the version the run settled on and opens the buffer that records diff --git a/bundle/phases/dms_test.go b/bundle/phases/dms_test.go index 6b045f863ae..9b3332c8cf5 100644 --- a/bundle/phases/dms_test.go +++ b/bundle/phases/dms_test.go @@ -17,6 +17,8 @@ func TestStagedOperationsCoversEveryTouchedResource(t *testing.T) { "resources.pipelines.bar": {Action: deployplan.Recreate}, "resources.schemas.baz": {Action: deployplan.Delete}, "resources.clusters.small": {Action: deployplan.Resize}, + "resources.jobs.bound": {Action: deployplan.Bind}, + "resources.jobs.updated": {Action: deployplan.BindAndUpdate}, }} staged, err := stagedOperations(plan) @@ -27,6 +29,8 @@ func TestStagedOperationsCoversEveryTouchedResource(t *testing.T) { {ResourceKey: "pipelines.bar", ActionType: bundledeployments.OperationActionTypeOperationActionTypeRecreate}, {ResourceKey: "schemas.baz", ActionType: bundledeployments.OperationActionTypeOperationActionTypeDelete}, {ResourceKey: "clusters.small", ActionType: bundledeployments.OperationActionTypeOperationActionTypeResize}, + {ResourceKey: "jobs.bound", ActionType: bundledeployments.OperationActionTypeOperationActionTypeBind}, + {ResourceKey: "jobs.updated", ActionType: bundledeployments.OperationActionTypeOperationActionTypeBindAndUpdate}, }, staged) } @@ -59,6 +63,8 @@ func TestActionToSDK(t *testing.T) { want bundledeployments.OperationActionType }{ {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {deployplan.Bind, bundledeployments.OperationActionTypeOperationActionTypeBind}, + {deployplan.BindAndUpdate, bundledeployments.OperationActionTypeOperationActionTypeBindAndUpdate}, {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, diff --git a/cmd/bundle/deployment/bind.go b/cmd/bundle/deployment/bind.go index 4b72cdd9e60..3d8c89fabe7 100644 --- a/cmd/bundle/deployment/bind.go +++ b/cmd/bundle/deployment/bind.go @@ -14,6 +14,7 @@ func newBindCommand() *cobra.Command { This command binds a workspace resource to a corresponding bundle resource definition. After binding, the workspace resource will be updated based on the bundle configuration on the next deployment. +For targets that record deployment history, binding applies these changes immediately. Arguments: KEY - The resource key defined in your bundle configuration @@ -50,12 +51,16 @@ Any manual changes made in the workspace UI may be overwritten on deployment.`, cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.RunE = func(cmd *cobra.Command, args []string) error { - err := BindResource(cmd, args[0], args[1], autoApprove, forceLock, false) + immediate, err := bindResource(cmd, args[0], args[1], autoApprove, forceLock, false) if err != nil { return err } - cmdio.LogString(cmd.Context(), "Run 'bundle deploy' to deploy changes to your workspace") + if immediate { + cmdio.LogString(cmd.Context(), "Bind changes have been applied to your workspace") + } else { + cmdio.LogString(cmd.Context(), "Run 'bundle deploy' to deploy changes to your workspace") + } return nil } diff --git a/cmd/bundle/deployment/bind_resource.go b/cmd/bundle/deployment/bind_resource.go index ef38c972362..0630633d219 100644 --- a/cmd/bundle/deployment/bind_resource.go +++ b/cmd/bundle/deployment/bind_resource.go @@ -15,6 +15,12 @@ import ( // BindResource binds a bundle resource to an existing workspace resource. // This function is shared between the bind command and generate commands with --bind flag. func BindResource(cmd *cobra.Command, resourceKey, resourceId string, autoApprove, forceLock, skipInitContext bool) error { + _, err := bindResource(cmd, resourceKey, resourceId, autoApprove, forceLock, skipInitContext) + return err +} + +// bindResource reports whether the bind applied its changes immediately. +func bindResource(cmd *cobra.Command, resourceKey, resourceId string, autoApprove, forceLock, skipInitContext bool) (bool, error) { b, stateDesc, err := utils.ProcessBundleRet(cmd, utils.ProcessOptions{ SkipInitContext: skipInitContext, AlwaysPull: true, @@ -23,23 +29,35 @@ func BindResource(cmd *cobra.Command, resourceKey, resourceId string, autoApprov }, }) if err != nil { - return err + return false, err } ctx := cmd.Context() resource, err := b.Config.Resources.FindResourceByConfigKey(resourceKey) if err != nil { - return err + return false, err } w := b.WorkspaceClient(ctx) exists, err := resource.Exists(ctx, w, resourceId) if err != nil { - return fmt.Errorf("failed to fetch the resource, err: %w", err) + return false, fmt.Errorf("failed to fetch the resource, err: %w", err) } if !exists { - return fmt.Errorf("%s with an id '%s' is not found", resource.ResourceDescription().SingularName, resourceId) + return false, fmt.Errorf("%s with an id '%s' is not found", resource.ResourceDescription().SingularName, resourceId) + } + + // A DMS bind plans against what the service records, so it reads that state first. + if stateDesc.IsDMS() { + if err := utils.OpenDirectStateForRead(ctx, b, stateDesc); err != nil { + return false, err + } + defer func() { + if _, err := b.DeploymentBundle.StateDB.Finalize(ctx); err != nil { + logdiag.LogError(ctx, err) + } + }() } phases.Bind(ctx, b, &phases.BindOptions{ @@ -49,9 +67,9 @@ func BindResource(cmd *cobra.Command, resourceKey, resourceId string, autoApprov ResourceId: resourceId, }, stateDesc) if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + return false, root.ErrAlreadyPrinted } cmdio.LogString(ctx, fmt.Sprintf("Successfully bound %s with an id '%s'", resource.ResourceDescription().SingularName, resourceId)) - return nil + return stateDesc.IsDMS(), nil }